diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 8c8c25440..3df42d8c1 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -153,6 +153,38 @@ from basic_memory.deps import * New code should import from specific submodules (`basic_memory.deps.services`) for clarity. +## Runtime Package Boundaries + +Accepted-note orchestration is shared by local and hosted runtimes. Its dependencies flow in one +direction: + +``` +schemas and runtime values + ↓ +repositories + ↓ +portable services and indexing workflows + ↓ +local or hosted composition roots and adapters +``` + +- `repository/` owns explicit-session persistence operations and persisted row values, including + accepted-note search rows and vector cleanup. It must not import `indexing/` workflows. +- `services/` owns runtime-neutral note preparation, note-content reads and writes, and delete + operations. These modules receive storage and repository capabilities explicitly. +- `indexing/` owns portable mutation, reconciliation, materialization, and project-index workflows. +- `index/` contains the local runtime's concrete adapters and composition helpers. Shared contracts + such as project-index requests and scheduler capabilities live in neutral modules within this + package; hosted code must not depend on `Local*` implementations. +- `cloud/` contains compatibility exports for downstream releases. Core production modules must + not import from it; new downstream code imports the neutral owner directly. + +Accepted Markdown create, replace, and edit operations cross one public persistence boundary: +`persist_accepted_note_snapshot`. That operation writes the entity snapshot, `NoteContent`, graph +rows, and entity search row inside the caller's transaction. Move intentionally uses the narrower +`persist_accepted_note_move` operation because changing a path must not replace observations or +relations. + ## MCP Tools Architecture ### Typed API Clients diff --git a/src/basic_memory/api/app.py b/src/basic_memory/api/app.py index 32a338f7e..0c9d7a91d 100644 --- a/src/basic_memory/api/app.py +++ b/src/basic_memory/api/app.py @@ -27,7 +27,7 @@ synchronize_projects, ) import logfire -from basic_memory.cloud.note_content_materialization import drain_pending_materializations +from basic_memory.index.note_content_materialization import drain_pending_materializations from basic_memory.config import init_api_logging from basic_memory.index.local_schedulers import drain_background_tasks from basic_memory.services.exceptions import EntityAlreadyExistsError diff --git a/src/basic_memory/api/v2/routers/knowledge_router.py b/src/basic_memory/api/v2/routers/knowledge_router.py index ec42eeeab..c70820e7b 100644 --- a/src/basic_memory/api/v2/routers/knowledge_router.py +++ b/src/basic_memory/api/v2/routers/knowledge_router.py @@ -21,7 +21,8 @@ import logfire from basic_memory import db -from basic_memory.cloud import DirectoryDeleteServiceError, NoteContentMutationServiceError +from basic_memory.services.directory_deletes import DirectoryDeleteServiceError +from basic_memory.services.note_content_writes import NoteContentMutationServiceError from basic_memory.ignore_utils import ( IGNORED_PATH_REJECTION_DETAIL, load_gitignore_patterns, diff --git a/src/basic_memory/cli/commands/command_utils.py b/src/basic_memory/cli/commands/command_utils.py index 7a07968a4..0360be1fe 100644 --- a/src/basic_memory/cli/commands/command_utils.py +++ b/src/basic_memory/cli/commands/command_utils.py @@ -32,7 +32,7 @@ def run_with_cleanup(coro: Coroutine[Any, Any, T]) -> T: # Deferred: basic_memory.db pulls SQLAlchemy + Alembic, which must not load # at CLI import time — only when a command actually runs (#886). from basic_memory import db - from basic_memory.cloud.note_content_materialization import drain_pending_materializations + from basic_memory.index.note_content_materialization import drain_pending_materializations from basic_memory.index.local_schedulers import drain_background_tasks async def _with_cleanup() -> T: diff --git a/src/basic_memory/cloud/directory_deletes.py b/src/basic_memory/cloud/directory_deletes.py index e1b6b1bf7..31866a3e2 100644 --- a/src/basic_memory/cloud/directory_deletes.py +++ b/src/basic_memory/cloud/directory_deletes.py @@ -1,118 +1,3 @@ -"""Shared directory-delete service facade. +"""Compatibility shim; import from ``basic_memory.services.directory_deletes``.""" -Runtime-specific callers provide the session boundary and the file cleanup -enqueuer. The core service owns request acceptance and response shaping. -""" - -from __future__ import annotations - -from contextlib import AbstractAsyncContextManager -from typing import Protocol - -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker - -from basic_memory import db -from basic_memory.indexing.directory_delete_runner import ( - DirectoryDeleteAcceptanceRequest, - DirectoryDeleteAcceptedResult, - DirectoryDeleteRejected, - DirectoryDeleteRejection, - DirectoryDeleteRuntime, - accept_directory_delete, - finish_directory_delete_acceptance, - normalize_directory_delete_path, -) - - -class DirectoryDeleteSessionMaker(Protocol): - """Session factory capability needed by directory-delete acceptance.""" - - def __call__(self) -> AbstractAsyncContextManager[AsyncSession]: ... - - -class DirectoryDeleteServiceError(Exception): - """Structured directory-delete service error for route adapters.""" - - def __init__(self, status_code: int, detail: str) -> None: - super().__init__(detail) - self.status_code = status_code - self.detail = detail - - -def directory_delete_service_error_from_rejection( - rejection: DirectoryDeleteRejection, -) -> DirectoryDeleteServiceError: - """Map core directory-delete rejections into route-facing errors.""" - return DirectoryDeleteServiceError( - rejection.kind.http_status_code, - rejection.detail, - ) - - -class DirectoryDeleteService: - """Accept directory deletes into project DB state before storage cleanup begins.""" - - def __init__( - self, - *, - session_maker: async_sessionmaker[AsyncSession], - runtime: DirectoryDeleteRuntime, - ) -> None: - self.session_maker = session_maker - self.runtime = runtime - - async def delete_directory( - self, - *, - project_external_id: str, - directory: str, - ) -> DirectoryDeleteAcceptedResult: - """Delete directory entities immediately and queue file cleanup in the background. - - The typed result carries the route status (``http_status_code``) and the - existing response contract (``to_response_payload``). - """ - request = DirectoryDeleteAcceptanceRequest( - project_external_id=project_external_id, - directory=directory, - ) - try: - # scoped_session enables `PRAGMA foreign_keys=ON` for SQLite; this bulk - # delete issues a Core DELETE on entity and relies on ON DELETE CASCADE - # for note_content/observations/relations, which a raw session_maker() - # connection (foreign_keys OFF by default) would leave orphaned. - async with db.scoped_session(self.session_maker) as session: - accepted = await accept_directory_delete( - session, - request=request, - store=self.runtime.store, - ) - except DirectoryDeleteRejected as error: - raise directory_delete_service_error_from_rejection(error.rejection) from error - - result = await finish_directory_delete_acceptance( - request=request, - accepted=accepted, - enqueuer=self.runtime.file_delete_enqueuer, - ) - - # Trigger: notes outside the deleted directory linked into it. - # Why: the delete cascaded their relation rows away, but those sources own - # matching search_index relation rows that now dangle; without a reindex - # they linger until an unrelated rebuild. - # Outcome: reindex each surviving source inline when the runtime provides a - # refresher (local); queued runtimes consume the ids from the result. - if accepted.relation_cleanup_entity_ids and self.runtime.relation_cleanup_refresher: - await self.runtime.relation_cleanup_refresher.refresh_relation_sources( - sorted(accepted.relation_cleanup_entity_ids) - ) - - return result - - @staticmethod - def normalize_directory_path(directory: str) -> str: - """Normalize a project-relative directory path or reject traversal.""" - try: - return normalize_directory_delete_path(directory) - except ValueError as exc: - raise DirectoryDeleteServiceError(400, "Invalid directory path") from exc +from basic_memory.services.directory_deletes import * # noqa: F403 diff --git a/src/basic_memory/cloud/note_content_materialization.py b/src/basic_memory/cloud/note_content_materialization.py index 26ff2dbf5..fd04698cd 100644 --- a/src/basic_memory/cloud/note_content_materialization.py +++ b/src/basic_memory/cloud/note_content_materialization.py @@ -1,560 +1,3 @@ -"""Local note-content materialization adapters.""" +"""Compatibility shim; import from ``basic_memory.index.note_content_materialization``.""" -from __future__ import annotations - -import asyncio -from collections.abc import Coroutine, Mapping -from contextlib import suppress -from dataclasses import dataclass, replace -from typing import Any, Protocol - -from loguru import logger -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker - -from basic_memory import db, file_utils -from basic_memory.indexing.index_file_runner import IndexFileExecutor -from basic_memory.indexing.note_file_delete_runner import run_note_file_delete -from basic_memory.indexing.note_materialization_runner import ( - ContentStoreNoteMaterializationFileWriter, - RepositoryNoteMaterializationPreflight, - RepositoryNoteMaterializationPublisher, - RepositoryNoteMaterializationStatusPublisher, - run_note_materialization, -) -from basic_memory.runtime.cleanup import ( - RuntimeNoteFileDeleteJobRequest, - plan_note_file_delete_job_request, -) -from basic_memory.runtime.note_content import ( - NOTE_CONTENT_EXTERNAL_CHANGE_SYNC_ERROR, - RuntimeAcceptedNoteChange, - RuntimeAcceptedNoteResponse, - RuntimeNoteContentResponsePayload, - RuntimeNoteMaterializationJobRequest, - RuntimeNoteMaterializationResult, - RuntimeNoteMaterializationStatus, - RuntimePendingNoteMaterialization, - plan_accepted_note_response, - plan_note_materialization_job_request, -) -from basic_memory.runtime.note_materialization import RuntimeFileMetadataSource -from basic_memory.runtime.storage import RuntimeFileChecksum, RuntimeFilePath -from basic_memory.models import Entity -from basic_memory.repository import EntityRepository, NoteContentRepository -from basic_memory.schemas.response import ObservationResponse, RelationResponse -from basic_memory.services.file_service import FileService - - -# The (project_id, entity_id) identity that pins all of one note's queued -# materializations to a single worker. -type _NoteRoutingKey = tuple[int, int] - - -class _MaterializationWorkerPool: - """Bounded in-process worker pool that drains queued note materializations. - - Mirrors the cloud's queue worker model locally: the accept enqueues a - materialization and returns; a fixed number of workers pull from per-worker - queues and run them. Bounding concurrency to `workers` is the point — - fire-and-forget `create_task` let every deferred file write + index run at - once, and at high write load they contended en masse for the single SQLite - writer and the event loop, collapsing the tail (p99) and throughput - (benchmarks/docs/write-load-benchmark.md). With N workers only N - materializations are in flight; the rest wait in the queues and drain over - time, so the accept path stays light AND the writer isn't thrashed. - - Jobs are routed to a worker by their note identity (project_id, entity_id), - so all jobs for one note run on the same worker FIFO in submission order. - Materializations for the same note must never run concurrently: the older - job's file write changes the on-disk checksum, so the newer job's writer - guard reads unexpected content and publishes a false - external_change_detected on the LATEST accepted row — the note is never - materialized and is falsely flagged as conflicted. - """ - - def __init__(self) -> None: - self._queues: list[asyncio.Queue[Coroutine[Any, Any, object]]] = [] - self._workers: list[asyncio.Task[None]] = [] - self._loop: asyncio.AbstractEventLoop | None = None - - def submit( - self, - work: Coroutine[Any, Any, object], - *, - workers: int, - key: _NoteRoutingKey, - ) -> None: - self._ensure_workers(workers) - self._queues[self._worker_index(key)].put_nowait(work) - - def _worker_index(self, key: _NoteRoutingKey) -> int: - # hash() of an int tuple is stable within one process, which is all - # routing needs: a job only has to serialize against other jobs queued - # in the same process lifetime. - return hash(key) % len(self._queues) - - def _ensure_workers(self, workers: int) -> None: - # Trigger: first submit, or submit on a different event loop than the one - # the workers were bound to (e.g. a fresh per-test loop). - # Why: workers are long-lived tasks bound to one loop; reusing queues - # whose workers live on a dead loop would hang. Outcome: (re)create one - # queue + worker task pair per worker on the current running loop. - # Orphaned workers on a closed loop are already dead, so dropping them - # is safe. - loop = asyncio.get_running_loop() - if self._queues and self._loop is loop: - return - self._loop = loop - self._queues = [asyncio.Queue() for _ in range(max(1, workers))] - self._workers = [asyncio.create_task(self._run(queue)) for queue in self._queues] - - async def _run(self, queue: asyncio.Queue[Coroutine[Any, Any, object]]) -> None: - while True: - work = await queue.get() - try: - await work - except Exception: # pragma: no cover - defensive worker guard - logger.exception("Local note materialization failed") - finally: - queue.task_done() - - async def join(self) -> None: - """Block until every queued materialization has completed.""" - for queue in self._queues: - await queue.join() - - async def aclose(self) -> None: - """Cancel workers and reset the pool (clean test teardown / shutdown).""" - workers = self._workers - self._workers = [] - self._queues = [] - self._loop = None - for worker in workers: - worker.cancel() - for worker in workers: - with suppress(asyncio.CancelledError): - await worker - - -_materialization_pool = _MaterializationWorkerPool() - - -async def drain_pending_materializations() -> None: - """Block until queued local materializations finish writing + indexing. - - One-shot clients (``bm tool write-note``, importers) return right after the - accept enqueues the markdown write/index; without this drain the event loop can - close before the worker writes the source-of-truth file, silently losing the - write even though the API already reported it accepted. Long-lived servers keep - the loop alive and don't need it. - """ - await _materialization_pool.join() - - -# --- Startup Recovery --- -# accept_write marks note_content "pending", then the materialization preflight -# flips it to "writing" before the file is written and the publisher records -# "synced". If the process dies anywhere between those points the row is stuck -# forever: the crash may land before the file write (nothing on disk) or after it -# but before publish (the correct accepted file is already on disk, row still -# "writing"). A transient write error (ENOSPC, permissions) publishes "failed" -# instead — equally terminal, since nothing else ever retries it. On the next -# startup we re-drive every stuck row. The write path short-circuits when the -# accepted content is already on disk, so the crash-after-write case publishes to -# "synced" instead of tripping the external-change guard. The db_version -# compare-and-set guard in the preflight and publisher makes recovery -# unconditionally safe: an older recovery attempt can never overwrite a newer -# accepted write or its file. - -# Synthetic provenance stamped on recovered writes so operators can tell a -# crash-recovery materialization apart from a normal accept-path write in logs -# and object metadata. -RECOVERY_NOTE_CHANGE_SOURCE = "note-content-materialization-recovery" -RECOVERY_NOTE_ACTOR_NAME = "startup-recovery" - - -async def run_recovery_materialization( - request: RuntimeNoteMaterializationJobRequest, - *, - session_maker: async_sessionmaker[AsyncSession], - file_service: FileService, -) -> RuntimeNoteMaterializationResult: - """Re-drive one stuck materialization through the standard guarded write path. - - Uses the same preflight/writer/publisher/status-publisher as an accept-path - write, so the db_version and file-conflict guards apply unchanged. No cleanup - paths: a recovery request carries no old-file move, so nothing is deleted. - """ - storage = LocalNoteContentStorage(file_service) - return await run_note_materialization( - request, - preflight=RepositoryNoteMaterializationPreflight(session_maker=session_maker), - writer=ContentStoreNoteMaterializationFileWriter(storage), - publisher=RepositoryNoteMaterializationPublisher(session_maker=session_maker), - status_publisher=RepositoryNoteMaterializationStatusPublisher(session_maker=session_maker), - cleanup_enqueuer=InlineNoteFileDeleteEnqueuer(storage), - ) - - -async def recover_stuck_materializations( - *, - session_maker: async_sessionmaker[AsyncSession], - file_service: FileService, - project_id: int, -) -> int: - """Re-drive every note materialization stuck in writing/pending/failed for a project. - - Meant to run once per project at startup, before serving. Non-fatal per row: - a single row that raises is logged and skipped so one poisoned note cannot - block startup recovery for the rest of the project. Returns the number of rows - that reached a written file state. - """ - async with db.scoped_session(session_maker) as session: - stuck_rows = await NoteContentRepository(project_id=project_id).find_stuck_materializations( - session - ) - - if not stuck_rows: - return 0 - - logger.info( - "Recovering stuck note materializations", - project_id=project_id, - stuck_count=len(stuck_rows), - ) - recovered = 0 - for row in stuck_rows: - # Rebuild the queue request from the row's own accepted db_version/db_checksum - # so the preflight guard matches the current accepted state; if a newer write - # has since advanced the row, the guard trips and this attempt no-ops. - request = RuntimeNoteMaterializationJobRequest( - project_id=project_id, - entity_id=row.entity_id, - db_version=int(row.db_version), - db_checksum=str(row.db_checksum), - actor_name=RECOVERY_NOTE_ACTOR_NAME, - source=RECOVERY_NOTE_CHANGE_SOURCE, - ) - try: - result = await run_recovery_materialization( - request, - session_maker=session_maker, - file_service=file_service, - ) - except Exception: - # Trigger: one row's materialization raised (storage/DB error). - # Why: recovery is best-effort startup cleanup; the version guard makes - # a later retry safe, so one bad row must not abort the whole sweep. - # Outcome: log and continue to the next stuck row. - logger.exception( - "Failed to recover stuck note materialization", - project_id=project_id, - entity_id=row.entity_id, - ) - continue - if result.status is RuntimeNoteMaterializationStatus.written: - recovered += 1 - return recovered - - -def note_content_payload_file_path( - payload: RuntimeNoteContentResponsePayload, -) -> RuntimeFilePath | None: - """Return the materialized file path carried by an accepted-note payload.""" - if isinstance(payload, RuntimeAcceptedNoteResponse): - return payload.file_path - if isinstance(payload, Mapping): - file_path = payload.get("file_path") - if isinstance(file_path, str) and file_path: - return file_path - return None - - -def file_write_status_from_materialization_result( - result: RuntimeNoteMaterializationResult, -) -> str: - """Return the response write marker for a terminal local materialization result.""" - if result.status is RuntimeNoteMaterializationStatus.conflict: - return "external_change_detected" - return "failed" - - -def note_content_payload_with_materialization_result( - payload: RuntimeNoteContentResponsePayload, - result: RuntimeNoteMaterializationResult, -) -> RuntimeNoteContentResponsePayload: - """Expose a failed local materialization result in the accepted-note response payload.""" - file_write_status = file_write_status_from_materialization_result(result) - - if isinstance(payload, RuntimeAcceptedNoteResponse): - return replace( - payload, - file_write_status=file_write_status, - file_checksum=result.file_checksum - if result.file_checksum is not None - else payload.file_checksum, - last_materialization_error=result.reason, - ) - - updated_payload = dict(payload) - updated_payload["file_write_status"] = file_write_status - updated_payload["last_materialization_error"] = result.reason - if result.file_checksum is not None: - updated_payload["file_checksum"] = result.file_checksum - if file_write_status == "external_change_detected": - updated_payload["sync_error"] = NOTE_CONTENT_EXTERNAL_CHANGE_SYNC_ERROR - return updated_payload - - -def indexed_observation_payloads(entity: Entity) -> tuple[dict[str, object], ...]: - """Serialize loaded observation rows into the v2 response shape.""" - return tuple( - ObservationResponse.model_validate(observation).model_dump(mode="json") - for observation in entity.observations - ) - - -def indexed_relation_payloads(entity: Entity) -> tuple[dict[str, object], ...]: - """Serialize loaded relation rows into the v2 response shape.""" - return tuple( - RelationResponse.model_validate(relation).model_dump(mode="json") - for relation in entity.relations - ) - - -async def load_indexed_note_content_response_payload( - *, - session_maker: async_sessionmaker[AsyncSession], - project_id: int, - entity_id: int, - fallback_source: str, -) -> RuntimeAcceptedNoteResponse: - """Reload the local indexed entity graph after inline materialization/indexing.""" - async with db.scoped_session(session_maker) as session: - entity = await EntityRepository(project_id=project_id).get_by_id( - session, - entity_id, - load_relations=True, - ) - if entity is None: - raise RuntimeError(f"Indexed entity {entity_id} was not found after materialization") - - note_content = await NoteContentRepository(project_id=project_id).get_by_entity_id( - session, - entity_id, - ) - if note_content is None: - raise RuntimeError( - f"Indexed note_content for entity {entity_id} was not found after materialization" - ) - - return replace( - plan_accepted_note_response( - entity=entity, - note_content=note_content, - fallback_source=fallback_source, - ), - observations=indexed_observation_payloads(entity), - relations=indexed_relation_payloads(entity), - ) - - -@dataclass(frozen=True, slots=True) -class LocalNoteContentStorage: - """Adapt the local FileService to note-content runtime storage protocols.""" - - file_service: FileService - - async def write_file( - self, - path: RuntimeFilePath, - content: str, - *, - metadata: dict[str, str] | None = None, - ) -> RuntimeFileChecksum: - _ = metadata - path_obj = self.file_service.base_path / path if isinstance(path, str) else path - full_path = path_obj if path_obj.is_absolute() else self.file_service.base_path / path_obj - - # Accepted-note materialization persists an already-accepted DB snapshot. - # Writing bytes keeps the materialized file checksum identical to the - # note_content checksum on Windows, where text mode would translate LF to CRLF. - await self.file_service.ensure_directory(full_path.parent) - await file_utils.write_file_atomic_bytes(full_path, content.encode("utf-8")) - return await self.file_service.compute_checksum(full_path) - - async def get_file_metadata(self, path: RuntimeFilePath) -> RuntimeFileMetadataSource: - return await self.file_service.get_file_metadata(path) - - async def exists(self, path: RuntimeFilePath) -> bool: - return await self.file_service.exists(path) - - async def compute_checksum(self, path: RuntimeFilePath) -> RuntimeFileChecksum: - return await self.file_service.compute_checksum(path) - - async def delete_file(self, path: RuntimeFilePath) -> None: - await self.file_service.delete_file(path) - - -@dataclass(frozen=True, slots=True) -class InlineNoteFileDeleteEnqueuer: - """Execute note-file cleanup immediately in the local runtime.""" - - storage: LocalNoteContentStorage - - async def enqueue_note_file_delete(self, request: RuntimeNoteFileDeleteJobRequest) -> None: - # Trigger: a move scheduled old-path cleanup whose old and new paths differ - # only by case (or otherwise alias the same inode) on a case-insensitive - # filesystem, so the old path now points at the just-written new file. - # Why: the checksum guard cannot tell "old file still present" from "old path - # aliases the new file" — both read the same bytes — so deleting the old path - # would destroy the note's only copy (then scan reconciliation removes the row). - # Outcome: skip the delete entirely; the paths are the same physical file. - if ( - request.live_file_path is not None - and self.storage.file_service.paths_share_storage_target( - request.file_path, request.live_file_path - ) - ): - logger.info( - "Skipping note-file cleanup that aliases the live file (case-only rename)", - entity_id=request.entity_id, - file_path=request.file_path, - live_file_path=request.live_file_path, - ) - return - await run_note_file_delete(request, storage=self.storage) - - -class RelationResolutionScheduling(Protocol): - """Capability to back-resolve forward references after a write is indexed.""" - - def schedule_relation_resolution(self, *, project_id: int) -> None: ... - - -@dataclass(frozen=True, slots=True) -class LocalNoteContentMaterializationProvider: - """Run accepted-note materialization inline for the local runtime.""" - - session_maker: async_sessionmaker[AsyncSession] - file_service: FileService - file_indexer: IndexFileExecutor | None = None - test_mode: bool = False - materialization_workers: int = 4 - relation_resolution_scheduler: RelationResolutionScheduling | None = None - - async def materialize_write_change( - self, - accepted: RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload], - ) -> RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload]: - """Materialize an accepted note write OFF the accept path. - - Cloud/local parity (DO NOT UNDO): cloud's materialize_write_change - enqueues a queue job and returns immediately, letting Tigris object storage - + indexing catch up asynchronously because S3 writes are slow. Locally we - mirror that with an in-process background task. The accept has already - persisted note_content (the write/read-through cache that serves reads); - here we only schedule writing the markdown file (the source of truth) and - indexing it. Writing + indexing the file is the heavy part of a write, so - doing it inline reintroduces a ~3x write-load regression - (benchmarks/docs/write-load-benchmark.md). - - PARITY INVARIANT: production must defer. Test mode runs inline ONLY so - tests can assert file/search state synchronously — never make the - production path synchronous to "simplify" this. - """ - materialization = accepted.materialization - if materialization is None: - return accepted - if self.test_mode: - return await self._materialize_write_now(accepted) - self._schedule_materialization(accepted, materialization) - return accepted - - def _schedule_materialization( - self, - accepted: RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload], - materialization: RuntimePendingNoteMaterialization, - ) -> None: - # Hand the materialization to the bounded worker pool instead of spawning - # an unbounded task per write — see _MaterializationWorkerPool for why. - # Keyed on the note's identity so two quick writes to the same note run - # sequentially on one worker instead of racing the writer guard into a - # false external_change_detected on the newer accepted row. - _materialization_pool.submit( - self._materialize_write_now(accepted), - workers=self.materialization_workers, - key=(materialization.project_id, materialization.entity_id), - ) - - async def _materialize_write_now( - self, - accepted: RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload], - ) -> RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload]: - if accepted.materialization is None: # pragma: no cover - guarded by caller - return accepted - storage = LocalNoteContentStorage(self.file_service) - cleanup_enqueuer = InlineNoteFileDeleteEnqueuer(storage) - result = await run_note_materialization( - plan_note_materialization_job_request(accepted.materialization), - preflight=RepositoryNoteMaterializationPreflight( - session_maker=self.session_maker, - ), - writer=ContentStoreNoteMaterializationFileWriter(storage), - publisher=RepositoryNoteMaterializationPublisher( - session_maker=self.session_maker, - ), - status_publisher=RepositoryNoteMaterializationStatusPublisher( - session_maker=self.session_maker, - ), - cleanup_enqueuer=cleanup_enqueuer, - ) - if result.status is not RuntimeNoteMaterializationStatus.written: - return replace( - accepted, - payload=note_content_payload_with_materialization_result( - accepted.payload, - result, - ), - ) - - file_path = note_content_payload_file_path(accepted.payload) - if file_path is not None and self.file_indexer is not None: - await self.file_indexer.index_file( - file_path, - source="note-content-materialization", - ) - # The deferred index has now inserted this note's entity/relation rows, - # so back-resolve inbound forward references. The router schedules an - # eager pass right after enqueue, but under load that pass can scan - # before this index lands; scheduling here (coalesced/re-armed by the - # resolution scheduler) guarantees a pass runs after indexing (#1002). - if self.relation_resolution_scheduler is not None: - self.relation_resolution_scheduler.schedule_relation_resolution( - project_id=accepted.materialization.project_id, - ) - return replace( - accepted, - payload=await load_indexed_note_content_response_payload( - session_maker=self.session_maker, - project_id=accepted.materialization.project_id, - entity_id=accepted.materialization.entity_id, - fallback_source=accepted.materialization.source - or "note-content-materialization", - ), - ) - return accepted - - async def materialize_delete_change( - self, - accepted: RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload], - ) -> RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload]: - """Delete materialized files immediately after local accepted-note deletes.""" - if accepted.file_delete is None: - return accepted - - storage = LocalNoteContentStorage(self.file_service) - await InlineNoteFileDeleteEnqueuer(storage).enqueue_note_file_delete( - plan_note_file_delete_job_request(accepted.file_delete) - ) - return accepted +from basic_memory.index.note_content_materialization import * # noqa: F403 diff --git a/src/basic_memory/cloud/note_content_reads.py b/src/basic_memory/cloud/note_content_reads.py index 0b562dbd6..92383856e 100644 --- a/src/basic_memory/cloud/note_content_reads.py +++ b/src/basic_memory/cloud/note_content_reads.py @@ -1,202 +1,3 @@ -"""Shared note-content read service facade.""" +"""Compatibility shim; import from ``basic_memory.services.note_content_reads``.""" -from __future__ import annotations - -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker - -from basic_memory import db -from basic_memory.indexing.note_content_read_repair_runner import ( - NoteContentReadRepairFileReader, - NoteContentReadView, - load_note_content_read_view_with_default_repositories, - note_content_resource_from_read_view, - note_content_response_payload_from_read_view, - prepare_note_content_read_repair_with_default_repositories, - run_note_content_read_repair_with_default_reconciler, -) -from basic_memory.models import Entity, NoteContent, Project -from basic_memory.runtime.note_content import ( - RuntimeNoteContentResource, - RuntimeNoteContentResponsePayload, -) - - -async def load_note_content_query_view( - *, - session_maker: async_sessionmaker[AsyncSession], - project_external_id: str, - entity_external_id: str, -) -> NoteContentReadView[Entity, NoteContent] | None: - """Load one project-scoped note view from current DB state.""" - async with db.scoped_session(session_maker) as session: - return await load_note_content_query_view_from_session( - session=session, - project_external_id=project_external_id, - entity_external_id=entity_external_id, - ) - - -async def load_note_content_query_view_from_session( - *, - session: AsyncSession, - project_external_id: str, - entity_external_id: str, -) -> NoteContentReadView[Entity, NoteContent] | None: - """Load one project-scoped note view using caller-owned session scope.""" - return await load_note_content_read_view_with_default_repositories( - session, - project_external_id=project_external_id, - entity_external_id=entity_external_id, - ) - - -class NoteContentQueryService: - """Load note-content rows and shape route-friendly read payloads.""" - - def __init__( - self, - *, - session_maker: async_sessionmaker[AsyncSession], - read_repair_file_reader: NoteContentReadRepairFileReader[Project, Entity] | None = None, - ) -> None: - self.session_maker = session_maker - self.read_repair_file_reader = read_repair_file_reader - - async def get_note_entity_payload( - self, - *, - project_external_id: str, - entity_external_id: str, - session: AsyncSession | None = None, - ) -> RuntimeNoteContentResponsePayload | None: - """Return the entity payload, enriching markdown notes from note_content.""" - if session is None: - note_view = await load_note_content_query_view( - session_maker=self.session_maker, - project_external_id=project_external_id, - entity_external_id=entity_external_id, - ) - else: - note_view = await load_note_content_query_view_from_session( - session=session, - project_external_id=project_external_id, - entity_external_id=entity_external_id, - ) - return note_content_response_payload_from_read_view(note_view) - - async def get_note_entity_payload_with_read_repair( - self, - *, - project_external_id: str, - entity_external_id: str, - session: AsyncSession | None = None, - source: str = "read_repair", - ) -> RuntimeNoteContentResponsePayload | None: - """Return entity payload, repairing missing note_content when a reader exists.""" - payload = await self.get_note_entity_payload( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - session=session, - ) - if payload is not None or self.read_repair_file_reader is None: - return payload - - repaired = await self.reconcile_note_content_from_file( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - source=source, - ) - if not repaired: - return None - # The repair commits through a separate scoped session, so reopen the read to - # avoid stale snapshots in caller-owned transactions. - return await self.get_note_entity_payload( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - ) - - async def get_note_resource( - self, - *, - project_external_id: str, - entity_external_id: str, - session: AsyncSession | None = None, - ) -> RuntimeNoteContentResource | None: - """Return full markdown content from note_content when available.""" - if session is None: - note_view = await load_note_content_query_view( - session_maker=self.session_maker, - project_external_id=project_external_id, - entity_external_id=entity_external_id, - ) - else: - note_view = await load_note_content_query_view_from_session( - session=session, - project_external_id=project_external_id, - entity_external_id=entity_external_id, - ) - if note_view is None: - return None - - return note_content_resource_from_read_view(note_view) - - async def get_note_resource_with_read_repair( - self, - *, - project_external_id: str, - entity_external_id: str, - session: AsyncSession | None = None, - source: str = "read_repair", - ) -> RuntimeNoteContentResource | None: - """Return markdown resource, repairing missing note_content when possible.""" - resource = await self.get_note_resource( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - session=session, - ) - if resource is not None or self.read_repair_file_reader is None: - return resource - - repaired = await self.reconcile_note_content_from_file( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - source=source, - ) - if not repaired: - return None - # The repair commits through a separate scoped session, so reopen the read to - # avoid stale snapshots in caller-owned transactions. - return await self.get_note_resource( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - ) - - async def reconcile_note_content_from_file( - self, - *, - project_external_id: str, - entity_external_id: str, - source: str, - ) -> bool: - """Repair a missing note_content row from the runtime's canonical file source.""" - async with db.scoped_session(self.session_maker) as session: - repair_preflight = await prepare_note_content_read_repair_with_default_repositories( - session, - project_external_id=project_external_id, - entity_external_id=entity_external_id, - ) - if not repair_preflight.should_read_file: - return repair_preflight.repaired - - repair_preflight.require_target() - - if self.read_repair_file_reader is None: - raise RuntimeError("note-content read repair requires a file reader") - - repair_run = await run_note_content_read_repair_with_default_reconciler( - repair_preflight, - session_maker=self.session_maker, - file_reader=self.read_repair_file_reader, - source=source, - ) - return repair_run.repaired +from basic_memory.services.note_content_reads import * # noqa: F403 diff --git a/src/basic_memory/cloud/note_content_writes.py b/src/basic_memory/cloud/note_content_writes.py index 85ec75971..fdb9ebe15 100644 --- a/src/basic_memory/cloud/note_content_writes.py +++ b/src/basic_memory/cloud/note_content_writes.py @@ -1,381 +1,3 @@ -"""Shared note-content mutation service facade.""" +"""Compatibility shim; import from ``basic_memory.services.note_content_writes``.""" -from __future__ import annotations - -from collections.abc import AsyncIterator -from contextlib import asynccontextmanager -from dataclasses import dataclass -from typing import Literal, Protocol -from uuid import UUID - -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker - -from basic_memory.indexing.accepted_note_mutation_runner import ( - AcceptedNoteCreateMutation, - AcceptedNoteDeleteMutation, - AcceptedNoteEditMutation, - AcceptedNoteMoveMutation, - AcceptedNoteMutationActor, - AcceptedNoteMutationDependencies, - AcceptedNoteMutationRejected, - AcceptedNoteMutationRejection, - AcceptedNoteUpdateMutation, - run_accepted_note_create, - run_accepted_note_delete, - run_accepted_note_edit, - run_accepted_note_move, - run_accepted_note_update, -) -from basic_memory.runtime.note_content import ( - RuntimeAcceptedNoteChange, - RuntimeNoteContentResponsePayload, -) -from basic_memory.schemas.base import Entity as EntitySchema -from basic_memory.schemas.request import EditEntityRequest - -AcceptedNoteChange = RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload] - - -class NoteContentMutationFreshener(Protocol): - """Refresh current runtime file state before mutating an existing note.""" - - async def freshen_note_content( - self, - *, - project_external_id: str, - entity_external_id: str, - ) -> None: ... - - -type NoteContentMutationKind = Literal["create", "update", "edit", "move"] - - -@dataclass(frozen=True, slots=True) -class NoteContentMutationActorContext: - """Who and what originated one accepted note mutation.""" - - user_profile_id: UUID | None - source: str - actor_kind: str | None = None - actor_name: str | None = None - - -class NoteContentMutationActorResolver(Protocol): - """Resolve the actor context for one mutation at the runtime boundary. - - Routes pass through whatever actor values they were called with; a runtime - adapter (e.g. cloud) can replace them with request-derived identity — user - profile, source header, MCP actor headers — without subclassing the service. - """ - - def resolve_mutation_actor( - self, - *, - mutation_kind: NoteContentMutationKind, - requested: NoteContentMutationActorContext, - ) -> NoteContentMutationActorContext: ... - - -# Route adapters place error.detail directly into the HTTP response body, so it -# is either a plain message string or an already-serialized structured detail -# (currently only the base-checksum conflict wire dict, issue #1445). -type NoteContentMutationErrorDetail = str | dict[str, str | None] - - -class NoteContentMutationServiceError(Exception): - """Structured note-content mutation service error for route adapters.""" - - def __init__(self, status_code: int, detail: NoteContentMutationErrorDetail) -> None: - super().__init__(str(detail)) - self.status_code = status_code - self.detail = detail - - -def note_content_mutation_error_from_rejection( - rejection: AcceptedNoteMutationRejection, -) -> NoteContentMutationServiceError: - """Map core accepted-note mutation rejections into route-facing errors.""" - detail = rejection.detail - # This mapping is the wire boundary: typed rejection details serialize to the - # JSON dict that HTTP routes place verbatim into the 4xx response body. - return NoteContentMutationServiceError( - rejection.kind.http_status_code, - detail if isinstance(detail, str) else detail.as_json_dict(), - ) - - -def accepted_note_mutation_actor( - *, - user_profile_id: UUID | None, - actor_kind: str | None, - actor_name: str | None, -) -> AcceptedNoteMutationActor: - """Build the typed accepted-note actor passed to core mutation runners.""" - return AcceptedNoteMutationActor( - user_profile_id=user_profile_id, - kind=actor_kind, - name=actor_name, - ) - - -@asynccontextmanager -async def accepted_note_transaction( - session_maker: async_sessionmaker[AsyncSession], -) -> AsyncIterator[AsyncSession]: - """Open one DB transaction for an accepted note mutation.""" - async with session_maker() as session: - async with session.begin(): - yield session - - -class NoteContentMutationService: - """Accept note mutations into DB state through core-owned mutation runners.""" - - def __init__( - self, - *, - session_maker: async_sessionmaker[AsyncSession], - mutation_dependencies: AcceptedNoteMutationDependencies, - content_freshener: NoteContentMutationFreshener | None = None, - actor_resolver: NoteContentMutationActorResolver | None = None, - ) -> None: - self.session_maker = session_maker - self.mutation_dependencies = mutation_dependencies - self.content_freshener = content_freshener - self.actor_resolver = actor_resolver - - def _resolve_actor( - self, - mutation_kind: NoteContentMutationKind, - *, - user_profile_id: UUID | None, - source: str, - actor_kind: str | None, - actor_name: str | None, - ) -> NoteContentMutationActorContext: - requested = NoteContentMutationActorContext( - user_profile_id=user_profile_id, - source=source, - actor_kind=actor_kind, - actor_name=actor_name, - ) - if self.actor_resolver is None: - return requested - return self.actor_resolver.resolve_mutation_actor( - mutation_kind=mutation_kind, - requested=requested, - ) - - async def freshen_existing_note_content( - self, - *, - project_external_id: str, - entity_external_id: str, - ) -> None: - """Let the runtime converge observed file state before an existing-note mutation.""" - if self.content_freshener is None: - return - await self.content_freshener.freshen_note_content( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - ) - - async def create_note( - self, - *, - project_external_id: str, - data: EntitySchema, - user_profile_id: UUID | None, - source: str, - actor_kind: str | None = None, - actor_name: str | None = None, - ) -> AcceptedNoteChange: - """POST a new markdown note into accepted DB state.""" - actor_context = self._resolve_actor( - "create", - user_profile_id=user_profile_id, - source=source, - actor_kind=actor_kind, - actor_name=actor_name, - ) - try: - async with accepted_note_transaction(self.session_maker) as session: - return await run_accepted_note_create( - session, - request=AcceptedNoteCreateMutation( - project_external_id=project_external_id, - data=data, - actor=accepted_note_mutation_actor( - user_profile_id=actor_context.user_profile_id, - actor_kind=actor_context.actor_kind, - actor_name=actor_context.actor_name, - ), - source=actor_context.source, - ), - dependencies=self.mutation_dependencies, - ) - except AcceptedNoteMutationRejected as error: - raise note_content_mutation_error_from_rejection(error.rejection) from error - - async def update_note( - self, - *, - project_external_id: str, - entity_external_id: str, - data: EntitySchema, - user_profile_id: UUID | None, - source: str, - base_checksum: str | None = None, - actor_kind: str | None = None, - actor_name: str | None = None, - ) -> AcceptedNoteChange: - """PUT a markdown note by creating or replacing accepted DB state. - - ``base_checksum`` is an optional optimistic-concurrency precondition: the - db_checksum the caller last synced. When supplied, the update runner - rejects the write with a structured 409 if the accepted checksum has - moved, so the caller rebases instead of clobbering the newer write - (issue #1445). It stays optional so callers without a synced base still - write. - """ - actor_context = self._resolve_actor( - "update", - user_profile_id=user_profile_id, - source=source, - actor_kind=actor_kind, - actor_name=actor_name, - ) - try: - await self.freshen_existing_note_content( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - ) - async with accepted_note_transaction(self.session_maker) as session: - return await run_accepted_note_update( - session, - request=AcceptedNoteUpdateMutation( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - data=data, - actor=accepted_note_mutation_actor( - user_profile_id=actor_context.user_profile_id, - actor_kind=actor_context.actor_kind, - actor_name=actor_context.actor_name, - ), - source=actor_context.source, - base_checksum=base_checksum, - ), - dependencies=self.mutation_dependencies, - ) - except AcceptedNoteMutationRejected as error: - raise note_content_mutation_error_from_rejection(error.rejection) from error - - async def edit_note( - self, - *, - project_external_id: str, - entity_external_id: str, - data: EditEntityRequest, - user_profile_id: UUID | None, - source: str, - actor_kind: str | None = None, - actor_name: str | None = None, - ) -> AcceptedNoteChange: - """PATCH a markdown note using the latest accepted DB content as the base.""" - actor_context = self._resolve_actor( - "edit", - user_profile_id=user_profile_id, - source=source, - actor_kind=actor_kind, - actor_name=actor_name, - ) - try: - await self.freshen_existing_note_content( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - ) - async with accepted_note_transaction(self.session_maker) as session: - return await run_accepted_note_edit( - session, - request=AcceptedNoteEditMutation( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - data=data, - actor=accepted_note_mutation_actor( - user_profile_id=actor_context.user_profile_id, - actor_kind=actor_context.actor_kind, - actor_name=actor_context.actor_name, - ), - source=actor_context.source, - ), - dependencies=self.mutation_dependencies, - ) - except AcceptedNoteMutationRejected as error: - raise note_content_mutation_error_from_rejection(error.rejection) from error - - async def move_note( - self, - *, - project_external_id: str, - entity_external_id: str, - destination_path: str, - user_profile_id: UUID | None, - source: str, - actor_kind: str | None = None, - actor_name: str | None = None, - ) -> AcceptedNoteChange: - """Move a note by accepting the new path before runtime materialization.""" - actor_context = self._resolve_actor( - "move", - user_profile_id=user_profile_id, - source=source, - actor_kind=actor_kind, - actor_name=actor_name, - ) - try: - await self.freshen_existing_note_content( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - ) - async with accepted_note_transaction(self.session_maker) as session: - return await run_accepted_note_move( - session, - request=AcceptedNoteMoveMutation( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - destination_path=destination_path, - actor=accepted_note_mutation_actor( - user_profile_id=actor_context.user_profile_id, - actor_kind=actor_context.actor_kind, - actor_name=actor_context.actor_name, - ), - source=actor_context.source, - ), - dependencies=self.mutation_dependencies, - ) - except AcceptedNoteMutationRejected as error: - raise note_content_mutation_error_from_rejection(error.rejection) from error - - async def delete_note( - self, - *, - project_external_id: str, - entity_external_id: str, - ) -> AcceptedNoteChange: - """DELETE the DB note and return the runtime follow-up change.""" - try: - await self.freshen_existing_note_content( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - ) - async with accepted_note_transaction(self.session_maker) as session: - return await run_accepted_note_delete( - session, - request=AcceptedNoteDeleteMutation( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - ), - dependencies=self.mutation_dependencies, - ) - except AcceptedNoteMutationRejected as error: - raise note_content_mutation_error_from_rejection(error.rejection) from error +from basic_memory.services.note_content_writes import * # noqa: F403 diff --git a/src/basic_memory/cloud/project_deletes.py b/src/basic_memory/cloud/project_deletes.py index b9a79a20b..6d2dbb7fd 100644 --- a/src/basic_memory/cloud/project_deletes.py +++ b/src/basic_memory/cloud/project_deletes.py @@ -1,130 +1,3 @@ -"""Route-facing project-delete acceptance orchestration.""" +"""Compatibility shim; import from ``basic_memory.services.project_deletes``.""" -from __future__ import annotations - -from dataclasses import dataclass -from typing import Protocol - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker - -from basic_memory.indexing.project_delete_acceptance import ProjectDeleteAcceptedResult -from basic_memory.models import Project -from basic_memory.runtime.jobs import RuntimeJobId, RuntimeProjectDeleteJobRequest -from basic_memory.schemas.project_info import ProjectItem - - -class ProjectDeleteAcceptanceError(Exception): - """Structured project-delete acceptance error for HTTP/API adapters.""" - - def __init__(self, status_code: int, detail: str) -> None: - super().__init__(detail) - self.status_code = status_code - self.detail = detail - - -class ProjectDeleteJobEnqueuer(Protocol): - """Capability that accepts background project-delete cleanup work.""" - - async def enqueue_project_delete( - self, - request: RuntimeProjectDeleteJobRequest, - ) -> RuntimeJobId: ... - - -@dataclass(frozen=True, slots=True) -class ProjectDeleteAcceptanceRequest: - """Route-level request for accepting a project delete.""" - - project_external_id: str - delete_notes: bool - - -async def load_project_for_delete_acceptance( - session: AsyncSession, - *, - project_external_id: str, -) -> Project | None: - """Load the project row that the request is about to hide.""" - result = await session.execute( - select(Project) - .where(Project.external_id == project_external_id) - .with_for_update(of=Project) - .limit(1) - ) - return result.scalars().one_or_none() - - -async def reactivate_accepted_project( - session_maker: async_sessionmaker[AsyncSession], - *, - project_id: int, -) -> None: - """Undo a soft delete when the background queue rejects the request.""" - async with session_maker() as session: - project = await session.get(Project, project_id) - if project is None: - return - project.is_active = True - await session.commit() - - -@dataclass(frozen=True, slots=True) -class ProjectDeleteAcceptanceService: - """Accept project deletes quickly and leave slow cleanup to a runtime adapter.""" - - session_maker: async_sessionmaker[AsyncSession] - job_enqueuer: ProjectDeleteJobEnqueuer - - async def delete_project( - self, - request: ProjectDeleteAcceptanceRequest, - ) -> ProjectDeleteAcceptedResult: - async with self.session_maker() as session: - project = await load_project_for_delete_acceptance( - session, - project_external_id=request.project_external_id, - ) - if project is None or not project.is_active: - raise ProjectDeleteAcceptanceError( - 404, - f"Project with external_id '{request.project_external_id}' not found", - ) - if project.is_default: - raise ProjectDeleteAcceptanceError( - 400, - f"Cannot delete default project '{project.name}'. " - "Set another project as default first.", - ) - - runtime_request = RuntimeProjectDeleteJobRequest( - project_id=project.id, - project_external_id=project.external_id, - project_name=project.name, - project_path=project.path, - delete_notes=request.delete_notes, - ) - old_project = ProjectItem( - id=project.id, - external_id=project.external_id, - name=project.name, - path=project.path, - is_default=project.is_default or False, - ) - project.is_active = False - await session.commit() - - try: - job_id = await self.job_enqueuer.enqueue_project_delete(runtime_request) - except Exception: - await reactivate_accepted_project( - self.session_maker, - project_id=runtime_request.project_id, - ) - raise - - return ProjectDeleteAcceptedResult.queued( - request=runtime_request, - job_id=job_id, - old_project=old_project, - ) +from basic_memory.services.project_deletes import * # noqa: F403 diff --git a/src/basic_memory/deps/services.py b/src/basic_memory/deps/services.py index 29996a624..18d07bfdb 100644 --- a/src/basic_memory/deps/services.py +++ b/src/basic_memory/deps/services.py @@ -40,23 +40,23 @@ SearchRepositoryV2ExternalDep, ) from basic_memory.indexing.relation_resolution import RepositoryRelationResolutionRuntime -from basic_memory.cloud import ( - DirectoryDeleteService, - LocalNoteContentMaterializationProvider, - NoteContentMutationService, - NoteContentQueryService, -) +from basic_memory.index.note_content_materialization import LocalNoteContentMaterializationProvider +from basic_memory.services.directory_deletes import DirectoryDeleteService +from basic_memory.services.note_content_reads import NoteContentQueryService +from basic_memory.services.note_content_writes import NoteContentMutationService from basic_memory.index.local_dependencies import build_local_markdown_file_indexer from basic_memory.index.local_notes import ( LocalAcceptedNotePreparerFactory, - LocalAcceptedNoteRepositories, LocalCurrentNoteContentFreshener, LocalDirectoryDeleteRelationCleanupRefresher, LocalDirectoryFileDeleteEnqueuer, ) +from basic_memory.repository.accepted_note_repositories import AcceptedNoteRepositories from basic_memory.index.local_project import ( LocalProjectIndexCommand, LocalProjectIndexRunner, +) +from basic_memory.index.project_indexing import ( ProjectIndexCommand, ProjectIndexObserver, ProjectIndexRunner, @@ -64,11 +64,13 @@ ) from basic_memory.index.local_schedulers import ( - EntityVectorSyncScheduler, LocalEntityVectorSyncScheduler, LocalProjectIndexScheduler, LocalRelationResolutionScheduler, LocalSearchReindexScheduler, +) +from basic_memory.index.schedulers import ( + EntityVectorSyncScheduler, RelationResolutionScheduler, SearchReindexScheduler, ) @@ -532,7 +534,7 @@ async def get_note_content_mutation_service( app_config: AppConfigDep, ) -> NoteContentMutationService: """Create the local accepted-note mutation facade for API routes.""" - accepted_note_repositories = LocalAcceptedNoteRepositories() + accepted_note_repositories = AcceptedNoteRepositories() return NoteContentMutationService( session_maker=session_maker, mutation_dependencies=AcceptedNoteMutationDependencies( diff --git a/src/basic_memory/index/local_notes.py b/src/basic_memory/index/local_notes.py index b4d7c490c..8f470a18e 100644 --- a/src/basic_memory/index/local_notes.py +++ b/src/basic_memory/index/local_notes.py @@ -23,21 +23,17 @@ from basic_memory.markdown import EntityParser from basic_memory.markdown.markdown_processor import MarkdownProcessor from basic_memory.models import Project -from basic_memory.repository import NoteContentRepository, ObservationRepository, RelationRepository -from basic_memory.repository.accepted_note_search_repository import AcceptedNoteSearchRepository +from basic_memory.repository.accepted_note_repositories import AcceptedNoteRepositories from basic_memory.repository.entity_repository import EntityRepository -from basic_memory.repository.search_repository import create_search_repository from basic_memory.runtime.cleanup import RuntimeFileDeleteResult, RuntimeNoteFileDeleteJobRequest from basic_memory.runtime.storage import ( - ProjectId, RuntimeFileChecksum, RuntimeFilePath, runtime_content_type_is_markdown, ) -from basic_memory.services import EntityService from basic_memory.services.exceptions import FileOperationError from basic_memory.services.file_service import FileService -from basic_memory.services.link_resolver import LinkResolver +from basic_memory.services.note_preparation import NotePreparation, NotePreparationDependencies from basic_memory.services.search_service import SearchService # --- Accepted-Note Mutations --- @@ -59,61 +55,18 @@ def create_note_preparer(self, project: Project) -> AcceptedNoteMutationPreparer app_config=self.app_config, ) entity_repository = EntityRepository(project_id=project.id) - search_repository = create_search_repository( - self.session_maker, - project_id=project.id, - app_config=self.app_config, - ) - search_service = SearchService( - search_repository, - entity_repository, - file_service, - self.session_maker, - ) - link_resolver = LinkResolver( - entity_repository=entity_repository, - search_service=search_service, - session_maker=self.session_maker, - ) - return EntityService( - entity_repository=entity_repository, - observation_repository=ObservationRepository(project_id=project.id), - relation_repository=RelationRepository(project_id=project.id), - entity_parser=entity_parser, - file_service=file_service, - link_resolver=link_resolver, - session_maker=self.session_maker, - search_service=search_service, - app_config=self.app_config, + return NotePreparation( + NotePreparationDependencies( + entity_parser=entity_parser, + entity_repository=entity_repository, + file_service=file_service, + session_maker=self.session_maker, + app_config=self.app_config, + ) ) -@dataclass(frozen=True, slots=True) -class LocalAcceptedNoteRepositories: - """Project-scoped core repositories for accepted-note mutations. - - One concrete bundle satisfies both the lookup and write repository - capabilities the accepted-note mutation runner consumes; cloud composes its - own tenant-scoped equivalent behind the same protocols. - """ - - def entity_repository(self, project_id: ProjectId) -> EntityRepository: - return EntityRepository(project_id=project_id) - - def pending_entity_repository(self, project_id: ProjectId) -> EntityRepository: - return EntityRepository(project_id=project_id) - - def note_content_repository(self, project_id: ProjectId) -> NoteContentRepository: - return NoteContentRepository(project_id=project_id) - - def search_repository(self, project_id: ProjectId) -> AcceptedNoteSearchRepository: - return AcceptedNoteSearchRepository(project_id=project_id) - - def observation_repository(self, project_id: ProjectId) -> ObservationRepository: - return ObservationRepository(project_id=project_id) - - def relation_repository(self, project_id: ProjectId) -> RelationRepository: - return RelationRepository(project_id=project_id) +LocalAcceptedNoteRepositories = AcceptedNoteRepositories # --- Current-Note Content Freshening --- diff --git a/src/basic_memory/index/local_project.py b/src/basic_memory/index/local_project.py index 748c5ae83..8826c99db 100644 --- a/src/basic_memory/index/local_project.py +++ b/src/basic_memory/index/local_project.py @@ -24,6 +24,12 @@ ) from basic_memory.index.local_moves import LocalProjectIndexMoveContentUpdater from basic_memory.index.local_runtime import LocalStorageFileMetadataSource +from basic_memory.index.project_indexing import ( + ProjectIndexObservation, + ProjectIndexRouteRequest, + ProjectIndexRunner, + ProjectIndexScheduler, +) from basic_memory.indexing.change_detector import ChangeDetector from basic_memory.indexing.embedding_index_planning import EmbeddingBatchVectorSync from basic_memory.indexing.file_batch_runner import ( @@ -449,15 +455,7 @@ class LocalProjectIndexRuntime: coordinator_job_id: RuntimeJobId | None = None -@dataclass(frozen=True, slots=True) -class LocalProjectIndexObservation: - """Current local project files observed through the project-index adapter.""" - - observed_files: tuple[RuntimeObservedIndexFile, ...] - - @property - def total_files(self) -> int: - return len(self.observed_files) +LocalProjectIndexObservation = ProjectIndexObservation class LocalProjectIndexRuntimeProvider(Protocol): @@ -709,51 +707,6 @@ async def index_project( ) -# --- Project-Index Route Commands --- - - -class ProjectIndexRunner(Protocol): - """Run project-wide indexing in the current process.""" - - async def index_project( - self, - project_id: int, - *, - force_full: bool = False, - ) -> ProjectIndexCoordinatorResult: ... - - -class ProjectIndexObserver(Protocol): - """Observe project files visible to the active runtime.""" - - async def observe_project(self, project_id: int) -> LocalProjectIndexObservation: ... - - -class ProjectIndexScheduler(Protocol): - """Schedule background project indexing.""" - - def schedule_project_index(self, *, project_id: int, force_full: bool = False) -> None: ... - - -@dataclass(frozen=True, slots=True) -class ProjectIndexRouteRequest: - """Route-level project-index command input.""" - - project_id: int - project_name: str - force_full: bool - run_in_background: bool - - -class ProjectIndexCommand(Protocol): - """Handle a project-index route request.""" - - async def index_project( - self, - request: ProjectIndexRouteRequest, - ) -> ProjectIndexResponse: ... - - @dataclass(frozen=True, slots=True) class LocalProjectIndexCommand: project_index_runner: ProjectIndexRunner diff --git a/src/basic_memory/index/local_schedulers.py b/src/basic_memory/index/local_schedulers.py index 1bc03f716..46c47c12f 100644 --- a/src/basic_memory/index/local_schedulers.py +++ b/src/basic_memory/index/local_schedulers.py @@ -10,45 +10,20 @@ import asyncio from dataclasses import dataclass -from typing import Any, Coroutine, Protocol +from typing import Any, Coroutine from loguru import logger -from basic_memory.index.local_project import ProjectIndexRunner +from basic_memory.index.project_indexing import ProjectIndexRunner +from basic_memory.index.schedulers import ( + EntityVectorSyncSearchService, + SearchReindexService, +) from basic_memory.indexing.relation_resolution import ( RelationResolutionRuntime, resolve_project_relations, ) -# --- Scheduler Capabilities --- - - -class EntityVectorSyncScheduler(Protocol): - """Schedule out-of-band semantic vector refreshes for note mutations.""" - - def schedule_entity_vector_sync(self, *, entity_id: int, project_id: int) -> None: ... - - -class SearchReindexScheduler(Protocol): - """Schedule a search-index rebuild for the active project.""" - - def schedule_search_reindex(self, *, project_id: int) -> None: ... - - -class RelationResolutionScheduler(Protocol): - """Schedule background forward-reference resolution after note mutations.""" - - def schedule_relation_resolution(self, *, project_id: int) -> None: ... - - -class EntityVectorSyncSearchService(Protocol): - async def sync_entity_vectors(self, entity_id: int) -> object: ... - - -class SearchReindexService(Protocol): - async def reindex_all(self) -> object: ... - - # --- Background Task Machinery --- diff --git a/src/basic_memory/index/note_content_materialization.py b/src/basic_memory/index/note_content_materialization.py new file mode 100644 index 000000000..26ff2dbf5 --- /dev/null +++ b/src/basic_memory/index/note_content_materialization.py @@ -0,0 +1,560 @@ +"""Local note-content materialization adapters.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Coroutine, Mapping +from contextlib import suppress +from dataclasses import dataclass, replace +from typing import Any, Protocol + +from loguru import logger +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from basic_memory import db, file_utils +from basic_memory.indexing.index_file_runner import IndexFileExecutor +from basic_memory.indexing.note_file_delete_runner import run_note_file_delete +from basic_memory.indexing.note_materialization_runner import ( + ContentStoreNoteMaterializationFileWriter, + RepositoryNoteMaterializationPreflight, + RepositoryNoteMaterializationPublisher, + RepositoryNoteMaterializationStatusPublisher, + run_note_materialization, +) +from basic_memory.runtime.cleanup import ( + RuntimeNoteFileDeleteJobRequest, + plan_note_file_delete_job_request, +) +from basic_memory.runtime.note_content import ( + NOTE_CONTENT_EXTERNAL_CHANGE_SYNC_ERROR, + RuntimeAcceptedNoteChange, + RuntimeAcceptedNoteResponse, + RuntimeNoteContentResponsePayload, + RuntimeNoteMaterializationJobRequest, + RuntimeNoteMaterializationResult, + RuntimeNoteMaterializationStatus, + RuntimePendingNoteMaterialization, + plan_accepted_note_response, + plan_note_materialization_job_request, +) +from basic_memory.runtime.note_materialization import RuntimeFileMetadataSource +from basic_memory.runtime.storage import RuntimeFileChecksum, RuntimeFilePath +from basic_memory.models import Entity +from basic_memory.repository import EntityRepository, NoteContentRepository +from basic_memory.schemas.response import ObservationResponse, RelationResponse +from basic_memory.services.file_service import FileService + + +# The (project_id, entity_id) identity that pins all of one note's queued +# materializations to a single worker. +type _NoteRoutingKey = tuple[int, int] + + +class _MaterializationWorkerPool: + """Bounded in-process worker pool that drains queued note materializations. + + Mirrors the cloud's queue worker model locally: the accept enqueues a + materialization and returns; a fixed number of workers pull from per-worker + queues and run them. Bounding concurrency to `workers` is the point — + fire-and-forget `create_task` let every deferred file write + index run at + once, and at high write load they contended en masse for the single SQLite + writer and the event loop, collapsing the tail (p99) and throughput + (benchmarks/docs/write-load-benchmark.md). With N workers only N + materializations are in flight; the rest wait in the queues and drain over + time, so the accept path stays light AND the writer isn't thrashed. + + Jobs are routed to a worker by their note identity (project_id, entity_id), + so all jobs for one note run on the same worker FIFO in submission order. + Materializations for the same note must never run concurrently: the older + job's file write changes the on-disk checksum, so the newer job's writer + guard reads unexpected content and publishes a false + external_change_detected on the LATEST accepted row — the note is never + materialized and is falsely flagged as conflicted. + """ + + def __init__(self) -> None: + self._queues: list[asyncio.Queue[Coroutine[Any, Any, object]]] = [] + self._workers: list[asyncio.Task[None]] = [] + self._loop: asyncio.AbstractEventLoop | None = None + + def submit( + self, + work: Coroutine[Any, Any, object], + *, + workers: int, + key: _NoteRoutingKey, + ) -> None: + self._ensure_workers(workers) + self._queues[self._worker_index(key)].put_nowait(work) + + def _worker_index(self, key: _NoteRoutingKey) -> int: + # hash() of an int tuple is stable within one process, which is all + # routing needs: a job only has to serialize against other jobs queued + # in the same process lifetime. + return hash(key) % len(self._queues) + + def _ensure_workers(self, workers: int) -> None: + # Trigger: first submit, or submit on a different event loop than the one + # the workers were bound to (e.g. a fresh per-test loop). + # Why: workers are long-lived tasks bound to one loop; reusing queues + # whose workers live on a dead loop would hang. Outcome: (re)create one + # queue + worker task pair per worker on the current running loop. + # Orphaned workers on a closed loop are already dead, so dropping them + # is safe. + loop = asyncio.get_running_loop() + if self._queues and self._loop is loop: + return + self._loop = loop + self._queues = [asyncio.Queue() for _ in range(max(1, workers))] + self._workers = [asyncio.create_task(self._run(queue)) for queue in self._queues] + + async def _run(self, queue: asyncio.Queue[Coroutine[Any, Any, object]]) -> None: + while True: + work = await queue.get() + try: + await work + except Exception: # pragma: no cover - defensive worker guard + logger.exception("Local note materialization failed") + finally: + queue.task_done() + + async def join(self) -> None: + """Block until every queued materialization has completed.""" + for queue in self._queues: + await queue.join() + + async def aclose(self) -> None: + """Cancel workers and reset the pool (clean test teardown / shutdown).""" + workers = self._workers + self._workers = [] + self._queues = [] + self._loop = None + for worker in workers: + worker.cancel() + for worker in workers: + with suppress(asyncio.CancelledError): + await worker + + +_materialization_pool = _MaterializationWorkerPool() + + +async def drain_pending_materializations() -> None: + """Block until queued local materializations finish writing + indexing. + + One-shot clients (``bm tool write-note``, importers) return right after the + accept enqueues the markdown write/index; without this drain the event loop can + close before the worker writes the source-of-truth file, silently losing the + write even though the API already reported it accepted. Long-lived servers keep + the loop alive and don't need it. + """ + await _materialization_pool.join() + + +# --- Startup Recovery --- +# accept_write marks note_content "pending", then the materialization preflight +# flips it to "writing" before the file is written and the publisher records +# "synced". If the process dies anywhere between those points the row is stuck +# forever: the crash may land before the file write (nothing on disk) or after it +# but before publish (the correct accepted file is already on disk, row still +# "writing"). A transient write error (ENOSPC, permissions) publishes "failed" +# instead — equally terminal, since nothing else ever retries it. On the next +# startup we re-drive every stuck row. The write path short-circuits when the +# accepted content is already on disk, so the crash-after-write case publishes to +# "synced" instead of tripping the external-change guard. The db_version +# compare-and-set guard in the preflight and publisher makes recovery +# unconditionally safe: an older recovery attempt can never overwrite a newer +# accepted write or its file. + +# Synthetic provenance stamped on recovered writes so operators can tell a +# crash-recovery materialization apart from a normal accept-path write in logs +# and object metadata. +RECOVERY_NOTE_CHANGE_SOURCE = "note-content-materialization-recovery" +RECOVERY_NOTE_ACTOR_NAME = "startup-recovery" + + +async def run_recovery_materialization( + request: RuntimeNoteMaterializationJobRequest, + *, + session_maker: async_sessionmaker[AsyncSession], + file_service: FileService, +) -> RuntimeNoteMaterializationResult: + """Re-drive one stuck materialization through the standard guarded write path. + + Uses the same preflight/writer/publisher/status-publisher as an accept-path + write, so the db_version and file-conflict guards apply unchanged. No cleanup + paths: a recovery request carries no old-file move, so nothing is deleted. + """ + storage = LocalNoteContentStorage(file_service) + return await run_note_materialization( + request, + preflight=RepositoryNoteMaterializationPreflight(session_maker=session_maker), + writer=ContentStoreNoteMaterializationFileWriter(storage), + publisher=RepositoryNoteMaterializationPublisher(session_maker=session_maker), + status_publisher=RepositoryNoteMaterializationStatusPublisher(session_maker=session_maker), + cleanup_enqueuer=InlineNoteFileDeleteEnqueuer(storage), + ) + + +async def recover_stuck_materializations( + *, + session_maker: async_sessionmaker[AsyncSession], + file_service: FileService, + project_id: int, +) -> int: + """Re-drive every note materialization stuck in writing/pending/failed for a project. + + Meant to run once per project at startup, before serving. Non-fatal per row: + a single row that raises is logged and skipped so one poisoned note cannot + block startup recovery for the rest of the project. Returns the number of rows + that reached a written file state. + """ + async with db.scoped_session(session_maker) as session: + stuck_rows = await NoteContentRepository(project_id=project_id).find_stuck_materializations( + session + ) + + if not stuck_rows: + return 0 + + logger.info( + "Recovering stuck note materializations", + project_id=project_id, + stuck_count=len(stuck_rows), + ) + recovered = 0 + for row in stuck_rows: + # Rebuild the queue request from the row's own accepted db_version/db_checksum + # so the preflight guard matches the current accepted state; if a newer write + # has since advanced the row, the guard trips and this attempt no-ops. + request = RuntimeNoteMaterializationJobRequest( + project_id=project_id, + entity_id=row.entity_id, + db_version=int(row.db_version), + db_checksum=str(row.db_checksum), + actor_name=RECOVERY_NOTE_ACTOR_NAME, + source=RECOVERY_NOTE_CHANGE_SOURCE, + ) + try: + result = await run_recovery_materialization( + request, + session_maker=session_maker, + file_service=file_service, + ) + except Exception: + # Trigger: one row's materialization raised (storage/DB error). + # Why: recovery is best-effort startup cleanup; the version guard makes + # a later retry safe, so one bad row must not abort the whole sweep. + # Outcome: log and continue to the next stuck row. + logger.exception( + "Failed to recover stuck note materialization", + project_id=project_id, + entity_id=row.entity_id, + ) + continue + if result.status is RuntimeNoteMaterializationStatus.written: + recovered += 1 + return recovered + + +def note_content_payload_file_path( + payload: RuntimeNoteContentResponsePayload, +) -> RuntimeFilePath | None: + """Return the materialized file path carried by an accepted-note payload.""" + if isinstance(payload, RuntimeAcceptedNoteResponse): + return payload.file_path + if isinstance(payload, Mapping): + file_path = payload.get("file_path") + if isinstance(file_path, str) and file_path: + return file_path + return None + + +def file_write_status_from_materialization_result( + result: RuntimeNoteMaterializationResult, +) -> str: + """Return the response write marker for a terminal local materialization result.""" + if result.status is RuntimeNoteMaterializationStatus.conflict: + return "external_change_detected" + return "failed" + + +def note_content_payload_with_materialization_result( + payload: RuntimeNoteContentResponsePayload, + result: RuntimeNoteMaterializationResult, +) -> RuntimeNoteContentResponsePayload: + """Expose a failed local materialization result in the accepted-note response payload.""" + file_write_status = file_write_status_from_materialization_result(result) + + if isinstance(payload, RuntimeAcceptedNoteResponse): + return replace( + payload, + file_write_status=file_write_status, + file_checksum=result.file_checksum + if result.file_checksum is not None + else payload.file_checksum, + last_materialization_error=result.reason, + ) + + updated_payload = dict(payload) + updated_payload["file_write_status"] = file_write_status + updated_payload["last_materialization_error"] = result.reason + if result.file_checksum is not None: + updated_payload["file_checksum"] = result.file_checksum + if file_write_status == "external_change_detected": + updated_payload["sync_error"] = NOTE_CONTENT_EXTERNAL_CHANGE_SYNC_ERROR + return updated_payload + + +def indexed_observation_payloads(entity: Entity) -> tuple[dict[str, object], ...]: + """Serialize loaded observation rows into the v2 response shape.""" + return tuple( + ObservationResponse.model_validate(observation).model_dump(mode="json") + for observation in entity.observations + ) + + +def indexed_relation_payloads(entity: Entity) -> tuple[dict[str, object], ...]: + """Serialize loaded relation rows into the v2 response shape.""" + return tuple( + RelationResponse.model_validate(relation).model_dump(mode="json") + for relation in entity.relations + ) + + +async def load_indexed_note_content_response_payload( + *, + session_maker: async_sessionmaker[AsyncSession], + project_id: int, + entity_id: int, + fallback_source: str, +) -> RuntimeAcceptedNoteResponse: + """Reload the local indexed entity graph after inline materialization/indexing.""" + async with db.scoped_session(session_maker) as session: + entity = await EntityRepository(project_id=project_id).get_by_id( + session, + entity_id, + load_relations=True, + ) + if entity is None: + raise RuntimeError(f"Indexed entity {entity_id} was not found after materialization") + + note_content = await NoteContentRepository(project_id=project_id).get_by_entity_id( + session, + entity_id, + ) + if note_content is None: + raise RuntimeError( + f"Indexed note_content for entity {entity_id} was not found after materialization" + ) + + return replace( + plan_accepted_note_response( + entity=entity, + note_content=note_content, + fallback_source=fallback_source, + ), + observations=indexed_observation_payloads(entity), + relations=indexed_relation_payloads(entity), + ) + + +@dataclass(frozen=True, slots=True) +class LocalNoteContentStorage: + """Adapt the local FileService to note-content runtime storage protocols.""" + + file_service: FileService + + async def write_file( + self, + path: RuntimeFilePath, + content: str, + *, + metadata: dict[str, str] | None = None, + ) -> RuntimeFileChecksum: + _ = metadata + path_obj = self.file_service.base_path / path if isinstance(path, str) else path + full_path = path_obj if path_obj.is_absolute() else self.file_service.base_path / path_obj + + # Accepted-note materialization persists an already-accepted DB snapshot. + # Writing bytes keeps the materialized file checksum identical to the + # note_content checksum on Windows, where text mode would translate LF to CRLF. + await self.file_service.ensure_directory(full_path.parent) + await file_utils.write_file_atomic_bytes(full_path, content.encode("utf-8")) + return await self.file_service.compute_checksum(full_path) + + async def get_file_metadata(self, path: RuntimeFilePath) -> RuntimeFileMetadataSource: + return await self.file_service.get_file_metadata(path) + + async def exists(self, path: RuntimeFilePath) -> bool: + return await self.file_service.exists(path) + + async def compute_checksum(self, path: RuntimeFilePath) -> RuntimeFileChecksum: + return await self.file_service.compute_checksum(path) + + async def delete_file(self, path: RuntimeFilePath) -> None: + await self.file_service.delete_file(path) + + +@dataclass(frozen=True, slots=True) +class InlineNoteFileDeleteEnqueuer: + """Execute note-file cleanup immediately in the local runtime.""" + + storage: LocalNoteContentStorage + + async def enqueue_note_file_delete(self, request: RuntimeNoteFileDeleteJobRequest) -> None: + # Trigger: a move scheduled old-path cleanup whose old and new paths differ + # only by case (or otherwise alias the same inode) on a case-insensitive + # filesystem, so the old path now points at the just-written new file. + # Why: the checksum guard cannot tell "old file still present" from "old path + # aliases the new file" — both read the same bytes — so deleting the old path + # would destroy the note's only copy (then scan reconciliation removes the row). + # Outcome: skip the delete entirely; the paths are the same physical file. + if ( + request.live_file_path is not None + and self.storage.file_service.paths_share_storage_target( + request.file_path, request.live_file_path + ) + ): + logger.info( + "Skipping note-file cleanup that aliases the live file (case-only rename)", + entity_id=request.entity_id, + file_path=request.file_path, + live_file_path=request.live_file_path, + ) + return + await run_note_file_delete(request, storage=self.storage) + + +class RelationResolutionScheduling(Protocol): + """Capability to back-resolve forward references after a write is indexed.""" + + def schedule_relation_resolution(self, *, project_id: int) -> None: ... + + +@dataclass(frozen=True, slots=True) +class LocalNoteContentMaterializationProvider: + """Run accepted-note materialization inline for the local runtime.""" + + session_maker: async_sessionmaker[AsyncSession] + file_service: FileService + file_indexer: IndexFileExecutor | None = None + test_mode: bool = False + materialization_workers: int = 4 + relation_resolution_scheduler: RelationResolutionScheduling | None = None + + async def materialize_write_change( + self, + accepted: RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload], + ) -> RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload]: + """Materialize an accepted note write OFF the accept path. + + Cloud/local parity (DO NOT UNDO): cloud's materialize_write_change + enqueues a queue job and returns immediately, letting Tigris object storage + + indexing catch up asynchronously because S3 writes are slow. Locally we + mirror that with an in-process background task. The accept has already + persisted note_content (the write/read-through cache that serves reads); + here we only schedule writing the markdown file (the source of truth) and + indexing it. Writing + indexing the file is the heavy part of a write, so + doing it inline reintroduces a ~3x write-load regression + (benchmarks/docs/write-load-benchmark.md). + + PARITY INVARIANT: production must defer. Test mode runs inline ONLY so + tests can assert file/search state synchronously — never make the + production path synchronous to "simplify" this. + """ + materialization = accepted.materialization + if materialization is None: + return accepted + if self.test_mode: + return await self._materialize_write_now(accepted) + self._schedule_materialization(accepted, materialization) + return accepted + + def _schedule_materialization( + self, + accepted: RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload], + materialization: RuntimePendingNoteMaterialization, + ) -> None: + # Hand the materialization to the bounded worker pool instead of spawning + # an unbounded task per write — see _MaterializationWorkerPool for why. + # Keyed on the note's identity so two quick writes to the same note run + # sequentially on one worker instead of racing the writer guard into a + # false external_change_detected on the newer accepted row. + _materialization_pool.submit( + self._materialize_write_now(accepted), + workers=self.materialization_workers, + key=(materialization.project_id, materialization.entity_id), + ) + + async def _materialize_write_now( + self, + accepted: RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload], + ) -> RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload]: + if accepted.materialization is None: # pragma: no cover - guarded by caller + return accepted + storage = LocalNoteContentStorage(self.file_service) + cleanup_enqueuer = InlineNoteFileDeleteEnqueuer(storage) + result = await run_note_materialization( + plan_note_materialization_job_request(accepted.materialization), + preflight=RepositoryNoteMaterializationPreflight( + session_maker=self.session_maker, + ), + writer=ContentStoreNoteMaterializationFileWriter(storage), + publisher=RepositoryNoteMaterializationPublisher( + session_maker=self.session_maker, + ), + status_publisher=RepositoryNoteMaterializationStatusPublisher( + session_maker=self.session_maker, + ), + cleanup_enqueuer=cleanup_enqueuer, + ) + if result.status is not RuntimeNoteMaterializationStatus.written: + return replace( + accepted, + payload=note_content_payload_with_materialization_result( + accepted.payload, + result, + ), + ) + + file_path = note_content_payload_file_path(accepted.payload) + if file_path is not None and self.file_indexer is not None: + await self.file_indexer.index_file( + file_path, + source="note-content-materialization", + ) + # The deferred index has now inserted this note's entity/relation rows, + # so back-resolve inbound forward references. The router schedules an + # eager pass right after enqueue, but under load that pass can scan + # before this index lands; scheduling here (coalesced/re-armed by the + # resolution scheduler) guarantees a pass runs after indexing (#1002). + if self.relation_resolution_scheduler is not None: + self.relation_resolution_scheduler.schedule_relation_resolution( + project_id=accepted.materialization.project_id, + ) + return replace( + accepted, + payload=await load_indexed_note_content_response_payload( + session_maker=self.session_maker, + project_id=accepted.materialization.project_id, + entity_id=accepted.materialization.entity_id, + fallback_source=accepted.materialization.source + or "note-content-materialization", + ), + ) + return accepted + + async def materialize_delete_change( + self, + accepted: RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload], + ) -> RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload]: + """Delete materialized files immediately after local accepted-note deletes.""" + if accepted.file_delete is None: + return accepted + + storage = LocalNoteContentStorage(self.file_service) + await InlineNoteFileDeleteEnqueuer(storage).enqueue_note_file_delete( + plan_note_file_delete_job_request(accepted.file_delete) + ) + return accepted diff --git a/src/basic_memory/index/project_indexing.py b/src/basic_memory/index/project_indexing.py new file mode 100644 index 000000000..3066ee433 --- /dev/null +++ b/src/basic_memory/index/project_indexing.py @@ -0,0 +1,48 @@ +"""Runtime-neutral route contracts for project indexing.""" + +from dataclasses import dataclass +from typing import Protocol + +from basic_memory.indexing.project_index_coordinator import ProjectIndexCoordinatorResult +from basic_memory.runtime.jobs import RuntimeObservedIndexFile +from basic_memory.schemas.v2.project_index import ProjectIndexResponse + + +@dataclass(frozen=True, slots=True) +class ProjectIndexObservation: + """Files visible to the active project-index runtime.""" + + observed_files: tuple[RuntimeObservedIndexFile, ...] + + @property + def total_files(self) -> int: + return len(self.observed_files) + + +class ProjectIndexRunner(Protocol): + async def index_project( + self, + project_id: int, + *, + force_full: bool = False, + ) -> ProjectIndexCoordinatorResult: ... + + +class ProjectIndexObserver(Protocol): + async def observe_project(self, project_id: int) -> ProjectIndexObservation: ... + + +class ProjectIndexScheduler(Protocol): + def schedule_project_index(self, *, project_id: int, force_full: bool = False) -> None: ... + + +@dataclass(frozen=True, slots=True) +class ProjectIndexRouteRequest: + project_id: int + project_name: str + force_full: bool + run_in_background: bool + + +class ProjectIndexCommand(Protocol): + async def index_project(self, request: ProjectIndexRouteRequest) -> ProjectIndexResponse: ... diff --git a/src/basic_memory/index/schedulers.py b/src/basic_memory/index/schedulers.py new file mode 100644 index 000000000..a180a9862 --- /dev/null +++ b/src/basic_memory/index/schedulers.py @@ -0,0 +1,23 @@ +"""Runtime-neutral capabilities for scheduling derived index work.""" + +from typing import Protocol + + +class EntityVectorSyncScheduler(Protocol): + def schedule_entity_vector_sync(self, *, entity_id: int, project_id: int) -> None: ... + + +class SearchReindexScheduler(Protocol): + def schedule_search_reindex(self, *, project_id: int) -> None: ... + + +class RelationResolutionScheduler(Protocol): + def schedule_relation_resolution(self, *, project_id: int) -> None: ... + + +class EntityVectorSyncSearchService(Protocol): + async def sync_entity_vectors(self, entity_id: int) -> object: ... + + +class SearchReindexService(Protocol): + async def reindex_all(self) -> object: ... diff --git a/src/basic_memory/indexing/accepted_note_mutation_runner.py b/src/basic_memory/indexing/accepted_note_mutation_runner.py index 348e592fe..0552de812 100644 --- a/src/basic_memory/indexing/accepted_note_mutation_runner.py +++ b/src/basic_memory/indexing/accepted_note_mutation_runner.py @@ -22,12 +22,12 @@ AcceptedNoteWriteRepositories, create_accepted_pending_entity, delete_accepted_note, - persist_accepted_note_write, + persist_accepted_note_move, + persist_accepted_note_snapshot, prepare_accepted_note_create, prepare_accepted_note_edit, prepare_accepted_note_move, prepare_accepted_note_replace, - replace_accepted_note_graph, ) from basic_memory.models import Entity, NoteContent, Project from basic_memory.repository import NoteContentVersionConflict @@ -486,26 +486,16 @@ async def _run_accepted_note_create( user_profile_value=user_profile_value, repositories=dependencies.write_repositories, ) - persisted = await persist_accepted_note_write( + persisted = await persist_accepted_note_snapshot( session, entity=entity, - markdown_content=prepared.markdown_content, + prepared=prepared, db_checksum=prepared_write.db_checksum, - search_content=prepared.search_content, + self_relation_resolver=preparer, last_source=request.source, updated_at=now, repositories=dependencies.write_repositories, ) - # Persist observations/relations in the same transaction as the entity and - # note_content. Skipping this left the graph tables empty until a later - # index_file pass reparsed the materialized file (issue #1076). - await replace_accepted_note_graph( - session, - entity=entity, - prepared=prepared, - self_relation_resolver=preparer, - repositories=dependencies.write_repositories, - ) return plan_accepted_note_write_change( status_code=201, entity=entity, @@ -631,12 +621,12 @@ async def _run_accepted_note_update( reject_accepted_note_mutation(AcceptedNoteMutationRejectKind.bad_request, str(error)) prepared = prepared_write.prepared - persisted = await persist_accepted_note_write( + persisted = await persist_accepted_note_snapshot( session, entity=entity, - markdown_content=prepared.markdown_content, + prepared=prepared, db_checksum=prepared_write.db_checksum, - search_content=prepared.search_content, + self_relation_resolver=preparer, last_source=request.source, updated_at=now, current_note_content=current_note_content, @@ -644,16 +634,6 @@ async def _run_accepted_note_update( accepted_file_path=entity.file_path, repositories=dependencies.write_repositories, ) - # Replace the graph atomically: a PUT create-or-replace owns the note's full - # observation/relation set, so stale rows from a prior write are dropped and - # the accepted markdown's rows land in the same transaction (issue #1076). - await replace_accepted_note_graph( - session, - entity=entity, - prepared=prepared, - self_relation_resolver=preparer, - repositories=dependencies.write_repositories, - ) return plan_accepted_note_write_change( status_code=201 if created else 200, entity=entity, @@ -701,28 +681,18 @@ async def _run_accepted_note_edit( reject_accepted_note_mutation(AcceptedNoteMutationRejectKind.bad_request, str(error)) prepared = prepared_write.prepared - persisted = await persist_accepted_note_write( + persisted = await persist_accepted_note_snapshot( session, entity=entity, - markdown_content=prepared.markdown_content, + prepared=prepared, db_checksum=prepared_write.db_checksum, - search_content=prepared.search_content, + self_relation_resolver=preparer, last_source=request.source, updated_at=now, current_note_content=current_note_content, accepted_file_path=entity.file_path, repositories=dependencies.write_repositories, ) - # An edit reparses the whole note, so its graph is authoritative: replace the - # observation/relation set so rows an edit removed are dropped and rows it - # added appear immediately, not after a later reindex (issue #1076). - await replace_accepted_note_graph( - session, - entity=entity, - prepared=prepared, - self_relation_resolver=preparer, - repositories=dependencies.write_repositories, - ) return plan_accepted_note_write_change( status_code=200, entity=entity, @@ -799,17 +769,14 @@ async def _run_accepted_note_move( except (ParseError, ValueError) as error: reject_accepted_note_mutation(AcceptedNoteMutationRejectKind.bad_request, str(error)) - persisted = await persist_accepted_note_write( + persisted = await persist_accepted_note_move( session, entity=entity, - markdown_content=prepared_move.markdown_content, - db_checksum=prepared_move.db_checksum, - search_content=prepared_move.search_content, + prepared=prepared_move, last_source=request.source, updated_at=now, current_note_content=current_note_content, existing_file_path=existing_file_path, - accepted_file_path=prepared_move.file_path, repositories=dependencies.write_repositories, ) return plan_accepted_note_write_change( diff --git a/src/basic_memory/indexing/accepted_note_search.py b/src/basic_memory/indexing/accepted_note_search.py index 9742aa342..856663e77 100644 --- a/src/basic_memory/indexing/accepted_note_search.py +++ b/src/basic_memory/indexing/accepted_note_search.py @@ -4,33 +4,15 @@ import ast from collections.abc import Iterable, Mapping -from dataclasses import dataclass from datetime import datetime from pathlib import Path from basic_memory.file_utils import ParseError, remove_frontmatter +from basic_memory.repository.accepted_note_search_row import AcceptedNoteSearchRow MAX_ACCEPTED_SEARCH_CONTENT_STEMS_SIZE = 6000 -@dataclass(frozen=True, slots=True) -class AcceptedNoteSearchRow: - """Entity-level search row for an accepted DB-first note snapshot.""" - - id: int - title: str - content_stems: str - content_snippet: str - permalink: str | None - file_path: str - item_type: str - note_type: str | None - entity_id: int - created_at: datetime - updated_at: datetime - project_id: int - - def strip_search_text(value: str | None) -> str: """Strip NUL bytes that PostgreSQL text columns cannot store.""" return (value or "").replace("\x00", "") diff --git a/src/basic_memory/indexing/accepted_note_write_runner.py b/src/basic_memory/indexing/accepted_note_write_runner.py index 076e610c3..b73594fb1 100644 --- a/src/basic_memory/indexing/accepted_note_write_runner.py +++ b/src/basic_memory/indexing/accepted_note_write_runner.py @@ -2,17 +2,15 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Sequence from dataclasses import dataclass from datetime import datetime -from pathlib import Path from typing import Protocol from sqlalchemy.ext.asyncio import AsyncSession from basic_memory import file_utils from basic_memory.indexing.accepted_note_search import ( - AcceptedNoteSearchRow, accepted_search_content_from_markdown, build_accepted_note_search_row, ) @@ -22,10 +20,8 @@ AcceptedObservationWrite, AcceptedRelationWrite, ) -from basic_memory.repository.entity_repository import ( - AcceptedPendingEntityWrite, - EntityMetadata, -) +from basic_memory.repository.accepted_note_search_row import AcceptedNoteSearchRow +from basic_memory.repository.entity_repository import AcceptedPendingEntityWrite from basic_memory.runtime.note_content import ( RuntimeAcceptedNoteChange, RuntimeAcceptedNoteContentWriteSource, @@ -44,78 +40,12 @@ RuntimeNoteContentVersion, ) from basic_memory.schemas.base import Entity as EntitySchema - - -class AcceptedPreparedEntityFields(Protocol): - """Prepared Basic Memory entity fields accepted before file materialization.""" - - @property - def title(self) -> str: ... - - @property - def note_type(self) -> str: ... - - @property - def entity_metadata(self) -> EntityMetadata: ... - - @property - def content_type(self) -> str: ... - - @property - def permalink(self) -> str | None: ... - - @property - def file_path(self) -> RuntimeFilePath: ... - - @property - def created_at(self) -> datetime: ... - - @property - def updated_at(self) -> datetime: ... - - -class AcceptedPreparedEntityWriteSource(Protocol): - """Prepared markdown/entity state produced by Basic Memory note semantics.""" - - @property - def entity_fields(self) -> AcceptedPreparedEntityFields: ... - - -class AcceptedPreparedMarkdownWriteSource(AcceptedPreparedEntityWriteSource, Protocol): - """Prepared accepted markdown produced before DB or storage persistence.""" - - @property - def markdown_content(self) -> str: ... - - @property - def search_content(self) -> str: ... - - @property - def observations(self) -> Sequence[AcceptedObservationWrite]: ... - - @property - def relations(self) -> Sequence[AcceptedRelationWrite]: ... - - -class AcceptedPreparedEntityTarget(Protocol): - """Mutable entity fields mirrored from one prepared accepted note.""" - - title: str - note_type: str - entity_metadata: EntityMetadata - content_type: str - permalink: str | None - file_path: RuntimeFilePath - created_at: datetime - updated_at: datetime - last_updated_by: str | None - - -class AcceptedNoteContentSource(Protocol): - """Current accepted markdown source used as a replacement or edit base.""" - - @property - def markdown_content(self) -> str: ... +from basic_memory.services.note_preparation import ( + PreparedEntityFields, + PreparedEntityMove, + PreparedEntityWrite, + apply_prepared_entity_fields, +) class AcceptedNoteCreatePreparer(Protocol): @@ -128,7 +58,7 @@ async def prepare_create_entity_content( check_storage_exists: bool = ..., skip_conflict_check: bool = ..., session: AsyncSession | None = ..., - ) -> AcceptedPreparedMarkdownWriteSource: ... + ) -> PreparedEntityWrite: ... class AcceptedNoteReplacePreparer(Protocol): @@ -141,7 +71,7 @@ async def prepare_update_entity_content( existing_content: str, *, session: AsyncSession | None = ..., - ) -> AcceptedPreparedMarkdownWriteSource: ... + ) -> PreparedEntityWrite: ... class AcceptedNoteEditPreparer(Protocol): @@ -159,7 +89,7 @@ async def prepare_edit_entity_content( expected_replacements: int = ..., replace_subsections: bool = ..., session: AsyncSession | None = ..., - ) -> AcceptedPreparedMarkdownWriteSource: ... + ) -> PreparedEntityWrite: ... class AcceptedNoteSelfRelationResolver(Protocol): @@ -173,22 +103,6 @@ async def resolve_deferred_self_relation( ) -> Entity | None: ... -class AcceptedPreparedMoveSource(Protocol): - """Prepared accepted markdown and permalink state for a note move.""" - - @property - def file_path(self) -> Path: ... - - @property - def markdown_content(self) -> str: ... - - @property - def search_content(self) -> str: ... - - @property - def permalink(self) -> str | None: ... - - class AcceptedNoteMovePreparer(Protocol): """Capability that derives accepted markdown for a note move.""" @@ -199,7 +113,7 @@ async def prepare_move_entity_content( destination_path: str, *, session: AsyncSession | None = ..., - ) -> AcceptedPreparedMoveSource: ... + ) -> PreparedEntityMove: ... async def verify_move_destination_absent( self, @@ -209,41 +123,6 @@ async def verify_move_destination_absent( ) -> None: ... -class AcceptedNoteContentEntitySource(Protocol): - """Entity identity required to accept one note_content snapshot.""" - - @property - def id(self) -> RuntimeEntityId: ... - - @property - def project_id(self) -> ProjectId: ... - - -class AcceptedNoteSearchEntitySource(AcceptedNoteContentEntitySource, Protocol): - """Entity fields required to refresh the hot accepted-note search row.""" - - @property - def title(self) -> str | None: ... - - @property - def note_type(self) -> str | None: ... - - @property - def entity_metadata(self) -> Mapping[str, object] | None: ... - - @property - def permalink(self) -> str | None: ... - - @property - def file_path(self) -> RuntimeFilePath: ... - - @property - def created_at(self) -> datetime: ... - - @property - def updated_at(self) -> datetime: ... - - class AcceptedNoteDeleteEntitySource(RuntimeDeletedNoteFileDeleteEntitySource, Protocol): """Entity identity required to delete one accepted note row.""" @@ -345,7 +224,7 @@ def relation_repository( class AcceptedPreparedNoteWrite: """Prepared accepted markdown plus the checksum of that exact markdown.""" - prepared: AcceptedPreparedMarkdownWriteSource + prepared: PreparedEntityWrite db_checksum: RuntimeNoteContentChecksum @@ -395,7 +274,7 @@ async def prepare_accepted_note_replace( *, entity: Entity, data: EntitySchema, - current_note_content: AcceptedNoteContentSource, + current_note_content: NoteContent, user_profile_value: str | None, ) -> AcceptedPreparedNoteWrite: """Prepare a full accepted replacement and apply its entity fields.""" @@ -423,7 +302,7 @@ async def prepare_accepted_note_edit( session: AsyncSession, *, entity: Entity, - current_note_content: AcceptedNoteContentSource, + current_note_content: NoteContent, operation: str, content: str, section: str | None, @@ -462,7 +341,7 @@ async def prepare_accepted_note_move( session: AsyncSession, *, entity: Entity, - current_note_content: AcceptedNoteContentSource, + current_note_content: NoteContent, accepted_file_path: RuntimeFilePath, should_update_permalink: bool, user_profile_value: str | None, @@ -503,25 +382,21 @@ async def prepare_accepted_note_move( def apply_accepted_prepared_entity_fields( - entity: AcceptedPreparedEntityTarget, - entity_fields: AcceptedPreparedEntityFields, + entity: Entity, + entity_fields: PreparedEntityFields, *, user_profile_value: str | None, ) -> None: """Copy prepared accepted markdown fields onto an entity row.""" - entity.title = entity_fields.title - entity.note_type = entity_fields.note_type - entity.entity_metadata = entity_fields.entity_metadata - entity.content_type = entity_fields.content_type - entity.permalink = entity_fields.permalink - entity.file_path = entity_fields.file_path - entity.created_at = entity_fields.created_at - entity.updated_at = entity_fields.updated_at - entity.last_updated_by = user_profile_value + apply_prepared_entity_fields( + entity, + entity_fields, + user_profile_value=user_profile_value, + ) def accepted_pending_entity_write_from_prepared( - prepared: AcceptedPreparedEntityWriteSource, + prepared: PreparedEntityWrite, *, user_profile_value: str | None, external_id: str | None = None, @@ -546,7 +421,7 @@ def accepted_pending_entity_write_from_prepared( async def create_accepted_pending_entity( session: AsyncSession, *, - prepared: AcceptedPreparedEntityWriteSource, + prepared: PreparedEntityWrite, project_id: ProjectId, user_profile_value: str | None, external_id: str | None = None, @@ -587,7 +462,7 @@ def accepted_note_content_write_from_markdown( async def accept_note_content_write( session: AsyncSession, *, - entity: AcceptedNoteContentEntitySource, + entity: Entity, markdown_content: str, db_version: RuntimeNoteContentVersion, db_checksum: RuntimeNoteContentChecksum, @@ -611,7 +486,7 @@ async def accept_note_content_write( def accepted_note_search_row_from_entity( - entity: AcceptedNoteSearchEntitySource, + entity: Entity, *, search_content: str, ) -> AcceptedNoteSearchRow: @@ -633,7 +508,7 @@ def accepted_note_search_row_from_entity( async def refresh_accepted_note_search_index( session: AsyncSession, *, - entity: AcceptedNoteSearchEntitySource, + entity: Entity, search_content: str, repositories: AcceptedNoteWriteRepositories, ) -> None: @@ -669,10 +544,10 @@ async def delete_accepted_note_vectors( await repository.delete_entity_vectors(session, entity_id) -async def persist_accepted_note_write( +async def _persist_accepted_note_content_and_search( session: AsyncSession, *, - entity: AcceptedNoteSearchEntitySource, + entity: Entity, markdown_content: str, search_content: str, db_checksum: RuntimeNoteContentChecksum, @@ -683,7 +558,7 @@ async def persist_accepted_note_write( accepted_file_path: RuntimeFilePath | None = None, repositories: AcceptedNoteWriteRepositories, ) -> AcceptedPersistedNoteWrite: - """Accept markdown into note_content and refresh search inside one transaction.""" + """Internal content/search phase shared by complete snapshots and moves.""" content_write = plan_accepted_note_content_write( project_id=entity.project_id, entity_id=entity.id, @@ -713,11 +588,11 @@ async def persist_accepted_note_write( ) -async def replace_accepted_note_graph( +async def _replace_accepted_note_graph( session: AsyncSession, *, entity: Entity, - prepared: AcceptedPreparedMarkdownWriteSource, + prepared: PreparedEntityWrite, self_relation_resolver: AcceptedNoteSelfRelationResolver, repositories: AcceptedNoteWriteRepositories, ) -> None: @@ -771,6 +646,71 @@ async def replace_accepted_note_graph( ) +async def persist_accepted_note_snapshot( + session: AsyncSession, + *, + entity: Entity, + prepared: PreparedEntityWrite, + db_checksum: RuntimeNoteContentChecksum, + self_relation_resolver: AcceptedNoteSelfRelationResolver, + last_source: RuntimeNoteChangeSource | None, + updated_at: datetime, + current_note_content: RuntimeAcceptedNoteContentWriteSource | None = None, + existing_file_path: RuntimeFilePath | None = None, + accepted_file_path: RuntimeFilePath | None = None, + repositories: AcceptedNoteWriteRepositories, +) -> AcceptedPersistedNoteWrite: + """Persist one complete accepted Markdown snapshot in the caller's transaction.""" + persisted = await _persist_accepted_note_content_and_search( + session, + entity=entity, + markdown_content=prepared.markdown_content, + search_content=prepared.search_content, + db_checksum=db_checksum, + last_source=last_source, + updated_at=updated_at, + current_note_content=current_note_content, + existing_file_path=existing_file_path, + accepted_file_path=accepted_file_path, + repositories=repositories, + ) + await _replace_accepted_note_graph( + session, + entity=entity, + prepared=prepared, + self_relation_resolver=self_relation_resolver, + repositories=repositories, + ) + return persisted + + +async def persist_accepted_note_move( + session: AsyncSession, + *, + entity: Entity, + prepared: AcceptedPreparedNoteMove, + last_source: RuntimeNoteChangeSource | None, + updated_at: datetime, + current_note_content: RuntimeAcceptedNoteContentWriteSource, + existing_file_path: RuntimeFilePath, + repositories: AcceptedNoteWriteRepositories, +) -> AcceptedPersistedNoteWrite: + """Persist the explicitly narrower content/search state for an accepted move.""" + return await _persist_accepted_note_content_and_search( + session, + entity=entity, + markdown_content=prepared.markdown_content, + search_content=prepared.search_content, + db_checksum=prepared.db_checksum, + last_source=last_source, + updated_at=updated_at, + current_note_content=current_note_content, + existing_file_path=existing_file_path, + accepted_file_path=prepared.file_path, + repositories=repositories, + ) + + async def delete_accepted_note_entity( session: AsyncSession, *, diff --git a/src/basic_memory/indexing/project_index_maintenance.py b/src/basic_memory/indexing/project_index_maintenance.py index 11bde0d59..428607dd4 100644 --- a/src/basic_memory/indexing/project_index_maintenance.py +++ b/src/basic_memory/indexing/project_index_maintenance.py @@ -12,7 +12,9 @@ from basic_memory import db from basic_memory.models import Entity, NoteContent, Relation -from basic_memory.repository.project_repository import _load_sqlite_vec_on_session +from basic_memory.repository.accepted_note_vector_cleanup import ( + delete_project_index_vector_rows, +) from basic_memory.runtime.storage import ProjectId @@ -320,46 +322,6 @@ def skipped_paths(self) -> tuple[str, ...]: ) """).bindparams(bindparam("deleted_entity_ids", expanding=True)) -DELETE_PROJECT_INDEX_VECTOR_CHUNKS_SQL = text(""" - DELETE FROM search_vector_chunks - WHERE project_id = :project_id - AND entity_id IN :deleted_entity_ids -""").bindparams(bindparam("deleted_entity_ids", expanding=True)) - -SELECT_PROJECT_INDEX_SQLITE_VECTOR_TABLES_SQL = text(""" - SELECT name - FROM sqlite_master - WHERE type = 'table' - AND name IN ('search_vector_chunks', 'search_vector_embeddings') -""") - -SELECT_PROJECT_INDEX_POSTGRES_VECTOR_TABLES_SQL = text(""" - SELECT table_name - FROM information_schema.tables - WHERE table_schema = ANY (current_schemas(false)) - AND table_name IN ('search_vector_chunks', 'search_vector_embeddings') -""") - -DELETE_PROJECT_INDEX_SQLITE_VECTOR_EMBEDDINGS_SQL = text(""" - DELETE FROM search_vector_embeddings - WHERE rowid IN ( - SELECT id - FROM search_vector_chunks - WHERE project_id = :project_id - AND entity_id IN :deleted_entity_ids - ) -""").bindparams(bindparam("deleted_entity_ids", expanding=True)) - -DELETE_PROJECT_INDEX_POSTGRES_VECTOR_EMBEDDINGS_SQL = text(""" - DELETE FROM search_vector_embeddings - WHERE chunk_id IN ( - SELECT id - FROM search_vector_chunks - WHERE project_id = :project_id - AND entity_id IN :deleted_entity_ids - ) -""").bindparams(bindparam("deleted_entity_ids", expanding=True)) - PROJECT_INDEX_SEARCH_INDEX_TABLE = table( "search_index", column("project_id"), @@ -370,62 +332,6 @@ def skipped_paths(self) -> tuple[str, ...]: ) -def project_index_session_dialect_name(session: AsyncSession) -> str: - """Return the SQLAlchemy dialect name for project-index maintenance.""" - return session.get_bind().dialect.name - - -async def project_index_vector_table_names(session: AsyncSession) -> frozenset[str]: - """Return available vector table names for the current database backend.""" - dialect_name = project_index_session_dialect_name(session) - if dialect_name == "sqlite": - result = await session.execute(SELECT_PROJECT_INDEX_SQLITE_VECTOR_TABLES_SQL) - elif dialect_name == "postgresql": - result = await session.execute(SELECT_PROJECT_INDEX_POSTGRES_VECTOR_TABLES_SQL) - else: - raise RuntimeError(f"Unsupported project-index database dialect: {dialect_name}") - - return frozenset(str(table_name) for table_name in result.scalars()) - - -async def delete_project_index_vector_rows( - session: AsyncSession, - *, - project_id: ProjectId, - entity_ids: Sequence[int], -) -> None: - """Delete backend vector rows for project-index entity deletes when tables exist.""" - deleted_entity_ids = tuple(entity_ids) - if not deleted_entity_ids: - return - - vector_table_names = await project_index_vector_table_names(session) - if "search_vector_chunks" not in vector_table_names: - return - - delete_params = { - "project_id": project_id, - "deleted_entity_ids": deleted_entity_ids, - } - if "search_vector_embeddings" in vector_table_names: - dialect_name = project_index_session_dialect_name(session) - if dialect_name == "sqlite": - if await _load_sqlite_vec_on_session(session): - await session.execute( - DELETE_PROJECT_INDEX_SQLITE_VECTOR_EMBEDDINGS_SQL, - delete_params, - ) - elif dialect_name == "postgresql": - await session.execute( - DELETE_PROJECT_INDEX_POSTGRES_VECTOR_EMBEDDINGS_SQL, - delete_params, - ) - else: - raise RuntimeError(f"Unsupported project-index database dialect: {dialect_name}") - - await session.execute(DELETE_PROJECT_INDEX_VECTOR_CHUNKS_SQL, delete_params) - - async def delete_project_index_entities( session: AsyncSession, *, diff --git a/src/basic_memory/mcp/server.py b/src/basic_memory/mcp/server.py index 98aff4684..188d6e1b2 100644 --- a/src/basic_memory/mcp/server.py +++ b/src/basic_memory/mcp/server.py @@ -12,7 +12,7 @@ from basic_memory import db from basic_memory.cli.auth import CLIAuth -from basic_memory.cloud.note_content_materialization import drain_pending_materializations +from basic_memory.index.note_content_materialization import drain_pending_materializations from basic_memory.db import scoped_session from basic_memory.index.local_schedulers import drain_background_tasks from basic_memory.mcp.client_info import MCPClientInfoMiddleware diff --git a/src/basic_memory/repository/accepted_note_repositories.py b/src/basic_memory/repository/accepted_note_repositories.py new file mode 100644 index 000000000..beb58efb2 --- /dev/null +++ b/src/basic_memory/repository/accepted_note_repositories.py @@ -0,0 +1,31 @@ +"""Project-scoped repositories for accepted-note mutations.""" + +from dataclasses import dataclass + +from basic_memory.repository import NoteContentRepository, ObservationRepository, RelationRepository +from basic_memory.repository.accepted_note_search_repository import AcceptedNoteSearchRepository +from basic_memory.repository.entity_repository import EntityRepository +from basic_memory.runtime.storage import ProjectId + + +@dataclass(frozen=True, slots=True) +class AcceptedNoteRepositories: + """Core repository bundle using the caller-owned transaction.""" + + def entity_repository(self, project_id: ProjectId) -> EntityRepository: + return EntityRepository(project_id=project_id) + + def pending_entity_repository(self, project_id: ProjectId) -> EntityRepository: + return EntityRepository(project_id=project_id) + + def note_content_repository(self, project_id: ProjectId) -> NoteContentRepository: + return NoteContentRepository(project_id=project_id) + + def search_repository(self, project_id: ProjectId) -> AcceptedNoteSearchRepository: + return AcceptedNoteSearchRepository(project_id=project_id) + + def observation_repository(self, project_id: ProjectId) -> ObservationRepository: + return ObservationRepository(project_id=project_id) + + def relation_repository(self, project_id: ProjectId) -> RelationRepository: + return RelationRepository(project_id=project_id) diff --git a/src/basic_memory/repository/accepted_note_search_repository.py b/src/basic_memory/repository/accepted_note_search_repository.py index e210f5209..a64ff3059 100644 --- a/src/basic_memory/repository/accepted_note_search_repository.py +++ b/src/basic_memory/repository/accepted_note_search_repository.py @@ -8,8 +8,8 @@ from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession -from basic_memory.indexing.accepted_note_search import AcceptedNoteSearchRow -from basic_memory.indexing.project_index_maintenance import delete_project_index_vector_rows +from basic_memory.repository.accepted_note_search_row import AcceptedNoteSearchRow +from basic_memory.repository.accepted_note_vector_cleanup import delete_project_index_vector_rows type SearchIndexSqlValue = str | int | datetime | None type SearchIndexSqlParams = dict[str, SearchIndexSqlValue] diff --git a/src/basic_memory/repository/accepted_note_search_row.py b/src/basic_memory/repository/accepted_note_search_row.py new file mode 100644 index 000000000..ff3348646 --- /dev/null +++ b/src/basic_memory/repository/accepted_note_search_row.py @@ -0,0 +1,22 @@ +"""Repository-level values for accepted-note search persistence.""" + +from dataclasses import dataclass +from datetime import datetime + + +@dataclass(frozen=True, slots=True) +class AcceptedNoteSearchRow: + """Entity-level search row for an accepted DB-first note snapshot.""" + + id: int + title: str + content_stems: str + content_snippet: str + permalink: str | None + file_path: str + item_type: str + note_type: str | None + entity_id: int + created_at: datetime + updated_at: datetime + project_id: int diff --git a/src/basic_memory/repository/accepted_note_vector_cleanup.py b/src/basic_memory/repository/accepted_note_vector_cleanup.py new file mode 100644 index 000000000..14f1f8b18 --- /dev/null +++ b/src/basic_memory/repository/accepted_note_vector_cleanup.py @@ -0,0 +1,106 @@ +"""Repository-owned cleanup for accepted-note vector search rows.""" + +from collections.abc import Sequence + +from sqlalchemy import bindparam, text +from sqlalchemy.ext.asyncio import AsyncSession + +from basic_memory.repository.project_repository import _load_sqlite_vec_on_session +from basic_memory.runtime.storage import ProjectId + + +DELETE_PROJECT_INDEX_VECTOR_CHUNKS_SQL = text(""" + DELETE FROM search_vector_chunks + WHERE project_id = :project_id + AND entity_id IN :deleted_entity_ids +""").bindparams(bindparam("deleted_entity_ids", expanding=True)) + +SELECT_PROJECT_INDEX_SQLITE_VECTOR_TABLES_SQL = text(""" + SELECT name + FROM sqlite_master + WHERE type = 'table' + AND name IN ('search_vector_chunks', 'search_vector_embeddings') +""") + +SELECT_PROJECT_INDEX_POSTGRES_VECTOR_TABLES_SQL = text(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = ANY (current_schemas(false)) + AND table_name IN ('search_vector_chunks', 'search_vector_embeddings') +""") + +DELETE_PROJECT_INDEX_SQLITE_VECTOR_EMBEDDINGS_SQL = text(""" + DELETE FROM search_vector_embeddings + WHERE rowid IN ( + SELECT id + FROM search_vector_chunks + WHERE project_id = :project_id + AND entity_id IN :deleted_entity_ids + ) +""").bindparams(bindparam("deleted_entity_ids", expanding=True)) + +DELETE_PROJECT_INDEX_POSTGRES_VECTOR_EMBEDDINGS_SQL = text(""" + DELETE FROM search_vector_embeddings + WHERE chunk_id IN ( + SELECT id + FROM search_vector_chunks + WHERE project_id = :project_id + AND entity_id IN :deleted_entity_ids + ) +""").bindparams(bindparam("deleted_entity_ids", expanding=True)) + + +def project_index_session_dialect_name(session: AsyncSession) -> str: + """Return the SQLAlchemy dialect name for project-index maintenance.""" + return session.get_bind().dialect.name + + +async def project_index_vector_table_names(session: AsyncSession) -> frozenset[str]: + """Return available vector table names for the current database backend.""" + dialect_name = project_index_session_dialect_name(session) + if dialect_name == "sqlite": + result = await session.execute(SELECT_PROJECT_INDEX_SQLITE_VECTOR_TABLES_SQL) + elif dialect_name == "postgresql": + result = await session.execute(SELECT_PROJECT_INDEX_POSTGRES_VECTOR_TABLES_SQL) + else: + raise RuntimeError(f"Unsupported project-index database dialect: {dialect_name}") + + return frozenset(str(table_name) for table_name in result.scalars()) + + +async def delete_project_index_vector_rows( + session: AsyncSession, + *, + project_id: ProjectId, + entity_ids: Sequence[int], +) -> None: + """Delete backend vector rows for project-index entity deletes when tables exist.""" + deleted_entity_ids = tuple(entity_ids) + if not deleted_entity_ids: + return + + vector_table_names = await project_index_vector_table_names(session) + if "search_vector_chunks" not in vector_table_names: + return + + delete_params = { + "project_id": project_id, + "deleted_entity_ids": deleted_entity_ids, + } + if "search_vector_embeddings" in vector_table_names: + dialect_name = project_index_session_dialect_name(session) + if dialect_name == "sqlite": + if await _load_sqlite_vec_on_session(session): + await session.execute( + DELETE_PROJECT_INDEX_SQLITE_VECTOR_EMBEDDINGS_SQL, + delete_params, + ) + elif dialect_name == "postgresql": + await session.execute( + DELETE_PROJECT_INDEX_POSTGRES_VECTOR_EMBEDDINGS_SQL, + delete_params, + ) + else: + raise RuntimeError(f"Unsupported project-index database dialect: {dialect_name}") + + await session.execute(DELETE_PROJECT_INDEX_VECTOR_CHUNKS_SQL, delete_params) diff --git a/src/basic_memory/schemas/project_index.py b/src/basic_memory/schemas/project_index.py index 4d3f7de48..dd7fa4584 100644 --- a/src/basic_memory/schemas/project_index.py +++ b/src/basic_memory/schemas/project_index.py @@ -5,7 +5,7 @@ from pydantic import BaseModel, Field if TYPE_CHECKING: - from basic_memory.index.local_project import LocalProjectIndexObservation + from basic_memory.index.project_indexing import ProjectIndexObservation from basic_memory.indexing.project_index_coordinator import ProjectIndexCoordinatorResult @@ -28,7 +28,7 @@ class ProjectIndexStatusResponse(BaseModel): @classmethod def from_observation( cls, - observation: "LocalProjectIndexObservation", + observation: "ProjectIndexObservation", ) -> "ProjectIndexStatusResponse": return cls( total_files=observation.total_files, diff --git a/src/basic_memory/services/directory_deletes.py b/src/basic_memory/services/directory_deletes.py new file mode 100644 index 000000000..2631141b1 --- /dev/null +++ b/src/basic_memory/services/directory_deletes.py @@ -0,0 +1,118 @@ +"""Runtime-neutral directory-delete service facade. + +Runtime-specific callers provide the session boundary and the file cleanup +enqueuer. The core service owns request acceptance and response shaping. +""" + +from __future__ import annotations + +from contextlib import AbstractAsyncContextManager +from typing import Protocol + +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from basic_memory import db +from basic_memory.indexing.directory_delete_runner import ( + DirectoryDeleteAcceptanceRequest, + DirectoryDeleteAcceptedResult, + DirectoryDeleteRejected, + DirectoryDeleteRejection, + DirectoryDeleteRuntime, + accept_directory_delete, + finish_directory_delete_acceptance, + normalize_directory_delete_path, +) + + +class DirectoryDeleteSessionMaker(Protocol): + """Session factory capability needed by directory-delete acceptance.""" + + def __call__(self) -> AbstractAsyncContextManager[AsyncSession]: ... + + +class DirectoryDeleteServiceError(Exception): + """Structured directory-delete service error for route adapters.""" + + def __init__(self, status_code: int, detail: str) -> None: + super().__init__(detail) + self.status_code = status_code + self.detail = detail + + +def directory_delete_service_error_from_rejection( + rejection: DirectoryDeleteRejection, +) -> DirectoryDeleteServiceError: + """Map core directory-delete rejections into route-facing errors.""" + return DirectoryDeleteServiceError( + rejection.kind.http_status_code, + rejection.detail, + ) + + +class DirectoryDeleteService: + """Accept directory deletes into project DB state before storage cleanup begins.""" + + def __init__( + self, + *, + session_maker: async_sessionmaker[AsyncSession], + runtime: DirectoryDeleteRuntime, + ) -> None: + self.session_maker = session_maker + self.runtime = runtime + + async def delete_directory( + self, + *, + project_external_id: str, + directory: str, + ) -> DirectoryDeleteAcceptedResult: + """Delete directory entities immediately and queue file cleanup in the background. + + The typed result carries the route status (``http_status_code``) and the + existing response contract (``to_response_payload``). + """ + request = DirectoryDeleteAcceptanceRequest( + project_external_id=project_external_id, + directory=directory, + ) + try: + # scoped_session enables `PRAGMA foreign_keys=ON` for SQLite; this bulk + # delete issues a Core DELETE on entity and relies on ON DELETE CASCADE + # for note_content/observations/relations, which a raw session_maker() + # connection (foreign_keys OFF by default) would leave orphaned. + async with db.scoped_session(self.session_maker) as session: + accepted = await accept_directory_delete( + session, + request=request, + store=self.runtime.store, + ) + except DirectoryDeleteRejected as error: + raise directory_delete_service_error_from_rejection(error.rejection) from error + + result = await finish_directory_delete_acceptance( + request=request, + accepted=accepted, + enqueuer=self.runtime.file_delete_enqueuer, + ) + + # Trigger: notes outside the deleted directory linked into it. + # Why: the delete cascaded their relation rows away, but those sources own + # matching search_index relation rows that now dangle; without a reindex + # they linger until an unrelated rebuild. + # Outcome: reindex each surviving source inline when the runtime provides a + # refresher (local); queued runtimes consume the ids from the result. + if accepted.relation_cleanup_entity_ids and self.runtime.relation_cleanup_refresher: + await self.runtime.relation_cleanup_refresher.refresh_relation_sources( + sorted(accepted.relation_cleanup_entity_ids) + ) + + return result + + @staticmethod + def normalize_directory_path(directory: str) -> str: + """Normalize a project-relative directory path or reject traversal.""" + try: + return normalize_directory_delete_path(directory) + except ValueError as exc: + raise DirectoryDeleteServiceError(400, "Invalid directory path") from exc diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index 3fb709c58..27bc673fa 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -1,43 +1,26 @@ """Service for managing entities in the database.""" from collections.abc import Callable -from copy import deepcopy from dataclasses import dataclass -from datetime import datetime from pathlib import Path -from typing import Any, List, Optional, Sequence, Tuple, Union +from typing import List, Optional, Sequence, Tuple, Union -import frontmatter -import yaml from loguru import logger from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from basic_memory import db from basic_memory.config import ProjectConfig, BasicMemoryConfig -from basic_memory.file_utils import ( - ParseError, - has_frontmatter, - parse_frontmatter, - remove_frontmatter, - dump_frontmatter, -) +from basic_memory.file_utils import remove_frontmatter from basic_memory.markdown import EntityMarkdown from basic_memory.markdown.entity_parser import ( EntityParser, - _coerce_to_string, normalize_frontmatter_metadata, ) -from basic_memory.markdown.utils import entity_model_from_markdown, schema_to_markdown +from basic_memory.markdown.utils import entity_model_from_markdown from basic_memory.models import Entity as EntityModel from basic_memory.models import Observation, Relation from basic_memory.models.knowledge import Entity -from basic_memory.repository import ( - AcceptedObservationWrite, - AcceptedRelationWrite, - ObservationRepository, - RelationRepository, -) -from basic_memory.repository.project_repository import ProjectRepository +from basic_memory.repository import ObservationRepository, RelationRepository from basic_memory.repository.entity_repository import EntityRepository from basic_memory.runtime.note_move import normalize_note_move_destination_path from basic_memory.schemas import Entity as EntitySchema @@ -55,9 +38,29 @@ EntityNotFoundError, ) from basic_memory.services.link_resolver import LinkResolver +from basic_memory.services.note_preparation import ( + NotePreparation, + NotePreparationDependencies, + PreparedEntityFields, + PreparedEntityMove, + PreparedEntityWrite, + _fenced_code_line_flags as _note_fenced_code_line_flags, + apply_prepared_entity_fields, + apply_edit_operation as apply_note_edit_operation, + insert_relative_to_section as insert_note_relative_to_section, + replace_section_content as replace_note_section_content, +) from basic_memory.services.search_service import SearchService -from basic_memory.utils import build_canonical_permalink -from basic_memory.workspace_context import workspace_slug_for_canonical_permalinks + +__all__ = [ + "EntityService", + "PreparedEntityFields", + "PreparedEntityMove", + "PreparedEntityWrite", + "apply_prepared_entity_fields", +] + +_fenced_code_line_flags = _note_fenced_code_line_flags @dataclass(frozen=True) @@ -69,233 +72,6 @@ class EntityWriteResult: search_content: str -@dataclass(frozen=True, slots=True) -class PreparedEntityFields: - """Entity row values that mirror one accepted markdown snapshot.""" - - title: str - note_type: str - entity_metadata: dict[str, Any] | None - content_type: str - permalink: str | None - file_path: str - created_at: datetime - updated_at: datetime - - -@dataclass(frozen=True) -class PreparedEntityWrite: - """Accepted note state before any persistence side effects happen. - - Prepare methods return this object after all note semantics have been resolved, - but before any file writes or database mutations occur. - - Attributes: - file_path: Canonical note path implied by the request. - markdown_content: Full markdown to persist, including frontmatter. - search_content: Frontmatter-stripped content for inline FTS indexing. - entity_fields: Typed entity row values that mirror the accepted markdown state. - entity_markdown: Parsed markdown reused by the local write path to update - entities, observations, and relations without reparsing a second time. - """ - - file_path: Path - markdown_content: str - search_content: str - entity_fields: PreparedEntityFields - entity_markdown: EntityMarkdown - - @property - def observations(self) -> list[AcceptedObservationWrite]: - """Accepted-write observations, mapped from the already-parsed markdown. - - Lets the DB-first accepted-write path persist the graph without a second - parse; the field-for-field mapping mirrors the ORM rows - ``update_entity_and_observations`` builds for the file-indexing path. - """ - return [ - AcceptedObservationWrite( - content=obs.content, - category=obs.category, - context=obs.context, - tags=obs.tags, - ) - for obs in self.entity_markdown.observations - ] - - @property - def relations(self) -> list[AcceptedRelationWrite]: - """Accepted-write relations, mapped from the already-parsed markdown. - - Targets stay unresolved here. The accepted runner resolves only safe - self-links; the forward-reference job links all other targets later. - """ - return [ - AcceptedRelationWrite( - relation_type=rel.type, - target_name=rel.target, - context=rel.context, - ) - for rel in self.entity_markdown.relations - ] - - -@dataclass(frozen=True, slots=True) -class PreparedEditTitleReconciliation: - """Markdown title state after reconciling an edit with the note's body heading.""" - - markdown_content: str - title: str - metadata: dict[str, Any] | None - - -@dataclass(frozen=True, slots=True) -class PreparedEntityMove: - """Accepted markdown state for a DB-first note move.""" - - file_path: Path - markdown_content: str - search_content: str - permalink: str | None - - -def _frontmatter_permalink(value: object) -> str | None: - """Return an explicit frontmatter permalink only when YAML parsed a real string.""" - return value if isinstance(value, str) and value else None - - -def reconcile_prepared_edit_title_from_h1( - *, - original_markdown: str, - markdown_content: str, - current_title: str | None, - prepared_title: str, - metadata: dict[str, Any] | None, -) -> PreparedEditTitleReconciliation: - """Keep PATCH title metadata in step when a direct H1 edit changes the note title.""" - # The indexing package imports services at module load time, so keep this shared helper import - # inside the function to avoid a startup cycle while still preserving one H1 parser. - from basic_memory.indexing.accepted_note_search import ( - accepted_search_content_from_markdown, - first_markdown_h1, - ) - - original_h1 = first_markdown_h1(accepted_search_content_from_markdown(original_markdown)) - prepared_h1 = first_markdown_h1(accepted_search_content_from_markdown(markdown_content)) - - if not prepared_h1 or prepared_h1 == prepared_title: - return PreparedEditTitleReconciliation( - markdown_content=markdown_content, - title=prepared_title, - metadata=metadata, - ) - - # Only infer a title edit when the note previously used the H1 as its title. - # If frontmatter and H1 intentionally differed, preserve that explicit metadata. - if original_h1 != current_title or prepared_title != current_title: - return PreparedEditTitleReconciliation( - markdown_content=markdown_content, - title=prepared_title, - metadata=metadata, - ) - - post = frontmatter.loads(markdown_content) - post.metadata["title"] = prepared_h1 - reconciled_metadata = metadata - if reconciled_metadata is not None: - reconciled_metadata = {**reconciled_metadata, "title": prepared_h1} - - return PreparedEditTitleReconciliation( - markdown_content=dump_frontmatter(post), - title=prepared_h1, - metadata=reconciled_metadata, - ) - - -def _markdown_heading_level(line: str) -> int | None: - """Return the ATX heading level of a line, or None when it is not a heading. - - After up to three leading spaces, only 1-6 '#' characters followed by - whitespace (or nothing) form a heading, per CommonMark. '#hashtag' text and - 7+ '#' runs are ordinary content, so boundaries never land on tag lines. - """ - indent = len(line) - len(line.lstrip(" ")) - if indent > 3: - return None - - candidate = line[indent:] - if not candidate.startswith("#"): - return None - level = len(candidate) - len(candidate.lstrip("#")) - if level > 6: - return None - rest = candidate[level:] - if rest and not rest.startswith((" ", "\t")): - return None - return level - - -def _fence_marker(line: str) -> tuple[str, int, str] | None: - """Return a CommonMark fence marker, length, and suffix for a delimiter candidate.""" - indent = len(line) - len(line.lstrip(" ")) - if indent > 3: - return None - - candidate = line[indent:] - if not candidate or candidate[0] not in ("`", "~"): - return None - - marker = candidate[0] - marker_length = len(candidate) - len(candidate.lstrip(marker)) - if marker_length < 3: - return None - return marker, marker_length, candidate[marker_length:] - - -def _fenced_code_line_flags(lines: list[str]) -> list[bool]: - """Mark which lines sit inside (or delimit) CommonMark fenced code blocks. - - Fenced code often contains '# comment' lines that look exactly like markdown - headings. Section matching and boundary detection must skip those lines, or a - code comment would match or terminate a section. CommonMark permits backtick - and tilde fences indented by at most three spaces; four-space-indented markers - are literal code and must not hide later real headings. - """ - flags: list[bool] = [] - open_marker: str | None = None - open_length = 0 - for line in lines: - marker = _fence_marker(line) - - if open_marker is None: - if marker is None: - flags.append(False) - continue - - marker_char, marker_length, suffix = marker - # Backtick fence info strings cannot contain a backtick. Treat such a - # line as ordinary text instead of opening a fence that never closes. - if marker_char == "`" and "`" in suffix: - flags.append(False) - continue - - flags.append(True) - open_marker = marker_char - open_length = marker_length - continue - - # Every line inside a fence, including its closing delimiter, is skipped - # by heading detection. A close must use the same marker, be at least as - # long as the opener, and contain only trailing whitespace. - flags.append(True) - if marker is not None: - marker_char, marker_length, suffix = marker - if marker_char == open_marker and marker_length >= open_length and not suffix.strip(): - open_marker = None - open_length = 0 - return flags - - class EntityService(BaseService[EntityModel]): """Service for managing entities in the database.""" @@ -320,7 +96,15 @@ def __init__( self.session_maker = session_maker self.search_service = search_service self.app_config = app_config - self._project_permalink: Optional[str] = None + self._note_preparation = NotePreparation( + NotePreparationDependencies( + entity_parser=entity_parser, + entity_repository=entity_repository, + file_service=file_service, + session_maker=session_maker, + app_config=app_config, + ) + ) # Callable that returns the current user ID (cloud user_profile_id UUID as string). # Default returns None for local/CLI usage. Cloud overrides this to read from UserContext. self.get_user_id: Callable[[], Optional[str]] = lambda: None @@ -331,33 +115,12 @@ async def detect_file_path_conflicts( skip_check: bool = False, session: AsyncSession | None = None, ) -> List[str]: - """Detect potential file path conflicts for a given file path. - - This checks for entities with similar file paths that might cause conflicts: - - Case sensitivity differences (Finance/file.md vs finance/file.md) - - Character encoding differences - - Hyphen vs space differences - - Unicode normalization differences - - Args: - file_path: The file path to check for conflicts - skip_check: If True, skip the check and return empty list (optimization for bulk operations) - - Returns: - List of file paths that might conflict with the given file path - """ - if skip_check: - return [] - - from basic_memory.utils import detect_potential_file_conflicts - - # Load only file paths. Conflict detection is on the hot write path and - # does not need observations or relations. - async with db.scoped_session(self.session_maker, session) as active_session: - existing_paths = await self.repository.get_all_file_paths(active_session) - - # Use the enhanced conflict detection utility - return detect_potential_file_conflicts(file_path, existing_paths) + """Delegate file-path conflict detection to the shared preparation capability.""" + return await self._note_preparation.detect_file_path_conflicts( + file_path, + skip_check=skip_check, + session=session, + ) async def resolve_permalink( self, @@ -366,121 +129,12 @@ async def resolve_permalink( skip_conflict_check: bool = False, session: AsyncSession | None = None, ) -> str: - """Get or generate unique permalink for an entity. - - Priority: - 1. If markdown has permalink and it's not used by another file -> use as is - 2. If markdown has permalink but it's used by another file -> make unique - 3. For existing files, keep current permalink from db - 4. Generate new unique permalink from file path - - Enhanced to detect and handle character-related conflicts. - - Note: Uses lightweight repository methods that skip eager loading of - observations and relations for better performance during bulk operations. - """ - file_path_str = Path(file_path).as_posix() - - # Check for potential file path conflicts before resolving permalink - async with db.scoped_session(self.session_maker, session) as active_session: - conflicts = await self.detect_file_path_conflicts( - file_path_str, skip_check=skip_conflict_check, session=active_session - ) - if conflicts: - logger.warning( - f"Detected potential file path conflicts for '{file_path_str}': {conflicts}" - ) - - # If markdown has explicit permalink, try to validate it - if markdown and markdown.frontmatter.permalink: - desired_permalink = markdown.frontmatter.permalink - # Use lightweight method - we only need to check file_path - existing_file_path = await self.repository.get_file_path_for_permalink( - active_session, desired_permalink - ) - - # If no conflict or it's our own file, use as is - if not existing_file_path or existing_file_path == file_path_str: - return desired_permalink - - # For existing files, try to find current permalink - # Use lightweight method - we only need the permalink - existing_permalink = await self.repository.get_permalink_for_file_path( - active_session, file_path_str - ) - if existing_permalink: - return existing_permalink - - # New file - generate permalink - if markdown and markdown.frontmatter.permalink: - desired_permalink = markdown.frontmatter.permalink - else: - # Trigger: generating a permalink for a new file - # Why: canonical permalinks may require project prefix for global addressing - # Outcome: include project slug when enabled in config - include_project = True - if self.app_config: - include_project = self.app_config.permalinks_include_project - - workspace_permalink = workspace_slug_for_canonical_permalinks() - project_permalink = None - # Trigger: project-prefixed permalinks are enabled, or organization workspace - # context requires a complete workspace/project canonical permalink. - # Why: project slug is the stable middle segment for globally addressable links. - # Outcome: fetch and cache the project's permalink before building the canonical URL. - if include_project or workspace_permalink: - project_permalink = await self._get_project_permalink(active_session) - - desired_permalink = build_canonical_permalink( - project_permalink, - file_path_str, - include_project=include_project, - workspace_permalink=workspace_permalink, - ) - - # Make unique if needed - enhanced to handle character conflicts - # Use lightweight existence check instead of loading full entity - permalink = desired_permalink - suffix = 1 - while await self.repository.permalink_exists(active_session, permalink): - permalink = f"{desired_permalink}-{suffix}" - suffix += 1 - logger.debug(f"creating unique permalink: {permalink}") - - return permalink - - async def _get_project_permalink(self, session: AsyncSession) -> Optional[str]: - """Get and cache the current project's permalink.""" - if self._project_permalink is not None: - return self._project_permalink - - project_id = self.repository.project_id - if project_id is None: # pragma: no cover - return None # pragma: no cover - - project_repository = ProjectRepository() - project = await project_repository.get_by_id(session, project_id) - if project: - self._project_permalink = project.permalink - return self._project_permalink - - def _build_frontmatter_markdown( - self, title: str, note_type: str, permalink: str - ) -> EntityMarkdown: - """Build a minimal EntityMarkdown object for permalink resolution.""" - from basic_memory.markdown.schemas import EntityFrontmatter - - frontmatter_metadata = { - "title": title, - "type": note_type, - "permalink": permalink, - } - frontmatter_obj = EntityFrontmatter(metadata=frontmatter_metadata) - return EntityMarkdown( - frontmatter=frontmatter_obj, - content="", - observations=[], - relations=[], + """Delegate permalink resolution to the shared preparation capability.""" + return await self._note_preparation.resolve_permalink( + file_path, + markdown, + skip_conflict_check=skip_conflict_check, + session=session, ) def _coerce_schema_input(self, schema: EntitySchema | EntityModel) -> EntitySchema: @@ -525,125 +179,6 @@ def _sync_prepared_schema_state( else: source_schema._permalink = prepared.entity_fields.permalink - def _apply_schema_frontmatter_overrides(self, schema: EntitySchema) -> EntityMarkdown | None: - """Apply schema content frontmatter overrides and return permalink resolution metadata.""" - if not schema.content or not has_frontmatter(schema.content): - return None - - # Trigger: callers supply markdown that already contains frontmatter. - # Why: user-authored frontmatter is part of accepted note semantics, not a persistence detail. - # Outcome: note_type/permalink derivation happens before any write path decides how to persist. - content_frontmatter = parse_frontmatter(schema.content) - - if "type" in content_frontmatter: - schema.note_type = _coerce_to_string(content_frontmatter["type"]) - - if "permalink" not in content_frontmatter: - return None - - content_permalink = _frontmatter_permalink(content_frontmatter["permalink"]) - if content_permalink is None: - return None - - return self._build_frontmatter_markdown( - schema.title, - schema.note_type, - content_permalink, - ) - - async def _resolve_schema_permalink( - self, - schema: EntitySchema, - *, - file_path: Path, - current_permalink: str | None = None, - content_markdown: EntityMarkdown | None = None, - skip_conflict_check: bool = False, - session: AsyncSession | None = None, - ) -> str | None: - """Resolve the canonical permalink for a create/update write.""" - if self.app_config and self.app_config.disable_permalinks: - if current_permalink is None: - schema._permalink = "" - return None - schema._permalink = current_permalink - return current_permalink - - if current_permalink and not (content_markdown and content_markdown.frontmatter.permalink): - schema._permalink = current_permalink - return current_permalink - - resolved_permalink = await self.resolve_permalink( - file_path, - content_markdown, - skip_conflict_check=skip_conflict_check, - session=session, - ) - schema._permalink = resolved_permalink - return resolved_permalink - - def _build_entity_fields( - self, - *, - file_path: Path, - content_type: str, - permalink: str | None, - entity_markdown: EntityMarkdown, - ) -> PreparedEntityFields: - """Build the entity row data that mirrors accepted markdown state.""" - if entity_markdown.created is None or entity_markdown.modified is None: # pragma: no cover - raise ValueError("Prepared markdown requires created and modified timestamps") - - normalized_metadata = normalize_frontmatter_metadata( - entity_markdown.frontmatter.metadata or {} - ) - entity_metadata = {k: v for k, v in normalized_metadata.items() if v is not None} - return PreparedEntityFields( - title=entity_markdown.frontmatter.title, - note_type=entity_markdown.frontmatter.type, - file_path=file_path.as_posix(), - content_type=content_type, - entity_metadata=entity_metadata or None, - permalink=permalink, - created_at=entity_markdown.created, - updated_at=entity_markdown.modified, - ) - - async def _build_prepared_write( - self, - *, - file_path: Path, - markdown_content: str, - content_type: str, - permalink: str | None, - preserved_created_at: datetime | None = None, - ) -> PreparedEntityWrite: - """Parse accepted markdown once so all persistence paths share the same state.""" - # Trigger: both local and cloud-style callers need the exact same accepted markdown. - # Why: parsing twice creates opportunities for drift between "what we accepted" and - # "what we indexed/persisted". - # Outcome: callers carry one prepared object through file writes, DB writes, and indexing. - entity_markdown = await self.entity_parser.parse_markdown_content( - file_path=file_path, - content=markdown_content, - # DB-first updates have no file ctime. Reuse the existing semantic creation - # time as that field's fallback so editing a legacy note cannot make it "new". - ctime=(preserved_created_at.timestamp() if preserved_created_at is not None else None), - ) - entity_fields = self._build_entity_fields( - file_path=file_path, - content_type=content_type, - permalink=permalink, - entity_markdown=entity_markdown, - ) - return PreparedEntityWrite( - file_path=file_path, - markdown_content=markdown_content, - search_content=remove_frontmatter(markdown_content), - entity_fields=entity_fields, - entity_markdown=entity_markdown, - ) - async def _read_persisted_write_content(self, file_path: Path) -> tuple[str, str]: """Read the stored markdown after write-time formatting has finished.""" # Trigger: format-on-save or platform-specific text writes can change the stored markdown @@ -687,37 +222,13 @@ async def prepare_create_entity_content( acceptance and must perform any external storage conflict handling themselves. """ - # Work on a copy so prepare methods are pure from the caller's perspective. - # The router/service layer still receives the same accepted result, but we avoid mutating - # the original schema instance in surprising ways. - schema = schema.model_copy(deep=True) - file_path = Path(schema.file_path) - - if check_storage_exists and await self.file_service.exists(file_path): - raise EntityAlreadyExistsError( - f"file for entity {schema.directory}/{schema.title} already exists: {file_path}" - ) - - content_markdown = self._apply_schema_frontmatter_overrides(schema) - permalink = await self._resolve_schema_permalink( + return await self._note_preparation.prepare_create_entity_content( schema, - file_path=file_path, - content_markdown=content_markdown, + check_storage_exists=check_storage_exists, skip_conflict_check=skip_conflict_check, session=session, ) - # Build the final markdown once here. Local mode will write it immediately; cloud mode can - # store it in note_content first and materialize later without re-deriving anything. - post = await schema_to_markdown(schema) - markdown_content = dump_frontmatter(post) - return await self._build_prepared_write( - file_path=file_path, - markdown_content=markdown_content, - content_type=schema.content_type, - permalink=permalink, - ) - async def prepare_update_entity_content( self, entity: EntityModel, @@ -734,68 +245,14 @@ async def prepare_update_entity_content( preserve unrecognized frontmatter keys from that explicit base content. No database rows are mutated here. """ - schema = schema.model_copy(deep=True) - file_path = Path(schema.file_path) - current_file_path = Path(entity.file_path) - existing_metadata: dict[str, Any] = {} - if has_frontmatter(existing_content): - try: - existing_metadata = parse_frontmatter(existing_content) - except ParseError: - # Trigger: the old note has frontmatter fences but malformed YAML. - # Why: a full replacement must be able to repair that note, and malformed - # metadata cannot be merged safely into the replacement. - # Outcome: discard only the invalid merge input; the final accepted markdown - # is still parsed and validated below. - pass - - content_markdown = self._apply_schema_frontmatter_overrides(schema) - # Trigger: a full replacement may also rename the note by changing title or directory. - # Why: cloud accepts the final markdown before S3 is updated, so the prepare contract must - # describe the requested destination instead of silently keeping the old path. - # Outcome: unchanged paths preserve the current permalink; renamed paths only rotate the - # permalink when move-policy allows it or frontmatter explicitly sets one. - update_permalink_on_rename = bool( - self.app_config and self.app_config.update_permalinks_on_move - ) - current_permalink = ( - entity.permalink - if file_path.as_posix() == current_file_path.as_posix() - or not update_permalink_on_rename - else None - ) - resolved_permalink = await self._resolve_schema_permalink( + return await self._note_preparation.prepare_update_entity_content( + entity, schema, - file_path=file_path, - current_permalink=current_permalink, - content_markdown=content_markdown, + existing_content, skip_conflict_check=skip_conflict_check, session=session, ) - post = await schema_to_markdown(schema) - - # Full updates preserve unrecognized frontmatter keys from the existing note. - # That keeps Basic Memory's write semantics stable for hand-authored metadata while still - # letting the incoming schema replace the fields it explicitly owns. - # Existing frontmatter is a merge input, not accepted state. Semantic validation happens - # after the incoming metadata has had a chance to repair invalid canonical values. - merged_metadata = deepcopy(existing_metadata) - merged_metadata.update(post.metadata) - merged_metadata["permalink"] = resolved_permalink - - merged_post = frontmatter.Post(post.content) - merged_post.metadata.update(merged_metadata) - - markdown_content = dump_frontmatter(merged_post) - return await self._build_prepared_write( - file_path=file_path, - markdown_content=markdown_content, - content_type=schema.content_type, - permalink=resolved_permalink, - preserved_created_at=entity.created_at, - ) - async def prepare_edit_entity_content( self, entity: EntityModel, @@ -817,68 +274,17 @@ async def prepare_edit_entity_content( edit base explicit so higher layers can reject stale content instead of silently editing whichever storage copy happens to be newest. """ - file_path = Path(entity.file_path) - # Edits are intentionally based on explicit caller-supplied content. That makes stale-base - # handling visible to the caller instead of quietly reading whatever persistence layer - # happens to be newest. - markdown_content = self.apply_edit_operation( + return await self._note_preparation.prepare_edit_entity_content( + entity, current_content, - operation, - content, - section, - find_text, - expected_replacements, - replace_subsections, - ) - - title = entity.title - note_type = entity.note_type - permalink = entity.permalink - metadata = entity.entity_metadata - - if has_frontmatter(markdown_content): - content_frontmatter = parse_frontmatter(markdown_content) - - if "title" in content_frontmatter: - title = _coerce_to_string(content_frontmatter["title"]) - if "type" in content_frontmatter: - note_type = _coerce_to_string(content_frontmatter["type"]) - - if self.app_config and self.app_config.disable_permalinks: - permalink = entity.permalink - elif "permalink" in content_frontmatter: - content_permalink = _frontmatter_permalink(content_frontmatter["permalink"]) - if content_permalink is not None: - content_markdown = self._build_frontmatter_markdown( - title, - note_type, - content_permalink, - ) - permalink = await self.resolve_permalink( - file_path, - content_markdown, - skip_conflict_check=skip_conflict_check, - session=session, - ) - - normalized_metadata = normalize_frontmatter_metadata(content_frontmatter or {}) - metadata = {k: v for k, v in normalized_metadata.items() if v is not None} or None - - title_reconciliation = reconcile_prepared_edit_title_from_h1( - original_markdown=current_content, - markdown_content=markdown_content, - current_title=entity.title, - prepared_title=title, - metadata=metadata, - ) - markdown_content = title_reconciliation.markdown_content - - return await self._build_prepared_write( - file_path=file_path, - markdown_content=markdown_content, - content_type=entity.content_type, - permalink=permalink, - preserved_created_at=entity.created_at, + operation=operation, + content=content, + section=section, + find_text=find_text, + expected_replacements=expected_replacements, + replace_subsections=replace_subsections, + skip_conflict_check=skip_conflict_check, + session=session, ) async def prepare_move_entity_content( @@ -895,31 +301,11 @@ async def prepare_move_entity_content( The caller supplies the current accepted markdown because cloud DB-first moves may need to use note_content rather than a materialized file. """ - # Keep the search helper import lazy for the same package-cycle reason as - # reconcile_prepared_edit_title_from_h1. - from basic_memory.indexing.accepted_note_search import ( - accepted_search_content_from_markdown, - ) - - file_path = Path(normalize_note_move_destination_path(destination_path)) - markdown_content = current_content - permalink = entity.permalink - disable_permalinks = bool(self.app_config and self.app_config.disable_permalinks) - update_permalinks_on_move = bool( - self.app_config and self.app_config.update_permalinks_on_move - ) - - if not disable_permalinks and (update_permalinks_on_move or entity.permalink is None): - permalink = await self.resolve_permalink(file_path, session=session) - post = frontmatter.loads(markdown_content) - post.metadata["permalink"] = permalink - markdown_content = dump_frontmatter(post) - - return PreparedEntityMove( - file_path=file_path, - markdown_content=markdown_content, - search_content=accepted_search_content_from_markdown(markdown_content), - permalink=permalink, + return await self._note_preparation.prepare_move_entity_content( + entity, + current_content, + destination_path, + session=session, ) async def verify_move_destination_absent( @@ -935,16 +321,10 @@ async def verify_move_destination_absent( a duplicate. A case-only rename or shared storage target (same physical file) is allowed. Cloud is DB-first and opts out via verify_storage_absent_on_create. """ - source = Path(source_file_path) - destination = Path(normalize_note_move_destination_path(destination_file_path)) - if ( - source != destination - and await self.file_service.exists(destination) - and not self._paths_share_storage_target(source, destination) - ): - raise EntityAlreadyExistsError( - f"file already exists at destination path: {destination.as_posix()}" - ) + await self._note_preparation.verify_move_destination_absent( + source_file_path=source_file_path, + destination_file_path=destination_file_path, + ) async def create_or_update_entity(self, schema: EntitySchema) -> Tuple[EntityModel, bool]: """Create new entity or update existing one. @@ -1424,35 +804,11 @@ async def resolve_deferred_self_relation( self, target: str, entity: EntityModel, session: AsyncSession | None = None ) -> EntityModel | None: """Resolve only self-relations that are safe to identify in deferred mode.""" - clean_target = target.strip() - if clean_target.startswith("[[") and clean_target.endswith("]]"): - clean_target = clean_target[2:-2].strip() - if "|" in clean_target: - clean_target = clean_target.split("|", 1)[0].strip() - - candidates = {entity.file_path} - if entity.permalink: - candidates.add(entity.permalink) - if entity.file_path.endswith(".md"): - candidates.add(entity.file_path[:-3]) - - if clean_target in candidates: - return entity - - if clean_target != entity.title: - return None - - # Title-only links are ambiguous because Basic Memory allows duplicate titles. - # Collapse them to self only when the title lookup proves this source is the sole candidate; - # otherwise leave the relation unresolved so we do not create a wrong permanent edge. - async with db.scoped_session(self.session_maker, session) as active_session: - title_matches = await self.repository.get_by_title( - active_session, clean_target, load_relations=False - ) - if len(title_matches) == 1 and title_matches[0].id == entity.id: - return entity - - return None + return await self._note_preparation.resolve_deferred_self_relation( + target, + entity, + session=session, + ) async def edit_entity( self, @@ -1572,60 +928,15 @@ def apply_edit_operation( replace_subsections: bool = True, ) -> str: """Apply the specified edit operation to the current content.""" - - if operation == "append": - # Ensure proper spacing - if current_content and not current_content.endswith("\n"): - return current_content + "\n" + content - return current_content + content # pragma: no cover - - elif operation == "prepend": - # Handle frontmatter-aware prepending - return self._prepend_after_frontmatter(current_content, content) - - elif operation == "find_replace": - if not find_text: - raise ValueError("find_text is required for find_replace operation") - if not find_text.strip(): - raise ValueError("find_text cannot be empty or whitespace only") - - # Count actual occurrences - actual_count = current_content.count(find_text) - - # Validate count matches expected - if actual_count != expected_replacements: - if actual_count == 0: - raise ValueError(f"Text to replace not found: '{find_text}'") - else: - raise ValueError( - f"Expected {expected_replacements} occurrences of '{find_text}', " - f"but found {actual_count}" - ) - - return current_content.replace(find_text, content) - - elif operation == "replace_section": - if not section: - raise ValueError("section is required for replace_section operation") - if not section.strip(): - raise ValueError("section cannot be empty or whitespace only") - return self.replace_section_content( - current_content, - section, - content, - replace_subsections=replace_subsections, - ) - - elif operation in ("insert_before_section", "insert_after_section"): - if not section: - raise ValueError("section is required for insert section operations") - if not section.strip(): - raise ValueError("section cannot be empty or whitespace only") - position = "before" if operation == "insert_before_section" else "after" - return self.insert_relative_to_section(current_content, section, content, position) - - else: - raise ValueError(f"Unsupported operation: {operation}") + return apply_note_edit_operation( + current_content, + operation, + content, + section, + find_text, + expected_replacements, + replace_subsections, + ) def replace_section_content( self, @@ -1665,67 +976,12 @@ def replace_section_content( Raises: ValueError: If multiple sections with the same header are found """ - # Normalize the section header (ensure it starts with #) - if not section_header.startswith("#"): - section_header = "## " + section_header - - # Strip duplicate header from new_content if present (fix for issue #390) - # LLMs sometimes include the section header in their content, which would create duplicates - new_content_lines = new_content.lstrip().split("\n") - if new_content_lines and new_content_lines[0].strip() == section_header.strip(): - # Remove the duplicate header line - new_content = "\n".join(new_content_lines[1:]).lstrip() - - lines = current_content.split("\n") - # Fenced code can contain lines identical to the requested header, so section - # matching must skip fenced lines or a code sample would trigger the - # duplicate-header error (or match as the section itself). - fenced = _fenced_code_line_flags(lines) - - # First pass: count matching sections to check for duplicates - matching_sections = [ - i - for i, line in enumerate(lines) - if not fenced[i] and line.strip() == section_header.strip() - ] - - # Handle multiple sections error - if len(matching_sections) > 1: - raise ValueError( - f"Multiple sections found with header '{section_header}'. " - f"Section replacement requires unique headers." - ) - - # If no section found, append it - if len(matching_sections) == 0: - logger.info(f"Section '{section_header}' not found, appending to end of document") - separator = "\n\n" if current_content and not current_content.endswith("\n\n") else "" - return current_content + separator + section_header + "\n" + new_content - - # Replace the single matching section. The replaced span always ends at a - # boundary computed from the ORIGINAL lines, so headings inside new_content - # cannot extend or shorten what gets consumed (issue #1012). - section_line_idx = matching_sections[0] - target_level = len(section_header) - len(section_header.lstrip("#")) - - end_idx = len(lines) - for i in range(section_line_idx + 1, len(lines)): - if fenced[i]: - continue - heading_level = _markdown_heading_level(lines[i]) - if heading_level is None: - continue - # Level-aware default: an h2 section owns its h3+ subsections, so only a - # heading at the same or higher level ends it. The opt-out stops at any - # heading, preserving subsections (pre-#1012 behavior). - if not replace_subsections or heading_level <= target_level: - end_idx = i - break - - result_lines = lines[: section_line_idx + 1] - result_lines.append(new_content) - result_lines.extend(lines[end_idx:]) - return "\n".join(result_lines) + return replace_note_section_content( + current_content, + section_header, + new_content, + replace_subsections=replace_subsections, + ) def insert_relative_to_section( self, @@ -1752,80 +1008,12 @@ def insert_relative_to_section( Raises: ValueError: If the section header is not found or appears more than once """ - # Normalize the section header (ensure it starts with #) - if not section_header.startswith("#"): - section_header = "## " + section_header - - lines = current_content.split("\n") - # Fenced code can contain a line identical to the requested heading; skip - # fenced lines so a code sample never anchors (or duplicates) the section. - fenced = _fenced_code_line_flags(lines) - matching_indices = [ - i - for i, line in enumerate(lines) - if not fenced[i] and line.strip() == section_header.strip() - ] - - if len(matching_indices) == 0: - raise ValueError( - f"Section '{section_header}' not found in document. " - f"Use replace_section to create a new section." - ) - if len(matching_indices) > 1: - raise ValueError( - f"Multiple sections found with header '{section_header}'. " - f"Section insertion requires unique headers." - ) - - idx = matching_indices[0] - - if position == "before": - # Insert new content before the section heading - before = lines[:idx] - after = lines[idx:] - # Ensure blank line separation - insert_lines = new_content.rstrip("\n").split("\n") - if before and before[-1].strip() != "": - insert_lines = [""] + insert_lines - return "\n".join(before + insert_lines + [""] + after) - else: - # Insert new content after the section heading line - before = lines[: idx + 1] - after = lines[idx + 1 :] - insert_lines = new_content.rstrip("\n").split("\n") - # Ensure blank line separation so inserted text doesn't merge - # with existing section content into a single paragraph - if after and after[0].strip() != "": - insert_lines = insert_lines + [""] - return "\n".join(before + insert_lines + after) - - def _prepend_after_frontmatter(self, current_content: str, content: str) -> str: - """Prepend content after frontmatter, preserving frontmatter structure.""" - - # Trigger: the note starts with frontmatter delimiters. - # Why: prepend must preserve the existing YAML block and insert content into the body, - # not silently rewrite malformed metadata into a corrupted accepted note state. - # Outcome: valid frontmatter is preserved, and malformed frontmatter fails fast. - if has_frontmatter(current_content): - # Parse and separate frontmatter from body. Parse errors are intentional caller-visible - # failures so prepare_edit_entity_content can reject unsafe accepted writes. - frontmatter_data = parse_frontmatter(current_content) - body_content = remove_frontmatter(current_content) - - # Prepend content to the body - if content and not content.endswith("\n"): - new_body = content + "\n" + body_content - else: - new_body = content + body_content - - # Reconstruct file with frontmatter + prepended body - yaml_fm = yaml.dump(frontmatter_data, sort_keys=False, allow_unicode=True) - return f"---\n{yaml_fm}---\n\n{new_body.strip()}" - - # No frontmatter means prepend is a plain text edit. - if content and not content.endswith("\n"): - return content + "\n" + current_content - return content + current_content + return insert_note_relative_to_section( + current_content, + section_header, + new_content, + position, + ) async def move_entity( self, diff --git a/src/basic_memory/services/initialization.py b/src/basic_memory/services/initialization.py index df35d1e6e..04f33ae88 100644 --- a/src/basic_memory/services/initialization.py +++ b/src/basic_memory/services/initialization.py @@ -68,7 +68,7 @@ async def recover_project_materializations( picked up by the initial project index. Non-fatal: a recovery failure must not block startup, so it is logged and startup continues. """ - from basic_memory.cloud.note_content_materialization import recover_stuck_materializations + from basic_memory.index.note_content_materialization import recover_stuck_materializations from basic_memory.services.file_service import FileService try: diff --git a/src/basic_memory/services/note_content_reads.py b/src/basic_memory/services/note_content_reads.py new file mode 100644 index 000000000..c409b66f6 --- /dev/null +++ b/src/basic_memory/services/note_content_reads.py @@ -0,0 +1,202 @@ +"""Runtime-neutral note-content read service facade.""" + +from __future__ import annotations + +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from basic_memory import db +from basic_memory.indexing.note_content_read_repair_runner import ( + NoteContentReadRepairFileReader, + NoteContentReadView, + load_note_content_read_view_with_default_repositories, + note_content_resource_from_read_view, + note_content_response_payload_from_read_view, + prepare_note_content_read_repair_with_default_repositories, + run_note_content_read_repair_with_default_reconciler, +) +from basic_memory.models import Entity, NoteContent, Project +from basic_memory.runtime.note_content import ( + RuntimeNoteContentResource, + RuntimeNoteContentResponsePayload, +) + + +async def load_note_content_query_view( + *, + session_maker: async_sessionmaker[AsyncSession], + project_external_id: str, + entity_external_id: str, +) -> NoteContentReadView[Entity, NoteContent] | None: + """Load one project-scoped note view from current DB state.""" + async with db.scoped_session(session_maker) as session: + return await load_note_content_query_view_from_session( + session=session, + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ) + + +async def load_note_content_query_view_from_session( + *, + session: AsyncSession, + project_external_id: str, + entity_external_id: str, +) -> NoteContentReadView[Entity, NoteContent] | None: + """Load one project-scoped note view using caller-owned session scope.""" + return await load_note_content_read_view_with_default_repositories( + session, + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ) + + +class NoteContentQueryService: + """Load note-content rows and shape route-friendly read payloads.""" + + def __init__( + self, + *, + session_maker: async_sessionmaker[AsyncSession], + read_repair_file_reader: NoteContentReadRepairFileReader[Project, Entity] | None = None, + ) -> None: + self.session_maker = session_maker + self.read_repair_file_reader = read_repair_file_reader + + async def get_note_entity_payload( + self, + *, + project_external_id: str, + entity_external_id: str, + session: AsyncSession | None = None, + ) -> RuntimeNoteContentResponsePayload | None: + """Return the entity payload, enriching markdown notes from note_content.""" + if session is None: + note_view = await load_note_content_query_view( + session_maker=self.session_maker, + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ) + else: + note_view = await load_note_content_query_view_from_session( + session=session, + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ) + return note_content_response_payload_from_read_view(note_view) + + async def get_note_entity_payload_with_read_repair( + self, + *, + project_external_id: str, + entity_external_id: str, + session: AsyncSession | None = None, + source: str = "read_repair", + ) -> RuntimeNoteContentResponsePayload | None: + """Return entity payload, repairing missing note_content when a reader exists.""" + payload = await self.get_note_entity_payload( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + session=session, + ) + if payload is not None or self.read_repair_file_reader is None: + return payload + + repaired = await self.reconcile_note_content_from_file( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + source=source, + ) + if not repaired: + return None + # The repair commits through a separate scoped session, so reopen the read to + # avoid stale snapshots in caller-owned transactions. + return await self.get_note_entity_payload( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ) + + async def get_note_resource( + self, + *, + project_external_id: str, + entity_external_id: str, + session: AsyncSession | None = None, + ) -> RuntimeNoteContentResource | None: + """Return full markdown content from note_content when available.""" + if session is None: + note_view = await load_note_content_query_view( + session_maker=self.session_maker, + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ) + else: + note_view = await load_note_content_query_view_from_session( + session=session, + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ) + if note_view is None: + return None + + return note_content_resource_from_read_view(note_view) + + async def get_note_resource_with_read_repair( + self, + *, + project_external_id: str, + entity_external_id: str, + session: AsyncSession | None = None, + source: str = "read_repair", + ) -> RuntimeNoteContentResource | None: + """Return markdown resource, repairing missing note_content when possible.""" + resource = await self.get_note_resource( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + session=session, + ) + if resource is not None or self.read_repair_file_reader is None: + return resource + + repaired = await self.reconcile_note_content_from_file( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + source=source, + ) + if not repaired: + return None + # The repair commits through a separate scoped session, so reopen the read to + # avoid stale snapshots in caller-owned transactions. + return await self.get_note_resource( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ) + + async def reconcile_note_content_from_file( + self, + *, + project_external_id: str, + entity_external_id: str, + source: str, + ) -> bool: + """Repair a missing note_content row from the runtime's canonical file source.""" + async with db.scoped_session(self.session_maker) as session: + repair_preflight = await prepare_note_content_read_repair_with_default_repositories( + session, + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ) + if not repair_preflight.should_read_file: + return repair_preflight.repaired + + repair_preflight.require_target() + + if self.read_repair_file_reader is None: + raise RuntimeError("note-content read repair requires a file reader") + + repair_run = await run_note_content_read_repair_with_default_reconciler( + repair_preflight, + session_maker=self.session_maker, + file_reader=self.read_repair_file_reader, + source=source, + ) + return repair_run.repaired diff --git a/src/basic_memory/services/note_content_writes.py b/src/basic_memory/services/note_content_writes.py new file mode 100644 index 000000000..e6c5e0ae1 --- /dev/null +++ b/src/basic_memory/services/note_content_writes.py @@ -0,0 +1,381 @@ +"""Runtime-neutral note-content mutation service facade.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass +from typing import Literal, Protocol +from uuid import UUID + +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from basic_memory.indexing.accepted_note_mutation_runner import ( + AcceptedNoteCreateMutation, + AcceptedNoteDeleteMutation, + AcceptedNoteEditMutation, + AcceptedNoteMoveMutation, + AcceptedNoteMutationActor, + AcceptedNoteMutationDependencies, + AcceptedNoteMutationRejected, + AcceptedNoteMutationRejection, + AcceptedNoteUpdateMutation, + run_accepted_note_create, + run_accepted_note_delete, + run_accepted_note_edit, + run_accepted_note_move, + run_accepted_note_update, +) +from basic_memory.runtime.note_content import ( + RuntimeAcceptedNoteChange, + RuntimeNoteContentResponsePayload, +) +from basic_memory.schemas.base import Entity as EntitySchema +from basic_memory.schemas.request import EditEntityRequest + +AcceptedNoteChange = RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload] + + +class NoteContentMutationFreshener(Protocol): + """Refresh current runtime file state before mutating an existing note.""" + + async def freshen_note_content( + self, + *, + project_external_id: str, + entity_external_id: str, + ) -> None: ... + + +type NoteContentMutationKind = Literal["create", "update", "edit", "move"] + + +@dataclass(frozen=True, slots=True) +class NoteContentMutationActorContext: + """Who and what originated one accepted note mutation.""" + + user_profile_id: UUID | None + source: str + actor_kind: str | None = None + actor_name: str | None = None + + +class NoteContentMutationActorResolver(Protocol): + """Resolve the actor context for one mutation at the runtime boundary. + + Routes pass through whatever actor values they were called with; a runtime + adapter (e.g. cloud) can replace them with request-derived identity — user + profile, source header, MCP actor headers — without subclassing the service. + """ + + def resolve_mutation_actor( + self, + *, + mutation_kind: NoteContentMutationKind, + requested: NoteContentMutationActorContext, + ) -> NoteContentMutationActorContext: ... + + +# Route adapters place error.detail directly into the HTTP response body, so it +# is either a plain message string or an already-serialized structured detail +# (currently only the base-checksum conflict wire dict, issue #1445). +type NoteContentMutationErrorDetail = str | dict[str, str | None] + + +class NoteContentMutationServiceError(Exception): + """Structured note-content mutation service error for route adapters.""" + + def __init__(self, status_code: int, detail: NoteContentMutationErrorDetail) -> None: + super().__init__(str(detail)) + self.status_code = status_code + self.detail = detail + + +def note_content_mutation_error_from_rejection( + rejection: AcceptedNoteMutationRejection, +) -> NoteContentMutationServiceError: + """Map core accepted-note mutation rejections into route-facing errors.""" + detail = rejection.detail + # This mapping is the wire boundary: typed rejection details serialize to the + # JSON dict that HTTP routes place verbatim into the 4xx response body. + return NoteContentMutationServiceError( + rejection.kind.http_status_code, + detail if isinstance(detail, str) else detail.as_json_dict(), + ) + + +def accepted_note_mutation_actor( + *, + user_profile_id: UUID | None, + actor_kind: str | None, + actor_name: str | None, +) -> AcceptedNoteMutationActor: + """Build the typed accepted-note actor passed to core mutation runners.""" + return AcceptedNoteMutationActor( + user_profile_id=user_profile_id, + kind=actor_kind, + name=actor_name, + ) + + +@asynccontextmanager +async def accepted_note_transaction( + session_maker: async_sessionmaker[AsyncSession], +) -> AsyncIterator[AsyncSession]: + """Open one DB transaction for an accepted note mutation.""" + async with session_maker() as session: + async with session.begin(): + yield session + + +class NoteContentMutationService: + """Accept note mutations into DB state through core-owned mutation runners.""" + + def __init__( + self, + *, + session_maker: async_sessionmaker[AsyncSession], + mutation_dependencies: AcceptedNoteMutationDependencies, + content_freshener: NoteContentMutationFreshener | None = None, + actor_resolver: NoteContentMutationActorResolver | None = None, + ) -> None: + self.session_maker = session_maker + self.mutation_dependencies = mutation_dependencies + self.content_freshener = content_freshener + self.actor_resolver = actor_resolver + + def _resolve_actor( + self, + mutation_kind: NoteContentMutationKind, + *, + user_profile_id: UUID | None, + source: str, + actor_kind: str | None, + actor_name: str | None, + ) -> NoteContentMutationActorContext: + requested = NoteContentMutationActorContext( + user_profile_id=user_profile_id, + source=source, + actor_kind=actor_kind, + actor_name=actor_name, + ) + if self.actor_resolver is None: + return requested + return self.actor_resolver.resolve_mutation_actor( + mutation_kind=mutation_kind, + requested=requested, + ) + + async def freshen_existing_note_content( + self, + *, + project_external_id: str, + entity_external_id: str, + ) -> None: + """Let the runtime converge observed file state before an existing-note mutation.""" + if self.content_freshener is None: + return + await self.content_freshener.freshen_note_content( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ) + + async def create_note( + self, + *, + project_external_id: str, + data: EntitySchema, + user_profile_id: UUID | None, + source: str, + actor_kind: str | None = None, + actor_name: str | None = None, + ) -> AcceptedNoteChange: + """POST a new markdown note into accepted DB state.""" + actor_context = self._resolve_actor( + "create", + user_profile_id=user_profile_id, + source=source, + actor_kind=actor_kind, + actor_name=actor_name, + ) + try: + async with accepted_note_transaction(self.session_maker) as session: + return await run_accepted_note_create( + session, + request=AcceptedNoteCreateMutation( + project_external_id=project_external_id, + data=data, + actor=accepted_note_mutation_actor( + user_profile_id=actor_context.user_profile_id, + actor_kind=actor_context.actor_kind, + actor_name=actor_context.actor_name, + ), + source=actor_context.source, + ), + dependencies=self.mutation_dependencies, + ) + except AcceptedNoteMutationRejected as error: + raise note_content_mutation_error_from_rejection(error.rejection) from error + + async def update_note( + self, + *, + project_external_id: str, + entity_external_id: str, + data: EntitySchema, + user_profile_id: UUID | None, + source: str, + base_checksum: str | None = None, + actor_kind: str | None = None, + actor_name: str | None = None, + ) -> AcceptedNoteChange: + """PUT a markdown note by creating or replacing accepted DB state. + + ``base_checksum`` is an optional optimistic-concurrency precondition: the + db_checksum the caller last synced. When supplied, the update runner + rejects the write with a structured 409 if the accepted checksum has + moved, so the caller rebases instead of clobbering the newer write + (issue #1445). It stays optional so callers without a synced base still + write. + """ + actor_context = self._resolve_actor( + "update", + user_profile_id=user_profile_id, + source=source, + actor_kind=actor_kind, + actor_name=actor_name, + ) + try: + await self.freshen_existing_note_content( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ) + async with accepted_note_transaction(self.session_maker) as session: + return await run_accepted_note_update( + session, + request=AcceptedNoteUpdateMutation( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + data=data, + actor=accepted_note_mutation_actor( + user_profile_id=actor_context.user_profile_id, + actor_kind=actor_context.actor_kind, + actor_name=actor_context.actor_name, + ), + source=actor_context.source, + base_checksum=base_checksum, + ), + dependencies=self.mutation_dependencies, + ) + except AcceptedNoteMutationRejected as error: + raise note_content_mutation_error_from_rejection(error.rejection) from error + + async def edit_note( + self, + *, + project_external_id: str, + entity_external_id: str, + data: EditEntityRequest, + user_profile_id: UUID | None, + source: str, + actor_kind: str | None = None, + actor_name: str | None = None, + ) -> AcceptedNoteChange: + """PATCH a markdown note using the latest accepted DB content as the base.""" + actor_context = self._resolve_actor( + "edit", + user_profile_id=user_profile_id, + source=source, + actor_kind=actor_kind, + actor_name=actor_name, + ) + try: + await self.freshen_existing_note_content( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ) + async with accepted_note_transaction(self.session_maker) as session: + return await run_accepted_note_edit( + session, + request=AcceptedNoteEditMutation( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + data=data, + actor=accepted_note_mutation_actor( + user_profile_id=actor_context.user_profile_id, + actor_kind=actor_context.actor_kind, + actor_name=actor_context.actor_name, + ), + source=actor_context.source, + ), + dependencies=self.mutation_dependencies, + ) + except AcceptedNoteMutationRejected as error: + raise note_content_mutation_error_from_rejection(error.rejection) from error + + async def move_note( + self, + *, + project_external_id: str, + entity_external_id: str, + destination_path: str, + user_profile_id: UUID | None, + source: str, + actor_kind: str | None = None, + actor_name: str | None = None, + ) -> AcceptedNoteChange: + """Move a note by accepting the new path before runtime materialization.""" + actor_context = self._resolve_actor( + "move", + user_profile_id=user_profile_id, + source=source, + actor_kind=actor_kind, + actor_name=actor_name, + ) + try: + await self.freshen_existing_note_content( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ) + async with accepted_note_transaction(self.session_maker) as session: + return await run_accepted_note_move( + session, + request=AcceptedNoteMoveMutation( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + destination_path=destination_path, + actor=accepted_note_mutation_actor( + user_profile_id=actor_context.user_profile_id, + actor_kind=actor_context.actor_kind, + actor_name=actor_context.actor_name, + ), + source=actor_context.source, + ), + dependencies=self.mutation_dependencies, + ) + except AcceptedNoteMutationRejected as error: + raise note_content_mutation_error_from_rejection(error.rejection) from error + + async def delete_note( + self, + *, + project_external_id: str, + entity_external_id: str, + ) -> AcceptedNoteChange: + """DELETE the DB note and return the runtime follow-up change.""" + try: + await self.freshen_existing_note_content( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ) + async with accepted_note_transaction(self.session_maker) as session: + return await run_accepted_note_delete( + session, + request=AcceptedNoteDeleteMutation( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ), + dependencies=self.mutation_dependencies, + ) + except AcceptedNoteMutationRejected as error: + raise note_content_mutation_error_from_rejection(error.rejection) from error diff --git a/src/basic_memory/services/note_preparation.py b/src/basic_memory/services/note_preparation.py new file mode 100644 index 000000000..662038b21 --- /dev/null +++ b/src/basic_memory/services/note_preparation.py @@ -0,0 +1,926 @@ +"""Prepare canonical Markdown values without persistence side effects.""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path + +import frontmatter +import yaml +from loguru import logger +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from basic_memory import db +from basic_memory.config import BasicMemoryConfig +from basic_memory.file_utils import ( + ParseError, + dump_frontmatter, + has_frontmatter, + parse_frontmatter, + remove_frontmatter, +) +from basic_memory.markdown import EntityMarkdown +from basic_memory.markdown.entity_parser import ( + EntityParser, + _coerce_to_string, + normalize_frontmatter_metadata, +) +from basic_memory.markdown.utils import schema_to_markdown +from basic_memory.models import Entity +from basic_memory.repository import AcceptedObservationWrite, AcceptedRelationWrite +from basic_memory.repository.entity_repository import EntityMetadata, EntityRepository +from basic_memory.repository.project_repository import ProjectRepository +from basic_memory.runtime.note_move import normalize_note_move_destination_path +from basic_memory.schemas import Entity as EntitySchema +from basic_memory.schemas.base import Permalink +from basic_memory.services.exceptions import EntityAlreadyExistsError +from basic_memory.services.file_service import FileService +from basic_memory.utils import build_canonical_permalink +from basic_memory.workspace_context import workspace_slug_for_canonical_permalinks + + +@dataclass(frozen=True, slots=True) +class PreparedEntityFields: + """Entity row values that mirror one accepted Markdown snapshot.""" + + title: str + note_type: str + entity_metadata: EntityMetadata + content_type: str + permalink: str | None + file_path: str + created_at: datetime + updated_at: datetime + + +@dataclass(frozen=True, slots=True) +class PreparedEntityWrite: + """Canonical Markdown and its parsed graph before persistence.""" + + file_path: Path + markdown_content: str + search_content: str + entity_fields: PreparedEntityFields + entity_markdown: EntityMarkdown + + @property + def observations(self) -> list[AcceptedObservationWrite]: + return [ + AcceptedObservationWrite( + content=observation.content, + category=observation.category, + context=observation.context, + tags=observation.tags, + ) + for observation in self.entity_markdown.observations + ] + + @property + def relations(self) -> list[AcceptedRelationWrite]: + return [ + AcceptedRelationWrite( + relation_type=relation.type, + target_name=relation.target, + context=relation.context, + ) + for relation in self.entity_markdown.relations + ] + + +@dataclass(frozen=True, slots=True) +class PreparedEntityMove: + """Canonical Markdown and identity state for an accepted note move.""" + + file_path: Path + markdown_content: str + search_content: str + permalink: str | None + + +@dataclass(frozen=True, slots=True) +class PreparedEditTitleReconciliation: + markdown_content: str + title: str + metadata: EntityMetadata + + +@dataclass(frozen=True, slots=True) +class NotePreparationDependencies: + """Only the dependencies required to derive accepted note state.""" + + entity_parser: EntityParser + entity_repository: EntityRepository + file_service: FileService + session_maker: async_sessionmaker[AsyncSession] + app_config: BasicMemoryConfig | None = None + + +def apply_prepared_entity_fields( + entity: Entity, + entity_fields: PreparedEntityFields, + *, + user_profile_value: str | None, +) -> None: + """Copy prepared accepted Markdown fields onto an entity row.""" + entity.title = entity_fields.title + entity.note_type = entity_fields.note_type + entity.entity_metadata = entity_fields.entity_metadata + entity.content_type = entity_fields.content_type + entity.permalink = entity_fields.permalink + entity.file_path = entity_fields.file_path + entity.created_at = entity_fields.created_at + entity.updated_at = entity_fields.updated_at + entity.last_updated_by = user_profile_value + + +def _frontmatter_permalink(value: object) -> str | None: + return value if isinstance(value, str) and value else None + + +def _build_frontmatter_markdown(title: str, note_type: str, permalink: str) -> EntityMarkdown: + from basic_memory.markdown.schemas import EntityFrontmatter + + return EntityMarkdown( + frontmatter=EntityFrontmatter( + metadata={"title": title, "type": note_type, "permalink": permalink} + ), + content="", + observations=[], + relations=[], + ) + + +async def detect_file_path_conflicts( + dependencies: NotePreparationDependencies, + file_path: str, + *, + skip_check: bool = False, + session: AsyncSession | None = None, +) -> list[str]: + if skip_check: + return [] + + from basic_memory.utils import detect_potential_file_conflicts + + async with db.scoped_session(dependencies.session_maker, session) as active_session: + existing_paths = await dependencies.entity_repository.get_all_file_paths(active_session) + return detect_potential_file_conflicts(file_path, existing_paths) + + +async def _project_permalink( + dependencies: NotePreparationDependencies, + session: AsyncSession, +) -> str | None: + project_id = dependencies.entity_repository.project_id + if project_id is None: # pragma: no cover + return None + project = await ProjectRepository().get_by_id(session, project_id) + return project.permalink if project else None + + +async def resolve_permalink( + dependencies: NotePreparationDependencies, + file_path: Permalink | Path, + markdown: EntityMarkdown | None = None, + *, + skip_conflict_check: bool = False, + session: AsyncSession | None = None, +) -> str: + """Resolve the unique canonical permalink for one prepared note.""" + file_path_str = Path(file_path).as_posix() + async with db.scoped_session(dependencies.session_maker, session) as active_session: + conflicts = await detect_file_path_conflicts( + dependencies, + file_path_str, + skip_check=skip_conflict_check, + session=active_session, + ) + if conflicts: + logger.warning( + f"Detected potential file path conflicts for '{file_path_str}': {conflicts}" + ) + + if markdown and markdown.frontmatter.permalink: + desired_permalink = markdown.frontmatter.permalink + existing_file_path = await dependencies.entity_repository.get_file_path_for_permalink( + active_session, desired_permalink + ) + if not existing_file_path or existing_file_path == file_path_str: + return desired_permalink + + existing_permalink = await dependencies.entity_repository.get_permalink_for_file_path( + active_session, file_path_str + ) + if existing_permalink: + return existing_permalink + + if markdown and markdown.frontmatter.permalink: + desired_permalink = markdown.frontmatter.permalink + else: + include_project = ( + dependencies.app_config.permalinks_include_project + if dependencies.app_config is not None + else True + ) + workspace_permalink = workspace_slug_for_canonical_permalinks() + project_permalink = None + if include_project or workspace_permalink: + project_permalink = await _project_permalink(dependencies, active_session) + desired_permalink = build_canonical_permalink( + project_permalink, + file_path_str, + include_project=include_project, + workspace_permalink=workspace_permalink, + ) + + permalink = desired_permalink + suffix = 1 + while await dependencies.entity_repository.permalink_exists(active_session, permalink): + permalink = f"{desired_permalink}-{suffix}" + suffix += 1 + return permalink + + +def _apply_schema_frontmatter_overrides(schema: EntitySchema) -> EntityMarkdown | None: + if not schema.content or not has_frontmatter(schema.content): + return None + content_frontmatter = parse_frontmatter(schema.content) + if "type" in content_frontmatter: + schema.note_type = _coerce_to_string(content_frontmatter["type"]) + content_permalink = _frontmatter_permalink(content_frontmatter.get("permalink")) + if content_permalink is None: + return None + return _build_frontmatter_markdown(schema.title, schema.note_type, content_permalink) + + +async def _resolve_schema_permalink( + dependencies: NotePreparationDependencies, + schema: EntitySchema, + *, + file_path: Path, + current_permalink: str | None = None, + content_markdown: EntityMarkdown | None = None, + skip_conflict_check: bool = False, + session: AsyncSession | None = None, +) -> str | None: + if dependencies.app_config and dependencies.app_config.disable_permalinks: + schema._permalink = current_permalink or "" + return current_permalink + if current_permalink and not (content_markdown and content_markdown.frontmatter.permalink): + schema._permalink = current_permalink + return current_permalink + resolved = await resolve_permalink( + dependencies, + file_path, + content_markdown, + skip_conflict_check=skip_conflict_check, + session=session, + ) + schema._permalink = resolved + return resolved + + +def _build_entity_fields( + *, + file_path: Path, + content_type: str, + permalink: str | None, + entity_markdown: EntityMarkdown, +) -> PreparedEntityFields: + if entity_markdown.created is None or entity_markdown.modified is None: # pragma: no cover + raise ValueError("Prepared Markdown requires created and modified timestamps") + + normalized_metadata = normalize_frontmatter_metadata(entity_markdown.frontmatter.metadata or {}) + entity_metadata = { + key: value for key, value in normalized_metadata.items() if value is not None + } + return PreparedEntityFields( + title=entity_markdown.frontmatter.title, + note_type=entity_markdown.frontmatter.type, + entity_metadata=entity_metadata or None, + content_type=content_type, + permalink=permalink, + file_path=file_path.as_posix(), + created_at=entity_markdown.created, + updated_at=entity_markdown.modified, + ) + + +async def _build_prepared_write( + dependencies: NotePreparationDependencies, + *, + file_path: Path, + markdown_content: str, + content_type: str, + permalink: str | None, + preserved_created_at: datetime | None = None, +) -> PreparedEntityWrite: + entity_markdown = await dependencies.entity_parser.parse_markdown_content( + file_path=file_path, + content=markdown_content, + # DB-first updates have no file ctime. Preserve the semantic creation time + # so editing a legacy note cannot make it appear newly created. + ctime=(preserved_created_at.timestamp() if preserved_created_at is not None else None), + ) + return PreparedEntityWrite( + file_path=file_path, + markdown_content=markdown_content, + search_content=remove_frontmatter(markdown_content), + entity_fields=_build_entity_fields( + file_path=file_path, + content_type=content_type, + permalink=permalink, + entity_markdown=entity_markdown, + ), + entity_markdown=entity_markdown, + ) + + +async def prepare_create_entity_content( + dependencies: NotePreparationDependencies, + schema: EntitySchema, + *, + check_storage_exists: bool = True, + skip_conflict_check: bool = False, + session: AsyncSession | None = None, +) -> PreparedEntityWrite: + schema = schema.model_copy(deep=True) + file_path = Path(schema.file_path) + if check_storage_exists and await dependencies.file_service.exists(file_path): + raise EntityAlreadyExistsError( + f"file for entity {schema.directory}/{schema.title} already exists: {file_path}" + ) + content_markdown = _apply_schema_frontmatter_overrides(schema) + permalink = await _resolve_schema_permalink( + dependencies, + schema, + file_path=file_path, + content_markdown=content_markdown, + skip_conflict_check=skip_conflict_check, + session=session, + ) + post = await schema_to_markdown(schema) + markdown_content = dump_frontmatter(post) + return await _build_prepared_write( + dependencies, + file_path=file_path, + markdown_content=markdown_content, + content_type=schema.content_type, + permalink=permalink, + ) + + +async def prepare_update_entity_content( + dependencies: NotePreparationDependencies, + entity: Entity, + schema: EntitySchema, + existing_content: str, + *, + skip_conflict_check: bool = False, + session: AsyncSession | None = None, +) -> PreparedEntityWrite: + schema = schema.model_copy(deep=True) + file_path = Path(schema.file_path) + current_file_path = Path(entity.file_path) + existing_metadata: dict[str, object] = {} + if has_frontmatter(existing_content): + try: + existing_metadata = parse_frontmatter(existing_content) + except ParseError: + # A replacement may repair malformed existing frontmatter. Discard only + # that invalid merge input; the final accepted Markdown is validated below. + pass + content_markdown = _apply_schema_frontmatter_overrides(schema) + update_permalink_on_rename = bool( + dependencies.app_config and dependencies.app_config.update_permalinks_on_move + ) + current_permalink = ( + entity.permalink + if file_path.as_posix() == current_file_path.as_posix() or not update_permalink_on_rename + else None + ) + resolved_permalink = await _resolve_schema_permalink( + dependencies, + schema, + file_path=file_path, + current_permalink=current_permalink, + content_markdown=content_markdown, + skip_conflict_check=skip_conflict_check, + session=session, + ) + post = await schema_to_markdown(schema) + merged_metadata = deepcopy(existing_metadata) + merged_metadata.update(post.metadata) + merged_metadata["permalink"] = resolved_permalink + merged_post = frontmatter.Post(post.content) + merged_post.metadata.update(merged_metadata) + markdown_content = dump_frontmatter(merged_post) + return await _build_prepared_write( + dependencies, + file_path=file_path, + markdown_content=markdown_content, + content_type=schema.content_type, + permalink=resolved_permalink, + preserved_created_at=entity.created_at, + ) + + +def reconcile_prepared_edit_title_from_h1( + *, + original_markdown: str, + markdown_content: str, + current_title: str | None, + prepared_title: str, + metadata: EntityMetadata, +) -> PreparedEditTitleReconciliation: + from basic_memory.indexing.accepted_note_search import ( + accepted_search_content_from_markdown, + first_markdown_h1, + ) + + original_h1 = first_markdown_h1(accepted_search_content_from_markdown(original_markdown)) + prepared_h1 = first_markdown_h1(accepted_search_content_from_markdown(markdown_content)) + if ( + not prepared_h1 + or prepared_h1 == prepared_title + or original_h1 != current_title + or prepared_title != current_title + ): + return PreparedEditTitleReconciliation(markdown_content, prepared_title, metadata) + post = frontmatter.loads(markdown_content) + post.metadata["title"] = prepared_h1 + reconciled_metadata = {**metadata, "title": prepared_h1} if metadata is not None else None + return PreparedEditTitleReconciliation(dump_frontmatter(post), prepared_h1, reconciled_metadata) + + +def _markdown_heading_level(line: str) -> int | None: + indent = len(line) - len(line.lstrip(" ")) + if indent > 3: + return None + candidate = line[indent:] + if not candidate.startswith("#"): + return None + level = len(candidate) - len(candidate.lstrip("#")) + if level > 6: + return None + rest = candidate[level:] + return level if not rest or rest.startswith((" ", "\t")) else None + + +def _fence_marker(line: str) -> tuple[str, int, str] | None: + indent = len(line) - len(line.lstrip(" ")) + if indent > 3: + return None + candidate = line[indent:] + if not candidate or candidate[0] not in ("`", "~"): + return None + marker = candidate[0] + marker_length = len(candidate) - len(candidate.lstrip(marker)) + if marker_length < 3: + return None + return marker, marker_length, candidate[marker_length:] + + +def _fenced_code_line_flags(lines: list[str]) -> list[bool]: + flags: list[bool] = [] + open_marker: str | None = None + open_length = 0 + for line in lines: + marker = _fence_marker(line) + if open_marker is None: + if marker is None: + flags.append(False) + continue + marker_char, marker_length, suffix = marker + if marker_char == "`" and "`" in suffix: + flags.append(False) + continue + flags.append(True) + open_marker = marker_char + open_length = marker_length + continue + flags.append(True) + if marker is not None: + marker_char, marker_length, suffix = marker + if marker_char == open_marker and marker_length >= open_length and not suffix.strip(): + open_marker = None + open_length = 0 + return flags + + +def replace_section_content( + current_content: str, + section_header: str, + new_content: str, + replace_subsections: bool = True, +) -> str: + if not section_header.startswith("#"): + section_header = "## " + section_header + new_content_lines = new_content.lstrip().split("\n") + if new_content_lines and new_content_lines[0].strip() == section_header.strip(): + new_content = "\n".join(new_content_lines[1:]).lstrip() + lines = current_content.split("\n") + fenced = _fenced_code_line_flags(lines) + matches = [ + index + for index, line in enumerate(lines) + if not fenced[index] and line.strip() == section_header.strip() + ] + if len(matches) > 1: + raise ValueError( + f"Multiple sections found with header '{section_header}'. " + "Section replacement requires unique headers." + ) + if not matches: + logger.info(f"Section '{section_header}' not found, appending to end of document") + separator = "\n\n" if current_content and not current_content.endswith("\n\n") else "" + return current_content + separator + section_header + "\n" + new_content + section_line_index = matches[0] + target_level = len(section_header) - len(section_header.lstrip("#")) + end_index = len(lines) + for index in range(section_line_index + 1, len(lines)): + if fenced[index]: + continue + heading_level = _markdown_heading_level(lines[index]) + if heading_level is not None and (not replace_subsections or heading_level <= target_level): + end_index = index + break + return "\n".join([*lines[: section_line_index + 1], new_content, *lines[end_index:]]) + + +def insert_relative_to_section( + current_content: str, + section_header: str, + new_content: str, + position: str, +) -> str: + if not section_header.startswith("#"): + section_header = "## " + section_header + lines = current_content.split("\n") + fenced = _fenced_code_line_flags(lines) + matches = [ + index + for index, line in enumerate(lines) + if not fenced[index] and line.strip() == section_header.strip() + ] + if not matches: + raise ValueError( + f"Section '{section_header}' not found in document. " + "Use replace_section to create a new section." + ) + if len(matches) > 1: + raise ValueError( + f"Multiple sections found with header '{section_header}'. " + "Section insertion requires unique headers." + ) + index = matches[0] + insert_lines = new_content.rstrip("\n").split("\n") + if position == "before": + before = lines[:index] + if before and before[-1].strip(): + insert_lines = ["", *insert_lines] + return "\n".join([*before, *insert_lines, "", *lines[index:]]) + after = lines[index + 1 :] + if after and after[0].strip(): + insert_lines.append("") + return "\n".join([*lines[: index + 1], *insert_lines, *after]) + + +def _prepend_after_frontmatter(current_content: str, content: str) -> str: + if has_frontmatter(current_content): + frontmatter_data = parse_frontmatter(current_content) + body_content = remove_frontmatter(current_content) + new_body = content + ("\n" if content and not content.endswith("\n") else "") + new_body += body_content + yaml_frontmatter = yaml.dump(frontmatter_data, sort_keys=False, allow_unicode=True) + return f"---\n{yaml_frontmatter}---\n\n{new_body.strip()}" + return content + ("\n" if content and not content.endswith("\n") else "") + current_content + + +def apply_edit_operation( + current_content: str, + operation: str, + content: str, + section: str | None = None, + find_text: str | None = None, + expected_replacements: int = 1, + replace_subsections: bool = True, +) -> str: + if operation == "append": + return ( + current_content + + ("\n" if current_content and not current_content.endswith("\n") else "") + + content + ) + if operation == "prepend": + return _prepend_after_frontmatter(current_content, content) + if operation == "find_replace": + if not find_text: + raise ValueError("find_text is required for find_replace operation") + if not find_text.strip(): + raise ValueError("find_text cannot be empty or whitespace only") + actual_count = current_content.count(find_text) + if actual_count != expected_replacements: + if actual_count == 0: + raise ValueError(f"Text to replace not found: '{find_text}'") + raise ValueError( + f"Expected {expected_replacements} occurrences of '{find_text}', " + f"but found {actual_count}" + ) + return current_content.replace(find_text, content) + if operation == "replace_section": + if not section: + raise ValueError("section is required for replace_section operation") + if not section.strip(): + raise ValueError("section cannot be empty or whitespace only") + return replace_section_content( + current_content, section, content, replace_subsections=replace_subsections + ) + if operation in ("insert_before_section", "insert_after_section"): + if not section: + raise ValueError("section is required for insert section operations") + if not section.strip(): + raise ValueError("section cannot be empty or whitespace only") + position = "before" if operation == "insert_before_section" else "after" + return insert_relative_to_section(current_content, section, content, position) + raise ValueError(f"Unsupported operation: {operation}") + + +async def prepare_edit_entity_content( + dependencies: NotePreparationDependencies, + entity: Entity, + current_content: str, + *, + operation: str, + content: str, + section: str | None = None, + find_text: str | None = None, + expected_replacements: int = 1, + replace_subsections: bool = True, + skip_conflict_check: bool = False, + session: AsyncSession | None = None, +) -> PreparedEntityWrite: + file_path = Path(entity.file_path) + markdown_content = apply_edit_operation( + current_content, + operation, + content, + section, + find_text, + expected_replacements, + replace_subsections, + ) + title = entity.title + note_type = entity.note_type + permalink = entity.permalink + metadata = entity.entity_metadata + if has_frontmatter(markdown_content): + content_frontmatter = parse_frontmatter(markdown_content) + if "title" in content_frontmatter: + title = _coerce_to_string(content_frontmatter["title"]) + if "type" in content_frontmatter: + note_type = _coerce_to_string(content_frontmatter["type"]) + if dependencies.app_config and dependencies.app_config.disable_permalinks: + permalink = entity.permalink + else: + content_permalink = _frontmatter_permalink(content_frontmatter.get("permalink")) + if content_permalink is not None: + permalink = await resolve_permalink( + dependencies, + file_path, + _build_frontmatter_markdown(title, note_type, content_permalink), + skip_conflict_check=skip_conflict_check, + session=session, + ) + normalized_metadata = normalize_frontmatter_metadata(content_frontmatter or {}) + metadata = { + key: value for key, value in normalized_metadata.items() if value is not None + } or None + reconciliation = reconcile_prepared_edit_title_from_h1( + original_markdown=current_content, + markdown_content=markdown_content, + current_title=entity.title, + prepared_title=title, + metadata=metadata, + ) + return await _build_prepared_write( + dependencies, + file_path=file_path, + markdown_content=reconciliation.markdown_content, + content_type=entity.content_type, + permalink=permalink, + preserved_created_at=entity.created_at, + ) + + +async def prepare_move_entity_content( + dependencies: NotePreparationDependencies, + entity: Entity, + current_content: str, + destination_path: str, + *, + session: AsyncSession | None = None, +) -> PreparedEntityMove: + from basic_memory.indexing.accepted_note_search import accepted_search_content_from_markdown + + file_path = Path(normalize_note_move_destination_path(destination_path)) + markdown_content = current_content + permalink = entity.permalink + disable_permalinks = bool( + dependencies.app_config and dependencies.app_config.disable_permalinks + ) + update_permalinks_on_move = bool( + dependencies.app_config and dependencies.app_config.update_permalinks_on_move + ) + if not disable_permalinks and (update_permalinks_on_move or entity.permalink is None): + permalink = await resolve_permalink(dependencies, file_path, session=session) + post = frontmatter.loads(markdown_content) + post.metadata["permalink"] = permalink + markdown_content = dump_frontmatter(post) + return PreparedEntityMove( + file_path=file_path, + markdown_content=markdown_content, + search_content=accepted_search_content_from_markdown(markdown_content), + permalink=permalink, + ) + + +def paths_share_storage_target(file_service: FileService, left: Path, right: Path) -> bool: + left_path = file_service.base_path / left + right_path = file_service.base_path / right + if not left_path.exists() or not right_path.exists(): + return False + try: + return left_path.samefile(right_path) + except OSError: + return False + + +async def verify_move_destination_absent( + dependencies: NotePreparationDependencies, + *, + source_file_path: str, + destination_file_path: str, +) -> None: + source = Path(source_file_path) + destination = Path(normalize_note_move_destination_path(destination_file_path)) + if ( + source != destination + and await dependencies.file_service.exists(destination) + and not paths_share_storage_target(dependencies.file_service, source, destination) + ): + raise EntityAlreadyExistsError( + f"file already exists at destination path: {destination.as_posix()}" + ) + + +async def resolve_deferred_self_relation( + dependencies: NotePreparationDependencies, + target: str, + entity: Entity, + session: AsyncSession | None = None, +) -> Entity | None: + clean_target = target.strip() + if clean_target.startswith("[[") and clean_target.endswith("]]"): + clean_target = clean_target[2:-2].strip() + if "|" in clean_target: + clean_target = clean_target.split("|", 1)[0].strip() + candidates = {entity.file_path} + if entity.permalink: + candidates.add(entity.permalink) + if entity.file_path.endswith(".md"): + candidates.add(entity.file_path[:-3]) + if clean_target in candidates: + return entity + if clean_target != entity.title: + return None + async with db.scoped_session(dependencies.session_maker, session) as active_session: + matches = await dependencies.entity_repository.get_by_title( + active_session, clean_target, load_relations=False + ) + return entity if len(matches) == 1 and matches[0].id == entity.id else None + + +@dataclass(frozen=True, slots=True) +class NotePreparation: + """Method-shaped adapter for accepted-note preparation protocols.""" + + dependencies: NotePreparationDependencies + + async def detect_file_path_conflicts( + self, file_path: str, skip_check: bool = False, session: AsyncSession | None = None + ) -> list[str]: + return await detect_file_path_conflicts( + self.dependencies, file_path, skip_check=skip_check, session=session + ) + + async def resolve_permalink( + self, + file_path: Permalink | Path, + markdown: EntityMarkdown | None = None, + skip_conflict_check: bool = False, + session: AsyncSession | None = None, + ) -> str: + return await resolve_permalink( + self.dependencies, + file_path, + markdown, + skip_conflict_check=skip_conflict_check, + session=session, + ) + + async def prepare_create_entity_content( + self, + schema: EntitySchema, + *, + check_storage_exists: bool = True, + skip_conflict_check: bool = False, + session: AsyncSession | None = None, + ) -> PreparedEntityWrite: + return await prepare_create_entity_content( + self.dependencies, + schema, + check_storage_exists=check_storage_exists, + skip_conflict_check=skip_conflict_check, + session=session, + ) + + async def prepare_update_entity_content( + self, + entity: Entity, + schema: EntitySchema, + existing_content: str, + *, + skip_conflict_check: bool = False, + session: AsyncSession | None = None, + ) -> PreparedEntityWrite: + return await prepare_update_entity_content( + self.dependencies, + entity, + schema, + existing_content, + skip_conflict_check=skip_conflict_check, + session=session, + ) + + async def prepare_edit_entity_content( + self, + entity: Entity, + current_content: str, + *, + operation: str, + content: str, + section: str | None = None, + find_text: str | None = None, + expected_replacements: int = 1, + replace_subsections: bool = True, + skip_conflict_check: bool = False, + session: AsyncSession | None = None, + ) -> PreparedEntityWrite: + return await prepare_edit_entity_content( + self.dependencies, + entity, + current_content, + operation=operation, + content=content, + section=section, + find_text=find_text, + expected_replacements=expected_replacements, + replace_subsections=replace_subsections, + skip_conflict_check=skip_conflict_check, + session=session, + ) + + async def prepare_move_entity_content( + self, + entity: Entity, + current_content: str, + destination_path: str, + *, + session: AsyncSession | None = None, + ) -> PreparedEntityMove: + return await prepare_move_entity_content( + self.dependencies, + entity, + current_content, + destination_path, + session=session, + ) + + async def verify_move_destination_absent( + self, *, source_file_path: str, destination_file_path: str + ) -> None: + await verify_move_destination_absent( + self.dependencies, + source_file_path=source_file_path, + destination_file_path=destination_file_path, + ) + + async def resolve_deferred_self_relation( + self, target: str, entity: Entity, session: AsyncSession | None = None + ) -> Entity | None: + return await resolve_deferred_self_relation( + self.dependencies, target, entity, session=session + ) diff --git a/src/basic_memory/services/project_deletes.py b/src/basic_memory/services/project_deletes.py new file mode 100644 index 000000000..cab780511 --- /dev/null +++ b/src/basic_memory/services/project_deletes.py @@ -0,0 +1,130 @@ +"""Runtime-neutral project-delete acceptance orchestration.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from basic_memory.indexing.project_delete_acceptance import ProjectDeleteAcceptedResult +from basic_memory.models import Project +from basic_memory.runtime.jobs import RuntimeJobId, RuntimeProjectDeleteJobRequest +from basic_memory.schemas.project_info import ProjectItem + + +class ProjectDeleteAcceptanceError(Exception): + """Structured project-delete acceptance error for HTTP/API adapters.""" + + def __init__(self, status_code: int, detail: str) -> None: + super().__init__(detail) + self.status_code = status_code + self.detail = detail + + +class ProjectDeleteJobEnqueuer(Protocol): + """Capability that accepts background project-delete cleanup work.""" + + async def enqueue_project_delete( + self, + request: RuntimeProjectDeleteJobRequest, + ) -> RuntimeJobId: ... + + +@dataclass(frozen=True, slots=True) +class ProjectDeleteAcceptanceRequest: + """Route-level request for accepting a project delete.""" + + project_external_id: str + delete_notes: bool + + +async def load_project_for_delete_acceptance( + session: AsyncSession, + *, + project_external_id: str, +) -> Project | None: + """Load the project row that the request is about to hide.""" + result = await session.execute( + select(Project) + .where(Project.external_id == project_external_id) + .with_for_update(of=Project) + .limit(1) + ) + return result.scalars().one_or_none() + + +async def reactivate_accepted_project( + session_maker: async_sessionmaker[AsyncSession], + *, + project_id: int, +) -> None: + """Undo a soft delete when the background queue rejects the request.""" + async with session_maker() as session: + project = await session.get(Project, project_id) + if project is None: + return + project.is_active = True + await session.commit() + + +@dataclass(frozen=True, slots=True) +class ProjectDeleteAcceptanceService: + """Accept project deletes quickly and leave slow cleanup to a runtime adapter.""" + + session_maker: async_sessionmaker[AsyncSession] + job_enqueuer: ProjectDeleteJobEnqueuer + + async def delete_project( + self, + request: ProjectDeleteAcceptanceRequest, + ) -> ProjectDeleteAcceptedResult: + async with self.session_maker() as session: + project = await load_project_for_delete_acceptance( + session, + project_external_id=request.project_external_id, + ) + if project is None or not project.is_active: + raise ProjectDeleteAcceptanceError( + 404, + f"Project with external_id '{request.project_external_id}' not found", + ) + if project.is_default: + raise ProjectDeleteAcceptanceError( + 400, + f"Cannot delete default project '{project.name}'. " + "Set another project as default first.", + ) + + runtime_request = RuntimeProjectDeleteJobRequest( + project_id=project.id, + project_external_id=project.external_id, + project_name=project.name, + project_path=project.path, + delete_notes=request.delete_notes, + ) + old_project = ProjectItem( + id=project.id, + external_id=project.external_id, + name=project.name, + path=project.path, + is_default=project.is_default or False, + ) + project.is_active = False + await session.commit() + + try: + job_id = await self.job_enqueuer.enqueue_project_delete(runtime_request) + except Exception: + await reactivate_accepted_project( + self.session_maker, + project_id=runtime_request.project_id, + ) + raise + + return ProjectDeleteAcceptedResult.queued( + request=runtime_request, + job_id=job_id, + old_project=old_project, + ) diff --git a/tests/api/v2/test_accepted_note_atomicity.py b/tests/api/v2/test_accepted_note_atomicity.py new file mode 100644 index 000000000..339f3a409 --- /dev/null +++ b/tests/api/v2/test_accepted_note_atomicity.py @@ -0,0 +1,210 @@ +"""Route regressions for indivisible accepted-note snapshots.""" + +from dataclasses import dataclass, field +from pathlib import Path + +import pytest +from fastapi import FastAPI +from httpx import AsyncClient +from sqlalchemy import select, text +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from basic_memory import db +from basic_memory.deps.services import get_note_content_materialization_provider +from basic_memory.models import Entity, NoteContent, Observation, Project, Relation +from basic_memory.runtime.note_content import ( + RuntimeAcceptedNoteChange, + RuntimeNoteContentResponsePayload, + runtime_note_content_payload_as_dict, +) +from basic_memory.schemas.v2 import EntityResponseV2 + + +@dataclass(frozen=True, slots=True) +class PersistedAcceptedSnapshot: + entity: Entity + note_content: NoteContent + observations: tuple[Observation, ...] + relations: tuple[Relation, ...] + search_content: str + + +async def _load_persisted_snapshot( + session_maker: async_sessionmaker[AsyncSession], + *, + project_id: int, + entity_id: int, +) -> PersistedAcceptedSnapshot: + async with db.scoped_session(session_maker) as session: + entity = await session.get(Entity, entity_id) + note_content = await session.get(NoteContent, entity_id) + observations = tuple( + ( + await session.scalars( + select(Observation) + .where( + Observation.project_id == project_id, + Observation.entity_id == entity_id, + ) + .order_by(Observation.id) + ) + ).all() + ) + relations = tuple( + ( + await session.scalars( + select(Relation) + .where( + Relation.project_id == project_id, + Relation.from_id == entity_id, + ) + .order_by(Relation.id) + ) + ).all() + ) + search_content = ( + await session.execute( + text(""" + SELECT content_stems + FROM search_index + WHERE project_id = :project_id + AND entity_id = :entity_id + AND type = 'entity' + """), + {"project_id": project_id, "entity_id": entity_id}, + ) + ).scalar_one() + + assert entity is not None + assert note_content is not None + return PersistedAcceptedSnapshot( + entity=entity, + note_content=note_content, + observations=observations, + relations=relations, + search_content=str(search_content), + ) + + +@dataclass(slots=True) +class InspectingNoteContentMaterializer: + """Capture committed DB state at the materialization boundary without writing a file.""" + + session_maker: async_sessionmaker[AsyncSession] + project_id: int + accepted_changes: list[RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload]] = field( + default_factory=list + ) + persisted_snapshots: list[PersistedAcceptedSnapshot] = field(default_factory=list) + + async def materialize_write_change( + self, + accepted: RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload], + ) -> RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload]: + payload = runtime_note_content_payload_as_dict(accepted.payload) + entity_id = payload.get("id") + assert isinstance(entity_id, int) + + self.persisted_snapshots.append( + await _load_persisted_snapshot( + self.session_maker, + project_id=self.project_id, + entity_id=entity_id, + ) + ) + self.accepted_changes.append(accepted) + return accepted + + +@pytest.mark.asyncio +async def test_create_and_update_persist_complete_snapshot_at_materialization_boundary( + app: FastAPI, + client: AsyncClient, + db_backend: str, + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, + v2_project_url: str, +) -> None: + """Create and replace expose complete committed state before the file write starts.""" + if db_backend != "sqlite": + pytest.skip("This regression intentionally inspects the SQLite FTS row") + + materializer = InspectingNoteContentMaterializer( + session_maker=session_maker, + project_id=test_project.id, + ) + app.dependency_overrides[get_note_content_materialization_provider] = lambda: materializer + + create_response = await client.post( + f"{v2_project_url}/knowledge/entities", + json={ + "title": "Atomic Snapshot", + "directory": "accepted", + "content": """ +# Atomic Snapshot + +## Facts +- [fact] Create snapshot observation +- "created link" [[Create Target]] +""", + }, + ) + + assert create_response.status_code == 202 + created = EntityResponseV2.model_validate(create_response.json()) + assert created.content is not None + assert created.file_write_status == "pending" + note_path = Path(test_project.path) / created.file_path + assert not note_path.exists() + + assert len(materializer.persisted_snapshots) == 1 + created_snapshot = materializer.persisted_snapshots[0] + assert created_snapshot.entity.id == created.id + assert created_snapshot.entity.title == "Atomic Snapshot" + assert created_snapshot.note_content.markdown_content == created.content + assert created_snapshot.note_content.db_version == 1 + assert created_snapshot.note_content.file_write_status == "pending" + assert [observation.content for observation in created_snapshot.observations] == [ + "Create snapshot observation" + ] + assert [ + (relation.relation_type, relation.to_name) for relation in created_snapshot.relations + ] == [("created link", "Create Target")] + assert "Create snapshot observation" in created_snapshot.search_content + + update_response = await client.put( + f"{v2_project_url}/knowledge/entities/{created.external_id}", + json={ + "title": "Atomic Snapshot", + "directory": "accepted", + "content": """ +# Atomic Snapshot + +## Replaced Facts +- [decision] Replacing update observation +- "updated link" [[Update Target]] +""", + }, + ) + + assert update_response.status_code == 202 + updated = EntityResponseV2.model_validate(update_response.json()) + assert updated.content is not None + assert updated.file_write_status == "pending" + assert not note_path.exists() + + assert len(materializer.persisted_snapshots) == 2 + updated_snapshot = materializer.persisted_snapshots[1] + assert updated_snapshot.entity.id == updated.id + assert updated_snapshot.note_content.markdown_content == updated.content + assert updated_snapshot.note_content.db_version == 2 + assert updated_snapshot.note_content.file_write_status == "pending" + assert [observation.content for observation in updated_snapshot.observations] == [ + "Replacing update observation" + ] + assert [ + (relation.relation_type, relation.to_name) for relation in updated_snapshot.relations + ] == [("updated link", "Update Target")] + assert "Replacing update observation" in updated_snapshot.search_content + assert "Create snapshot observation" not in updated_snapshot.search_content + assert len(materializer.accepted_changes) == 2 diff --git a/tests/cli/test_command_utils.py b/tests/cli/test_command_utils.py index c8b6d1e40..55e1bff69 100644 --- a/tests/cli/test_command_utils.py +++ b/tests/cli/test_command_utils.py @@ -1,6 +1,6 @@ """Tests for CLI command utilities.""" -import basic_memory.cloud.note_content_materialization as note_content_materialization +import basic_memory.index.note_content_materialization as note_content_materialization import basic_memory.db as db import basic_memory.index.local_schedulers as local_schedulers from basic_memory.cli.commands.command_utils import run_with_cleanup diff --git a/tests/cloud/test_cloud_services.py b/tests/cloud/test_cloud_services.py index 0766d7e54..7063093bf 100644 --- a/tests/cloud/test_cloud_services.py +++ b/tests/cloud/test_cloud_services.py @@ -36,14 +36,14 @@ from basic_memory.schemas.request import EditEntityRequest from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker -import basic_memory.cloud.note_content_reads as note_content_reads -import basic_memory.cloud.note_content_writes as note_content_writes -from basic_memory.cloud.directory_deletes import ( +import basic_memory.services.note_content_reads as note_content_reads +import basic_memory.services.note_content_writes as note_content_writes +from basic_memory.services.directory_deletes import ( DirectoryDeleteService, DirectoryDeleteServiceError, ) -from basic_memory.cloud.note_content_reads import NoteContentQueryService -from basic_memory.cloud.note_content_writes import ( +from basic_memory.services.note_content_reads import NoteContentQueryService +from basic_memory.services.note_content_writes import ( NoteContentMutationActorContext, NoteContentMutationService, NoteContentMutationServiceError, diff --git a/tests/cloud/test_note_content_materialization.py b/tests/cloud/test_note_content_materialization.py index ede08c57a..f4c2c9bf6 100644 --- a/tests/cloud/test_note_content_materialization.py +++ b/tests/cloud/test_note_content_materialization.py @@ -11,8 +11,8 @@ import pytest from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker -import basic_memory.cloud.note_content_materialization as note_content_materialization -from basic_memory.cloud.note_content_materialization import ( +import basic_memory.index.note_content_materialization as note_content_materialization +from basic_memory.index.note_content_materialization import ( InlineNoteFileDeleteEnqueuer, LocalNoteContentMaterializationProvider, LocalNoteContentStorage, diff --git a/tests/cloud/test_note_content_read_service.py b/tests/cloud/test_note_content_read_service.py index 8892e6a7f..247c34330 100644 --- a/tests/cloud/test_note_content_read_service.py +++ b/tests/cloud/test_note_content_read_service.py @@ -9,8 +9,8 @@ import pytest from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker -import basic_memory.cloud.note_content_reads as note_content_reads -from basic_memory.cloud.note_content_reads import NoteContentQueryService +import basic_memory.services.note_content_reads as note_content_reads +from basic_memory.services.note_content_reads import NoteContentQueryService from basic_memory.runtime.note_content import ( NOTE_CONTENT_EXTERNAL_CHANGE_SYNC_ERROR, runtime_note_content_payload_as_dict, diff --git a/tests/cloud/test_project_deletes.py b/tests/cloud/test_project_deletes.py index a768bbfbe..9c1913bfa 100644 --- a/tests/cloud/test_project_deletes.py +++ b/tests/cloud/test_project_deletes.py @@ -4,7 +4,7 @@ import pytest import pytest_asyncio -from basic_memory.cloud.project_deletes import ( +from basic_memory.services.project_deletes import ( ProjectDeleteAcceptanceError, ProjectDeleteAcceptanceRequest, ProjectDeleteAcceptanceService, diff --git a/tests/indexing/test_accepted_note_mutation_runner.py b/tests/indexing/test_accepted_note_mutation_runner.py index c6f0115c0..dac1f9ca9 100644 --- a/tests/indexing/test_accepted_note_mutation_runner.py +++ b/tests/indexing/test_accepted_note_mutation_runner.py @@ -8,8 +8,10 @@ from pathlib import Path from types import SimpleNamespace from typing import cast +from unittest.mock import AsyncMock from uuid import UUID +import basic_memory.indexing.accepted_note_mutation_runner as accepted_note_mutation_module import pytest from sqlalchemy.ext.asyncio import AsyncSession @@ -32,6 +34,12 @@ run_accepted_note_update, ) from basic_memory.indexing.accepted_note_search import AcceptedNoteSearchRow +from basic_memory.markdown.schemas import ( + EntityFrontmatter, + EntityMarkdown, + Observation as MarkdownObservation, + Relation as MarkdownRelation, +) from basic_memory.models import Entity, NoteContent, Project from basic_memory.repository import ( AcceptedNoteContentWrite, @@ -43,6 +51,11 @@ from basic_memory.schemas.base import Entity as EntitySchema from basic_memory.schemas.request import EditEntityRequest from basic_memory.services.exceptions import EntityAlreadyExistsError +from basic_memory.services.note_preparation import ( + PreparedEntityFields, + PreparedEntityMove, + PreparedEntityWrite, +) _NOW = datetime(2026, 6, 20, 14, 30, tzinfo=UTC) @@ -51,33 +64,48 @@ _ACTOR_ID = UUID("11111111-1111-4111-8111-111111111111") -@dataclass(frozen=True, slots=True) -class _PreparedFields: - title: str - note_type: str - entity_metadata: dict[str, object] | None - content_type: str - permalink: str | None - file_path: str - created_at: datetime = _PREPARED_CREATED_AT - updated_at: datetime = _PREPARED_UPDATED_AT - - -@dataclass(frozen=True, slots=True) -class _PreparedWrite: - markdown_content: str - search_content: str - entity_fields: _PreparedFields - observations: Sequence[AcceptedObservationWrite] = () - relations: Sequence[AcceptedRelationWrite] = () - - -@dataclass(frozen=True, slots=True) -class _PreparedMove: - file_path: Path - markdown_content: str - search_content: str - permalink: str | None +def _prepared_write( + *, + markdown_content: str, + search_content: str, + entity_fields: PreparedEntityFields, + observations: Sequence[AcceptedObservationWrite] = (), + relations: Sequence[AcceptedRelationWrite] = (), +) -> PreparedEntityWrite: + entity_markdown = EntityMarkdown( + frontmatter=EntityFrontmatter( + metadata={ + "title": entity_fields.title, + "type": entity_fields.note_type, + "permalink": entity_fields.permalink, + } + ), + content=markdown_content, + observations=[ + MarkdownObservation( + content=observation.content, + category=observation.category, + context=observation.context, + tags=observation.tags, + ) + for observation in observations + ], + relations=[ + MarkdownRelation( + type=relation.relation_type, + target=relation.target_name, + context=relation.context, + ) + for relation in relations + ], + ) + return PreparedEntityWrite( + file_path=Path(entity_fields.file_path), + markdown_content=markdown_content, + search_content=search_content, + entity_fields=entity_fields, + entity_markdown=entity_markdown, + ) @pytest.fixture(autouse=True) @@ -89,6 +117,16 @@ def _freeze_mutation_clock(monkeypatch: pytest.MonkeyPatch) -> None: ) +@pytest.fixture +def persistence_calls(monkeypatch: pytest.MonkeyPatch) -> tuple[AsyncMock, AsyncMock]: + """Record which complete or move-only persistence boundary each mutation uses.""" + snapshot = AsyncMock(wraps=accepted_note_mutation_module.persist_accepted_note_snapshot) + move = AsyncMock(wraps=accepted_note_mutation_module.persist_accepted_note_move) + monkeypatch.setattr(accepted_note_mutation_module, "persist_accepted_note_snapshot", snapshot) + monkeypatch.setattr(accepted_note_mutation_module, "persist_accepted_note_move", move) + return snapshot, move + + class _MutationSession: def __init__(self) -> None: self.deleted: list[object] = [] @@ -104,16 +142,16 @@ async def flush(self) -> None: class _CreatePreparer: def __init__( self, - prepared: _PreparedWrite, + prepared: PreparedEntityWrite, *, - prepared_move: _PreparedMove | None = None, + prepared_move: PreparedEntityMove | None = None, move_destination_error: EntityAlreadyExistsError | None = None, filename_conflicts: list[str] | None = None, ) -> None: self.prepared = prepared self.move_destination_error = move_destination_error self.filename_conflicts = filename_conflicts or [] - self.prepared_move = prepared_move or _PreparedMove( + self.prepared_move = prepared_move or PreparedEntityMove( file_path=Path(prepared.entity_fields.file_path), markdown_content=prepared.markdown_content, search_content=prepared.search_content, @@ -136,7 +174,7 @@ async def prepare_create_entity_content( check_storage_exists: bool = True, skip_conflict_check: bool = False, session: AsyncSession | None = None, - ) -> _PreparedWrite: + ) -> PreparedEntityWrite: self.calls.append((schema, check_storage_exists, session)) self.skip_conflict_checks.append(skip_conflict_check) return self.prepared @@ -157,7 +195,7 @@ async def prepare_update_entity_content( existing_content: str, *, session: AsyncSession | None = None, - ) -> _PreparedWrite: + ) -> PreparedEntityWrite: self.replace_calls.append((entity, schema, existing_content, session)) return self.prepared @@ -173,7 +211,7 @@ async def prepare_edit_entity_content( expected_replacements: int = 1, replace_subsections: bool = True, session: AsyncSession | None = None, - ) -> _PreparedWrite: + ) -> PreparedEntityWrite: self.edit_calls.append( ( entity, @@ -196,7 +234,7 @@ async def prepare_move_entity_content( destination_path: str, *, session: AsyncSession | None = None, - ) -> _PreparedMove: + ) -> PreparedEntityMove: self.move_calls.append((entity, current_content, destination_path, session)) return self.prepared_move @@ -455,38 +493,42 @@ def _schema() -> EntitySchema: ) -def _prepared() -> _PreparedWrite: - return _PreparedWrite( +def _prepared() -> PreparedEntityWrite: + return _prepared_write( markdown_content="# Accepted\n", search_content="Accepted", - entity_fields=_PreparedFields( + entity_fields=PreparedEntityFields( title="Accepted", note_type="note", entity_metadata={"status": "draft"}, content_type="text/markdown", permalink="accepted", file_path="notes/accepted.md", + created_at=_PREPARED_CREATED_AT, + updated_at=_PREPARED_UPDATED_AT, ), ) -def _prepared_replacement() -> _PreparedWrite: - return _PreparedWrite( +def _prepared_replacement() -> PreparedEntityWrite: + return _prepared_write( markdown_content="# Replacement\n", search_content="Replacement", - entity_fields=_PreparedFields( + entity_fields=PreparedEntityFields( title="Replacement", note_type="note", entity_metadata={"status": "updated"}, content_type="text/markdown", permalink="replacement", file_path="notes/replacement.md", + created_at=_PREPARED_CREATED_AT, + updated_at=_PREPARED_UPDATED_AT, ), ) -def _prepared_move() -> _PreparedMove: - return _PreparedMove( +def _prepared_move() -> PreparedEntityMove: + return PreparedEntityMove( file_path=Path("archive/accepted.md"), markdown_content="# Moved\n", search_content="Moved", @@ -570,7 +612,9 @@ def _dependencies( @pytest.mark.asyncio -async def test_run_accepted_note_create_persists_prepared_markdown() -> None: +async def test_run_accepted_note_create_persists_prepared_markdown( + persistence_calls: tuple[AsyncMock, AsyncMock], +) -> None: session = cast(AsyncSession, object()) schema = _schema() project = _project() @@ -631,6 +675,8 @@ async def test_run_accepted_note_create_persists_prepared_markdown() -> None: assert change.materialization.actor_kind == "user" assert change.materialization.actor_name == "Ada" assert change.materialization.previous_file_path is None + assert persistence_calls[0].await_count == 1 + assert persistence_calls[1].await_count == 0 @pytest.mark.asyncio @@ -708,7 +754,9 @@ async def test_run_accepted_note_create_allows_equivalent_non_markdown_resource_ @pytest.mark.asyncio -async def test_run_accepted_note_update_replaces_existing_note_content() -> None: +async def test_run_accepted_note_update_replaces_existing_note_content( + persistence_calls: tuple[AsyncMock, AsyncMock], +) -> None: session = _MutationSession() schema = _schema() project = _project() @@ -760,6 +808,8 @@ async def test_run_accepted_note_update_replaces_existing_note_content() -> None assert change.materialization is not None assert change.materialization.db_version == 2 assert change.materialization.previous_file_path is None + assert persistence_calls[0].await_count == 1 + assert persistence_calls[1].await_count == 0 @pytest.mark.asyncio @@ -1067,7 +1117,9 @@ async def test_run_accepted_note_update_rejects_non_markdown_existing_entity() - @pytest.mark.asyncio -async def test_run_accepted_note_edit_applies_patch_against_db_content() -> None: +async def test_run_accepted_note_edit_applies_patch_against_db_content( + persistence_calls: tuple[AsyncMock, AsyncMock], +) -> None: session = _MutationSession() project = _project() prepared = _prepared_replacement() @@ -1126,12 +1178,15 @@ async def test_run_accepted_note_edit_applies_patch_against_db_content() -> None assert change.status_code == 200 assert change.materialization is not None assert change.materialization.source == "mcp" + assert persistence_calls[0].await_count == 1 + assert persistence_calls[1].await_count == 0 @pytest.mark.asyncio @pytest.mark.parametrize("file_checksum", ["file-checksum", None]) async def test_run_accepted_note_move_carries_previous_path_and_materialized_cleanup( file_checksum: str | None, + persistence_calls: tuple[AsyncMock, AsyncMock], ) -> None: session = _MutationSession() project = _project() @@ -1194,6 +1249,8 @@ async def test_run_accepted_note_move_carries_previous_path_and_materialized_cle assert cleanup is not None assert cleanup.file_path == "notes/accepted.md" assert cleanup.file_checksum == "file-checksum" + assert persistence_calls[0].await_count == 0 + assert persistence_calls[1].await_count == 1 @pytest.mark.asyncio @@ -1282,18 +1339,20 @@ def _prepared_with_graph( *, observations: Sequence[AcceptedObservationWrite], relations: Sequence[AcceptedRelationWrite], -) -> _PreparedWrite: +) -> PreparedEntityWrite: """A prepared accepted write carrying a parsed observation/relation graph.""" - return _PreparedWrite( + return _prepared_write( markdown_content="# Accepted\n", search_content="Accepted", - entity_fields=_PreparedFields( + entity_fields=PreparedEntityFields( title="Accepted", note_type="dev_accept_person", entity_metadata={"type": "dev_accept_person"}, content_type="text/markdown", permalink="accepted", file_path="notes/accepted.md", + created_at=_PREPARED_CREATED_AT, + updated_at=_PREPARED_UPDATED_AT, ), observations=observations, relations=relations, diff --git a/tests/indexing/test_accepted_note_write_runner.py b/tests/indexing/test_accepted_note_write_runner.py index 53d9ed60d..d824d105f 100644 --- a/tests/indexing/test_accepted_note_write_runner.py +++ b/tests/indexing/test_accepted_note_write_runner.py @@ -23,16 +23,22 @@ create_accepted_pending_entity, delete_accepted_note, delete_accepted_note_entity, - persist_accepted_note_write, + persist_accepted_note_move, + persist_accepted_note_snapshot, prepare_accepted_note_create, prepare_accepted_note_edit, prepare_accepted_note_move, prepare_accepted_note_replace, refresh_accepted_note_search_index, - replace_accepted_note_graph, delete_accepted_note_search_index, ) from basic_memory.models import Entity, NoteContent +from basic_memory.markdown.schemas import ( + EntityFrontmatter, + EntityMarkdown, + Observation as MarkdownObservation, + Relation as MarkdownRelation, +) from basic_memory.repository import ( AcceptedNoteContentWrite, AcceptedObservationWrite, @@ -40,41 +46,21 @@ ) from basic_memory.repository.entity_repository import AcceptedPendingEntityWrite from basic_memory.schemas.base import Entity as EntitySchema +from basic_memory.services.note_preparation import ( + PreparedEntityFields, + PreparedEntityMove, + PreparedEntityWrite, +) +_PreparedFields = PreparedEntityFields +_PreparedWrite = PreparedEntityWrite +_PreparedMove = PreparedEntityMove + _PREPARED_CREATED_AT = datetime(2024, 1, 15, 10, 30, tzinfo=UTC) _PREPARED_UPDATED_AT = datetime(2024, 1, 16, 11, 45, tzinfo=UTC) -@dataclass(frozen=True, slots=True) -class _PreparedFields: - title: str - note_type: str - entity_metadata: dict[str, object] | None - content_type: str - permalink: str | None - file_path: str - created_at: datetime = _PREPARED_CREATED_AT - updated_at: datetime = _PREPARED_UPDATED_AT - - -@dataclass(frozen=True, slots=True) -class _PreparedWrite: - markdown_content: str - search_content: str - entity_fields: _PreparedFields - observations: Sequence[AcceptedObservationWrite] = () - relations: Sequence[AcceptedRelationWrite] = () - - -@dataclass(frozen=True, slots=True) -class _PreparedMove: - file_path: Path - markdown_content: str - search_content: str - permalink: str | None - - class _FlushSession: def __init__(self) -> None: self.flush_count = 0 @@ -230,7 +216,7 @@ async def delete(self, entity: object) -> None: class _CreatePreparer: - def __init__(self, prepared: _PreparedWrite) -> None: + def __init__(self, prepared: PreparedEntityWrite) -> None: self.prepared = prepared self.calls: list[tuple[EntitySchema, bool, AsyncSession | None]] = [] self.skip_conflict_checks: list[bool] = [] @@ -242,14 +228,14 @@ async def prepare_create_entity_content( check_storage_exists: bool = True, skip_conflict_check: bool = False, session: AsyncSession | None = None, - ) -> _PreparedWrite: + ) -> PreparedEntityWrite: self.calls.append((schema, check_storage_exists, session)) self.skip_conflict_checks.append(skip_conflict_check) return self.prepared class _ReplacePreparer: - def __init__(self, prepared: _PreparedWrite) -> None: + def __init__(self, prepared: PreparedEntityWrite) -> None: self.prepared = prepared self.calls: list[tuple[Entity, EntitySchema, str, AsyncSession | None]] = [] @@ -260,13 +246,13 @@ async def prepare_update_entity_content( existing_content: str, *, session: AsyncSession | None = None, - ) -> _PreparedWrite: + ) -> PreparedEntityWrite: self.calls.append((entity, schema, existing_content, session)) return self.prepared class _EditPreparer: - def __init__(self, prepared: _PreparedWrite) -> None: + def __init__(self, prepared: PreparedEntityWrite) -> None: self.prepared = prepared self.calls: list[ tuple[Entity, str, str, str, str | None, str | None, int, bool, AsyncSession | None] @@ -284,7 +270,7 @@ async def prepare_edit_entity_content( expected_replacements: int = 1, replace_subsections: bool = True, session: AsyncSession | None = None, - ) -> _PreparedWrite: + ) -> PreparedEntityWrite: self.calls.append( ( entity, @@ -302,7 +288,7 @@ async def prepare_edit_entity_content( class _MovePreparer: - def __init__(self, prepared: _PreparedMove) -> None: + def __init__(self, prepared: PreparedEntityMove) -> None: self.prepared = prepared self.calls: list[tuple[Entity, str, str, AsyncSession | None]] = [] @@ -313,7 +299,7 @@ async def prepare_move_entity_content( destination_path: str, *, session: AsyncSession | None = None, - ) -> _PreparedMove: + ) -> PreparedEntityMove: self.calls.append((entity, current_content, destination_path, session)) return self.prepared @@ -402,19 +388,51 @@ def _prepared( *, markdown_content: str = "# Accepted\n", search_content: str = "Accepted", - fields: _PreparedFields | None = None, -) -> _PreparedWrite: - return _PreparedWrite( + fields: PreparedEntityFields | None = None, + observations: Sequence[AcceptedObservationWrite] = (), + relations: Sequence[AcceptedRelationWrite] = (), +) -> PreparedEntityWrite: + prepared_fields = fields or PreparedEntityFields( + title="Accepted", + note_type="note", + entity_metadata={"status": "draft"}, + content_type="text/markdown", + permalink="accepted", + file_path="notes/accepted.md", + created_at=_PREPARED_CREATED_AT, + updated_at=_PREPARED_UPDATED_AT, + ) + return PreparedEntityWrite( + file_path=Path(prepared_fields.file_path), markdown_content=markdown_content, search_content=search_content, - entity_fields=fields - or _PreparedFields( - title="Accepted", - note_type="note", - entity_metadata={"status": "draft"}, - content_type="text/markdown", - permalink="accepted", - file_path="notes/accepted.md", + entity_fields=prepared_fields, + entity_markdown=EntityMarkdown( + frontmatter=EntityFrontmatter( + metadata={ + "title": prepared_fields.title, + "type": prepared_fields.note_type, + "permalink": prepared_fields.permalink, + } + ), + content=search_content, + observations=[ + MarkdownObservation( + content=observation.content, + category=observation.category, + context=observation.context, + tags=observation.tags, + ) + for observation in observations + ], + relations=[ + MarkdownRelation( + type=relation.relation_type, + target=relation.target_name, + context=relation.context, + ) + for relation in relations + ], ), ) @@ -461,7 +479,7 @@ def _note_content() -> NoteContent: @pytest.mark.asyncio async def test_prepare_accepted_note_create_hashes_prepared_markdown() -> None: - session = cast(AsyncSession, object()) + session = cast(AsyncSession, _FlushSession()) schema = _schema() prepared = _prepared(markdown_content="# Created\n") preparer = _CreatePreparer(prepared) @@ -491,6 +509,8 @@ async def test_prepare_accepted_note_replace_applies_entity_fields() -> None: content_type="text/markdown", permalink="replacement", file_path="notes/replacement.md", + created_at=_PREPARED_CREATED_AT, + updated_at=_PREPARED_UPDATED_AT, ) prepared = _prepared(markdown_content="# Replacement\n", fields=fields) preparer = _ReplacePreparer(prepared) @@ -530,6 +550,8 @@ async def test_prepare_accepted_note_edit_applies_entity_fields() -> None: content_type="text/markdown", permalink="edited", file_path="notes/edited.md", + created_at=_PREPARED_CREATED_AT, + updated_at=_PREPARED_UPDATED_AT, ) prepared = _prepared(markdown_content="# Edited\n", fields=fields) preparer = _EditPreparer(prepared) @@ -584,6 +606,8 @@ def test_apply_accepted_prepared_entity_fields_updates_mutable_entity() -> None: content_type="text/markdown", permalink="applied", file_path="schemas/applied.md", + created_at=_PREPARED_CREATED_AT, + updated_at=_PREPARED_UPDATED_AT, ), user_profile_value="user-3", ) @@ -698,6 +722,7 @@ async def test_create_accepted_pending_entity_uses_repository_protocol() -> None session = cast(AsyncSession, object()) entity = _entity() repository = _PendingEntityRepository(entity) + result = await create_accepted_pending_entity( session, prepared=_prepared(), @@ -819,7 +844,7 @@ async def test_delete_accepted_note_search_index_uses_repository_protocol() -> N @pytest.mark.asyncio -async def test_persist_accepted_note_write_plans_content_and_refreshes_search() -> None: +async def test_persist_accepted_note_snapshot_persists_content_search_and_graph() -> None: session = cast(AsyncSession, object()) entity = _entity() entity.file_path = "notes/new.md" @@ -832,13 +857,32 @@ async def test_persist_accepted_note_write_plans_content_and_refreshes_search() persisted_note_content = _note_content() content_repository = _NoteContentRepository(persisted_note_content) search_repository = _SearchRepository() + observation_repository = _ObservationRepository() + relation_repository = _RelationRepository() + observation = AcceptedObservationWrite( + content="Snapshot is complete", + category="status", + context=None, + tags=None, + ) + relation = AcceptedRelationWrite( + relation_type="documents", + target_name="Another Note", + context=None, + ) + prepared = _prepared( + markdown_content="# New\n", + search_content="New body", + observations=(observation,), + relations=(relation,), + ) - result = await persist_accepted_note_write( + result = await persist_accepted_note_snapshot( session, entity=entity, - markdown_content="# New\n", - search_content="New body", + prepared=prepared, db_checksum="new-db-checksum", + self_relation_resolver=_SelfRelationResolver(), last_source="api", updated_at=updated_at, current_note_content=current_note_content, @@ -847,6 +891,8 @@ async def test_persist_accepted_note_write_plans_content_and_refreshes_search() repositories=_repository_provider( note_content_repository=content_repository, search_repository=search_repository, + observation_repository=observation_repository, + relation_repository=relation_repository, ), ) @@ -872,6 +918,45 @@ async def test_persist_accepted_note_write_plans_content_and_refreshes_search() assert len(search_repository.calls) == 1 assert search_repository.calls[0].entity_id == entity.id assert search_repository.calls[0].content_snippet == "New body" + assert observation_repository.calls == [(entity.id, prepared.observations)] + assert relation_repository.calls == [(entity.id, prepared.relations)] + + +@pytest.mark.asyncio +async def test_persist_accepted_note_move_is_explicitly_content_and_search_only() -> None: + session = cast(AsyncSession, _FlushSession()) + entity = _entity() + entity.file_path = "notes/new.md" + current_note_content = _note_content() + current_note_content.file_path = "notes/old.md" + content_repository = _NoteContentRepository(_note_content()) + search_repository = _SearchRepository() + prepared = await prepare_accepted_note_move( + None, + session, + entity=entity, + current_note_content=current_note_content, + accepted_file_path="notes/new.md", + should_update_permalink=False, + user_profile_value=None, + ) + + await persist_accepted_note_move( + session, + entity=entity, + prepared=prepared, + last_source="api", + updated_at=datetime(2026, 6, 19, 14, 0, tzinfo=UTC), + current_note_content=current_note_content, + existing_file_path="notes/old.md", + repositories=_repository_provider( + note_content_repository=content_repository, + search_repository=search_repository, + ), + ) + + assert len(content_repository.calls) == 1 + assert len(search_repository.calls) == 1 @pytest.mark.asyncio @@ -946,73 +1031,22 @@ async def test_delete_accepted_note_plans_cleanup_and_deletes_entity() -> None: @pytest.mark.asyncio -async def test_replace_accepted_note_graph_persists_observations_and_relations() -> None: - """The graph handoff forwards the prepared observation/relation set to the repos.""" - observation_repository = _ObservationRepository() - relation_repository = _RelationRepository() - repositories = _repository_provider( - observation_repository=observation_repository, - relation_repository=relation_repository, - ) - prepared = _PreparedWrite( - markdown_content="# Accepted\n", - search_content="Accepted", - entity_fields=_PreparedFields( - title="Accepted", - note_type="note", - entity_metadata=None, - content_type="text/markdown", - permalink="accepted", - file_path="notes/accepted.md", - ), - observations=[ - AcceptedObservationWrite( - content="Ada Acceptance", - category="name", - context=None, - tags=None, - ) - ], - relations=[ - AcceptedRelationWrite( - relation_type="works_at", - target_name="XSYS Target", - context=None, - ) - ], - ) - resolver = _SelfRelationResolver() - session = cast(AsyncSession, _FlushSession()) - - await replace_accepted_note_graph( - session, - entity=_entity(), - prepared=prepared, - self_relation_resolver=resolver, - repositories=repositories, - ) - - # Both repos are scoped to the entity's project (7) and receive the parsed set. - assert observation_repository.calls == [(42, prepared.observations)] - assert relation_repository.calls == [(42, prepared.relations)] - assert [call[0] for call in resolver.calls] == ["XSYS Target"] - - -@pytest.mark.asyncio -async def test_replace_accepted_note_graph_resolves_safe_self_relation() -> None: +async def test_persist_accepted_note_snapshot_resolves_safe_self_relation() -> None: """A safe self-link carries its ID because deferred resolution skips self targets.""" relation_repository = _RelationRepository() entity = _entity() - prepared = _PreparedWrite( + prepared = _prepared( markdown_content="# Accepted\n", search_content="Accepted", - entity_fields=_PreparedFields( + fields=_PreparedFields( title="Accepted", note_type="note", entity_metadata=None, content_type="text/markdown", permalink="accepted", file_path="notes/accepted.md", + created_at=_PREPARED_CREATED_AT, + updated_at=_PREPARED_UPDATED_AT, ), relations=[ AcceptedRelationWrite( @@ -1024,12 +1058,17 @@ async def test_replace_accepted_note_graph_resolves_safe_self_relation() -> None ) resolver = _SelfRelationResolver(entity) - await replace_accepted_note_graph( - cast(AsyncSession, _FlushSession()), + await persist_accepted_note_snapshot( + cast(AsyncSession, object()), entity=entity, prepared=prepared, + db_checksum="snapshot-checksum", self_relation_resolver=resolver, + last_source="api", + updated_at=entity.updated_at, repositories=_repository_provider( + note_content_repository=_NoteContentRepository(_note_content()), + search_repository=_SearchRepository(), observation_repository=_ObservationRepository(), relation_repository=relation_repository, ), @@ -1051,23 +1090,29 @@ async def test_replace_accepted_note_graph_resolves_safe_self_relation() -> None @pytest.mark.asyncio -async def test_replace_accepted_note_graph_forwards_empty_sets() -> None: +async def test_persist_accepted_note_snapshot_forwards_empty_graph_sets() -> None: """A note with no observations/relations still clears the graph (empty replace).""" observation_repository = _ObservationRepository() relation_repository = _RelationRepository() repositories = _repository_provider( + note_content_repository=_NoteContentRepository(_note_content()), + search_repository=_SearchRepository(), observation_repository=observation_repository, relation_repository=relation_repository, ) prepared = _prepared() - await replace_accepted_note_graph( - cast(AsyncSession, _FlushSession()), - entity=_entity(), + entity = _entity() + await persist_accepted_note_snapshot( + cast(AsyncSession, object()), + entity=entity, prepared=prepared, + db_checksum="snapshot-checksum", self_relation_resolver=_SelfRelationResolver(), + last_source="api", + updated_at=entity.updated_at, repositories=repositories, ) - assert observation_repository.calls == [(42, ())] + assert observation_repository.calls == [(42, [])] assert relation_repository.calls == [(42, [])] diff --git a/tests/indexing/test_project_index_maintenance.py b/tests/indexing/test_project_index_maintenance.py index e83bd635a..6f06eafc9 100644 --- a/tests/indexing/test_project_index_maintenance.py +++ b/tests/indexing/test_project_index_maintenance.py @@ -7,6 +7,7 @@ from typing import cast import basic_memory.indexing.project_index_maintenance as project_index_maintenance_module +import basic_memory.repository.accepted_note_vector_cleanup as accepted_note_vector_cleanup_module import pytest from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -1255,7 +1256,7 @@ async def fake_load_sqlite_vec_on_session( return True monkeypatch.setattr( - project_index_maintenance_module, + accepted_note_vector_cleanup_module, "_load_sqlite_vec_on_session", fake_load_sqlite_vec_on_session, ) diff --git a/tests/mcp/test_server_lifespan_branches.py b/tests/mcp/test_server_lifespan_branches.py index aec8f6db4..26aabc1b1 100644 --- a/tests/mcp/test_server_lifespan_branches.py +++ b/tests/mcp/test_server_lifespan_branches.py @@ -2,7 +2,7 @@ import pytest -import basic_memory.cloud.note_content_materialization as note_content_materialization +import basic_memory.index.note_content_materialization as note_content_materialization import basic_memory.mcp.server as server_module from basic_memory import db from basic_memory.mcp.server import lifespan, mcp diff --git a/tests/test_architecture_boundaries.py b/tests/test_architecture_boundaries.py new file mode 100644 index 000000000..a14328a53 --- /dev/null +++ b/tests/test_architecture_boundaries.py @@ -0,0 +1,42 @@ +"""Static assertions for core runtime dependency direction.""" + +import ast +from pathlib import Path + + +PACKAGE_ROOT = Path(__file__).parents[1] / "src" / "basic_memory" + + +def _imported_modules(path: Path) -> set[str]: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + modules: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + modules.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module is not None: + modules.add(node.module) + return modules + + +def test_core_production_modules_do_not_import_cloud_compatibility_package() -> None: + violations = { + path.relative_to(PACKAGE_ROOT).as_posix(): sorted( + module for module in _imported_modules(path) if module.startswith("basic_memory.cloud") + ) + for path in PACKAGE_ROOT.rglob("*.py") + if "cloud" not in path.relative_to(PACKAGE_ROOT).parts + } + assert not {path: modules for path, modules in violations.items() if modules} + + +def test_repositories_do_not_import_indexing_workflows() -> None: + repository_root = PACKAGE_ROOT / "repository" + violations = { + path.relative_to(PACKAGE_ROOT).as_posix(): sorted( + module + for module in _imported_modules(path) + if module.startswith("basic_memory.indexing") + ) + for path in repository_root.rglob("*.py") + } + assert not {path: modules for path, modules in violations.items() if modules}