From 8bff587dd7eeaf09ae8d088f7e861bb60208727d Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 28 Jul 2026 23:41:32 -0500 Subject: [PATCH 01/28] perf(api): add optional Redis read caching Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 314 +++++++++ pyproject.toml | 5 + src/basic_memory/api/container.py | 9 +- .../api/v2/routers/knowledge_router.py | 234 ++++--- .../api/v2/routers/resource_router.py | 162 +++-- src/basic_memory/deps/__init__.py | 8 + src/basic_memory/deps/read_cache.py | 21 + src/basic_memory/deps/services.py | 6 + src/basic_memory/index/local_project.py | 36 +- src/basic_memory/index/local_runtime.py | 64 +- src/basic_memory/index/watch_coordinator.py | 3 + src/basic_memory/read_cache/__init__.py | 31 + src/basic_memory/read_cache/contract.py | 106 +++ src/basic_memory/read_cache/invalidation.py | 45 ++ src/basic_memory/read_cache/keys.py | 79 +++ src/basic_memory/read_cache/null.py | 28 + src/basic_memory/read_cache/policy.py | 4 + src/basic_memory/read_cache/read_through.py | 111 ++++ src/basic_memory/read_cache/redis.py | 191 ++++++ src/basic_memory/services/initialization.py | 10 +- .../services/note_content_writes.py | 27 +- test-int/read_cache/conftest.py | 101 +++ test-int/read_cache/test_api_read_cache.py | 206 ++++++ .../read_cache/test_read_cache_benchmark.py | 108 +++ test-int/read_cache/test_redis_read_cache.py | 620 ++++++++++++++++++ tests/index/test_watch_coordinator.py | 6 +- tests/services/test_initialization.py | 3 + uv.lock | 17 +- 28 files changed, 2381 insertions(+), 174 deletions(-) create mode 100644 docs/REDIS_READ_CACHE_PLAN.md create mode 100644 src/basic_memory/deps/read_cache.py create mode 100644 src/basic_memory/read_cache/__init__.py create mode 100644 src/basic_memory/read_cache/contract.py create mode 100644 src/basic_memory/read_cache/invalidation.py create mode 100644 src/basic_memory/read_cache/keys.py create mode 100644 src/basic_memory/read_cache/null.py create mode 100644 src/basic_memory/read_cache/policy.py create mode 100644 src/basic_memory/read_cache/read_through.py create mode 100644 src/basic_memory/read_cache/redis.py create mode 100644 test-int/read_cache/conftest.py create mode 100644 test-int/read_cache/test_api_read_cache.py create mode 100644 test-int/read_cache/test_read_cache_benchmark.py create mode 100644 test-int/read_cache/test_redis_read_cache.py diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md new file mode 100644 index 000000000..57652d3d6 --- /dev/null +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -0,0 +1,314 @@ +# Redis Read Cache Plan + +## Status + +Accepted implementation plan. Basic Memory owns semantic read caching; Basic Memory Cloud owns +tenant and principal rate limiting. The datasets are logically isolated even when they share a +Redis instance. + +## Goals + +- Reduce latency and database work for repeated entity and note reads. +- Keep Redis optional so the default local-first installation has no external service + requirement. +- Make cache semantics, typed serialization, TTLs, and invalidation part of Basic Memory. +- Let a host such as Basic Memory Cloud inject an existing Redis client and opaque namespace. +- Preserve independent clients, key prefixes, metrics, and failure behavior for read caching and + rate limiting. + +## Ownership Boundary + +Basic Memory owns: + +- cacheable read operations; +- canonical request keys; +- typed serialization; +- TTL and payload-size policy; +- project-scoped invalidation; +- the no-op and Redis read-cache implementations. + +Cloud owns: + +- tenant and principal identity; +- rate-limit policy and enforcement; +- the Redis deployment topology; +- the opaque cache namespace supplied to Basic Memory; +- capacity, eviction, and availability decisions when Redis is shared. + +Physical topology is deliberately outside the Basic Memory cache contract. Cloud may point the +read cache and limiter at separate Redis instances or at separately configured clients on one +instance. + +### Tenant isolation on shared Redis + +Cloud does not create a Redis instance or connection pool per tenant. It reuses one long-lived, +Basic Memory-specific Redis client and constructs a lightweight `RedisReadCache` adapter from +trusted request or worker context: + +- `namespace`: the stable tenant/workspace UUID; +- `project_id`: the Basic Memory project external UUID; +- `prefix`: the Basic Memory read-cache keyspace, separate from rate limiting. + +The adapter hashes `(namespace, project_id)` into the Redis Cluster scope. Two tenants with the +same project identifier therefore cannot share a generation or data key. Tenant isolation must +not rely on project UUID uniqueness alone: every Cloud read and invalidation supplies the +authenticated tenant/workspace UUID, even when project UUIDs are globally unique in practice. + +The namespace is host-owned isolation context, not an API input. Never derive it from a display +name or slug, accept it from a caller, or include API keys and other secrets in it. API requests +and background workers must use one canonical tenant-to-namespace function so an index worker +invalidates the exact scope populated by the request path. + +### Cloud host requirements + +Basic Memory Cloud must satisfy all of these requirements before enabling cached reads for a +tenant: + +1. Derive one stable namespace from authenticated Cloud context. The preferred value is the + tenant UUID or workspace UUID already owned by Cloud. Project UUID alone is not a tenant + boundary, even if current database constraints make it globally unique. +1. Complete authorization and tenant database/schema selection before cache lookup. The + namespace prevents key collisions; it does not replace Cloud's access-control boundary. +1. Override `get_read_cache` at the Cloud composition root. Construct + `RedisReadCache(client=shared_basic_memory_client, namespace=trusted_namespace)` as a + lightweight request-scoped adapter; reuse the long-lived client and connection pool. +1. Pass the trusted tenant/workspace identity through internal queue payloads, or include enough + trusted identifiers for workers to derive the exact same namespace. Never copy a namespace + from a public request field. +1. Inject the namespace-bound cache into every mutation-producing runtime: accepted note + materialization, object-storage events, direct and project indexing, directory moves/deletes, + and relation-resolution workers. Request-path invalidation alone is insufficient because a + worker can update a cached entity after the request returns. +1. Keep `bm:read:v1` separate from rate-limit and Cloud control-plane prefixes, metrics, + timeouts, and failure policies. The clients may target one Redis deployment, but a read-cache + timeout must bypass while a rate-limit decision keeps its Cloud-owned security behavior. +1. Enable reads only after request and worker invalidation use the same namespace in the target + environment. Roll out by tenant cohort, watch hit/bypass/invalidation outcomes and database + queries, then remove overlapping Cloud gateway response caches only after parity. +1. Coordinate rolling deployments around the `bm:read:v1` payload/key contract. Bump the prefix + for incompatible serialized response changes so mixed application versions never interpret + one another's payloads with different schemas. + +If Cloud ever changes the namespace source, treat that as a cache-key migration. A new namespace +is safe because it cannot read the old tenant scope, but old keys remain until their TTLs expire +and every worker must switch atomically enough to avoid missing invalidations. + +## Architecture + +```mermaid +flowchart LR + H["Cloud or standalone API host"] -->|"client plus opaque namespace"| RC["Basic Memory ReadCache"] + API["Basic Memory read routes"] --> RC + RC -->|"hit"| API + RC -->|"miss"| DB["Services, repositories, and storage"] + DB -->|"successful result"| RC + W["Writes, indexing, and storage events"] -->|"invalidate after commit"| RC + + RL["Cloud tenant rate limiter"] --> RLD["Cloud rate-limit keyspace"] + RC --> BMD["Basic Memory read-cache keyspace"] + RLD -. "same or separate instance" .-> R["Redis"] + BMD -. "same or separate instance" .-> R +``` + +## Core Contract + +Introduce `src/basic_memory/read_cache/` with: + +- a narrow `ReadCache` protocol; +- immutable request/key values; +- a `NullReadCache` default; +- canonical key construction; +- typed Pydantic read-through helpers; +- an optional `RedisReadCache` adapter. + +The cache is namespace-bound at construction. Its public operations are: + +- `lookup(key)`, which returns the generation observed with a hit or miss; +- `store(key, lookup, payload, ttl)`, which reports stored, superseded, or disabled; +- `invalidate_project(project_id)`. + +Cloud can create a lightweight namespace-bound adapter around a long-lived, Basic +Memory-specific async Redis client. Basic Memory does not receive tenant, subscription, or +rate-limit concepts. + +## Keys And Invalidation + +Use versioned, cluster-compatible keys: + +```text +bm:read:v1:{scope_digest}:generation +bm:read:v1:{scope_digest}:: +``` + +`scope_digest` hashes the host-supplied tenant/workspace namespace and project external ID. The +Redis Cluster hash tag keeps that tenant-project scope's generation and data keys in one slot. + +Each value records the random generation token under which it was created: + +1. Read the generation and data key together. +1. Accept the cached value only when its embedded generation matches. +1. After a successful mutation commit, replace the generation with a new random token. +1. Let unreachable entries expire; never scan or bulk-delete keys. + +Random tokens prevent an evicted generation key from returning to an old integer generation and +reviving stale data. A read that fills after concurrent invalidation also remains safe because its +old token no longer matches. + +## Initial Cache Surface + +Phase one: + +| Operation | Initial TTL | Constraints | +| ---------------------- | ----------: | --------------------------------------- | +| Entity by external ID | 60 seconds | Cache validated `EntityResponseV2` JSON | +| Identifier resolution | 60 seconds | Cache successful same-project results | +| Markdown note resource | 60 seconds | Cache only below an explicit size limit | + +Phase two, after measuring phase one: + +| Operation | Initial TTL | Constraints | +| --------------------------- | ------------: | ---------------------------------------------- | +| Search | 30 seconds | Canonicalize the complete query and pagination | +| Directory reads | 30-60 seconds | Key every filtering and pagination input | +| Context and recent activity | 15-30 seconds | Normalize or bound time-relative inputs | + +Do not initially cache failures, missing entities, graph/orphan responses, large or arbitrary +binary resources, schema inference, writes, or Cloud control-plane data. + +Caching is semantic rather than HTTP-method based. The POST identifier-resolution and search +operations can be cached without changing their public API. + +## Placement + +Cache typed boundary values rather than SQLAlchemy models. Use an explicit read-through helper in +the API routes so hit, miss, serialization, and fallback behavior remain visible. + +Primary integration points: + +- `src/basic_memory/api/container.py` +- `src/basic_memory/api/app.py` +- `src/basic_memory/deps/read_cache.py` +- `src/basic_memory/api/v2/routers/knowledge_router.py` +- `src/basic_memory/api/v2/routers/resource_router.py` +- later, `src/basic_memory/api/v2/routers/search_router.py` + +Invalidation belongs at portable mutation and indexing completion boundaries, not only in +FastAPI routes. It must cover accepted note writes, direct file indexing, filesystem watcher +updates, project indexing, directory mutations, Cloud storage events, and later search or +relation changes that affect cached responses. + +## Dependency And Lifecycle + +Use the official asynchronous `redis-py` client behind the Basic Memory protocol. Add it only as +an optional package extra. A host may instead supply a compatible, already-owned client. + +The Core `ApiContainer` carries `NullReadCache` by default. A managed host activates caching by +injecting or dependency-overriding a namespace-bound implementation and owns that client's +lifecycle; Cloud therefore reuses its long-lived Basic Memory cache client. Local CLI, MCP +in-process ASGI routing, and the standalone API remain on `NullReadCache` in the first rollout. +A later standalone Redis setting can create and close a client in the FastAPI lifespan without +changing the cache contract. + +The FastAPI Redis SDK is not the foundational dependency for this work. The cache contract must +also participate in portable indexing and hosted storage-event invalidation, and Basic Memory's +local ASGI transport does not run FastAPI lifespan. + +## Failure Behavior + +- Connection and timeout failures are represented explicitly as cache-unavailable outcomes. +- Reads bypass Redis and use the authoritative path when the cache is unavailable. +- Cache-store failures do not fail an otherwise successful read. +- Cache-invalidation failures do not fail committed writes, but they emit prominent telemetry. +- Short initial TTLs bound stale-data exposure after an invalidation failure and Redis recovery. +- Serialization and programming errors fail fast rather than masquerading as cache misses. + +Rate-limit failure behavior remains entirely Cloud-owned. + +## Observability + +Record: + +- hit, miss, bypass, store, invalidation, unavailable, and oversize outcomes; +- operation name without tenant or project metric labels; +- Redis operation latency; +- cached payload size; +- authoritative read latency on misses; +- hashed scope and request identifiers on diagnostic spans only. + +Do not add public cache headers in the first version. + +## Integration Tests + +Redis behavior must be tested against a real Redis server, not a mocked or in-memory substitute. +Integration tests will start Redis through testcontainers or use an explicitly configured CI +Redis URL. + +Run the focused suite with: + +```bash +LOGFIRE_IGNORE_NO_CONFIG=1 uv run pytest -p pytest_mock --no-cov -q test-int/read_cache +``` + +`BASIC_MEMORY_TEST_REDIS_URL` selects an externally managed test server. Otherwise the fixture +starts `redis:8.8-alpine`; `BASIC_MEMORY_TEST_REDIS_IMAGE` can override that image without +changing the test contract. + +The real-Redis suite must prove: + +- namespace, project, operation, and request isolation; +- deterministic canonical keys; +- cache hit and TTL expiry behavior; +- project invalidation; +- a fill that completes after invalidation is never served; +- loss of the generation key cannot revive an older value; +- Redis restart or unavailability produces explicit bypass behavior; +- no invalidation operation touches keys outside the Basic Memory prefix; +- payload size limits; +- repeated API entity reads use the real cached representation; +- successful writes invalidate while rejected or rolled-back writes do not. + +Run route behavior against both SQLite and Postgres where persistence behavior differs. Redis +semantics themselves are asserted only against the real Redis integration fixture. + +## Delivery Sequence + +### 1. Cache infrastructure + +- Add the protocol, key values, no-op backend, Redis adapter, typed helper, optional dependency, + telemetry, and real Redis integration tests. +- Do not cache production routes yet. + +### 2. Hot entity reads + +- Cache entity, resolution, and bounded markdown-resource reads behind default-off configuration. +- Wire project invalidation through accepted writes and indexing paths. +- Add full-stack API and repeated `read_note` integration coverage. + +### 3. Cloud rollout + +- Inject a Basic Memory-specific Redis client and tenant namespace. +- Derive that namespace from trusted request and worker context with one canonical function. +- Invalidate from materialization, storage-event, and relation-resolution workers using the + same tenant namespace as the request path before enabling reads for a tenant. +- Start with shadow telemetry or a limited tenant cohort. +- Compare hit rate, Redis latency, database query volume, and end-to-end tool latency. + +### 4. Expand from evidence + +- Add search, directory, and graph-context reads when measured reuse supports them. +- Refine project-wide invalidation only if unrelated writes materially reduce the entity hit + rate. + +### 5. Remove overlap + +- Remove matching Cloud gateway response-cache families once Basic Memory caching reaches + behavioral and observability parity. + +## Verification Gates + +- Focused unit and real Redis integration tests. +- Entity/read API integration tests on SQLite and Postgres. +- `just fast-check`. +- `just doctor`. +- The appropriate broader SQLite and Postgres suites before opening a pull request. +- A before/after benchmark showing cache-hit latency and reduced authoritative reads. diff --git a/pyproject.toml b/pyproject.toml index 8e8d8650b..7d1a322b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,9 @@ milvus = [ "pymilvus>=3.0.0,<4; sys_platform == 'win32'", "pymilvus[milvus-lite]>=3.0.0,<4; sys_platform != 'win32'", ] +redis = [ + "redis>=8.0.0,<9", +] [project.urls] Homepage = "https://github.com/basicmachines-co/basic-memory" @@ -103,6 +106,7 @@ markers = [ "benchmark: Performance benchmark tests (deselect with '-m \"not benchmark\"')", "slow: Slow-running tests (deselect with '-m \"not slow\"')", "postgres: Tests that run against Postgres backend (deselect with '-m \"not postgres\"')", + "redis: Tests that run against a real Redis server", "windows: Windows-specific tests (deselect with '-m \"not windows\"')", "smoke: Fast end-to-end smoke tests for MCP flows", "semantic: Tests requiring semantic dependencies (fastembed, sqlite-vec, openai)", @@ -131,6 +135,7 @@ dev = [ "ruff>=0.16.0", "freezegun>=1.5.5", "testcontainers[postgres]>=4.0.0", + "redis>=8.0.0,<9", "psycopg>=3.2.0", "pyright>=1.1.408", "pytest-testmon>=2.2.0", diff --git a/src/basic_memory/api/container.py b/src/basic_memory/api/container.py index 4bf18da38..18bdb17ea 100644 --- a/src/basic_memory/api/container.py +++ b/src/basic_memory/api/container.py @@ -10,13 +10,14 @@ - Factories for services are provided, not singletons """ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import TYPE_CHECKING from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker, AsyncSession from basic_memory import db from basic_memory.config import BasicMemoryConfig, ConfigManager +from basic_memory.read_cache import NullReadCache, ReadCache from basic_memory.runtime.mode import RuntimeMode, resolve_runtime_mode if TYPE_CHECKING: # pragma: no cover @@ -39,6 +40,11 @@ class ApiContainer: engine: AsyncEngine | None = None session_maker: async_sessionmaker[AsyncSession] | None = None + # --- Optional semantic read cache --- + # Hosts inject a namespace-bound implementation; local and off-lifespan + # ASGI requests deliberately stay dependency-free by default. + read_cache: ReadCache = field(default_factory=NullReadCache) + @classmethod def create(cls) -> "ApiContainer": # pragma: no cover """Create container by reading ConfigManager. @@ -91,6 +97,7 @@ def create_watch_coordinator(self) -> "WatchCoordinator": # pragma: no cover config=self.config, should_watch=self.should_watch_files, skip_reason=self.watch_skip_reason, + read_cache=self.read_cache, ) # --- Database Factory --- diff --git a/src/basic_memory/api/v2/routers/knowledge_router.py b/src/basic_memory/api/v2/routers/knowledge_router.py index 3ecf5874b..a8ed7a66a 100644 --- a/src/basic_memory/api/v2/routers/knowledge_router.py +++ b/src/basic_memory/api/v2/routers/knowledge_router.py @@ -44,12 +44,24 @@ EntityRepositoryV2ExternalDep, RelationRepositoryV2ExternalDep, ProjectExternalIdPathDep, + ReadCacheDep, IndexFileExecutorV2ExternalDep, EntityVectorSyncSchedulerDep, RelationResolutionSchedulerDep, SessionDep, SessionMakerDep, ) +from basic_memory.read_cache import ( + ReadCacheKey, + ReadCacheOperation, + invalidate_project_read_cache, + read_cache_request_digest, + read_through_model, +) +from basic_memory.read_cache.policy import ( + READ_CACHE_MAX_PAYLOAD_BYTES, + READ_CACHE_TTL_SECONDS, +) from basic_memory.runtime.note_content import ( NOTE_CONTENT_BASE_CHECKSUM_HEADER, runtime_note_content_payload_as_dict, @@ -230,11 +242,15 @@ async def get_orphan_entities( @router.post("/resolve", response_model=EntityResolveResponse) async def resolve_identifier( project_id: ProjectExternalIdPathDep, + project_external_id: Annotated[ + str, Path(alias="project_id", description="Project external UUID") + ], data: EntityResolveRequest, link_resolver: LinkResolverV2ExternalDep, entity_repository: EntityRepositoryV2ExternalDep, project_repository: ProjectRepositoryDep, session: SessionDep, + read_cache: ReadCacheDep, ) -> EntityResolveResponse: """Resolve a string identifier (external_id, permalink, title, or path) to entity info. @@ -273,60 +289,77 @@ async def resolve_identifier( ): logger.info(f"API v2 request: resolve_identifier for '{data.identifier}'") - entity = await entity_repository.get_by_external_id(session, data.identifier) - resolution_method = "external_id" if entity else "search" - - if not entity: - try: - entity = await link_resolver.resolve_link( - data.identifier, - source_path=data.source_path, - strict=data.strict, - session=session, + async def load() -> EntityResolveResponse: + entity = await entity_repository.get_by_external_id(session, data.identifier) + resolution_method = "external_id" if entity else "search" + + if not entity: + try: + entity = await link_resolver.resolve_link( + data.identifier, + source_path=data.source_path, + strict=data.strict, + session=session, + ) + except AmbiguousIdentifierError as exc: + # A strict resolve refused to guess between several same-title notes (#1148). + # Surface it as 409 so edit/move report ambiguity and ask for an exact id. + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=str(exc), + ) from exc + if entity: + if entity.permalink == data.identifier: + resolution_method = "permalink" + elif entity.title == data.identifier: + resolution_method = "title" + elif entity.file_path == data.identifier: + resolution_method = "path" + + if not entity: + raise HTTPException( + status_code=404, + detail=f"Entity not found: '{data.identifier}'", ) - except AmbiguousIdentifierError as exc: - # A strict resolve refused to guess between several same-title notes (#1148). - # Surface it as 409 so edit/move report the ambiguity and ask for an exact id. + + owner_project = await project_repository.get_by_id(session, entity.project_id) + if not owner_project: # pragma: no cover raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=str(exc), - ) from exc - if entity: - if entity.permalink == data.identifier: - resolution_method = "permalink" - elif entity.title == data.identifier: - resolution_method = "title" - elif entity.file_path == data.identifier: - resolution_method = "path" - else: - resolution_method = "search" - - if not entity: - raise HTTPException(status_code=404, detail=f"Entity not found: '{data.identifier}'") - - owner_project = await project_repository.get_by_id(session, entity.project_id) - if not owner_project: # pragma: no cover - raise HTTPException( - status_code=500, - detail="Resolved entity references an unknown project", - ) + status_code=500, + detail="Resolved entity references an unknown project", + ) - result = EntityResolveResponse( - external_id=entity.external_id, - entity_id=entity.id, - project_external_id=owner_project.external_id, - permalink=entity.permalink, - file_path=entity.file_path, - title=entity.title, - resolution_method=resolution_method, - ) + result = EntityResolveResponse( + external_id=entity.external_id, + entity_id=entity.id, + project_external_id=owner_project.external_id, + permalink=entity.permalink, + file_path=entity.file_path, + title=entity.title, + resolution_method=resolution_method, + ) + logger.debug( + f"API v2 response: resolved '{data.identifier}' " + f"to external_id={result.external_id} via {resolution_method}" + ) + return result - logger.debug( - f"API v2 response: resolved '{data.identifier}' to external_id={result.external_id} via {resolution_method}" + return await read_through_model( + cache=read_cache, + key=ReadCacheKey( + project_id=project_external_id, + operation=ReadCacheOperation.resolve, + request_digest=read_cache_request_digest(data.model_dump_json()), + ), + model_type=EntityResolveResponse, + load=load, + ttl_seconds=READ_CACHE_TTL_SECONDS, + max_payload_bytes=READ_CACHE_MAX_PAYLOAD_BYTES, + # Cross-project references depend on two projects. Keep phase-one + # generation invalidation exact by caching only local resolutions. + should_store=lambda resolved: resolved.project_external_id == project_external_id, ) - return result - ## Single-file indexing endpoint @@ -381,6 +414,9 @@ def _canonical_file_path(home: pathlib.Path, segments: list[str]) -> str | None: async def index_file( data: IndexFileRequest, project_id: ProjectExternalIdPathDep, + project_external_id: Annotated[ + str, Path(alias="project_id", description="Project external UUID") + ], file_service: FileServiceV2ExternalDep, file_indexer: IndexFileExecutorV2ExternalDep, entity_repository: EntityRepositoryV2ExternalDep, @@ -388,6 +424,7 @@ async def index_file( search_service: SearchServiceV2ExternalDep, app_config: AppConfigDep, session_maker: SessionMakerDep, + read_cache: ReadCacheDep, ) -> EntityResponseV2: """Index a single markdown file that exists on disk but is not indexed yet. @@ -488,6 +525,7 @@ async def index_file( ) indexed = await file_indexer.index_file(file_path, source="api-index-file") + await invalidate_project_read_cache(read_cache, project_external_id) async with db.scoped_session(session_maker) as session: entity = await entity_repository.get_by_id(session, indexed.entity_id) if entity is None: # pragma: no cover @@ -518,10 +556,13 @@ async def index_file( @router.get("/entities/{entity_id}", response_model=EntityResponseV2) async def get_entity_by_id( project_id: ProjectExternalIdPathDep, - project_repository: ProjectRepositoryDep, + project_external_id: Annotated[ + str, Path(alias="project_id", description="Project external UUID") + ], entity_repository: EntityRepositoryV2ExternalDep, note_content_query_service: NoteContentQueryServiceDep, session: SessionDep, + read_cache: ReadCacheDep, entity_id: str = Path(..., description="Entity external ID (UUID)"), ) -> EntityResponseV2: """Get an entity by its external ID (UUID). @@ -546,30 +587,42 @@ async def get_entity_by_id( ): logger.info(f"API v2 request: get_entity_by_id entity_id={entity_id}") - project = await project_repository.get_by_id(session, project_id) - if project is None: # pragma: no cover - raise HTTPException(status_code=404, detail=f"Project with ID {project_id} not found") + async def load() -> EntityResponseV2: + note_payload = ( + await note_content_query_service.get_note_entity_payload_with_read_repair( + project_external_id=project_external_id, + entity_external_id=entity_id, + session=session, + ) + ) + if note_payload is not None: + result = entity_response_from_note_content_payload(note_payload) + logger.info(f"API v2 response: external_id={entity_id}, title='{result.title}'") + return result - note_payload = await note_content_query_service.get_note_entity_payload_with_read_repair( - project_external_id=project.external_id, - entity_external_id=entity_id, - session=session, - ) - if note_payload is not None: - result = entity_response_from_note_content_payload(note_payload) + entity = await entity_repository.get_by_external_id(session, entity_id) + if not entity: + raise HTTPException( + status_code=404, + detail=f"Entity with external_id '{entity_id}' not found", + ) + + result = EntityResponseV2.model_validate(entity) logger.info(f"API v2 response: external_id={entity_id}, title='{result.title}'") return result - entity = await entity_repository.get_by_external_id(session, entity_id) - if not entity: - raise HTTPException( - status_code=404, detail=f"Entity with external_id '{entity_id}' not found" - ) - - result = EntityResponseV2.model_validate(entity) - logger.info(f"API v2 response: external_id={entity_id}, title='{result.title}'") - - return result + return await read_through_model( + cache=read_cache, + key=ReadCacheKey( + project_id=project_external_id, + operation=ReadCacheOperation.entity, + request_digest=read_cache_request_digest(entity_id), + ), + model_type=EntityResponseV2, + load=load, + ttl_seconds=READ_CACHE_TTL_SECONDS, + max_payload_bytes=READ_CACHE_MAX_PAYLOAD_BYTES, + ) ## Create endpoints @@ -907,6 +960,9 @@ async def move_entity( async def move_directory( data: MoveDirectoryRequestV2, project_id: ProjectExternalIdPathDep, + project_external_id: Annotated[ + str, Path(alias="project_id", description="Project external UUID") + ], entity_service: EntityServiceV2ExternalDep, project_config: ProjectConfigV2ExternalDep, app_config: AppConfigDep, @@ -914,6 +970,7 @@ async def move_directory( vector_sync_scheduler: EntityVectorSyncSchedulerDep, relation_resolution_scheduler: RelationResolutionSchedulerDep, session_maker: SessionMakerDep, + read_cache: ReadCacheDep, ) -> DirectoryMoveResult: """Move all entities in a directory to a new location. @@ -946,22 +1003,29 @@ async def move_directory( project_config=project_config, app_config=app_config, ) + await invalidate_project_read_cache(read_cache, project_external_id) - # Reindex moved entities - for file_path in result.moved_files: - async with db.scoped_session(session_maker) as session: - entity = await entity_service.link_resolver.resolve_link( - file_path, session=session - ) - if entity: - await search_service.index_entity(entity) - _schedule_post_write_followups( - vector_sync_scheduler=vector_sync_scheduler, - relation_resolution_scheduler=relation_resolution_scheduler, - app_config=app_config, - entity_id=entity.id, - project_id=project_id, - ) + try: + # Reindex moved entities + for file_path in result.moved_files: + async with db.scoped_session(session_maker) as session: + entity = await entity_service.link_resolver.resolve_link( + file_path, session=session + ) + if entity: + await search_service.index_entity(entity) + _schedule_post_write_followups( + vector_sync_scheduler=vector_sync_scheduler, + relation_resolution_scheduler=relation_resolution_scheduler, + app_config=app_config, + entity_id=entity.id, + project_id=project_id, + ) + finally: + # Reindexing can alter entity responses after the move was first + # invalidated. Close that fill window even after partial + # follow-up failure. + await invalidate_project_read_cache(read_cache, project_external_id) logger.info( f"API v2 response: move_directory " @@ -984,6 +1048,7 @@ async def delete_directory( str, Path(alias="project_id", description="Project external UUID") ], directory_delete_service: DirectoryDeleteServiceDep, + read_cache: ReadCacheDep, ) -> Response: """Delete all entities in a directory. @@ -1011,6 +1076,7 @@ async def delete_directory( project_external_id=project_external_id, directory=data.directory, ) + await invalidate_project_read_cache(read_cache, project_external_id) payload = result.to_response_payload() logger.info( f"API v2 response: delete_directory " diff --git a/src/basic_memory/api/v2/routers/resource_router.py b/src/basic_memory/api/v2/routers/resource_router.py index fbdebe2c8..9f265f1da 100644 --- a/src/basic_memory/api/v2/routers/resource_router.py +++ b/src/basic_memory/api/v2/routers/resource_router.py @@ -13,6 +13,7 @@ from fastapi import APIRouter, HTTPException, Response, Path from loguru import logger +from pydantic import BaseModel, ConfigDict import logfire from basic_memory import db @@ -21,19 +22,44 @@ FileServiceV2ExternalDep, EntityRepositoryV2ExternalDep, NoteContentQueryServiceDep, + ReadCacheDep, SessionMakerDep, ) +from basic_memory.read_cache import ( + ReadCacheKey, + ReadCacheOperation, + read_cache_request_digest, + read_through_model, +) +from basic_memory.read_cache.policy import ( + READ_CACHE_MAX_PAYLOAD_BYTES, + READ_CACHE_TTL_SECONDS, +) from basic_memory.utils import validate_project_path router = APIRouter(prefix="/resource", tags=["resources-v2"]) +class CachedResourceResponse(BaseModel): + """Typed wire value for one cacheable resource response.""" + + content: bytes + media_type: str + + model_config = ConfigDict(ser_json_bytes="base64", val_json_bytes="base64") + + +def _is_markdown_resource(resource: CachedResourceResponse) -> bool: + return resource.media_type.partition(";")[0].strip().lower() == "text/markdown" + + @router.get("/{entity_id}") async def get_resource_content( config: ProjectConfigV2ExternalDep, entity_repository: EntityRepositoryV2ExternalDep, file_service: FileServiceV2ExternalDep, note_content_query_service: NoteContentQueryServiceDep, + read_cache: ReadCacheDep, session_maker: SessionMakerDep, project_id: str = Path(..., description="Project external UUID"), entity_id: str = Path(..., description="Entity external UUID"), @@ -61,69 +87,91 @@ async def get_resource_content( ): logger.debug(f"V2 Getting content for project {project_id}, entity_id: {entity_id}") - # Keep the DB session open only for the lookups; close it before the - # filesystem I/O below so large/slow resource reads don't pin a pooled - # connection (and an open read transaction on Postgres) for their duration. - async with db.scoped_session(session_maker) as session: - note_resource = await note_content_query_service.get_note_resource_with_read_repair( - project_external_id=project_id, - entity_external_id=entity_id, - session=session, - ) - if note_resource is not None: - return Response( - content=note_resource.content, - media_type=note_resource.content_type, + async def load() -> CachedResourceResponse: + # Keep the DB session open only for the lookups; close it before the + # filesystem I/O below so large/slow resource reads don't pin a pooled + # connection (and an open read transaction on Postgres) for their duration. + async with db.scoped_session(session_maker) as session: + note_resource = await note_content_query_service.get_note_resource_with_read_repair( + project_external_id=project_id, + entity_external_id=entity_id, + session=session, ) + if note_resource is not None: + return CachedResourceResponse( + content=note_resource.content.encode("utf-8"), + media_type=note_resource.content_type, + ) + + with logfire.span( + "api.resource.get_content.load_entity", + domain="resource", + action="get_content", + phase="load_entity", + ): + entity = await entity_repository.get_by_external_id(session, entity_id) + if not entity: + raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found") + # Copy the scalar columns needed for file I/O so the session can close. + entity_file_path = entity.file_path + entity_db_id = entity.id with logfire.span( - "api.resource.get_content.load_entity", + "api.resource.get_content.validate_path", domain="resource", action="get_content", - phase="load_entity", + phase="validate_path", ): - entity = await entity_repository.get_by_external_id(session, entity_id) - if not entity: - raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found") - # Copy the scalar columns needed for file I/O so the session can close. - entity_file_path = entity.file_path - entity_db_id = entity.id - - with logfire.span( - "api.resource.get_content.validate_path", - domain="resource", - action="get_content", - phase="validate_path", - ): - project_path = PathLib(config.home) - if not validate_project_path(entity_file_path, project_path): - logger.error( # pragma: no cover - f"Invalid file path in entity {entity_db_id}: {entity_file_path}" - ) - raise HTTPException( # pragma: no cover - status_code=500, - detail="Entity contains invalid file path", - ) + project_path = PathLib(config.home) + if not validate_project_path(entity_file_path, project_path): + logger.error( # pragma: no cover + f"Invalid file path in entity {entity_db_id}: {entity_file_path}" + ) + raise HTTPException( # pragma: no cover + status_code=500, + detail="Entity contains invalid file path", + ) - with logfire.span( - "api.resource.get_content.ensure_exists", - domain="resource", - action="get_content", - phase="ensure_exists", - ): - if not await file_service.exists(entity_file_path): - raise HTTPException( # pragma: no cover - status_code=404, - detail=f"File not found: {entity_file_path}", - ) + with logfire.span( + "api.resource.get_content.ensure_exists", + domain="resource", + action="get_content", + phase="ensure_exists", + ): + if not await file_service.exists(entity_file_path): + raise HTTPException( # pragma: no cover + status_code=404, + detail=f"File not found: {entity_file_path}", + ) - with logfire.span( - "api.resource.get_content.read_content", - domain="resource", - action="get_content", - phase="read_content", - ): - content = await file_service.read_file_bytes(entity_file_path) - content_type = file_service.content_type(entity_file_path) + with logfire.span( + "api.resource.get_content.read_content", + domain="resource", + action="get_content", + phase="read_content", + ): + content = await file_service.read_file_bytes(entity_file_path) + content_type = file_service.content_type(entity_file_path) + + return CachedResourceResponse( + content=content, + media_type=content_type, + ) - return Response(content=content, media_type=content_type) + resource = await read_through_model( + cache=read_cache, + key=ReadCacheKey( + project_id=project_id, + operation=ReadCacheOperation.resource, + request_digest=read_cache_request_digest(entity_id), + ), + model_type=CachedResourceResponse, + load=load, + ttl_seconds=READ_CACHE_TTL_SECONDS, + max_payload_bytes=READ_CACHE_MAX_PAYLOAD_BYTES, + should_store=_is_markdown_resource, + ) + return Response( + content=resource.content, + media_type=resource.media_type, + ) diff --git a/src/basic_memory/deps/__init__.py b/src/basic_memory/deps/__init__.py index 82dbe7cbd..f2dca6f7a 100644 --- a/src/basic_memory/deps/__init__.py +++ b/src/basic_memory/deps/__init__.py @@ -38,6 +38,11 @@ ProjectConfigV2ExternalDep, ) +from basic_memory.deps.read_cache import ( + get_read_cache, + ReadCacheDep, +) + from basic_memory.deps.repositories import ( get_entity_repository_v2_external, EntityRepositoryV2ExternalDep, @@ -123,6 +128,9 @@ "ProjectExternalIdPathDep", "get_project_config_v2_external", "ProjectConfigV2ExternalDep", + # Read cache + "get_read_cache", + "ReadCacheDep", # Repositories "get_entity_repository_v2_external", "EntityRepositoryV2ExternalDep", diff --git a/src/basic_memory/deps/read_cache.py b/src/basic_memory/deps/read_cache.py new file mode 100644 index 000000000..6cf95c1e3 --- /dev/null +++ b/src/basic_memory/deps/read_cache.py @@ -0,0 +1,21 @@ +"""Optional semantic read-cache dependency.""" + +from typing import Annotated + +from fastapi import Depends, Request + +from basic_memory.read_cache import ReadCache + + +def get_read_cache(request: Request) -> ReadCache: + """Return the host-injected cache or the container's no-op default.""" + container = getattr(request.app.state, "container", None) + if container is not None: + return container.read_cache + # Deferred import avoids api.app -> routers -> deps circular initialization. + from basic_memory.api.container import resolve_container + + return resolve_container().read_cache + + +ReadCacheDep = Annotated[ReadCache, Depends(get_read_cache)] diff --git a/src/basic_memory/deps/services.py b/src/basic_memory/deps/services.py index d5996b103..94198d260 100644 --- a/src/basic_memory/deps/services.py +++ b/src/basic_memory/deps/services.py @@ -23,6 +23,7 @@ ProjectConfigV2ExternalDep, ProjectRepositoryDep, ) +from basic_memory.deps.read_cache import ReadCacheDep from basic_memory.deps.repositories import ( EntityRepositoryV2ExternalDep, ObservationRepositoryV2ExternalDep, @@ -47,6 +48,7 @@ from basic_memory.index.local_project import ( LocalProjectIndexCommand, LocalProjectIndexRunner, + LocalProjectIndexRuntimeFactory, ) from basic_memory.index.project_indexing import ( ProjectIndexCommand, @@ -313,6 +315,7 @@ async def get_note_content_mutation_service( file_indexer: IndexFileExecutorV2ExternalDep, session_maker: SessionMakerDep, app_config: AppConfigDep, + read_cache: ReadCacheDep, ) -> NoteContentMutationService: """Create the local accepted-note mutation facade for API routes.""" accepted_note_repositories = AcceptedNoteRepositories( @@ -347,6 +350,7 @@ async def get_note_content_mutation_service( file_indexer=file_indexer, session_maker=session_maker, ), + read_cache=read_cache, ) @@ -361,11 +365,13 @@ async def get_note_content_mutation_service( async def get_project_index_runner( project_repository: ProjectRepositoryDep, session_maker: SessionMakerDep, + read_cache: ReadCacheDep, ) -> LocalProjectIndexRunner: """Create the local project-index runner used by API routes and tasks.""" return LocalProjectIndexRunner( project_repository=project_repository, session_maker=session_maker, + runtime_factory=LocalProjectIndexRuntimeFactory(read_cache=read_cache), ) diff --git a/src/basic_memory/index/local_project.py b/src/basic_memory/index/local_project.py index fb3a0b41f..8572941df 100644 --- a/src/basic_memory/index/local_project.py +++ b/src/basic_memory/index/local_project.py @@ -5,7 +5,7 @@ import asyncio import os from collections.abc import Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Any, override, Protocol @@ -79,6 +79,7 @@ resolve_project_index_completion_relations, ) from basic_memory.models import Entity, Project +from basic_memory.read_cache import NullReadCache, ReadCache, invalidate_project_read_cache from basic_memory.repository import NoteContentRepository from basic_memory.runtime.jobs import ( RuntimeIndexFileBatchJobRequest, @@ -459,6 +460,7 @@ class LocalProjectIndexRuntime: embedding_vector_sync: EmbeddingBatchVectorSync | None = None batch_size: int = 100 coordinator_job_id: RuntimeJobId | None = None + read_cache: ReadCache = field(default_factory=NullReadCache) LocalProjectIndexObservation = ProjectIndexObservation @@ -573,6 +575,7 @@ class LocalProjectIndexRuntimeFactory: batch_size: int = 100 read_max_concurrent: int = 8 index_max_concurrent: int = 8 + read_cache: ReadCache = field(default_factory=NullReadCache) async def dependencies_for_project(self, project: Project) -> LocalIndexProjectDependencies: return await self.dependency_provider.dependencies_for_project(project) @@ -652,6 +655,7 @@ def runtime_from_dependencies( ), embedding_vector_sync=local_project_embedding_vector_sync(dependencies), batch_size=self.batch_size, + read_cache=self.read_cache, ) async def runtime_for_project(self, project: Project) -> LocalProjectIndexRuntime: @@ -770,12 +774,28 @@ async def run_local_project_index( batch_size=runtime.batch_size, embedding_vector_sync=runtime.embedding_vector_sync, ) + # Project indexing has already committed entity/search changes. Invalidate + # before relation repair so a later failure cannot leave pre-index values + # reachable for the full TTL. + await invalidate_project_read_cache( + runtime.read_cache, + request.project.project_external_id, + ) if runtime.completion_relation_runtime is not None: - await resolve_project_index_completion_relations( - ProjectIndexRelationResolutionContext( - project_id=request.project.project_id, - project_path=request.project.project_path, - ), - runtime.completion_relation_runtime, - ) + try: + await resolve_project_index_completion_relations( + ProjectIndexRelationResolutionContext( + project_id=request.project.project_id, + project_path=request.project.project_path, + ), + runtime.completion_relation_runtime, + ) + finally: + # Relation repair can mutate cached entity responses after the first + # invalidation. Clear any value filled during that window, including + # when repair commits partial progress before raising. + await invalidate_project_read_cache( + runtime.read_cache, + request.project.project_external_id, + ) return result diff --git a/src/basic_memory/index/local_runtime.py b/src/basic_memory/index/local_runtime.py index 4a2dc8c96..87da36b27 100644 --- a/src/basic_memory/index/local_runtime.py +++ b/src/basic_memory/index/local_runtime.py @@ -2,7 +2,7 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from loguru import logger @@ -57,6 +57,7 @@ RepositoryExternalFileDeleteEntities, ) from basic_memory.models import Entity, Project +from basic_memory.read_cache import NullReadCache, ReadCache, invalidate_project_read_cache from basic_memory.repository import NoteContentRepository from basic_memory.runtime.projects import ProjectRuntimeReference from basic_memory.runtime.storage import ( @@ -144,6 +145,7 @@ class LocalInlineStorageEventResultRecorder: relation_cleanup_search_refresher: ProjectIndexMovedEntitySearchRefresher relation_runtime: RelationResolutionRuntime index_embeddings: bool + read_cache: ReadCache = field(default_factory=NullReadCache) async def index_file_completed( self, @@ -158,6 +160,12 @@ async def index_file_completed( entity_id=result.entity_id, ) + if result.status == IndexFileJobStatus.processed: + await invalidate_project_read_cache( + self.read_cache, + self.project.project_external_id, + ) + # --- Relation repair --- # Back-resolve forward references now that this file is indexed. relation_request = plan_index_file_relation_resolution( @@ -168,15 +176,24 @@ async def index_file_completed( ) ) if relation_request is not None: - relation_result = await resolve_project_relations(self.relation_runtime) - logger.info( - "Local event-index relation repair completed", - project_id=relation_request.project_id, - project_path=relation_request.project_path, - resolved=relation_result.resolved, - remaining=relation_result.remaining, - passes=relation_result.passes, - ) + try: + relation_result = await resolve_project_relations(self.relation_runtime) + logger.info( + "Local event-index relation repair completed", + project_id=relation_request.project_id, + project_path=relation_request.project_path, + resolved=relation_result.resolved, + remaining=relation_result.remaining, + passes=relation_result.passes, + ) + finally: + # Relation repair changes cached entity payloads after indexing. + # A second generation bump closes the fill window opened by the + # first post-index invalidation, even after partial failure. + await invalidate_project_read_cache( + self.read_cache, + self.project.project_external_id, + ) # --- Semantic embedding --- # Trigger: a file was (re)indexed and semantic embeddings are enabled. @@ -209,12 +226,27 @@ async def delete_file_completed( ) if not result.entity_deleted: return - if not isinstance(result.deleted_entity, Entity): - raise RuntimeError("Local external file delete returned an incomplete entity result") - await self.search_service.handle_delete(result.deleted_entity) - await self.relation_cleanup_search_refresher.refresh_moved_entities( - tuple(sorted(result.relation_cleanup_entity_ids)), + await invalidate_project_read_cache( + self.read_cache, + self.project.project_external_id, ) + try: + if not isinstance(result.deleted_entity, Entity): + raise RuntimeError( + "Local external file delete returned an incomplete entity result" + ) + await self.search_service.handle_delete(result.deleted_entity) + await self.relation_cleanup_search_refresher.refresh_moved_entities( + tuple(sorted(result.relation_cleanup_entity_ids)), + ) + finally: + # Cleanup may rewrite relations on surviving entities. Invalidate + # values filled after the delete became visible, including partial + # cleanup progress followed by an error. + await invalidate_project_read_cache( + self.read_cache, + self.project.project_external_id, + ) async def skip_event(self, operation: RuntimeStorageEventOperation) -> None: logger.debug( @@ -264,6 +296,7 @@ class LocalWatchEventIndexRuntimeFactory: # let runtime construction opt in via semantic_search_enabled (#1016). index_embeddings: bool = False move_batch_size: int = 100 + read_cache: ReadCache = field(default_factory=NullReadCache) async def runtime_for_project(self, project: Project) -> StorageEventIndexRuntime: dependencies = await self.dependency_provider.dependencies_for_project(project) @@ -332,6 +365,7 @@ async def runtime_for_project(self, project: Project) -> StorageEventIndexRuntim entity_indexer=dependencies.search_service, ), index_embeddings=self.index_embeddings, + read_cache=self.read_cache, ), index_embeddings=self.index_embeddings, ) diff --git a/src/basic_memory/index/watch_coordinator.py b/src/basic_memory/index/watch_coordinator.py index eba94eea3..4c527343b 100644 --- a/src/basic_memory/index/watch_coordinator.py +++ b/src/basic_memory/index/watch_coordinator.py @@ -9,6 +9,7 @@ from loguru import logger from basic_memory.config import BasicMemoryConfig +from basic_memory.read_cache import NullReadCache, ReadCache class WatchStatus(Enum): @@ -30,6 +31,7 @@ class WatchCoordinator: should_watch: bool = True skip_reason: str | None = None quiet: bool = True + read_cache: ReadCache = field(default_factory=NullReadCache) _status: WatchStatus = field(default=WatchStatus.NOT_STARTED, init=False) _watch_task: asyncio.Task[None] | None = field(default=None, init=False) @@ -72,6 +74,7 @@ async def _watch_runner() -> None: # pragma: no cover self.config, quiet=self.quiet, recovery_complete=recovery_complete, + read_cache=self.read_cache, ) except asyncio.CancelledError: logger.debug("Local event-index watcher cancelled") diff --git a/src/basic_memory/read_cache/__init__.py b/src/basic_memory/read_cache/__init__.py new file mode 100644 index 000000000..a5bc41832 --- /dev/null +++ b/src/basic_memory/read_cache/__init__.py @@ -0,0 +1,31 @@ +"""Optional semantic read caching for Basic Memory.""" + +from basic_memory.read_cache.contract import ( + ReadCache, + ReadCacheDataError, + ReadCacheInvalidationStatus, + ReadCacheKey, + ReadCacheLookup, + ReadCacheOperation, + ReadCacheStoreStatus, + ReadCacheUnavailable, +) +from basic_memory.read_cache.invalidation import invalidate_project_read_cache +from basic_memory.read_cache.keys import read_cache_request_digest +from basic_memory.read_cache.null import NullReadCache +from basic_memory.read_cache.read_through import read_through_model + +__all__ = [ + "NullReadCache", + "ReadCache", + "ReadCacheDataError", + "ReadCacheInvalidationStatus", + "ReadCacheKey", + "ReadCacheLookup", + "ReadCacheOperation", + "ReadCacheStoreStatus", + "ReadCacheUnavailable", + "invalidate_project_read_cache", + "read_cache_request_digest", + "read_through_model", +] diff --git a/src/basic_memory/read_cache/contract.py b/src/basic_memory/read_cache/contract.py new file mode 100644 index 000000000..4f855dd1c --- /dev/null +++ b/src/basic_memory/read_cache/contract.py @@ -0,0 +1,106 @@ +"""Portable contract for best-effort semantic read caching.""" + +from dataclasses import dataclass +from enum import StrEnum +from typing import Protocol +from uuid import UUID + + +class ReadCacheOperation(StrEnum): + """Read operations supported by the initial cache rollout.""" + + entity = "entity" + resolve = "resolve" + resource = "resource" + + +class ReadCacheStoreStatus(StrEnum): + """Outcome of one best-effort cache store.""" + + stored = "stored" + superseded = "superseded" + disabled = "disabled" + + +class ReadCacheInvalidationStatus(StrEnum): + """Outcome of one project-generation invalidation attempt.""" + + invalidated = "invalidated" + unavailable = "unavailable" + disabled = "disabled" + + +def canonical_read_cache_project_id(project_id: str) -> str: + """Return one canonical UUID spelling for project cache scope.""" + if not project_id: + raise ValueError("read-cache project_id must not be empty") + try: + return str(UUID(project_id)) + except ValueError as error: + raise ValueError("read-cache project_id must be a valid UUID") from error + + +@dataclass(frozen=True, slots=True) +class ReadCacheKey: + """Project-scoped identity for one canonical read request.""" + + project_id: str + operation: ReadCacheOperation + request_digest: str + + def __post_init__(self) -> None: + object.__setattr__( + self, + "project_id", + canonical_read_cache_project_id(self.project_id), + ) + if len(self.request_digest) != 64: + raise ValueError("read-cache request_digest must be a SHA-256 hex digest") + try: + bytes.fromhex(self.request_digest) + except ValueError as error: + raise ValueError("read-cache request_digest must be a SHA-256 hex digest") from error + + +@dataclass(frozen=True, slots=True) +class ReadCacheLookup: + """Cache lookup result plus the generation observed by that read. + + A missing generation means the cache implementation is disabled. Read-through + callers can then skip serialization and the store call entirely. + """ + + generation: str | None + payload: bytes | None = None + + @property + def is_hit(self) -> bool: + return self.payload is not None + + +class ReadCacheUnavailable(RuntimeError): + """The cache backend could not complete an optional operation.""" + + +class ReadCacheDataError(RuntimeError): + """A cached value violated the Basic Memory cache encoding contract.""" + + +class ReadCache(Protocol): + """Best-effort read cache with project-generation invalidation.""" + + async def lookup(self, key: ReadCacheKey) -> ReadCacheLookup: + """Return a cached payload and the generation observed by this lookup.""" + + async def store( + self, + key: ReadCacheKey, + lookup: ReadCacheLookup, + payload: bytes, + *, + ttl_seconds: int, + ) -> ReadCacheStoreStatus: + """Store a payload under the generation observed by ``lookup``.""" + + async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: + """Make every existing cached value for one project unreachable.""" diff --git a/src/basic_memory/read_cache/invalidation.py b/src/basic_memory/read_cache/invalidation.py new file mode 100644 index 000000000..3d6c07c6a --- /dev/null +++ b/src/basic_memory/read_cache/invalidation.py @@ -0,0 +1,45 @@ +"""Best-effort project invalidation shared by mutation and indexing runtimes.""" + +from loguru import logger + +import logfire +from basic_memory.read_cache.contract import ( + ReadCache, + ReadCacheInvalidationStatus, + ReadCacheUnavailable, +) + + +def _record_invalidation_event(event: ReadCacheInvalidationStatus) -> None: + logfire.metric_counter("basic_memory_read_cache_events_total").add( + 1, + attributes={ + "operation": "project", + "event": event.value, + }, + ) + + +async def invalidate_project_read_cache( + cache: ReadCache, + project_id: str, +) -> ReadCacheInvalidationStatus: + """Invalidate one project without failing an already-committed mutation.""" + with logfire.span("read_cache.invalidate_project") as span: + try: + status = await cache.invalidate_project(project_id) + except ReadCacheUnavailable as error: + # Trigger: an authoritative mutation committed while Redis was unavailable. + # Why: failing the request cannot roll the mutation back and would invite + # duplicate retries; the 60-second TTL already bounds stale exposure. + # Outcome: surface prominent telemetry and let the committed write succeed. + status = ReadCacheInvalidationStatus.unavailable + logger.error( + "Read cache project invalidation unavailable; cached values may remain " + "reachable until TTL expiry", + error=str(error), + ) + + _record_invalidation_event(status) + span.set_attribute("cache.outcome", status.value) + return status diff --git a/src/basic_memory/read_cache/keys.py b/src/basic_memory/read_cache/keys.py new file mode 100644 index 000000000..66ac1f080 --- /dev/null +++ b/src/basic_memory/read_cache/keys.py @@ -0,0 +1,79 @@ +"""Canonical request digests and Redis key construction.""" + +from dataclasses import dataclass +from hashlib import sha256 + +from basic_memory.read_cache.contract import ( + ReadCacheKey, + canonical_read_cache_project_id, +) + +DEFAULT_READ_CACHE_PREFIX = "bm:read:v1" + + +def read_cache_request_digest(*parts: str) -> str: + """Hash length-delimited request parts into one stable cache identity.""" + digest = sha256() + for part in parts: + encoded = part.encode("utf-8") + digest.update(len(encoded).to_bytes(8, byteorder="big")) + digest.update(encoded) + return digest.hexdigest() + + +@dataclass(frozen=True, slots=True) +class RedisReadCacheKeys: + """Redis keys for one project generation and canonical request.""" + + generation: str + data: str + + +def _redis_read_cache_cluster_scope(*, namespace: str, project_id: str) -> str: + if not namespace: + raise ValueError("read-cache namespace must not be empty") + + scope_digest = read_cache_request_digest( + namespace, + canonical_read_cache_project_id(project_id), + ) + return f"{{{scope_digest}}}" + + +def redis_read_cache_generation_key( + *, + prefix: str, + namespace: str, + project_id: str, +) -> str: + """Build the generation key shared by every cached read in one project.""" + if not prefix or any(character.isspace() for character in prefix): + raise ValueError("read-cache prefix must be non-empty and contain no whitespace") + + cluster_scope = _redis_read_cache_cluster_scope( + namespace=namespace, + project_id=project_id, + ) + return f"{prefix}:{cluster_scope}:generation" + + +def redis_read_cache_keys( + *, + prefix: str, + namespace: str, + key: ReadCacheKey, +) -> RedisReadCacheKeys: + """Build versioned Redis keys without exposing namespace values.""" + cluster_scope = _redis_read_cache_cluster_scope( + namespace=namespace, + project_id=key.project_id, + ) + generation = redis_read_cache_generation_key( + prefix=prefix, + namespace=namespace, + project_id=key.project_id, + ) + return RedisReadCacheKeys( + generation=generation, + data=f"{prefix}:{cluster_scope}:{key.operation.value}:{key.request_digest}", + ) diff --git a/src/basic_memory/read_cache/null.py b/src/basic_memory/read_cache/null.py new file mode 100644 index 000000000..e7dc957e3 --- /dev/null +++ b/src/basic_memory/read_cache/null.py @@ -0,0 +1,28 @@ +"""No-op cache used by default local-first installations.""" + +from basic_memory.read_cache.contract import ( + ReadCacheInvalidationStatus, + ReadCacheKey, + ReadCacheLookup, + ReadCacheStoreStatus, +) + + +class NullReadCache: + """A disabled cache implementation with no external dependencies.""" + + async def lookup(self, key: ReadCacheKey) -> ReadCacheLookup: + return ReadCacheLookup(generation=None) + + async def store( + self, + key: ReadCacheKey, + lookup: ReadCacheLookup, + payload: bytes, + *, + ttl_seconds: int, + ) -> ReadCacheStoreStatus: + return ReadCacheStoreStatus.disabled + + async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: + return ReadCacheInvalidationStatus.disabled diff --git a/src/basic_memory/read_cache/policy.py b/src/basic_memory/read_cache/policy.py new file mode 100644 index 000000000..6ee02bffb --- /dev/null +++ b/src/basic_memory/read_cache/policy.py @@ -0,0 +1,4 @@ +"""Initial semantic read-cache policy.""" + +READ_CACHE_TTL_SECONDS = 60 +READ_CACHE_MAX_PAYLOAD_BYTES = 1024 * 1024 diff --git a/src/basic_memory/read_cache/read_through.py b/src/basic_memory/read_cache/read_through.py new file mode 100644 index 000000000..7549a7a33 --- /dev/null +++ b/src/basic_memory/read_cache/read_through.py @@ -0,0 +1,111 @@ +"""Typed read-through behavior shared by cacheable API boundaries.""" + +from collections.abc import Awaitable, Callable + +import logfire +from pydantic import BaseModel + +from basic_memory.read_cache.contract import ( + ReadCache, + ReadCacheKey, + ReadCacheUnavailable, +) + + +def _record_event(key: ReadCacheKey, event: str) -> None: + logfire.metric_counter("basic_memory_read_cache_events_total").add( + 1, + attributes={ + "operation": key.operation.value, + "event": event, + }, + ) + + +async def read_through_model[ModelT: BaseModel]( + *, + cache: ReadCache, + key: ReadCacheKey, + model_type: type[ModelT], + load: Callable[[], Awaitable[ModelT]], + ttl_seconds: int, + max_payload_bytes: int, + should_store: Callable[[ModelT], bool] | None = None, +) -> ModelT: + """Return a validated cached model or load and best-effort cache it.""" + if ttl_seconds <= 0: + raise ValueError("read-cache ttl_seconds must be positive") + if max_payload_bytes <= 0: + raise ValueError("read-cache max_payload_bytes must be positive") + + with logfire.span( + "read_cache.read_through", + operation=key.operation.value, + ) as span: + try: + lookup = await cache.lookup(key) + except ReadCacheUnavailable: + # Trigger: Redis is unreachable or timed out. + # Why: the database or storage path remains authoritative. + # Outcome: return fresh data without attempting another cache operation. + _record_event(key, "bypass") + span.set_attribute("cache.outcome", "bypass") + return await load() + + if lookup.generation is None: + # Trigger: the host selected the no-op cache implementation. + # Why: an optional cache must not serialize every response merely to + # discover that storage is disabled. + # Outcome: execute only the authoritative read path. + _record_event(key, "disabled") + span.set_attribute("cache.outcome", "disabled") + return await load() + + if lookup.payload is not None: + _record_event(key, "hit") + span.set_attributes( + { + "cache.outcome": "hit", + "cache.payload_bytes": len(lookup.payload), + } + ) + return model_type.model_validate_json(lookup.payload) + + _record_event(key, "miss") + value = await load() + if should_store is not None and not should_store(value): + _record_event(key, "ineligible") + span.set_attribute("cache.outcome", "ineligible") + return value + + payload = value.model_dump_json().encode("utf-8") + if len(payload) > max_payload_bytes: + _record_event(key, "oversize") + span.set_attributes( + { + "cache.outcome": "oversize", + "cache.payload_bytes": len(payload), + } + ) + return value + + try: + store_status = await cache.store( + key, + lookup, + payload, + ttl_seconds=ttl_seconds, + ) + except ReadCacheUnavailable: + _record_event(key, "store_unavailable") + span.set_attribute("cache.outcome", "store_unavailable") + return value + + _record_event(key, store_status.value) + span.set_attributes( + { + "cache.outcome": store_status.value, + "cache.payload_bytes": len(payload), + } + ) + return value diff --git a/src/basic_memory/read_cache/redis.py b/src/basic_memory/read_cache/redis.py new file mode 100644 index 000000000..b3f26e5da --- /dev/null +++ b/src/basic_memory/read_cache/redis.py @@ -0,0 +1,191 @@ +"""Redis implementation of Basic Memory semantic read caching. + +This module is imported only when the optional ``redis`` dependency is installed. +The caller owns the Redis client lifecycle. +""" + +from uuid import uuid4 + +from redis.asyncio import Redis +from redis.exceptions import ConnectionError as RedisConnectionError +from redis.exceptions import TimeoutError as RedisTimeoutError + +from basic_memory.read_cache.contract import ( + ReadCacheDataError, + ReadCacheInvalidationStatus, + ReadCacheKey, + ReadCacheLookup, + ReadCacheStoreStatus, + ReadCacheUnavailable, +) +from basic_memory.read_cache.keys import ( + DEFAULT_READ_CACHE_PREFIX, + RedisReadCacheKeys, + redis_read_cache_generation_key, + redis_read_cache_keys, +) + +_ENVELOPE_SEPARATOR = b"\n" +_INITIALIZE_GENERATION_SCRIPT = """ +local generation = redis.call("GET", KEYS[1]) +if generation then + return generation +end +redis.call("SET", KEYS[1], ARGV[1]) +return ARGV[1] +""" +_STORE_IF_CURRENT_SCRIPT = """ +if redis.call("GET", KEYS[1]) ~= ARGV[1] then + return 0 +end +redis.call("SET", KEYS[2], ARGV[2], "EX", ARGV[3]) +return 1 +""" + + +def create_redis_read_cache_client( + url: str, + *, + max_connections: int = 20, + socket_timeout: float = 0.1, +) -> Redis: + """Create an async Redis client whose lifecycle remains caller-owned.""" + return Redis.from_url( + url, + decode_responses=False, + max_connections=max_connections, + socket_connect_timeout=socket_timeout, + socket_timeout=socket_timeout, + ) + + +def _required_bytes(value: object, *, field: str) -> bytes: + if isinstance(value, bytes): + return value + if isinstance(value, str): + return value.encode("utf-8") + raise ReadCacheDataError(f"Redis returned an invalid {field} value") + + +def _decode_generation(value: object) -> tuple[bytes, str]: + encoded = _required_bytes(value, field="generation") + try: + generation = encoded.decode("ascii") + decoded = bytes.fromhex(generation) + except (UnicodeDecodeError, ValueError) as error: + raise ReadCacheDataError("Redis returned an invalid generation token") from error + if len(decoded) != 16: + raise ReadCacheDataError("Redis returned an invalid generation token") + return encoded, generation + + +def _store_status(value: object) -> ReadCacheStoreStatus: + if value == 1: + return ReadCacheStoreStatus.stored + if value == 0: + return ReadCacheStoreStatus.superseded + raise ReadCacheDataError("Redis returned an invalid cache store result") + + +class RedisReadCache: + """Namespace-bound Redis cache with race-safe generation invalidation. + + A managed host may reuse one client and Redis deployment across tenants. + It must construct this lightweight adapter from trusted tenant/workspace + context so the mandatory namespace cannot be supplied by an API caller. + """ + + def __init__( + self, + *, + client: Redis, + namespace: str, + prefix: str = DEFAULT_READ_CACHE_PREFIX, + ) -> None: + namespace = namespace.strip() + if not namespace: + raise ValueError("read-cache namespace must not be empty") + self._client = client + self._namespace = namespace + self._prefix = prefix + + def _keys(self, key: ReadCacheKey) -> RedisReadCacheKeys: + return redis_read_cache_keys( + prefix=self._prefix, + namespace=self._namespace, + key=key, + ) + + async def _initialize_generation(self, generation_key: str) -> bytes: + token = uuid4().hex.encode("ascii") + generation = await self._client.eval( + _INITIALIZE_GENERATION_SCRIPT, + 1, + generation_key, + token, + ) + return _required_bytes(generation, field="generation") + + async def lookup(self, key: ReadCacheKey) -> ReadCacheLookup: + keys = self._keys(key) + try: + generation_value, cached_value = await self._client.mget([keys.generation, keys.data]) + if generation_value is None: + generation_value = await self._initialize_generation(keys.generation) + except (RedisConnectionError, RedisTimeoutError) as error: + raise ReadCacheUnavailable("Redis cache lookup failed") from error + + generation, generation_text = _decode_generation(generation_value) + if cached_value is None: + return ReadCacheLookup(generation=generation_text) + + encoded = _required_bytes(cached_value, field="cached payload") + cached_generation, separator, payload = encoded.partition(_ENVELOPE_SEPARATOR) + if not separator or not cached_generation: + raise ReadCacheDataError("Redis cached payload has an invalid generation envelope") + if cached_generation != generation: + return ReadCacheLookup(generation=generation_text) + return ReadCacheLookup(generation=generation_text, payload=payload) + + async def store( + self, + key: ReadCacheKey, + lookup: ReadCacheLookup, + payload: bytes, + *, + ttl_seconds: int, + ) -> ReadCacheStoreStatus: + if lookup.generation is None: + raise ValueError("Redis cache store requires a lookup generation") + if ttl_seconds <= 0: + raise ValueError("read-cache ttl_seconds must be positive") + + keys = self._keys(key) + generation, _ = _decode_generation(lookup.generation) + encoded = generation + _ENVELOPE_SEPARATOR + payload + try: + stored = await self._client.eval( + _STORE_IF_CURRENT_SCRIPT, + 2, + keys.generation, + keys.data, + generation, + encoded, + ttl_seconds, + ) + except (RedisConnectionError, RedisTimeoutError) as error: + raise ReadCacheUnavailable("Redis cache store failed") from error + + return _store_status(stored) + + async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: + generation_key = redis_read_cache_generation_key( + prefix=self._prefix, + namespace=self._namespace, + project_id=project_id, + ) + try: + await self._client.set(generation_key, uuid4().hex.encode("ascii")) + except (RedisConnectionError, RedisTimeoutError) as error: + raise ReadCacheUnavailable("Redis project invalidation failed") from error + return ReadCacheInvalidationStatus.invalidated diff --git a/src/basic_memory/services/initialization.py b/src/basic_memory/services/initialization.py index 7ce0d7815..cc27ba520 100644 --- a/src/basic_memory/services/initialization.py +++ b/src/basic_memory/services/initialization.py @@ -23,6 +23,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from basic_memory.index.local_project import LocalProjectIndexRuntimeProvider + from basic_memory.read_cache import ReadCache async def run_initial_project_index( @@ -151,6 +152,7 @@ async def initialize_file_indexing( quiet: bool = True, *, recovery_complete: asyncio.Event | None = None, + read_cache: "ReadCache | None" = None, ) -> None: """Initialize file indexing services. @@ -160,6 +162,7 @@ async def initialize_file_indexing( app_config: The Basic Memory project configuration quiet: Whether to suppress Rich console output (True for MCP, False for CLI watch) recovery_complete: Optional startup barrier set after durable recovery finishes. + read_cache: Optional host-injected semantic read cache. Returns: The watch service task that's monitoring file changes @@ -177,6 +180,7 @@ async def initialize_file_indexing( from basic_memory.index.local_project import LocalProjectIndexRuntimeFactory from basic_memory.index.local_runtime import LocalWatchEventIndexRuntimeFactory from basic_memory.index.watch_service import WatchService + from basic_memory.read_cache import NullReadCache # Get database session (migrations already run if needed) _, session_maker = await db.get_or_create_db( @@ -190,11 +194,15 @@ async def initialize_file_indexing( # running multiple `basic-memory mcp --project X` processes does not produce # duplicate watchers fighting over the same files. constrained_project = os.environ.get("BASIC_MEMORY_MCP_PROJECT") + active_read_cache = read_cache if read_cache is not None else NullReadCache() event_index_runtime_factory = LocalWatchEventIndexRuntimeFactory( index_embeddings=app_config.semantic_search_enabled, + read_cache=active_read_cache, + ) + project_index_runtime_factory = LocalProjectIndexRuntimeFactory( + read_cache=active_read_cache, ) - project_index_runtime_factory = LocalProjectIndexRuntimeFactory() # Initialize watch service watch_service = WatchService( diff --git a/src/basic_memory/services/note_content_writes.py b/src/basic_memory/services/note_content_writes.py index e6c5e0ae1..d7993792b 100644 --- a/src/basic_memory/services/note_content_writes.py +++ b/src/basic_memory/services/note_content_writes.py @@ -30,6 +30,7 @@ RuntimeAcceptedNoteChange, RuntimeNoteContentResponsePayload, ) +from basic_memory.read_cache import NullReadCache, ReadCache, invalidate_project_read_cache from basic_memory.schemas.base import Entity as EntitySchema from basic_memory.schemas.request import EditEntityRequest @@ -138,11 +139,17 @@ def __init__( mutation_dependencies: AcceptedNoteMutationDependencies, content_freshener: NoteContentMutationFreshener | None = None, actor_resolver: NoteContentMutationActorResolver | None = None, + read_cache: ReadCache | None = None, ) -> None: self.session_maker = session_maker self.mutation_dependencies = mutation_dependencies self.content_freshener = content_freshener self.actor_resolver = actor_resolver + self.read_cache = read_cache if read_cache is not None else NullReadCache() + + async def _invalidate_project(self, project_external_id: str) -> None: + """Invalidate semantic reads after the accepted transaction commits.""" + await invalidate_project_read_cache(self.read_cache, project_external_id) def _resolve_actor( self, @@ -200,7 +207,7 @@ async def create_note( ) try: async with accepted_note_transaction(self.session_maker) as session: - return await run_accepted_note_create( + accepted = await run_accepted_note_create( session, request=AcceptedNoteCreateMutation( project_external_id=project_external_id, @@ -214,6 +221,8 @@ async def create_note( ), dependencies=self.mutation_dependencies, ) + await self._invalidate_project(project_external_id) + return accepted except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error @@ -251,7 +260,7 @@ async def update_note( entity_external_id=entity_external_id, ) async with accepted_note_transaction(self.session_maker) as session: - return await run_accepted_note_update( + accepted = await run_accepted_note_update( session, request=AcceptedNoteUpdateMutation( project_external_id=project_external_id, @@ -267,6 +276,8 @@ async def update_note( ), dependencies=self.mutation_dependencies, ) + await self._invalidate_project(project_external_id) + return accepted except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error @@ -295,7 +306,7 @@ async def edit_note( entity_external_id=entity_external_id, ) async with accepted_note_transaction(self.session_maker) as session: - return await run_accepted_note_edit( + accepted = await run_accepted_note_edit( session, request=AcceptedNoteEditMutation( project_external_id=project_external_id, @@ -310,6 +321,8 @@ async def edit_note( ), dependencies=self.mutation_dependencies, ) + await self._invalidate_project(project_external_id) + return accepted except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error @@ -338,7 +351,7 @@ async def move_note( entity_external_id=entity_external_id, ) async with accepted_note_transaction(self.session_maker) as session: - return await run_accepted_note_move( + accepted = await run_accepted_note_move( session, request=AcceptedNoteMoveMutation( project_external_id=project_external_id, @@ -353,6 +366,8 @@ async def move_note( ), dependencies=self.mutation_dependencies, ) + await self._invalidate_project(project_external_id) + return accepted except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error @@ -369,7 +384,7 @@ async def delete_note( entity_external_id=entity_external_id, ) async with accepted_note_transaction(self.session_maker) as session: - return await run_accepted_note_delete( + accepted = await run_accepted_note_delete( session, request=AcceptedNoteDeleteMutation( project_external_id=project_external_id, @@ -377,5 +392,7 @@ async def delete_note( ), dependencies=self.mutation_dependencies, ) + await self._invalidate_project(project_external_id) + return accepted except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error diff --git a/test-int/read_cache/conftest.py b/test-int/read_cache/conftest.py new file mode 100644 index 000000000..f9c6cc97c --- /dev/null +++ b/test-int/read_cache/conftest.py @@ -0,0 +1,101 @@ +"""Real Redis fixtures for read-cache integration tests.""" + +from __future__ import annotations + +import os +import shutil +import time +from collections.abc import AsyncGenerator, Generator +from dataclasses import dataclass +from uuid import uuid4 + +import pytest +import pytest_asyncio +from redis import Redis as SyncRedis +from redis.asyncio import Redis +from redis.exceptions import RedisError +from testcontainers.core.container import DockerContainer + +from basic_memory.read_cache.redis import RedisReadCache, create_redis_read_cache_client + + +def _wait_for_redis(url: str) -> None: + client = SyncRedis.from_url( + url, + socket_connect_timeout=0.1, + socket_timeout=0.1, + ) + deadline = time.monotonic() + 10 + last_error: RedisError | None = None + try: + while time.monotonic() < deadline: + try: + client.ping() + return + except RedisError as error: + last_error = error + time.sleep(0.05) + finally: + client.close() + + raise RuntimeError("Redis test server did not become ready") from last_error + + +@pytest.fixture(scope="session") +def redis_url() -> Generator[str]: + """Use a configured Redis server or start a real Redis 8 testcontainer.""" + configured_url = os.environ.get("BASIC_MEMORY_TEST_REDIS_URL") + if configured_url: + _wait_for_redis(configured_url) + yield configured_url + return + + if shutil.which("docker") is None: + pytest.skip("Docker is required for real Redis integration tests") + + image = os.environ.get("BASIC_MEMORY_TEST_REDIS_IMAGE", "redis:8.8-alpine") + with DockerContainer(image).with_exposed_ports(6379) as container: + host = container.get_container_host_ip() + port = container.get_exposed_port(6379) + url = f"redis://{host}:{port}/0" + _wait_for_redis(url) + yield url + + +@dataclass(frozen=True, slots=True) +class RedisCacheHarness: + """One isolated Redis prefix and the client that owns it.""" + + cache: RedisReadCache + client: Redis + namespace: str + prefix: str + + +@pytest_asyncio.fixture +async def redis_cache(redis_url: str) -> AsyncGenerator[RedisCacheHarness]: + """Yield an isolated cache over the suite's real Redis server.""" + prefix = f"bm:test:read:{uuid4().hex}" + namespace = f"tenant-{uuid4().hex}" + # Container scheduling can make the first async handshake exceed the + # production cache's intentionally tight timeout. Behavioral timeout tests + # construct their own 50ms clients, so keep this shared harness stable. + client = create_redis_read_cache_client(redis_url, socket_timeout=1.0) + cache = RedisReadCache( + client=client, + namespace=namespace, + prefix=prefix, + ) + try: + await client.ping() + yield RedisCacheHarness( + cache=cache, + client=client, + namespace=namespace, + prefix=prefix, + ) + finally: + keys = [key async for key in client.scan_iter(match=f"{prefix}:*")] + if keys: + await client.delete(*keys) + await client.aclose() diff --git a/test-int/read_cache/test_api_read_cache.py b/test-int/read_cache/test_api_read_cache.py new file mode 100644 index 000000000..781a9b21f --- /dev/null +++ b/test-int/read_cache/test_api_read_cache.py @@ -0,0 +1,206 @@ +"""Full-stack API coverage against the real Redis read cache.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path +from typing import Protocol + +import pytest +from fastapi import FastAPI +from httpx import AsyncClient +from redis.asyncio import Redis +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker + +from basic_memory import db +from basic_memory.deps import get_read_cache +from basic_memory.models import Project +from basic_memory.models.knowledge import Entity +from basic_memory.read_cache import ReadCacheKey, ReadCacheOperation, read_cache_request_digest +from basic_memory.read_cache.keys import ( + redis_read_cache_generation_key, + redis_read_cache_keys, +) +from basic_memory.read_cache.redis import RedisReadCache +from basic_memory.repository import EntityRepository +from basic_memory.runtime.note_content import NOTE_CONTENT_BASE_CHECKSUM_HEADER +from basic_memory.schemas.v2 import EntityResolveRequest + + +class RedisCacheHarness(Protocol): + """Structural type for the real Redis fixture.""" + + cache: RedisReadCache + client: Redis + namespace: str + prefix: str + + +pytestmark = pytest.mark.redis + + +def _cache_key( + *, + project_id: str, + operation: ReadCacheOperation, + request: str, +) -> ReadCacheKey: + return ReadCacheKey( + project_id=project_id, + operation=operation, + request_digest=read_cache_request_digest(request), + ) + + +@pytest.mark.asyncio +async def test_entity_resolve_and_markdown_reads_cache_then_write_invalidates( + app: FastAPI, + client: AsyncClient, + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """Successful reads populate Redis; rejected writes preserve and accepted writes bump.""" + app.dependency_overrides[get_read_cache] = lambda: redis_cache.cache + project_external_id = str(test_project.external_id) + project_url = f"/v2/projects/{project_external_id}" + + created_response = await client.post( + f"{project_url}/knowledge/entities", + json={ + "title": "Redis Cached Note", + "directory": "cache", + "content": "# Redis Cached Note\n\nVersion one.", + }, + ) + assert created_response.status_code == 202 + created = created_response.json() + entity_id = created["external_id"] + + entity_response = await client.get(f"{project_url}/knowledge/entities/{entity_id}") + resolve_request = EntityResolveRequest(identifier=created["permalink"]) + resolve_response = await client.post( + f"{project_url}/knowledge/resolve", + json=resolve_request.model_dump(mode="json"), + ) + resource_response = await client.get(f"{project_url}/resource/{entity_id}") + + assert entity_response.status_code == 200 + assert resolve_response.status_code == 200 + assert resource_response.status_code == 200 + assert "Version one." in resource_response.text + + keys = ( + _cache_key( + project_id=project_external_id, + operation=ReadCacheOperation.entity, + request=entity_id, + ), + _cache_key( + project_id=project_external_id, + operation=ReadCacheOperation.resolve, + request=resolve_request.model_dump_json(), + ), + _cache_key( + project_id=project_external_id, + operation=ReadCacheOperation.resource, + request=entity_id, + ), + ) + for key in keys: + redis_keys = redis_read_cache_keys( + prefix=redis_cache.prefix, + namespace=redis_cache.namespace, + key=key, + ) + assert await redis_cache.client.exists(redis_keys.data) == 1 + + generation_key = redis_read_cache_generation_key( + prefix=redis_cache.prefix, + namespace=redis_cache.namespace, + project_id=project_external_id, + ) + populated_generation = await redis_cache.client.get(generation_key) + assert populated_generation is not None + + rejected_response = await client.put( + f"{project_url}/knowledge/entities/{entity_id}", + headers={NOTE_CONTENT_BASE_CHECKSUM_HEADER: "stale-checksum"}, + json={ + "title": "Redis Cached Note", + "directory": "cache", + "content": "# Redis Cached Note\n\nRejected replacement.", + }, + ) + assert rejected_response.status_code == 409 + assert await redis_cache.client.get(generation_key) == populated_generation + + edited_response = await client.patch( + f"{project_url}/knowledge/entities/{entity_id}", + json={ + "operation": "append", + "content": "\n\nVersion two.", + }, + ) + assert edited_response.status_code == 202 + invalidated_generation = await redis_cache.client.get(generation_key) + assert invalidated_generation is not None + assert invalidated_generation != populated_generation + + refreshed_entity = await client.get(f"{project_url}/knowledge/entities/{entity_id}") + refreshed_resource = await client.get(f"{project_url}/resource/{entity_id}") + assert refreshed_entity.status_code == 200 + assert "Version two." in refreshed_entity.json()["content"] + assert refreshed_resource.status_code == 200 + assert "Version two." in refreshed_resource.text + + +@pytest.mark.asyncio +async def test_non_markdown_resource_is_never_cached( + app: FastAPI, + client: AsyncClient, + test_project: Project, + redis_cache: RedisCacheHarness, + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], +) -> None: + """Arbitrary files remain authoritative even when they fit under the payload cap.""" + app.dependency_overrides[get_read_cache] = lambda: redis_cache.cache + project_external_id = str(test_project.external_id) + file_path = "cache/plain.txt" + disk_path = Path(test_project.path) / file_path + disk_path.parent.mkdir(parents=True, exist_ok=True) + disk_path.write_text("Plain text remains uncached.", encoding="utf-8") + + repository = EntityRepository(project_id=test_project.id) + _, session_maker = engine_factory + async with db.scoped_session(session_maker) as session: + entity = await repository.add( + session, + Entity( + title="plain.txt", + note_type="file", + content_type="text/plain", + file_path=file_path, + checksum="plain-checksum", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + ) + + resource_url = f"/v2/projects/{project_external_id}/resource/{entity.external_id}" + first = await client.get(resource_url) + second = await client.get(resource_url) + assert first.status_code == 200 + assert second.status_code == 200 + assert second.text == "Plain text remains uncached." + + key = _cache_key( + project_id=project_external_id, + operation=ReadCacheOperation.resource, + request=entity.external_id, + ) + redis_keys = redis_read_cache_keys( + prefix=redis_cache.prefix, + namespace=redis_cache.namespace, + key=key, + ) + assert await redis_cache.client.exists(redis_keys.data) == 0 diff --git a/test-int/read_cache/test_read_cache_benchmark.py b/test-int/read_cache/test_read_cache_benchmark.py new file mode 100644 index 000000000..647901264 --- /dev/null +++ b/test-int/read_cache/test_read_cache_benchmark.py @@ -0,0 +1,108 @@ +"""Small end-to-end benchmark for warmed entity reads.""" + +from __future__ import annotations + +from statistics import median +from time import perf_counter +from typing import Protocol + +import pytest +from fastapi import FastAPI +from httpx import AsyncClient +from redis.asyncio import Redis +from sqlalchemy import event +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker + +from basic_memory.deps import get_read_cache +from basic_memory.models import Project +from basic_memory.read_cache import NullReadCache +from basic_memory.read_cache.redis import RedisReadCache + + +class RedisCacheHarness(Protocol): + """Structural type for the real Redis fixture.""" + + cache: RedisReadCache + client: Redis + namespace: str + prefix: str + + +pytestmark = [pytest.mark.redis, pytest.mark.benchmark] + + +async def _entity_read_latencies( + client: AsyncClient, + url: str, + *, + samples: int, +) -> list[float]: + latencies: list[float] = [] + for _ in range(samples): + started = perf_counter() + response = await client.get(url) + latencies.append(perf_counter() - started) + assert response.status_code == 200 + return latencies + + +@pytest.mark.asyncio +async def test_benchmark_warmed_entity_reads_reduce_authoritative_queries( + app: FastAPI, + client: AsyncClient, + test_project: Project, + redis_cache: RedisCacheHarness, + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], +) -> None: + """Report HTTP latency and prove cache hits remove authoritative SQL work.""" + project_external_id = str(test_project.external_id) + project_url = f"/v2/projects/{project_external_id}" + created_response = await client.post( + f"{project_url}/knowledge/entities", + json={ + "title": "Redis Benchmark Note", + "directory": "cache", + "content": "# Redis Benchmark Note\n\nRepeated read benchmark.", + }, + ) + assert created_response.status_code == 202 + entity_url = f"{project_url}/knowledge/entities/{created_response.json()['external_id']}" + + query_count = 0 + + def count_query(*_: object) -> None: + nonlocal query_count + query_count += 1 + + engine, _ = engine_factory + event.listen(engine.sync_engine, "before_cursor_execute", count_query) + try: + app.dependency_overrides[get_read_cache] = NullReadCache + await client.get(entity_url) + query_count = 0 + authoritative_latencies = await _entity_read_latencies( + client, + entity_url, + samples=20, + ) + authoritative_queries = query_count + + app.dependency_overrides[get_read_cache] = lambda: redis_cache.cache + await client.get(entity_url) + query_count = 0 + cached_latencies = await _entity_read_latencies( + client, + entity_url, + samples=20, + ) + cached_queries = query_count + finally: + event.remove(engine.sync_engine, "before_cursor_execute", count_query) + + assert cached_queries < authoritative_queries + print( + "\nRedis entity-read benchmark: " + f"authoritative_median_ms={median(authoritative_latencies) * 1_000:.3f}, " + f"cached_median_ms={median(cached_latencies) * 1_000:.3f}, " + f"authoritative_sql={authoritative_queries}, cached_sql={cached_queries}" + ) diff --git a/test-int/read_cache/test_redis_read_cache.py b/test-int/read_cache/test_redis_read_cache.py new file mode 100644 index 000000000..c2cdb3b25 --- /dev/null +++ b/test-int/read_cache/test_redis_read_cache.py @@ -0,0 +1,620 @@ +"""Integration coverage for the Redis semantic read cache.""" + +from __future__ import annotations + +import asyncio +import socket +from typing import override, Protocol +from uuid import uuid4 + +import pytest +from pydantic import BaseModel, ValidationError +from redis.asyncio import Redis + +from basic_memory.read_cache import ( + NullReadCache, + ReadCacheDataError, + ReadCacheInvalidationStatus, + ReadCacheKey, + ReadCacheLookup, + ReadCacheOperation, + ReadCacheStoreStatus, + ReadCacheUnavailable, + invalidate_project_read_cache, + read_cache_request_digest, + read_through_model, +) +from basic_memory.read_cache.keys import ( + redis_read_cache_generation_key, + redis_read_cache_keys, +) +from basic_memory.read_cache.redis import ( + RedisReadCache, + _required_bytes, + _store_status, + create_redis_read_cache_client, +) + + +class RedisCacheHarness(Protocol): + """Structural type for the real Redis fixture.""" + + cache: RedisReadCache + client: Redis + namespace: str + prefix: str + + +class CachedEntity(BaseModel): + """Small typed boundary value used by read-through tests.""" + + external_id: str + title: str + + +pytestmark = pytest.mark.redis + +PROJECT_ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" +OTHER_PROJECT_ID = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" + + +def _key( + *, + project_id: str = PROJECT_ID, + operation: ReadCacheOperation = ReadCacheOperation.entity, + request: str = "entity-1", +) -> ReadCacheKey: + return ReadCacheKey( + project_id=project_id, + operation=operation, + request_digest=read_cache_request_digest(request), + ) + + +@pytest.mark.asyncio +async def test_round_trip_and_ttl_expiry(redis_cache: RedisCacheHarness) -> None: + key = _key() + + miss = await redis_cache.cache.lookup(key) + assert miss.generation is not None + assert not miss.is_hit + + await redis_cache.cache.store(key, miss, b"cached entity", ttl_seconds=1) + hit = await redis_cache.cache.lookup(key) + assert hit.is_hit + assert hit.payload == b"cached entity" + assert hit.generation == miss.generation + + await asyncio.sleep(1.1) + expired = await redis_cache.cache.lookup(key) + assert not expired.is_hit + assert expired.generation == miss.generation + + +@pytest.mark.asyncio +async def test_cache_identity_isolates_every_key_dimension( + redis_cache: RedisCacheHarness, +) -> None: + key = _key() + miss = await redis_cache.cache.lookup(key) + await redis_cache.cache.store(key, miss, b"only this key", ttl_seconds=60) + + variants = [ + _key(project_id=OTHER_PROJECT_ID), + _key(operation=ReadCacheOperation.resolve), + _key(request="entity-2"), + ] + for variant in variants: + assert not (await redis_cache.cache.lookup(variant)).is_hit + + other_namespace = RedisReadCache( + client=redis_cache.client, + namespace="another-tenant", + prefix=redis_cache.prefix, + ) + assert not (await other_namespace.lookup(key)).is_hit + await other_namespace.invalidate_project(key.project_id) + assert (await redis_cache.cache.lookup(key)).payload == b"only this key" + + +@pytest.mark.asyncio +async def test_project_invalidation_rejects_a_concurrent_stale_fill( + redis_cache: RedisCacheHarness, +) -> None: + project_key = _key() + other_project_key = _key(project_id=OTHER_PROJECT_ID) + + stale_lookup = await redis_cache.cache.lookup(project_key) + other_lookup = await redis_cache.cache.lookup(other_project_key) + other_status = await redis_cache.cache.store( + other_project_key, + other_lookup, + b"other project", + ttl_seconds=60, + ) + + invalidation_status = await redis_cache.cache.invalidate_project(project_key.project_id) + current_lookup = await redis_cache.cache.lookup(project_key) + current_status = await redis_cache.cache.store( + project_key, + current_lookup, + b"current fill", + ttl_seconds=60, + ) + stale_status = await redis_cache.cache.store( + project_key, + stale_lookup, + b"stale fill", + ttl_seconds=60, + ) + + assert other_status is ReadCacheStoreStatus.stored + assert invalidation_status is ReadCacheInvalidationStatus.invalidated + assert current_status is ReadCacheStoreStatus.stored + assert stale_status is ReadCacheStoreStatus.superseded + assert (await redis_cache.cache.lookup(project_key)).payload == b"current fill" + assert (await redis_cache.cache.lookup(other_project_key)).payload == b"other project" + + +@pytest.mark.asyncio +async def test_lost_generation_key_cannot_revive_old_data( + redis_cache: RedisCacheHarness, +) -> None: + key = _key() + miss = await redis_cache.cache.lookup(key) + await redis_cache.cache.store(key, miss, b"old data", ttl_seconds=60) + redis_keys = redis_read_cache_keys( + prefix=redis_cache.prefix, + namespace=redis_cache.namespace, + key=key, + ) + + await redis_cache.client.delete(redis_keys.generation) + after_eviction = await redis_cache.cache.lookup(key) + + assert not after_eviction.is_hit + assert after_eviction.generation != miss.generation + + +@pytest.mark.asyncio +async def test_invalidation_leaves_unrelated_redis_data_untouched( + redis_cache: RedisCacheHarness, +) -> None: + unrelated_key = f"unrelated:{uuid4().hex}" + await redis_cache.client.set(unrelated_key, b"keep") + try: + await redis_cache.cache.invalidate_project(PROJECT_ID) + assert await redis_cache.client.get(unrelated_key) == b"keep" + finally: + await redis_cache.client.delete(unrelated_key) + + +@pytest.mark.asyncio +async def test_corrupt_redis_values_fail_fast(redis_cache: RedisCacheHarness) -> None: + key = _key() + await redis_cache.cache.lookup(key) + redis_keys = redis_read_cache_keys( + prefix=redis_cache.prefix, + namespace=redis_cache.namespace, + key=key, + ) + + await redis_cache.client.set(redis_keys.data, b"missing envelope") + with pytest.raises(ReadCacheDataError, match="invalid generation envelope"): + await redis_cache.cache.lookup(key) + + await redis_cache.client.set(redis_keys.data, b"\npayload") + with pytest.raises(ReadCacheDataError, match="invalid generation envelope"): + await redis_cache.cache.lookup(key) + + await redis_cache.client.set(redis_keys.generation, b"\xff") + with pytest.raises(ReadCacheDataError, match="invalid generation token"): + await redis_cache.cache.lookup(key) + + await redis_cache.client.set(redis_keys.generation, b"abcd") + with pytest.raises(ReadCacheDataError, match="invalid generation token"): + await redis_cache.cache.lookup(key) + + +@pytest.mark.asyncio +async def test_decode_responses_client_remains_compatible(redis_url: str) -> None: + prefix = f"bm:test:read:{uuid4().hex}" + client = Redis.from_url(redis_url, decode_responses=True) + cache = RedisReadCache( + client=client, + namespace="decoded-client", + prefix=prefix, + ) + key = _key() + try: + miss = await cache.lookup(key) + await cache.store(key, miss, b"text payload", ttl_seconds=60) + assert (await cache.lookup(key)).payload == b"text payload" + finally: + keys = [key async for key in client.scan_iter(match=f"{prefix}:*")] + if keys: + await client.delete(*keys) + await client.aclose() + + +@pytest.mark.asyncio +async def test_unavailable_redis_is_explicit_for_every_operation() -> None: + with socket.socket() as reserved_port: + reserved_port.bind(("127.0.0.1", 0)) + port = reserved_port.getsockname()[1] + + client = create_redis_read_cache_client( + f"redis://127.0.0.1:{port}/0", + socket_timeout=0.05, + ) + cache = RedisReadCache(client=client, namespace="unavailable") + key = _key() + try: + with pytest.raises(ReadCacheUnavailable, match="lookup failed"): + await cache.lookup(key) + with pytest.raises(ReadCacheUnavailable, match="store failed"): + await cache.store( + key, + ReadCacheLookup(generation="0" * 32), + b"payload", + ttl_seconds=60, + ) + with pytest.raises(ReadCacheUnavailable, match="invalidation failed"): + await cache.invalidate_project(key.project_id) + finally: + await client.aclose() + + +@pytest.mark.asyncio +async def test_null_cache_preserves_disabled_semantics() -> None: + class StoreMustNotRun(NullReadCache): + @override + async def store( + self, + key: ReadCacheKey, + lookup: ReadCacheLookup, + payload: bytes, + *, + ttl_seconds: int, + ) -> ReadCacheStoreStatus: + raise AssertionError("disabled read-through must skip serialization and storage") + + cache = StoreMustNotRun() + key = _key() + lookup = await cache.lookup(key) + + assert lookup == ReadCacheLookup(generation=None) + store_status = await NullReadCache().store(key, lookup, b"ignored", ttl_seconds=60) + assert store_status is ReadCacheStoreStatus.disabled + status = await cache.invalidate_project(key.project_id) + assert status is ReadCacheInvalidationStatus.disabled + + async def load() -> CachedEntity: + return CachedEntity(external_id="entity-1", title="Authoritative") + + result = await read_through_model( + cache=cache, + key=key, + model_type=CachedEntity, + load=load, + ttl_seconds=60, + max_payload_bytes=1_024, + ) + assert result.title == "Authoritative" + + +@pytest.mark.asyncio +async def test_typed_read_through_uses_real_cached_representation( + redis_cache: RedisCacheHarness, +) -> None: + loads = 0 + + async def load() -> CachedEntity: + nonlocal loads + loads += 1 + return CachedEntity(external_id="entity-1", title="First") + + first = await read_through_model( + cache=redis_cache.cache, + key=_key(), + model_type=CachedEntity, + load=load, + ttl_seconds=60, + max_payload_bytes=1_024, + ) + second = await read_through_model( + cache=redis_cache.cache, + key=_key(), + model_type=CachedEntity, + load=load, + ttl_seconds=60, + max_payload_bytes=1_024, + ) + + assert first == CachedEntity(external_id="entity-1", title="First") + assert second == first + assert loads == 1 + + +@pytest.mark.asyncio +async def test_typed_read_through_does_not_cache_oversize_models( + redis_cache: RedisCacheHarness, +) -> None: + loads = 0 + + async def load() -> CachedEntity: + nonlocal loads + loads += 1 + return CachedEntity(external_id="entity-1", title="Too large") + + for _ in range(2): + await read_through_model( + cache=redis_cache.cache, + key=_key(), + model_type=CachedEntity, + load=load, + ttl_seconds=60, + max_payload_bytes=1, + ) + + assert loads == 2 + + +@pytest.mark.asyncio +async def test_typed_read_through_does_not_cache_ineligible_models( + redis_cache: RedisCacheHarness, +) -> None: + loads = 0 + + async def load() -> CachedEntity: + nonlocal loads + loads += 1 + return CachedEntity(external_id="cross-project", title="Other tenant") + + for _ in range(2): + await read_through_model( + cache=redis_cache.cache, + key=_key(operation=ReadCacheOperation.resolve), + model_type=CachedEntity, + load=load, + ttl_seconds=60, + max_payload_bytes=1_024, + should_store=lambda entity: entity.external_id != "cross-project", + ) + + assert loads == 2 + + +@pytest.mark.asyncio +async def test_typed_read_through_rejects_invalid_cached_models( + redis_cache: RedisCacheHarness, +) -> None: + key = _key() + miss = await redis_cache.cache.lookup(key) + await redis_cache.cache.store(key, miss, b'{"wrong":"shape"}', ttl_seconds=60) + + async def load() -> CachedEntity: + raise AssertionError("invalid cache data must not fall through to the loader") + + with pytest.raises(ValidationError): + await read_through_model( + cache=redis_cache.cache, + key=key, + model_type=CachedEntity, + load=load, + ttl_seconds=60, + max_payload_bytes=1_024, + ) + + +@pytest.mark.asyncio +async def test_typed_read_through_bypasses_unavailable_real_redis() -> None: + with socket.socket() as reserved_port: + reserved_port.bind(("127.0.0.1", 0)) + port = reserved_port.getsockname()[1] + + client = create_redis_read_cache_client( + f"redis://127.0.0.1:{port}/0", + socket_timeout=0.05, + ) + cache = RedisReadCache(client=client, namespace="unavailable") + + async def load() -> CachedEntity: + return CachedEntity(external_id="entity-1", title="Authoritative") + + try: + result = await read_through_model( + cache=cache, + key=_key(), + model_type=CachedEntity, + load=load, + ttl_seconds=60, + max_payload_bytes=1_024, + ) + finally: + await client.aclose() + + assert result.title == "Authoritative" + + +@pytest.mark.asyncio +async def test_invalidation_helper_preserves_committed_write_when_redis_is_unavailable() -> None: + with socket.socket() as reserved_port: + reserved_port.bind(("127.0.0.1", 0)) + port = reserved_port.getsockname()[1] + + client = create_redis_read_cache_client( + f"redis://127.0.0.1:{port}/0", + socket_timeout=0.05, + ) + cache = RedisReadCache(client=client, namespace="unavailable") + try: + status = await invalidate_project_read_cache(cache, PROJECT_ID) + finally: + await client.aclose() + + assert status is ReadCacheInvalidationStatus.unavailable + + +@pytest.mark.asyncio +async def test_typed_read_through_returns_data_when_real_redis_store_times_out( + redis_cache: RedisCacheHarness, + redis_url: str, +) -> None: + prefix = f"bm:test:read:{uuid4().hex}" + client = create_redis_read_cache_client(redis_url, socket_timeout=0.05) + cache = RedisReadCache( + client=client, + namespace="paused-store", + prefix=prefix, + ) + + async def load() -> CachedEntity: + await redis_cache.client.execute_command("CLIENT", "PAUSE", 200, "WRITE") + return CachedEntity(external_id="entity-1", title="Authoritative") + + try: + result = await read_through_model( + cache=cache, + key=_key(), + model_type=CachedEntity, + load=load, + ttl_seconds=60, + max_payload_bytes=1_024, + ) + assert result.title == "Authoritative" + finally: + await asyncio.sleep(0.25) + keys = [key async for key in client.scan_iter(match=f"{prefix}:*")] + if keys: + await client.delete(*keys) + await client.aclose() + + +@pytest.mark.asyncio +async def test_typed_read_through_validates_policy_before_loading() -> None: + async def load() -> CachedEntity: + raise AssertionError("invalid policy must fail before loading") + + with pytest.raises(ValueError, match="ttl_seconds"): + await read_through_model( + cache=NullReadCache(), + key=_key(), + model_type=CachedEntity, + load=load, + ttl_seconds=0, + max_payload_bytes=1, + ) + with pytest.raises(ValueError, match="max_payload_bytes"): + await read_through_model( + cache=NullReadCache(), + key=_key(), + model_type=CachedEntity, + load=load, + ttl_seconds=1, + max_payload_bytes=0, + ) + + +def test_key_validation_and_canonicalization() -> None: + assert read_cache_request_digest("ab", "c") != read_cache_request_digest("a", "bc") + assert read_cache_request_digest("same") == read_cache_request_digest("same") + + key = _key() + assert _key(project_id=PROJECT_ID.upper()).project_id == PROJECT_ID + generation_key = redis_read_cache_generation_key( + prefix="bm:read:v1", + namespace="tenant", + project_id=key.project_id, + ) + redis_keys = redis_read_cache_keys( + prefix="bm:read:v1", + namespace="tenant", + key=key, + ) + assert redis_keys.generation == generation_key + assert f"{{{read_cache_request_digest('tenant', key.project_id)}}}" in redis_keys.data + assert generation_key == redis_read_cache_generation_key( + prefix="bm:read:v1", + namespace="tenant", + project_id=PROJECT_ID.upper(), + ) + + with pytest.raises(ValueError, match="project_id"): + _key(project_id="") + with pytest.raises(ValueError, match="valid UUID"): + _key(project_id="not-a-uuid") + with pytest.raises(ValueError, match="SHA-256"): + ReadCacheKey( + project_id=PROJECT_ID, + operation=ReadCacheOperation.entity, + request_digest="short", + ) + with pytest.raises(ValueError, match="SHA-256"): + ReadCacheKey( + project_id=PROJECT_ID, + operation=ReadCacheOperation.entity, + request_digest="z" * 64, + ) + with pytest.raises(ValueError, match="prefix"): + redis_read_cache_generation_key( + prefix="", + namespace="tenant", + project_id=PROJECT_ID, + ) + with pytest.raises(ValueError, match="prefix"): + redis_read_cache_generation_key( + prefix="bad prefix", + namespace="tenant", + project_id=PROJECT_ID, + ) + with pytest.raises(ValueError, match="namespace"): + redis_read_cache_generation_key( + prefix="bm:read:v1", + namespace="", + project_id=PROJECT_ID, + ) + with pytest.raises(ValueError, match="project_id"): + redis_read_cache_generation_key(prefix="bm:read:v1", namespace="tenant", project_id="") + + +@pytest.mark.asyncio +async def test_invalid_store_inputs_fail_before_redis( + redis_cache: RedisCacheHarness, +) -> None: + key = _key() + + with pytest.raises(ValueError, match="lookup generation"): + await redis_cache.cache.store( + key, + ReadCacheLookup(generation=None), + b"payload", + ttl_seconds=60, + ) + with pytest.raises(ValueError, match="positive"): + await redis_cache.cache.store( + key, + ReadCacheLookup(generation="0" * 32), + b"payload", + ttl_seconds=0, + ) + with pytest.raises(ReadCacheDataError, match="generation token"): + await redis_cache.cache.store( + key, + ReadCacheLookup(generation="invalid"), + b"payload", + ttl_seconds=60, + ) + with pytest.raises(ValueError, match="namespace"): + RedisReadCache(client=redis_cache.client, namespace="") + with pytest.raises(ValueError, match="namespace"): + RedisReadCache(client=redis_cache.client, namespace=" ") + with pytest.raises(ValueError, match="project_id"): + await redis_cache.cache.invalidate_project("") + + +def test_required_bytes_rejects_non_string_values() -> None: + with pytest.raises(ReadCacheDataError, match="invalid test value"): + _required_bytes(1, field="test") + with pytest.raises(ReadCacheDataError, match="invalid cache store result"): + _store_status(2) diff --git a/tests/index/test_watch_coordinator.py b/tests/index/test_watch_coordinator.py index d9ecfc3be..00f1bf194 100644 --- a/tests/index/test_watch_coordinator.py +++ b/tests/index/test_watch_coordinator.py @@ -24,8 +24,9 @@ async def initialize_after_recovery( quiet: bool = True, *, recovery_complete: asyncio.Event | None = None, + read_cache: object | None = None, ) -> None: - del config, quiet + del config, quiet, read_cache await allow_recovery_to_finish.wait() assert recovery_complete is not None recovery_complete.set() @@ -62,8 +63,9 @@ async def fail_before_recovery( quiet: bool = True, *, recovery_complete: asyncio.Event | None = None, + read_cache: object | None = None, ) -> None: - del config, quiet, recovery_complete + del config, quiet, recovery_complete, read_cache raise RuntimeError("recovery boom") monkeypatch.setattr( diff --git a/tests/services/test_initialization.py b/tests/services/test_initialization.py index 6ad901862..55707c052 100644 --- a/tests/services/test_initialization.py +++ b/tests/services/test_initialization.py @@ -315,6 +315,9 @@ def capture_task(coro): return original_create_task(coro) class RecordingProjectIndexRuntimeFactory: + def __init__(self, *, read_cache: object) -> None: + self.read_cache = read_cache + async def runtime_for_project(self, project): # noqa: ANN001 return f"runtime:{project.name}" diff --git a/uv.lock b/uv.lock index 6940a9a2d..df31cd178 100644 --- a/uv.lock +++ b/uv.lock @@ -332,6 +332,9 @@ milvus = [ { name = "pymilvus" }, { name = "pymilvus", extra = ["milvus-lite"], marker = "sys_platform != 'win32'" }, ] +redis = [ + { name = "redis" }, +] [package.dev-dependencies] dev = [ @@ -351,6 +354,7 @@ dev = [ { name = "pytest-testmon" }, { name = "pytest-timeout" }, { name = "pytest-xdist" }, + { name = "redis" }, { name = "ruff" }, { name = "testcontainers" }, { name = "ty" }, @@ -395,6 +399,7 @@ requires-dist = [ { name = "python-dotenv", specifier = ">=1.1.0" }, { name = "python-frontmatter", specifier = ">=1.1.0" }, { name = "pyyaml", specifier = ">=6.0.1" }, + { name = "redis", marker = "extra == 'redis'", specifier = ">=8.0.0,<9" }, { name = "rich", specifier = ">=13.9.4" }, { name = "sniffio", specifier = ">=1.3.1" }, { name = "sqlalchemy", specifier = ">=2.0.0" }, @@ -404,7 +409,7 @@ requires-dist = [ { name = "uvloop", marker = "sys_platform != 'win32'", specifier = ">=0.21.0" }, { name = "watchfiles", specifier = ">=1.0.4" }, ] -provides-extras = ["milvus"] +provides-extras = ["milvus", "redis"] [package.metadata.requires-dev] dev = [ @@ -424,6 +429,7 @@ dev = [ { name = "pytest-testmon", specifier = ">=2.2.0" }, { name = "pytest-timeout", specifier = ">=2.4.0" }, { name = "pytest-xdist", specifier = ">=3.0.0" }, + { name = "redis", specifier = ">=8.0.0,<9" }, { name = "ruff", specifier = ">=0.16.0" }, { name = "testcontainers", extras = ["postgres"], specifier = ">=4.0.0" }, { name = "ty", specifier = ">=0.0.64" }, @@ -3556,6 +3562,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/28/26534bed77109632a956977f60d8519049f545abc39215d086e33a61f1f2/pyyaml_ft-8.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:de04cfe9439565e32f178106c51dd6ca61afaa2907d143835d501d84703d3793", size = 171579, upload-time = "2025-06-10T15:32:14.34Z" }, ] +[[package]] +name = "redis" +version = "8.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/c3/928b290c2c0ca99ab96eea5b4ff8f30be8112b075301a7d3ba214a3c8c12/redis-8.0.1.tar.gz", hash = "sha256:afc5a7a2f5a084f5b1880dec548dd45be17db7e43c82a30d84f952aefb05cfb0", size = 5114170, upload-time = "2026-06-23T14:52:37.728Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/0a/c2345ebf1ebe70840ce3f6c6ee612f8fa749cfbd1b03069c53bf0c62aaad/redis-8.0.1-py3-none-any.whl", hash = "sha256:47daa35a058c23468d6437f17a8c76882cb316b838ef763036af99b96cedd743", size = 502406, upload-time = "2026-06-23T14:52:36.137Z" }, +] + [[package]] name = "referencing" version = "0.37.0" From 1f146493aea821548f10ae541f74f9461e841bc4 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 29 Jul 2026 00:02:30 -0500 Subject: [PATCH 02/28] fix(api): close Redis cache freshness gaps Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 23 ++-- .../api/v2/routers/knowledge_router.py | 8 +- src/basic_memory/deps/services.py | 14 ++- src/basic_memory/index/local_schedulers.py | 23 +++- .../index/note_content_materialization.py | 110 ++++++++++-------- test-int/read_cache/test_api_read_cache.py | 23 +++- .../test_note_content_materialization.py | 33 +++++- tests/index/test_local_schedulers.py | 37 +++++- 8 files changed, 204 insertions(+), 67 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index 57652d3d6..62566ecf6 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -79,6 +79,10 @@ tenant: materialization, object-storage events, direct and project indexing, directory moves/deletes, and relation-resolution workers. Request-path invalidation alone is insufficient because a worker can update a cached entity after the request returns. +1. Treat each asynchronous state transition as a separate freshness boundary. Invalidate after + the accepted-note transaction commits, again after terminal materialization/status publication + and indexing, and again after relation resolution completes. This prevents a read filled + between phases from surviving the later worker commit. 1. Keep `bm:read:v1` separate from rate-limit and Cloud control-plane prefixes, metrics, timeouts, and failure policies. The clients may target one Redis deployment, but a read-cache timeout must bypass while a rate-limit decision keeps its Cloud-owned security behavior. @@ -161,7 +165,7 @@ Phase one: | Operation | Initial TTL | Constraints | | ---------------------- | ----------: | --------------------------------------- | | Entity by external ID | 60 seconds | Cache validated `EntityResponseV2` JSON | -| Identifier resolution | 60 seconds | Cache successful same-project results | +| Identifier resolution | 60 seconds | Include body and workspace context | | Markdown note resource | 60 seconds | Cache only below an explicit size limit | Phase two, after measuring phase one: @@ -176,7 +180,9 @@ Do not initially cache failures, missing entities, graph/orphan responses, large binary resources, schema inference, writes, or Cloud control-plane data. Caching is semantic rather than HTTP-method based. The POST identifier-resolution and search -operations can be cached without changing their public API. +operations can be cached without changing their public API. Identifier resolution depends on the +request-local workspace permalink context, so its request digest includes the workspace slug and +workspace type in addition to the validated request body. ## Placement @@ -193,9 +199,11 @@ Primary integration points: - later, `src/basic_memory/api/v2/routers/search_router.py` Invalidation belongs at portable mutation and indexing completion boundaries, not only in -FastAPI routes. It must cover accepted note writes, direct file indexing, filesystem watcher -updates, project indexing, directory mutations, Cloud storage events, and later search or -relation changes that affect cached responses. +FastAPI routes. It must cover accepted note writes, terminal deferred materialization and status +publication, direct file indexing, filesystem watcher updates, project indexing, directory +mutations, Cloud storage events, and relation-resolution changes that affect cached responses. +Each later phase invalidates again so a value filled after an earlier generation bump cannot +outlive the state that phase publishes. ## Dependency And Lifecycle @@ -288,8 +296,9 @@ semantics themselves are asserted only against the real Redis integration fixtur - Inject a Basic Memory-specific Redis client and tenant namespace. - Derive that namespace from trusted request and worker context with one canonical function. -- Invalidate from materialization, storage-event, and relation-resolution workers using the - same tenant namespace as the request path before enabling reads for a tenant. +- Invalidate after accepted-note commit, terminal materialization/indexing, storage events, and + relation-resolution workers using the same tenant namespace as the request path before enabling + reads for a tenant. - Start with shadow telemetry or a limited tenant cohort. - Compare hit rate, Redis latency, database query volume, and end-to-end tool latency. diff --git a/src/basic_memory/api/v2/routers/knowledge_router.py b/src/basic_memory/api/v2/routers/knowledge_router.py index a8ed7a66a..810bdd6e6 100644 --- a/src/basic_memory/api/v2/routers/knowledge_router.py +++ b/src/basic_memory/api/v2/routers/knowledge_router.py @@ -83,6 +83,7 @@ OrphanEntitiesResponse, IndexFileRequest, ) +from basic_memory.workspace_context import current_workspace_permalink_context from basic_memory.schemas.response import DirectoryMoveResult from basic_memory.utils import validate_project_path @@ -344,12 +345,17 @@ async def load() -> EntityResolveResponse: ) return result + workspace_context = current_workspace_permalink_context() return await read_through_model( cache=read_cache, key=ReadCacheKey( project_id=project_external_id, operation=ReadCacheOperation.resolve, - request_digest=read_cache_request_digest(data.model_dump_json()), + request_digest=read_cache_request_digest( + data.model_dump_json(), + workspace_context.workspace_slug if workspace_context else "", + workspace_context.workspace_type if workspace_context else "", + ), ), model_type=EntityResolveResponse, load=load, diff --git a/src/basic_memory/deps/services.py b/src/basic_memory/deps/services.py index 94198d260..50395ec87 100644 --- a/src/basic_memory/deps/services.py +++ b/src/basic_memory/deps/services.py @@ -14,7 +14,7 @@ from pathlib import Path from typing import Annotated -from fastapi import Depends +from fastapi import Depends, Path as FastAPIPath from loguru import logger from basic_memory.deps.config import AppConfigDep @@ -427,12 +427,16 @@ async def get_search_reindex_scheduler( async def get_relation_resolution_scheduler( + project_external_id: Annotated[ + str, FastAPIPath(alias="project_id", description="Project external UUID") + ], session_maker: SessionMakerDep, entity_repository: EntityRepositoryV2ExternalDep, relation_repository: RelationRepositoryV2ExternalDep, link_resolver: LinkResolverV2ExternalDep, search_service: SearchServiceV2ExternalDep, app_config: AppConfigDep, + read_cache: ReadCacheDep, ) -> RelationResolutionScheduler: # Build the project-scoped resolution runtime. It owns its own sessions via # session_maker, so it is safe to run from a detached background task. @@ -449,6 +453,8 @@ async def get_relation_resolution_scheduler( ) return LocalRelationResolutionScheduler( relation_runtime=runtime, + project_external_id=project_external_id, + read_cache=read_cache, test_mode=app_config.is_test_env, ) @@ -480,11 +486,15 @@ async def get_relation_resolution_scheduler( async def get_note_content_materialization_provider( + project_external_id: Annotated[ + str, FastAPIPath(alias="project_id", description="Project external UUID") + ], file_service: FileServiceV2ExternalDep, file_indexer: IndexFileExecutorV2ExternalDep, session_maker: SessionMakerDep, app_config: AppConfigDep, relation_resolution_scheduler: RelationResolutionSchedulerDep, + read_cache: ReadCacheDep, ) -> LocalNoteContentMaterializationProvider: """Create the local materializer for accepted-note route writes. @@ -495,6 +505,8 @@ async def get_note_content_materialization_provider( return LocalNoteContentMaterializationProvider( session_maker=session_maker, file_service=file_service, + project_external_id=project_external_id, + read_cache=read_cache, file_indexer=file_indexer, test_mode=app_config.is_test_env, materialization_workers=app_config.materialization_workers, diff --git a/src/basic_memory/index/local_schedulers.py b/src/basic_memory/index/local_schedulers.py index 99dd6ca4c..5e32cd382 100644 --- a/src/basic_memory/index/local_schedulers.py +++ b/src/basic_memory/index/local_schedulers.py @@ -20,6 +20,7 @@ RelationResolutionRuntime, resolve_project_relations, ) +from basic_memory.read_cache import ReadCache, invalidate_project_read_cache from basic_memory.runtime.vector_sync import EntityVectorSync # --- Background Task Machinery --- @@ -192,6 +193,8 @@ class LocalRelationResolutionScheduler: """ relation_runtime: RelationResolutionRuntime + project_external_id: str + read_cache: ReadCache test_mode: bool debounce_seconds: float = 0.5 @@ -220,12 +223,22 @@ async def _resolve_after_debounce(self, project_id: int) -> None: # Writes up to here are covered by the scan we are about to run, so only # writes that land DURING the scan should force a re-run. _dirty_relation_resolution.discard(project_id) - await resolve_project_relations(self.relation_runtime) + try: + await resolve_project_relations(self.relation_runtime) + finally: + # Relation resolution commits entity changes after the index pass. + # A second bump closes the window in which an intermediate entity + # response could have populated the current generation. + await invalidate_project_read_cache( + self.read_cache, + self.project_external_id, + ) finally: rerun = project_id in _dirty_relation_resolution _dirty_relation_resolution.discard(project_id) _pending_relation_resolution.discard(project_id) - # Re-arm outside the in-flight window (pending now cleared) so a write that - # raced the scan gets its own pass. Bounded to one extra pass per burst. - if rerun: - self.schedule_relation_resolution(project_id=project_id) + # Re-arm after clearing the in-flight marker so a write that raced the + # scan gets its own pass. Keep this in the cleanup path so a cache + # failure cannot drop the required relation rerun. + if rerun: + self.schedule_relation_resolution(project_id=project_id) diff --git a/src/basic_memory/index/note_content_materialization.py b/src/basic_memory/index/note_content_materialization.py index 172558963..f19484a9a 100644 --- a/src/basic_memory/index/note_content_materialization.py +++ b/src/basic_memory/index/note_content_materialization.py @@ -51,6 +51,7 @@ NoteFileVacateRepository, RecoverableVacate, ) +from basic_memory.read_cache import ReadCache, invalidate_project_read_cache from basic_memory.schemas.response import ObservationResponse, RelationResponse from basic_memory.services.file_service import FileService @@ -571,6 +572,8 @@ class LocalNoteContentMaterializationProvider: session_maker: async_sessionmaker[AsyncSession] file_service: FileService + project_external_id: str + read_cache: ReadCache file_indexer: IndexFileExecutor | None = None test_mode: bool = False materialization_workers: int = 4 @@ -626,60 +629,69 @@ async def _materialize_write_now( ) -> RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload]: if accepted.materialization is None: # pragma: no cover - guarded by caller return accepted - storage = LocalNoteContentStorage(self.file_service) - cleanup_enqueuer = InlineNoteFileDeleteEnqueuer( - storage, - vacate_clearer=RepositoryMoveVacateClearer(session_maker=self.session_maker), - ) - result = await run_note_materialization( - plan_note_materialization_job_request(accepted.materialization), - preflight=RepositoryNoteMaterializationPreflight( - session_maker=self.session_maker, - ), - writer=ContentStoreNoteMaterializationFileWriter(storage), - publisher=RepositoryNoteMaterializationPublisher( - session_maker=self.session_maker, - ), - status_publisher=RepositoryNoteMaterializationStatusPublisher( - session_maker=self.session_maker, - ), - cleanup_enqueuer=cleanup_enqueuer, - ) - if result.status is not RuntimeNoteMaterializationStatus.written: - return replace( - accepted, - payload=note_content_payload_with_materialization_result( - accepted.payload, - result, + try: + storage = LocalNoteContentStorage(self.file_service) + cleanup_enqueuer = InlineNoteFileDeleteEnqueuer( + storage, + vacate_clearer=RepositoryMoveVacateClearer(session_maker=self.session_maker), + ) + result = await run_note_materialization( + plan_note_materialization_job_request(accepted.materialization), + preflight=RepositoryNoteMaterializationPreflight( + session_maker=self.session_maker, ), + writer=ContentStoreNoteMaterializationFileWriter(storage), + publisher=RepositoryNoteMaterializationPublisher( + session_maker=self.session_maker, + ), + status_publisher=RepositoryNoteMaterializationStatusPublisher( + session_maker=self.session_maker, + ), + cleanup_enqueuer=cleanup_enqueuer, ) + if result.status is not RuntimeNoteMaterializationStatus.written: + return replace( + accepted, + payload=note_content_payload_with_materialization_result( + accepted.payload, + result, + ), + ) - file_path = note_content_payload_file_path(accepted.payload) - if file_path is not None and self.file_indexer is not None: - await self.file_indexer.index_file( - file_path, - source="note-content-materialization", - ) - # The deferred index has now inserted this note's entity/relation rows, - # so back-resolve inbound forward references. The router schedules an - # eager pass right after enqueue, but under load that pass can scan - # before this index lands; scheduling here (coalesced/re-armed by the - # resolution scheduler) guarantees a pass runs after indexing (#1002). - if self.relation_resolution_scheduler is not None: - self.relation_resolution_scheduler.schedule_relation_resolution( - project_id=accepted.materialization.project_id, + file_path = note_content_payload_file_path(accepted.payload) + if file_path is not None and self.file_indexer is not None: + await self.file_indexer.index_file( + file_path, + source="note-content-materialization", ) - return replace( - accepted, - payload=await load_indexed_note_content_response_payload( - session_maker=self.session_maker, - project_id=accepted.materialization.project_id, - entity_id=accepted.materialization.entity_id, - fallback_source=accepted.materialization.source - or "note-content-materialization", - ), + # The deferred index has now inserted this note's entity/relation rows, + # so back-resolve inbound forward references. The router schedules an + # eager pass right after enqueue, but under load that pass can scan + # before this index lands; scheduling here (coalesced/re-armed by the + # resolution scheduler) guarantees a pass runs after indexing (#1002). + if self.relation_resolution_scheduler is not None: + self.relation_resolution_scheduler.schedule_relation_resolution( + project_id=accepted.materialization.project_id, + ) + return replace( + accepted, + payload=await load_indexed_note_content_response_payload( + session_maker=self.session_maker, + project_id=accepted.materialization.project_id, + entity_id=accepted.materialization.entity_id, + fallback_source=accepted.materialization.source + or "note-content-materialization", + ), + ) + return accepted + finally: + # The accepted-write invalidation runs before deferred materialization. + # Invalidate again after status publication and indexing so a read + # filled during that window cannot survive the terminal state. + await invalidate_project_read_cache( + self.read_cache, + self.project_external_id, ) - return accepted async def materialize_delete_change( self, diff --git a/test-int/read_cache/test_api_read_cache.py b/test-int/read_cache/test_api_read_cache.py index 781a9b21f..e51d848b2 100644 --- a/test-int/read_cache/test_api_read_cache.py +++ b/test-int/read_cache/test_api_read_cache.py @@ -25,6 +25,10 @@ from basic_memory.repository import EntityRepository from basic_memory.runtime.note_content import NOTE_CONTENT_BASE_CHECKSUM_HEADER from basic_memory.schemas.v2 import EntityResolveRequest +from basic_memory.workspace_context import ( + WORKSPACE_SLUG_HEADER, + WORKSPACE_TYPE_HEADER, +) class RedisCacheHarness(Protocol): @@ -44,11 +48,12 @@ def _cache_key( project_id: str, operation: ReadCacheOperation, request: str, + request_context: tuple[str, ...] = (), ) -> ReadCacheKey: return ReadCacheKey( project_id=project_id, operation=operation, - request_digest=read_cache_request_digest(request), + request_digest=read_cache_request_digest(request, *request_context), ) @@ -82,10 +87,19 @@ async def test_entity_resolve_and_markdown_reads_cache_then_write_invalidates( f"{project_url}/knowledge/resolve", json=resolve_request.model_dump(mode="json"), ) + workspace_resolve_response = await client.post( + f"{project_url}/knowledge/resolve", + headers={ + WORKSPACE_SLUG_HEADER: "team-paul", + WORKSPACE_TYPE_HEADER: "organization", + }, + json=resolve_request.model_dump(mode="json"), + ) resource_response = await client.get(f"{project_url}/resource/{entity_id}") assert entity_response.status_code == 200 assert resolve_response.status_code == 200 + assert workspace_resolve_response.status_code == 200 assert resource_response.status_code == 200 assert "Version one." in resource_response.text @@ -99,6 +113,13 @@ async def test_entity_resolve_and_markdown_reads_cache_then_write_invalidates( project_id=project_external_id, operation=ReadCacheOperation.resolve, request=resolve_request.model_dump_json(), + request_context=("", ""), + ), + _cache_key( + project_id=project_external_id, + operation=ReadCacheOperation.resolve, + request=resolve_request.model_dump_json(), + request_context=("team-paul", "organization"), ), _cache_key( project_id=project_external_id, diff --git a/tests/cloud/test_note_content_materialization.py b/tests/cloud/test_note_content_materialization.py index 7ad9c5b3d..de3ea5754 100644 --- a/tests/cloud/test_note_content_materialization.py +++ b/tests/cloud/test_note_content_materialization.py @@ -6,7 +6,7 @@ import os from datetime import UTC, datetime from hashlib import sha256 -from typing import Any, cast +from typing import Any, cast, override import pytest from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -27,6 +27,11 @@ NoteContentRepository, ) from basic_memory.repository.note_file_vacate_repository import NoteFileVacateRepository +from basic_memory.read_cache import ( + NullReadCache, + ReadCache, + ReadCacheInvalidationStatus, +) from basic_memory.runtime.cleanup import RuntimeNoteFileDeleteJobRequest from basic_memory.indexing.models import FileIndexOperation, FileIndexResult from basic_memory.runtime.note_content import ( @@ -42,6 +47,8 @@ ) from basic_memory.services.file_service import FileService +PROJECT_EXTERNAL_ID = "00000000-0000-0000-0000-000000000007" + class RecordingFileIndexer: def __init__(self) -> None: @@ -60,6 +67,16 @@ async def index_file(self, file_path: str, *, source: str) -> FileIndexResult: ) +class RecordingReadCache(NullReadCache): + def __init__(self) -> None: + self.invalidated_project_ids: list[str] = [] + + @override + async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: + self.invalidated_project_ids.append(project_id) + return ReadCacheInvalidationStatus.invalidated + + def accepted_materialization_change() -> RuntimeAcceptedNoteChange[ RuntimeNoteContentResponsePayload ]: @@ -102,12 +119,15 @@ def local_materialization_provider( indexer: RecordingFileIndexer, *, test_mode: bool = True, + read_cache: ReadCache | None = None, ) -> LocalNoteContentMaterializationProvider: # test_mode=True keeps materialization inline so these tests can assert the # result synchronously; production defers it to a background task. return LocalNoteContentMaterializationProvider( session_maker=cast(async_sessionmaker[AsyncSession], object()), file_service=cast(FileService, object()), + project_external_id=PROJECT_EXTERNAL_ID, + read_cache=read_cache if read_cache is not None else NullReadCache(), file_indexer=indexer, test_mode=test_mode, ) @@ -277,19 +297,24 @@ async def fake_run_note_materialization( pool = note_content_materialization._MaterializationWorkerPool() monkeypatch.setattr(note_content_materialization, "_materialization_pool", pool) indexer = RecordingFileIndexer() + read_cache = RecordingReadCache() accepted = accepted_materialization_change() result = await local_materialization_provider( - indexer, test_mode=False + indexer, + test_mode=False, + read_cache=read_cache, ).materialize_write_change(accepted) # Returned immediately with the accepted DB state — no inline write yet. assert result is accepted assert requests == [] + assert read_cache.invalidated_project_ids == [] # The write happens off the accept path via the bounded pool; drain to confirm. await pool.join() assert len(requests) == 1 + assert read_cache.invalidated_project_ids == [PROJECT_EXTERNAL_ID] await pool.aclose() @@ -350,9 +375,12 @@ class RecordingScheduler: def schedule_relation_resolution(self, *, project_id: int) -> None: scheduled.append(project_id) + read_cache = RecordingReadCache() provider = LocalNoteContentMaterializationProvider( session_maker=cast(async_sessionmaker[AsyncSession], object()), file_service=cast(FileService, object()), + project_external_id=PROJECT_EXTERNAL_ID, + read_cache=read_cache, file_indexer=RecordingFileIndexer(), test_mode=True, relation_resolution_scheduler=RecordingScheduler(), @@ -362,6 +390,7 @@ def schedule_relation_resolution(self, *, project_id: int) -> None: assert accepted.materialization is not None assert scheduled == [accepted.materialization.project_id] + assert read_cache.invalidated_project_ids == [PROJECT_EXTERNAL_ID] @pytest.mark.asyncio diff --git a/tests/index/test_local_schedulers.py b/tests/index/test_local_schedulers.py index 75bcd154f..53fa40ad6 100644 --- a/tests/index/test_local_schedulers.py +++ b/tests/index/test_local_schedulers.py @@ -1,7 +1,7 @@ """Typed scheduler tests for derived async work.""" import asyncio -from typing import cast +from typing import cast, override import pytest @@ -13,6 +13,12 @@ LocalSearchReindexScheduler, drain_background_tasks, ) +from basic_memory.read_cache import ( + NullReadCache, + ReadCacheInvalidationStatus, +) + +PROJECT_EXTERNAL_ID = "00000000-0000-0000-0000-000000000013" class StubProjectIndexRunner: @@ -41,6 +47,16 @@ async def reindex_all(self) -> None: self.reindexed_project = True +class RecordingReadCache(NullReadCache): + def __init__(self) -> None: + self.invalidated_project_ids: list[str] = [] + + @override + async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: + self.invalidated_project_ids.append(project_id) + return ReadCacheInvalidationStatus.invalidated + + @pytest.mark.asyncio async def test_entity_vector_scheduler_maps_to_search_service(): """Entity vector scheduling should call the semantic vector sync method.""" @@ -243,9 +259,12 @@ async def test_relation_resolution_scheduler_runs_project_resolution(): _pending_relation_resolution.clear() runtime = StubRelationResolutionRuntime() + read_cache = RecordingReadCache() scheduler = LocalRelationResolutionScheduler( relation_runtime=runtime, + project_external_id=PROJECT_EXTERNAL_ID, + read_cache=read_cache, test_mode=False, debounce_seconds=0.0, ) @@ -253,6 +272,7 @@ async def test_relation_resolution_scheduler_runs_project_resolution(): await asyncio.sleep(0.05) assert runtime.resolve_calls == 1 + assert read_cache.invalidated_project_ids == [PROJECT_EXTERNAL_ID] # The pending marker is cleared after the pass so later writes can schedule. assert 13 not in _pending_relation_resolution @@ -267,6 +287,8 @@ async def test_relation_resolution_scheduler_coalesces_a_burst(): scheduler = LocalRelationResolutionScheduler( relation_runtime=runtime, + project_external_id=PROJECT_EXTERNAL_ID, + read_cache=NullReadCache(), test_mode=False, debounce_seconds=0.02, ) @@ -289,6 +311,7 @@ async def test_relation_resolution_scheduler_reruns_for_write_during_pass(): _pending_relation_resolution.clear() _dirty_relation_resolution.clear() + read_cache = RecordingReadCache() class WriteDuringScanRuntime: def __init__(self) -> None: @@ -309,6 +332,8 @@ async def resolve_relations(self, entity_id: int | None = None) -> set[int]: runtime = WriteDuringScanRuntime() scheduler = LocalRelationResolutionScheduler( relation_runtime=runtime, + project_external_id=PROJECT_EXTERNAL_ID, + read_cache=read_cache, test_mode=False, debounce_seconds=0.0, ) @@ -318,6 +343,10 @@ async def resolve_relations(self, entity_id: int | None = None) -> set[int]: await asyncio.sleep(0.05) assert runtime.resolve_calls == 2 + assert read_cache.invalidated_project_ids == [ + PROJECT_EXTERNAL_ID, + PROJECT_EXTERNAL_ID, + ] assert 21 not in _pending_relation_resolution assert 21 not in _dirty_relation_resolution @@ -369,6 +398,8 @@ async def resolve_relations(self, entity_id: int | None = None) -> set[int]: runtime = WriteDuringScanRuntime() scheduler = LocalRelationResolutionScheduler( relation_runtime=runtime, + project_external_id=PROJECT_EXTERNAL_ID, + read_cache=NullReadCache(), test_mode=False, debounce_seconds=0.0, ) @@ -388,14 +419,18 @@ async def test_relation_resolution_scheduler_is_noop_in_test_mode(): _pending_relation_resolution.clear() runtime = StubRelationResolutionRuntime() + read_cache = RecordingReadCache() scheduler = LocalRelationResolutionScheduler( relation_runtime=runtime, + project_external_id=PROJECT_EXTERNAL_ID, + read_cache=read_cache, test_mode=True, ) scheduler.schedule_relation_resolution(project_id=13) await asyncio.sleep(0.05) assert runtime.resolve_calls == 0 + assert read_cache.invalidated_project_ids == [] # Test mode must not leak a pending marker (it never runs the clearer). assert 13 not in _pending_relation_resolution From 621f642efce3a1d1bdd4fa996637597beabba74b Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 29 Jul 2026 00:18:46 -0500 Subject: [PATCH 03/28] fix(sync): invalidate cache after recovery and moves Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 30 +- src/basic_memory/index/local_moves.py | 30 +- src/basic_memory/index/local_runtime.py | 2 + src/basic_memory/services/initialization.py | 36 ++- .../read_cache/test_runtime_invalidation.py | 261 ++++++++++++++++++ 5 files changed, 334 insertions(+), 25 deletions(-) create mode 100644 test-int/read_cache/test_runtime_invalidation.py diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index 62566ecf6..adfe534f9 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -77,12 +77,18 @@ tenant: from a public request field. 1. Inject the namespace-bound cache into every mutation-producing runtime: accepted note materialization, object-storage events, direct and project indexing, directory moves/deletes, - and relation-resolution workers. Request-path invalidation alone is insufficient because a - worker can update a cached entity after the request returns. + watcher-detected paired moves, relation-resolution workers, and startup recovery or + reconciliation. Request-path invalidation alone is insufficient because a worker can update a + cached entity after the request returns. 1. Treat each asynchronous state transition as a separate freshness boundary. Invalidate after the accepted-note transaction commits, again after terminal materialization/status publication and indexing, and again after relation resolution completes. This prevents a read filled between phases from surviving the later worker commit. +1. Invalidate watcher-detected moves at their own completion boundary. Paired delete/create + events are consumed by move processing and therefore bypass the ordinary watcher callbacks. +1. If startup recovery or reconciliation changes materialization, vacate, index, or relation + state, invalidate through the same namespace-bound cache before releasing the serving barrier + or resuming tenant traffic. 1. Keep `bm:read:v1` separate from rate-limit and Cloud control-plane prefixes, metrics, timeouts, and failure policies. The clients may target one Redis deployment, but a read-cache timeout must bypass while a rate-limit decision keeps its Cloud-owned security behavior. @@ -106,7 +112,7 @@ flowchart LR RC -->|"hit"| API RC -->|"miss"| DB["Services, repositories, and storage"] DB -->|"successful result"| RC - W["Writes, indexing, and storage events"] -->|"invalidate after commit"| RC + W["Writes, indexing, recovery, and storage events"] -->|"invalidate after commit"| RC RL["Cloud tenant rate limiter"] --> RLD["Cloud rate-limit keyspace"] RC --> BMD["Basic Memory read-cache keyspace"] @@ -201,9 +207,10 @@ Primary integration points: Invalidation belongs at portable mutation and indexing completion boundaries, not only in FastAPI routes. It must cover accepted note writes, terminal deferred materialization and status publication, direct file indexing, filesystem watcher updates, project indexing, directory -mutations, Cloud storage events, and relation-resolution changes that affect cached responses. -Each later phase invalidates again so a value filled after an earlier generation bump cannot -outlive the state that phase publishes. +mutations, watcher-detected paired moves, startup recovery or reconciliation, Cloud storage +events, and relation-resolution changes that affect cached responses. Each later phase +invalidates again so a value filled after an earlier generation bump cannot outlive the state +that phase publishes. Recovery invalidation runs before the serving barrier is released. ## Dependency And Lifecycle @@ -273,7 +280,9 @@ The real-Redis suite must prove: - no invalidation operation touches keys outside the Basic Memory prefix; - payload size limits; - repeated API entity reads use the real cached representation; -- successful writes invalidate while rejected or rolled-back writes do not. +- successful writes invalidate while rejected or rolled-back writes do not; +- watcher-detected paired moves invalidate even though their events bypass ordinary callbacks; +- startup recovery that changes materialization state invalidates before serving resumes. Run route behavior against both SQLite and Postgres where persistence behavior differs. Redis semantics themselves are asserted only against the real Redis integration fixture. @@ -297,8 +306,11 @@ semantics themselves are asserted only against the real Redis integration fixtur - Inject a Basic Memory-specific Redis client and tenant namespace. - Derive that namespace from trusted request and worker context with one canonical function. - Invalidate after accepted-note commit, terminal materialization/indexing, storage events, and - relation-resolution workers using the same tenant namespace as the request path before enabling - reads for a tenant. + relation-resolution workers using the same tenant namespace as the request path. +- Invalidate watcher-detected paired moves at move completion, and invalidate any recovery or + reconciliation state change before releasing the serving barrier or resuming tenant traffic. +- Enable reads for a tenant only after every request, worker, move, and recovery boundary has + namespace and invalidation parity. - Start with shadow telemetry or a limited tenant cohort. - Compare hit rate, Redis latency, database query volume, and end-to-end tool latency. diff --git a/src/basic_memory/index/local_moves.py b/src/basic_memory/index/local_moves.py index 2f71275e8..91af49dd3 100644 --- a/src/basic_memory/index/local_moves.py +++ b/src/basic_memory/index/local_moves.py @@ -35,6 +35,7 @@ STORAGE_OBJECT_CREATED_EVENTS, STORAGE_OBJECT_DELETED_EVENT, ) +from basic_memory.read_cache import ReadCache, invalidate_project_read_cache from basic_memory.services import FileService @@ -171,6 +172,8 @@ class LocalWatchMoveProcessor: entity_repository: LocalMoveEntityRepository maintenance_runner: ProjectIndexMaintenanceRunner moved_entity_search_refresher: ProjectIndexMovedEntitySearchRefresher + project_external_id: str + read_cache: ReadCache batch_size: int = 100 async def process_moves( @@ -182,14 +185,25 @@ async def process_moves( removed_event_indexes: set[int] = set() if moved_files: - move_run = await self.maintenance_runner.run_move_batches( - moved_files=moved_files, - batch_size=self.batch_size, - ) - refresh_entity_ids = move_run.moved_entity_ids | move_run.relation_cleanup_entity_ids - if refresh_entity_ids: - await self.moved_entity_search_refresher.refresh_moved_entities( - sorted(refresh_entity_ids) + try: + move_run = await self.maintenance_runner.run_move_batches( + moved_files=moved_files, + batch_size=self.batch_size, + ) + refresh_entity_ids = ( + move_run.moved_entity_ids | move_run.relation_cleanup_entity_ids + ) + if refresh_entity_ids: + await self.moved_entity_search_refresher.refresh_moved_entities( + sorted(refresh_entity_ids) + ) + finally: + # Move pairs bypass ordinary file/delete completion callbacks. + # Invalidate after maintenance and search refresh so cached paths, + # permalinks, and relations cannot retain the pre-move state. + await invalidate_project_read_cache( + self.read_cache, + self.project_external_id, ) moved_old_paths = set(moved_files) - set(move_run.missing_paths) diff --git a/src/basic_memory/index/local_runtime.py b/src/basic_memory/index/local_runtime.py index 87da36b27..3db1ffe8d 100644 --- a/src/basic_memory/index/local_runtime.py +++ b/src/basic_memory/index/local_runtime.py @@ -383,6 +383,8 @@ async def runtime_for_project(self, project: Project) -> StorageEventIndexRuntim entity_repository=dependencies.entity_repository, maintenance_runner=maintenance_runner, moved_entity_search_refresher=moved_entity_search_refresher, + project_external_id=project_ref.project_external_id, + read_cache=self.read_cache, batch_size=self.move_batch_size, ), ) diff --git a/src/basic_memory/services/initialization.py b/src/basic_memory/services/initialization.py index cc27ba520..d7d8b63a8 100644 --- a/src/basic_memory/services/initialization.py +++ b/src/basic_memory/services/initialization.py @@ -51,6 +51,8 @@ async def run_initial_project_index( async def recover_project_materializations( project: Project, session_maker: "async_sessionmaker[AsyncSession]", + *, + read_cache: "ReadCache | None" = None, ) -> None: """Re-drive note materialization and move cleanup lost across a process exit. @@ -82,15 +84,29 @@ async def recover_project_materializations( file_service=file_service, project_id=project.id, ) - if recovered or recovered_vacates: - logger.info( - "Recovered note materialization state on startup", - project=project.name, - recovered_materializations=recovered, - recovered_move_vacates=recovered_vacates, - ) except Exception as e: # pragma: no cover - defensive startup guard logger.error(f"Error recovering stuck materializations for project {project.name}: {e}") + return + + if not recovered and not recovered_vacates: + return + + logger.info( + "Recovered note materialization state on startup", + project=project.name, + recovered_materializations=recovered, + recovered_move_vacates=recovered_vacates, + ) + + # Redis can outlive the process that left this materialization unfinished. + # Invalidate before releasing the startup barrier so no pre-crash pending or + # failed payload remains reachable while background indexing catches up. + from basic_memory.read_cache import NullReadCache, invalidate_project_read_cache + + await invalidate_project_read_cache( + read_cache if read_cache is not None else NullReadCache(), + str(project.external_id), + ) async def initialize_database(app_config: BasicMemoryConfig) -> None: @@ -235,7 +251,11 @@ async def initialize_file_indexing( # cleanup whose in-process enqueue was lost. Runs synchronously so the files # converge before the initial project index scans them. for project in active_projects: - await recover_project_materializations(project, session_maker) + await recover_project_materializations( + project, + session_maker, + read_cache=active_read_cache, + ) # Trigger: the API/MCP lifespan is waiting for durable startup recovery. # Why: serving accepted writes while recovery holds cached vacate work can diff --git a/test-int/read_cache/test_runtime_invalidation.py b/test-int/read_cache/test_runtime_invalidation.py new file mode 100644 index 000000000..daa6cbbe7 --- /dev/null +++ b/test-int/read_cache/test_runtime_invalidation.py @@ -0,0 +1,261 @@ +"""Real Redis coverage for non-request cache invalidation boundaries.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime +from pathlib import Path +from typing import Protocol, cast, override + +import pytest +from redis.asyncio import Redis +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker + +from basic_memory import db +from basic_memory.index.local_moves import ( + LocalMoveEntityRepository, + LocalWatchMoveProcessor, +) +from basic_memory.indexing.project_index_maintenance import ( + ProjectIndexDeleteRun, + ProjectIndexMoveRun, +) +from basic_memory.models import Project +from basic_memory.models.knowledge import Entity +from basic_memory.read_cache import ( + ReadCacheKey, + ReadCacheOperation, + read_cache_request_digest, +) +from basic_memory.read_cache.keys import redis_read_cache_generation_key +from basic_memory.read_cache.redis import RedisReadCache +from basic_memory.repository import EntityRepository +from basic_memory.repository.note_content_repository import ( + AcceptedNoteContentWrite, + NoteContentRepository, +) +from basic_memory.runtime.storage import ( + STORAGE_OBJECT_DELETED_EVENT, + StorageEventPayload, + StorageObjectIdentity, + StorageObjectVersion, +) +from basic_memory.services.file_service import FileService +from basic_memory.services.initialization import recover_project_materializations + +pytestmark = pytest.mark.redis + + +class RedisCacheHarness(Protocol): + cache: RedisReadCache + client: Redis + namespace: str + prefix: str + + +class DetectedMoveProcessor(LocalWatchMoveProcessor): + """Exercise move completion without coupling the test to detection I/O.""" + + @override + async def detect_moves( + self, + events: Sequence[StorageEventPayload], + ) -> tuple[dict[str, str], set[int]]: + del events + return {"notes/old.md": "notes/new.md"}, {0, 1} + + @override + async def detect_transient_missing_events( + self, + events: Sequence[StorageEventPayload], + *, + exclude_indexes: set[int], + ) -> set[int]: + del events, exclude_indexes + return set() + + @override + async def detect_missing_entity_delete_events( + self, + events: Sequence[StorageEventPayload], + *, + exclude_indexes: set[int], + ) -> set[int]: + del events, exclude_indexes + return set() + + +class RecordingMoveMaintenance: + def __init__(self) -> None: + self.calls: list[tuple[dict[str, str], int]] = [] + + async def run_move_batches( + self, + *, + moved_files: Mapping[str, str], + batch_size: int, + ) -> ProjectIndexMoveRun: + self.calls.append((dict(moved_files), batch_size)) + return ProjectIndexMoveRun( + total_moves=1, + total_updated_files=1, + records=(), + moved_entity_ids=frozenset({17}), + ) + + async def run_delete_batches( + self, + *, + deleted_paths: Sequence[str], + batch_size: int, + ) -> ProjectIndexDeleteRun: + del deleted_paths, batch_size + raise AssertionError("watcher move completion must not run delete maintenance") + + +class RecordingMovedEntitySearchRefresher: + def __init__(self) -> None: + self.calls: list[list[int]] = [] + + async def refresh_moved_entities(self, entity_ids: Sequence[int]) -> None: + self.calls.append(list(entity_ids)) + + +async def _initialized_generation( + redis_cache: RedisCacheHarness, + project_external_id: str, + *, + request: str, +) -> bytes | str: + await redis_cache.cache.lookup( + ReadCacheKey( + project_id=project_external_id, + operation=ReadCacheOperation.entity, + request_digest=read_cache_request_digest(request), + ) + ) + generation = await redis_cache.client.get( + redis_read_cache_generation_key( + prefix=redis_cache.prefix, + namespace=redis_cache.namespace, + project_id=project_external_id, + ) + ) + assert generation is not None + return generation + + +def _move_event(event_name: str, path: str) -> StorageEventPayload: + return StorageEventPayload( + event_name=event_name, + event_time="2026-07-29T00:00:00Z", + object_version=StorageObjectVersion( + identity=StorageObjectIdentity( + bucket_name="local-filesystem", + key=f"project/{path}", + ), + etag="move-etag", + ), + ) + + +@pytest.mark.asyncio +async def test_watcher_move_completion_invalidates_real_redis( + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="watcher-move", + ) + maintenance = RecordingMoveMaintenance() + search_refresher = RecordingMovedEntitySearchRefresher() + processor = DetectedMoveProcessor( + session_maker=cast(async_sessionmaker[AsyncSession], object()), + file_service=cast(FileService, object()), + entity_repository=cast(LocalMoveEntityRepository, object()), + maintenance_runner=maintenance, + moved_entity_search_refresher=search_refresher, + project_external_id=project_external_id, + read_cache=redis_cache.cache, + ) + + result = await processor.process_moves( + ( + _move_event(STORAGE_OBJECT_DELETED_EVENT, "notes/old.md"), + _move_event("OBJECT_CREATED_PUT", "notes/new.md"), + ) + ) + + assert result.remaining_events == () + assert result.processed_moves == 1 + assert maintenance.calls == [({"notes/old.md": "notes/new.md"}, 100)] + assert search_refresher.calls == [[17]] + generation_after = await _initialized_generation( + redis_cache, + project_external_id, + request="watcher-move", + ) + assert generation_after != generation_before + + +@pytest.mark.asyncio +async def test_startup_materialization_recovery_invalidates_real_redis( + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + _, session_maker = engine_factory + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="startup-recovery", + ) + entity_repository = EntityRepository(project_id=test_project.id) + content_repository = NoteContentRepository(project_id=test_project.id) + async with db.scoped_session(session_maker) as session: + entity = await entity_repository.add( + session, + Entity( + title="Recovered Note", + note_type="note", + content_type="text/markdown", + file_path="notes/recovered.md", + checksum="entity-checksum-1", + created_at=datetime.now(UTC), + updated_at=datetime.now(UTC), + ), + ) + await content_repository.accept_write( + session, + AcceptedNoteContentWrite( + entity_id=entity.id, + markdown_content="# Recovered with cache invalidation\n", + db_version=1, + db_checksum="db-checksum-1", + last_source="api", + updated_at=datetime.now(UTC), + ), + ) + row = await content_repository.select_by_id(session, entity.id) + assert row is not None + row.file_write_status = "writing" + await session.flush() + + await recover_project_materializations( + test_project, + session_maker, + read_cache=redis_cache.cache, + ) + + written = Path(test_project.path) / entity.file_path + assert written.read_text(encoding="utf-8") == "# Recovered with cache invalidation\n" + generation_after = await _initialized_generation( + redis_cache, + project_external_id, + request="startup-recovery", + ) + assert generation_after != generation_before From 87ee016a53ecf92b0804834d2fadc0517191e53a Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 29 Jul 2026 00:36:32 -0500 Subject: [PATCH 04/28] fix(sync): close cache invalidation failure paths Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 30 +- .../api/v2/routers/knowledge_router.py | 2 +- src/basic_memory/index/local_project.py | 42 +-- .../index/note_content_materialization.py | 25 +- .../services/directory_deletes.py | 51 ++- src/basic_memory/services/initialization.py | 7 +- .../read_cache/test_runtime_invalidation.py | 299 ++++++++++++++++-- .../test_note_content_materialization.py | 31 +- 8 files changed, 393 insertions(+), 94 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index adfe534f9..9f8862eeb 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -84,11 +84,17 @@ tenant: the accepted-note transaction commits, again after terminal materialization/status publication and indexing, and again after relation resolution completes. This prevents a read filled between phases from surviving the later worker commit. +1. Put project-index invalidation in a failure-safe completion boundary. Move, delete, file-index, + and vector batches can commit incrementally before a later batch raises. +1. Invalidate directory deletion immediately after its acceptance transaction commits, then + again after file cleanup and surviving-relation refresh. A slow or failed cleanup must not + keep deleted entities reachable through the pre-acceptance generation. 1. Invalidate watcher-detected moves at their own completion boundary. Paired delete/create events are consumed by move processing and therefore bypass the ordinary watcher callbacks. -1. If startup recovery or reconciliation changes materialization, vacate, index, or relation - state, invalidate through the same namespace-bound cache before releasing the serving barrier - or resuming tenant traffic. +1. If startup recovery or reconciliation attempts can publish materialization, vacate, index, or + relation state, invalidate through the same namespace-bound cache before releasing the serving + barrier or resuming tenant traffic. Terminal conflict and failure publication count even when + the recovery did not produce a written file. 1. Keep `bm:read:v1` separate from rate-limit and Cloud control-plane prefixes, metrics, timeouts, and failure policies. The clients may target one Redis deployment, but a read-cache timeout must bypass while a rate-limit decision keeps its Cloud-owned security behavior. @@ -210,7 +216,9 @@ publication, direct file indexing, filesystem watcher updates, project indexing, mutations, watcher-detected paired moves, startup recovery or reconciliation, Cloud storage events, and relation-resolution changes that affect cached responses. Each later phase invalidates again so a value filled after an earlier generation bump cannot outlive the state -that phase publishes. Recovery invalidation runs before the serving barrier is released. +that phase publishes. Directory deletion invalidates after acceptance and after cleanup. Project +indexing invalidates even after a partial failure. Recovery invalidation runs before the serving +barrier is released and includes terminal conflict or failure publication. ## Dependency And Lifecycle @@ -282,7 +290,10 @@ The real-Redis suite must prove: - repeated API entity reads use the real cached representation; - successful writes invalidate while rejected or rolled-back writes do not; - watcher-detected paired moves invalidate even though their events bypass ordinary callbacks; -- startup recovery that changes materialization state invalidates before serving resumes. +- startup recovery that publishes written, conflict, or failed materialization state invalidates + before serving resumes; +- project-index failures invalidate any earlier committed batches; +- directory deletion invalidates before cleanup starts and again after cleanup completes. Run route behavior against both SQLite and Postgres where persistence behavior differs. Redis semantics themselves are asserted only against the real Redis integration fixture. @@ -308,9 +319,12 @@ semantics themselves are asserted only against the real Redis integration fixtur - Invalidate after accepted-note commit, terminal materialization/indexing, storage events, and relation-resolution workers using the same tenant namespace as the request path. - Invalidate watcher-detected paired moves at move completion, and invalidate any recovery or - reconciliation state change before releasing the serving barrier or resuming tenant traffic. -- Enable reads for a tenant only after every request, worker, move, and recovery boundary has - namespace and invalidation parity. + reconciliation attempt that can publish terminal state before releasing the serving barrier or + resuming tenant traffic. +- Invalidate project indexing from a failure-safe completion boundary, and invalidate directory + deletion both after acceptance commit and after cleanup/relation refresh. +- Enable reads for a tenant only after every request, worker, partial-index, accepted-delete, move, + and recovery boundary has namespace and invalidation parity. - Start with shadow telemetry or a limited tenant cohort. - Compare hit rate, Redis latency, database query volume, and end-to-end tool latency. diff --git a/src/basic_memory/api/v2/routers/knowledge_router.py b/src/basic_memory/api/v2/routers/knowledge_router.py index 810bdd6e6..7f4db361b 100644 --- a/src/basic_memory/api/v2/routers/knowledge_router.py +++ b/src/basic_memory/api/v2/routers/knowledge_router.py @@ -1081,8 +1081,8 @@ async def delete_directory( result = await directory_delete_service.delete_directory( project_external_id=project_external_id, directory=data.directory, + read_cache=read_cache, ) - await invalidate_project_read_cache(read_cache, project_external_id) payload = result.to_response_payload() logger.info( f"API v2 response: delete_directory " diff --git a/src/basic_memory/index/local_project.py b/src/basic_memory/index/local_project.py index 8572941df..a3919efb4 100644 --- a/src/basic_memory/index/local_project.py +++ b/src/basic_memory/index/local_project.py @@ -761,26 +761,28 @@ async def run_local_project_index( runtime: LocalProjectIndexRuntime, ) -> ProjectIndexCoordinatorResult: """Run project-wide local indexing through the storage-neutral coordinator.""" - result = await run_project_index_coordinator( - request, - coordinator_job_id=runtime.coordinator_job_id, - observed_file_source=runtime.observed_file_source, - change_detector=runtime.change_detector, - maintenance_runner=runtime.maintenance_runner, - moved_entity_search_refresher=runtime.moved_entity_search_refresher, - workflow_starter=runtime.workflow_starter, - batch_enqueuer=runtime.batch_enqueuer, - fanout_failure_recorder=runtime.fanout_failure_recorder, - batch_size=runtime.batch_size, - embedding_vector_sync=runtime.embedding_vector_sync, - ) - # Project indexing has already committed entity/search changes. Invalidate - # before relation repair so a later failure cannot leave pre-index values - # reachable for the full TTL. - await invalidate_project_read_cache( - runtime.read_cache, - request.project.project_external_id, - ) + try: + result = await run_project_index_coordinator( + request, + coordinator_job_id=runtime.coordinator_job_id, + observed_file_source=runtime.observed_file_source, + change_detector=runtime.change_detector, + maintenance_runner=runtime.maintenance_runner, + moved_entity_search_refresher=runtime.moved_entity_search_refresher, + workflow_starter=runtime.workflow_starter, + batch_enqueuer=runtime.batch_enqueuer, + fanout_failure_recorder=runtime.fanout_failure_recorder, + batch_size=runtime.batch_size, + embedding_vector_sync=runtime.embedding_vector_sync, + ) + finally: + # The coordinator commits moves, deletes, and file batches incrementally. + # Invalidate even when a later batch or vector sync raises so already + # published changes cannot remain behind the previous generation. + await invalidate_project_read_cache( + runtime.read_cache, + request.project.project_external_id, + ) if runtime.completion_relation_runtime is not None: try: await resolve_project_index_completion_relations( diff --git a/src/basic_memory/index/note_content_materialization.py b/src/basic_memory/index/note_content_materialization.py index f19484a9a..e4e7ef73f 100644 --- a/src/basic_memory/index/note_content_materialization.py +++ b/src/basic_memory/index/note_content_materialization.py @@ -184,6 +184,14 @@ async def drain_pending_materializations() -> None: RECOVERY_NOTE_ACTOR_NAME = "startup-recovery" +@dataclass(frozen=True, slots=True) +class MaterializationRecoverySummary: + """State-publication summary for one startup recovery sweep.""" + + attempted: int = 0 + written: int = 0 + + async def run_recovery_materialization( request: RuntimeNoteMaterializationJobRequest, *, @@ -215,13 +223,13 @@ async def recover_stuck_materializations( session_maker: async_sessionmaker[AsyncSession], file_service: FileService, project_id: int, -) -> int: +) -> MaterializationRecoverySummary: """Re-drive every note materialization stuck in writing/pending/failed for a project. Meant to run once per project at startup, before serving. Non-fatal per row: a single row that raises is logged and skipped so one poisoned note cannot - block startup recovery for the rest of the project. Returns the number of rows - that reached a written file state. + block startup recovery for the rest of the project. Every attempt can publish + terminal status, so the summary distinguishes attempted rows from writes. """ async with db.scoped_session(session_maker) as session: stuck_rows = await NoteContentRepository(project_id=project_id).find_stuck_materializations( @@ -229,14 +237,14 @@ async def recover_stuck_materializations( ) if not stuck_rows: - return 0 + return MaterializationRecoverySummary() logger.info( "Recovering stuck note materializations", project_id=project_id, stuck_count=len(stuck_rows), ) - recovered = 0 + written = 0 for row in stuck_rows: # Rebuild the queue request from the row's own accepted db_version/db_checksum # so the preflight guard matches the current accepted state; if a newer write @@ -267,8 +275,11 @@ async def recover_stuck_materializations( ) continue if result.status is RuntimeNoteMaterializationStatus.written: - recovered += 1 - return recovered + written += 1 + return MaterializationRecoverySummary( + attempted=len(stuck_rows), + written=written, + ) async def _recover_deleted_destination_vacate( diff --git a/src/basic_memory/services/directory_deletes.py b/src/basic_memory/services/directory_deletes.py index 75a476a97..4248640ce 100644 --- a/src/basic_memory/services/directory_deletes.py +++ b/src/basic_memory/services/directory_deletes.py @@ -19,6 +19,7 @@ finish_directory_delete_acceptance, normalize_directory_delete_path, ) +from basic_memory.read_cache import NullReadCache, ReadCache, invalidate_project_read_cache class DirectoryDeleteServiceError(Exception): @@ -57,6 +58,7 @@ async def delete_directory( *, project_external_id: str, directory: str, + read_cache: ReadCache | None = None, ) -> DirectoryDeleteAcceptedResult: """Delete directory entities immediately and queue file cleanup in the background. @@ -81,24 +83,43 @@ async def delete_directory( except DirectoryDeleteRejected as error: raise directory_delete_service_error_from_rejection(error.rejection) from error - result = await finish_directory_delete_acceptance( - request=request, - accepted=accepted, - enqueuer=self.runtime.file_delete_enqueuer, - ) + active_read_cache = read_cache if read_cache is not None else NullReadCache() + if accepted.files: + # Acceptance commits entity and search deletion before storage cleanup. + # Invalidate now so deleted reads cannot survive a slow or failed + # follow-up phase. + await invalidate_project_read_cache( + active_read_cache, + project_external_id, + ) - # Trigger: notes outside the deleted directory linked into it. - # Why: the delete cascaded their relation rows away, but those sources own - # matching search_index relation rows that now dangle; without a reindex - # they linger until an unrelated rebuild. - # Outcome: reindex each surviving source inline when the runtime provides a - # refresher (local); queued runtimes consume the ids from the result. - if accepted.relation_cleanup_entity_ids and self.runtime.relation_cleanup_refresher: - await self.runtime.relation_cleanup_refresher.refresh_relation_sources( - sorted(accepted.relation_cleanup_entity_ids) + try: + result = await finish_directory_delete_acceptance( + request=request, + accepted=accepted, + enqueuer=self.runtime.file_delete_enqueuer, ) - return result + # Trigger: notes outside the deleted directory linked into it. + # Why: the delete cascaded their relation rows away, but those sources own + # matching search_index relation rows that now dangle; without a reindex + # they linger until an unrelated rebuild. + # Outcome: reindex each surviving source inline when the runtime provides a + # refresher (local); queued runtimes consume the ids from the result. + if accepted.relation_cleanup_entity_ids and self.runtime.relation_cleanup_refresher: + await self.runtime.relation_cleanup_refresher.refresh_relation_sources( + sorted(accepted.relation_cleanup_entity_ids) + ) + + return result + finally: + if accepted.files: + # Cleanup and relation refresh can publish additional state or fail + # after partial progress. Close the fill window in either case. + await invalidate_project_read_cache( + active_read_cache, + project_external_id, + ) @staticmethod def normalize_directory_path(directory: str) -> str: diff --git a/src/basic_memory/services/initialization.py b/src/basic_memory/services/initialization.py index d7d8b63a8..6df2b326a 100644 --- a/src/basic_memory/services/initialization.py +++ b/src/basic_memory/services/initialization.py @@ -74,7 +74,7 @@ async def recover_project_materializations( # FileService needs only base_path to write the accepted markdown bytes; # the markdown_processor/app_config are unused on the materialization path. file_service = FileService(Path(project.path)) - recovered = await recover_stuck_materializations( + materialization_recovery = await recover_stuck_materializations( session_maker=session_maker, file_service=file_service, project_id=project.id, @@ -88,13 +88,14 @@ async def recover_project_materializations( logger.error(f"Error recovering stuck materializations for project {project.name}: {e}") return - if not recovered and not recovered_vacates: + if not materialization_recovery.attempted and not recovered_vacates: return logger.info( "Recovered note materialization state on startup", project=project.name, - recovered_materializations=recovered, + attempted_materializations=materialization_recovery.attempted, + recovered_materializations=materialization_recovery.written, recovered_move_vacates=recovered_vacates, ) diff --git a/test-int/read_cache/test_runtime_invalidation.py b/test-int/read_cache/test_runtime_invalidation.py index daa6cbbe7..ec3810c85 100644 --- a/test-int/read_cache/test_runtime_invalidation.py +++ b/test-int/read_cache/test_runtime_invalidation.py @@ -16,6 +16,12 @@ LocalMoveEntityRepository, LocalWatchMoveProcessor, ) +from basic_memory.index.local_project import LocalProjectIndexRuntime, run_local_project_index +from basic_memory.indexing.change_planning import ChangeReport +from basic_memory.indexing.directory_delete_runner import ( + DirectoryDeleteRuntime, + RepositoryDirectoryDeleteAcceptanceStore, +) from basic_memory.indexing.project_index_maintenance import ( ProjectIndexDeleteRun, ProjectIndexMoveRun, @@ -34,12 +40,20 @@ AcceptedNoteContentWrite, NoteContentRepository, ) +from basic_memory.runtime.cleanup import RuntimeFileDeleteResult, RuntimeNoteFileDeleteJobRequest +from basic_memory.runtime.jobs import ( + RuntimeIndexFileBatchJobRequest, + RuntimeObservedIndexFile, + RuntimeProjectIndexJobRequest, +) +from basic_memory.runtime.projects import ProjectRuntimeReference from basic_memory.runtime.storage import ( STORAGE_OBJECT_DELETED_EVENT, StorageEventPayload, StorageObjectIdentity, StorageObjectVersion, ) +from basic_memory.services.directory_deletes import DirectoryDeleteService from basic_memory.services.file_service import FileService from basic_memory.services.initialization import recover_project_materializations @@ -121,6 +135,82 @@ async def refresh_moved_entities(self, entity_ids: Sequence[int]) -> None: self.calls.append(list(entity_ids)) +class EmptyObservedFileSource: + async def list_observed_index_files(self) -> tuple[RuntimeObservedIndexFile, ...]: + return () + + +class DeletedFileChangeDetector: + async def detect_all_changes( + self, + storage_files: Mapping[str, RuntimeObservedIndexFile], + ) -> ChangeReport: + del storage_files + return ChangeReport(deleted_files=["notes/stale.md"]) + + +class FailingDeleteMaintenance: + async def run_move_batches( + self, + *, + moved_files: Mapping[str, str], + batch_size: int, + ) -> ProjectIndexMoveRun: + del moved_files, batch_size + return ProjectIndexMoveRun( + total_moves=0, + total_updated_files=0, + records=(), + ) + + async def run_delete_batches( + self, + *, + deleted_paths: Sequence[str], + batch_size: int, + ) -> ProjectIndexDeleteRun: + del deleted_paths, batch_size + raise RuntimeError("partial project index failure") + + +class UnusedBatchEnqueuer: + async def enqueue_index_file_batch( + self, + request: RuntimeIndexFileBatchJobRequest, + ) -> None: + del request + raise AssertionError("failing maintenance must stop before file batches") + + +class GenerationObservingDirectoryDeleteEnqueuer: + def __init__( + self, + redis_cache: RedisCacheHarness, + project_external_id: str, + ) -> None: + self.redis_cache = redis_cache + self.project_external_id = project_external_id + self.observed_generations: list[bytes | str] = [] + + async def enqueue_directory_file_delete( + self, + request: RuntimeNoteFileDeleteJobRequest, + ) -> RuntimeFileDeleteResult: + generation = await self.redis_cache.client.get( + redis_read_cache_generation_key( + prefix=self.redis_cache.prefix, + namespace=self.redis_cache.namespace, + project_id=self.project_external_id, + ) + ) + assert generation is not None + self.observed_generations.append(generation) + return RuntimeFileDeleteResult.already_absent( + entity_id=request.entity_id, + file_path=request.file_path, + ) + + async def _initialized_generation( redis_cache: RedisCacheHarness, project_external_id: str, @@ -145,6 +235,47 @@ async def _initialized_generation( return generation +async def _seed_recovery_note( + session_maker: async_sessionmaker[AsyncSession], + project: Project, + *, + title: str, + file_path: str, + markdown_content: str, +) -> Entity: + entity_repository = EntityRepository(project_id=project.id) + content_repository = NoteContentRepository(project_id=project.id) + async with db.scoped_session(session_maker) as session: + entity = await entity_repository.add( + session, + Entity( + title=title, + note_type="note", + content_type="text/markdown", + file_path=file_path, + checksum="entity-checksum-1", + created_at=datetime.now(UTC), + updated_at=datetime.now(UTC), + ), + ) + await content_repository.accept_write( + session, + AcceptedNoteContentWrite( + entity_id=entity.id, + markdown_content=markdown_content, + db_version=1, + db_checksum="db-checksum-1", + last_source="api", + updated_at=datetime.now(UTC), + ), + ) + row = await content_repository.select_by_id(session, entity.id) + assert row is not None + row.file_write_status = "writing" + await session.flush() + return entity + + def _move_event(event_name: str, path: str) -> StorageEventPayload: return StorageEventPayload( event_name=event_name, @@ -214,48 +345,162 @@ async def test_startup_materialization_recovery_invalidates_real_redis( project_external_id, request="startup-recovery", ) - entity_repository = EntityRepository(project_id=test_project.id) + entity = await _seed_recovery_note( + session_maker, + test_project, + title="Recovered Note", + file_path="notes/recovered.md", + markdown_content="# Recovered with cache invalidation\n", + ) + + await recover_project_materializations( + test_project, + session_maker, + read_cache=redis_cache.cache, + ) + + written = Path(test_project.path) / entity.file_path + assert written.read_text(encoding="utf-8") == "# Recovered with cache invalidation\n" + generation_after = await _initialized_generation( + redis_cache, + project_external_id, + request="startup-recovery", + ) + assert generation_after != generation_before + + +@pytest.mark.asyncio +async def test_startup_recovery_conflict_invalidates_published_failure_in_real_redis( + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + _, session_maker = engine_factory + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="startup-recovery-conflict", + ) + entity = await _seed_recovery_note( + session_maker, + test_project, + title="Conflicted Recovery", + file_path="notes/conflicted-recovery.md", + markdown_content="# Accepted recovery content\n", + ) + target = Path(test_project.path) / entity.file_path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("# External edit\n", encoding="utf-8") + + await recover_project_materializations( + test_project, + session_maker, + read_cache=redis_cache.cache, + ) + content_repository = NoteContentRepository(project_id=test_project.id) async with db.scoped_session(session_maker) as session: - entity = await entity_repository.add( + row = await content_repository.get_by_entity_id(session, entity.id) + assert row is not None + assert row.file_write_status == "external_change_detected" + generation_after = await _initialized_generation( + redis_cache, + project_external_id, + request="startup-recovery-conflict", + ) + assert generation_after != generation_before + + +@pytest.mark.asyncio +async def test_project_index_failure_invalidates_real_redis( + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="project-index-failure", + ) + + with pytest.raises(RuntimeError, match="partial project index failure"): + await run_local_project_index( + RuntimeProjectIndexJobRequest( + project=ProjectRuntimeReference.from_project(test_project), + embeddings=False, + ), + runtime=LocalProjectIndexRuntime( + observed_file_source=EmptyObservedFileSource(), + change_detector=DeletedFileChangeDetector(), + maintenance_runner=FailingDeleteMaintenance(), + moved_entity_search_refresher=RecordingMovedEntitySearchRefresher(), + batch_enqueuer=UnusedBatchEnqueuer(), + read_cache=redis_cache.cache, + ), + ) + + generation_after = await _initialized_generation( + redis_cache, + project_external_id, + request="project-index-failure", + ) + assert generation_after != generation_before + + +@pytest.mark.asyncio +async def test_directory_delete_invalidates_before_and_after_cleanup_in_real_redis( + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + _, session_maker = engine_factory + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="directory-delete", + ) + entity_repository = EntityRepository(project_id=test_project.id) + async with db.scoped_session(session_maker) as session: + await entity_repository.add( session, Entity( - title="Recovered Note", + title="Deleted During Cleanup", note_type="note", content_type="text/markdown", - file_path="notes/recovered.md", - checksum="entity-checksum-1", + file_path="delete-with-cache/note.md", + checksum="directory-delete-checksum", created_at=datetime.now(UTC), updated_at=datetime.now(UTC), ), ) - await content_repository.accept_write( - session, - AcceptedNoteContentWrite( - entity_id=entity.id, - markdown_content="# Recovered with cache invalidation\n", - db_version=1, - db_checksum="db-checksum-1", - last_source="api", - updated_at=datetime.now(UTC), - ), - ) - row = await content_repository.select_by_id(session, entity.id) - assert row is not None - row.file_write_status = "writing" - await session.flush() - await recover_project_materializations( - test_project, - session_maker, + enqueuer = GenerationObservingDirectoryDeleteEnqueuer( + redis_cache, + project_external_id, + ) + service = DirectoryDeleteService( + session_maker=session_maker, + runtime=DirectoryDeleteRuntime( + store=RepositoryDirectoryDeleteAcceptanceStore(), + file_delete_enqueuer=enqueuer, + ), + ) + + result = await service.delete_directory( + project_external_id=project_external_id, + directory="delete-with-cache", read_cache=redis_cache.cache, ) - written = Path(test_project.path) / entity.file_path - assert written.read_text(encoding="utf-8") == "# Recovered with cache invalidation\n" + assert result.deleted_files == ("delete-with-cache/note.md",) + assert len(enqueuer.observed_generations) == 1 + generation_during_cleanup = enqueuer.observed_generations[0] + assert generation_during_cleanup != generation_before generation_after = await _initialized_generation( redis_cache, project_external_id, - request="startup-recovery", + request="directory-delete", ) - assert generation_after != generation_before + assert generation_after != generation_during_cleanup diff --git a/tests/cloud/test_note_content_materialization.py b/tests/cloud/test_note_content_materialization.py index de3ea5754..7587dc6d5 100644 --- a/tests/cloud/test_note_content_materialization.py +++ b/tests/cloud/test_note_content_materialization.py @@ -560,7 +560,8 @@ async def test_recover_stuck_materializations_writes_file_and_marks_synced( project_id=test_project.id, ) - assert recovered == 1 + assert recovered.attempted == 1 + assert recovered.written == 1 written = file_service.base_path / sample_entity.file_path assert written.read_text(encoding="utf-8") == "# Recovered\n\nThe crash left this unwritten.\n" @@ -616,14 +617,13 @@ async def test_move_vacate_recovery_waits_for_destination_then_cleans_source( ) assert source.exists() - assert ( - await recover_stuck_materializations( - session_maker=session_maker, - file_service=file_service, - project_id=test_project.id, - ) - == 1 + recovered = await recover_stuck_materializations( + session_maker=session_maker, + file_service=file_service, + project_id=test_project.id, ) + assert recovered.attempted == 1 + assert recovered.written == 1 assert ( await recover_move_vacates( session_maker=session_maker, @@ -668,7 +668,8 @@ async def test_recover_stuck_materializations_re_drives_failed_row( project_id=test_project.id, ) - assert recovered == 1 + assert recovered.attempted == 1 + assert recovered.written == 1 written = file_service.base_path / sample_entity.file_path assert written.read_text(encoding="utf-8") == "# Recovered after transient failure\n" @@ -703,7 +704,8 @@ async def test_recover_stuck_materializations_returns_zero_when_none_stuck( project_id=test_project.id, ) - assert recovered == 0 + assert recovered.attempted == 0 + assert recovered.written == 0 @pytest.mark.asyncio @@ -736,7 +738,8 @@ async def boom(*_: Any, **__: Any) -> None: project_id=test_project.id, ) - assert recovered == 0 + assert recovered.attempted == 1 + assert recovered.written == 0 @pytest.mark.asyncio @@ -768,7 +771,8 @@ async def test_recover_stuck_materializations_does_not_overwrite_unexpected_file project_id=test_project.id, ) - assert recovered == 0 + assert recovered.attempted == 1 + assert recovered.written == 0 assert target.read_text(encoding="utf-8") == "# External edit\n" repository = NoteContentRepository(project_id=test_project.id) @@ -817,7 +821,8 @@ async def test_recover_stuck_materializations_publishes_already_written_file( project_id=test_project.id, ) - assert recovered == 1 + assert recovered.attempted == 1 + assert recovered.written == 1 assert target.read_text(encoding="utf-8") == markdown_content repository = NoteContentRepository(project_id=test_project.id) From c80d63121239dca44483abe53b6aeaf457876cbb Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 29 Jul 2026 00:47:34 -0500 Subject: [PATCH 05/28] fix(sync): invalidate partial single-file indexes Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 16 +++- .../api/v2/routers/knowledge_router.py | 9 ++- src/basic_memory/index/local_runtime.py | 9 +++ test-int/read_cache/test_api_read_cache.py | 80 ++++++++++++++++++- .../read_cache/test_runtime_invalidation.py | 50 ++++++++++++ 5 files changed, 156 insertions(+), 8 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index 9f8862eeb..4ed3c733d 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -86,6 +86,9 @@ tenant: between phases from surviving the later worker commit. 1. Put project-index invalidation in a failure-safe completion boundary. Move, delete, file-index, and vector batches can commit incrementally before a later batch raises. +1. Put direct single-file and watcher file-index invalidation in failure-safe boundaries. Entity + transactions can commit before search refresh or note-content reconciliation raises, so a + failed index attempt can still publish cache-relevant state. 1. Invalidate directory deletion immediately after its acceptance transaction commits, then again after file cleanup and surviving-relation refresh. A slow or failed cleanup must not keep deleted entities reachable through the pre-acceptance generation. @@ -217,8 +220,9 @@ mutations, watcher-detected paired moves, startup recovery or reconciliation, Cl events, and relation-resolution changes that affect cached responses. Each later phase invalidates again so a value filled after an earlier generation bump cannot outlive the state that phase publishes. Directory deletion invalidates after acceptance and after cleanup. Project -indexing invalidates even after a partial failure. Recovery invalidation runs before the serving -barrier is released and includes terminal conflict or failure publication. +indexing invalidates even after a partial failure. Direct single-file and watcher file indexing +invalidate even when a follow-up fails after the entity commit. Recovery invalidation runs before +the serving barrier is released and includes terminal conflict or failure publication. ## Dependency And Lifecycle @@ -293,6 +297,8 @@ The real-Redis suite must prove: - startup recovery that publishes written, conflict, or failed materialization state invalidates before serving resumes; - project-index failures invalidate any earlier committed batches; +- direct and watcher file-index failures invalidate any entity state committed before failed + search or reconciliation follow-ups; - directory deletion invalidates before cleanup starts and again after cleanup completes. Run route behavior against both SQLite and Postgres where persistence behavior differs. Redis @@ -323,8 +329,10 @@ semantics themselves are asserted only against the real Redis integration fixtur resuming tenant traffic. - Invalidate project indexing from a failure-safe completion boundary, and invalidate directory deletion both after acceptance commit and after cleanup/relation refresh. -- Enable reads for a tenant only after every request, worker, partial-index, accepted-delete, move, - and recovery boundary has namespace and invalidation parity. +- Invalidate direct and watcher file indexing from failure-safe boundaries because entity commits + precede some search and reconciliation follow-ups. +- Enable reads for a tenant only after every request, worker, partial-index, direct-index, + accepted-delete, move, and recovery boundary has namespace and invalidation parity. - Start with shadow telemetry or a limited tenant cohort. - Compare hit rate, Redis latency, database query volume, and end-to-end tool latency. diff --git a/src/basic_memory/api/v2/routers/knowledge_router.py b/src/basic_memory/api/v2/routers/knowledge_router.py index 7f4db361b..582131a20 100644 --- a/src/basic_memory/api/v2/routers/knowledge_router.py +++ b/src/basic_memory/api/v2/routers/knowledge_router.py @@ -530,8 +530,13 @@ async def index_file( detail=f"Only markdown files can be indexed: '{data.file_path}'", ) - indexed = await file_indexer.index_file(file_path, source="api-index-file") - await invalidate_project_read_cache(read_cache, project_external_id) + try: + indexed = await file_indexer.index_file(file_path, source="api-index-file") + finally: + # The file indexer commits entity state before search and reconciliation + # follow-ups finish. Invalidate even when a later phase raises so those + # partial commits cannot remain reachable through the old generation. + await invalidate_project_read_cache(read_cache, project_external_id) async with db.scoped_session(session_maker) as session: entity = await entity_repository.get_by_id(session, indexed.entity_id) if entity is None: # pragma: no cover diff --git a/src/basic_memory/index/local_runtime.py b/src/basic_memory/index/local_runtime.py index 3db1ffe8d..8906beedf 100644 --- a/src/basic_memory/index/local_runtime.py +++ b/src/basic_memory/index/local_runtime.py @@ -65,6 +65,7 @@ RuntimeFileChecksum, RuntimeFilePath, RuntimeStorageEventOperation, + RuntimeStorageEventOperationKind, ) from basic_memory.services import FileService from basic_memory.services.exceptions import FileOperationError @@ -265,6 +266,14 @@ async def event_failed( file_path=operation.relative_path, error=str(exc), ) + if operation.kind == RuntimeStorageEventOperationKind.index_file: + # LocalMarkdownFileIndexer commits the entity before all search and + # reconciliation follow-ups complete. A watcher failure can therefore + # publish partial state even though the success callback never runs. + await invalidate_project_read_cache( + self.read_cache, + self.project.project_external_id, + ) @dataclass(frozen=True, slots=True) diff --git a/test-int/read_cache/test_api_read_cache.py b/test-int/read_cache/test_api_read_cache.py index e51d848b2..fe9f7c13b 100644 --- a/test-int/read_cache/test_api_read_cache.py +++ b/test-int/read_cache/test_api_read_cache.py @@ -8,12 +8,13 @@ import pytest from fastapi import FastAPI -from httpx import AsyncClient +from httpx import ASGITransport, AsyncClient from redis.asyncio import Redis from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker from basic_memory import db -from basic_memory.deps import get_read_cache +from basic_memory.deps import get_index_file_executor_v2_external, get_read_cache +from basic_memory.indexing.models import FileIndexResult from basic_memory.models import Project from basic_memory.models.knowledge import Entity from basic_memory.read_cache import ReadCacheKey, ReadCacheOperation, read_cache_request_digest @@ -43,6 +44,19 @@ class RedisCacheHarness(Protocol): pytestmark = pytest.mark.redis +class FailingIndexFileExecutor: + """Represent an indexer that fails after it may have committed entity state.""" + + async def index_file( + self, + file_path: str, + *, + source: str, + ) -> FileIndexResult: + del file_path, source + raise RuntimeError("partial direct index failure") + + def _cache_key( *, project_id: str, @@ -57,6 +71,30 @@ def _cache_key( ) +async def _initialized_generation( + redis_cache: RedisCacheHarness, + project_id: str, + *, + request: str, +) -> bytes | str: + await redis_cache.cache.lookup( + _cache_key( + project_id=project_id, + operation=ReadCacheOperation.entity, + request=request, + ) + ) + generation = await redis_cache.client.get( + redis_read_cache_generation_key( + prefix=redis_cache.prefix, + namespace=redis_cache.namespace, + project_id=project_id, + ) + ) + assert generation is not None + return generation + + @pytest.mark.asyncio async def test_entity_resolve_and_markdown_reads_cache_then_write_invalidates( app: FastAPI, @@ -225,3 +263,41 @@ async def test_non_markdown_resource_is_never_cached( key=key, ) assert await redis_cache.client.exists(redis_keys.data) == 0 + + +@pytest.mark.asyncio +async def test_direct_index_failure_invalidates_real_redis( + app: FastAPI, + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """A partial direct index commit cannot retain the previous cache generation.""" + app.dependency_overrides[get_read_cache] = lambda: redis_cache.cache + app.dependency_overrides[get_index_file_executor_v2_external] = FailingIndexFileExecutor + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="direct-index-failure", + ) + file_path = "cache/direct-index-failure.md" + disk_path = Path(test_project.path) / file_path + disk_path.parent.mkdir(parents=True, exist_ok=True) + disk_path.write_text("# Partial direct index\n", encoding="utf-8") + + async with AsyncClient( + transport=ASGITransport(app=app, raise_app_exceptions=False), + base_url="http://test", + ) as failure_client: + response = await failure_client.post( + f"/v2/projects/{project_external_id}/knowledge/index-file", + json={"file_path": file_path}, + ) + + assert response.status_code == 500 + generation_after = await _initialized_generation( + redis_cache, + project_external_id, + request="direct-index-failure", + ) + assert generation_after != generation_before diff --git a/test-int/read_cache/test_runtime_invalidation.py b/test-int/read_cache/test_runtime_invalidation.py index ec3810c85..5e2dfe05a 100644 --- a/test-int/read_cache/test_runtime_invalidation.py +++ b/test-int/read_cache/test_runtime_invalidation.py @@ -16,7 +16,9 @@ LocalMoveEntityRepository, LocalWatchMoveProcessor, ) +from basic_memory.index.local_dependencies import LocalIndexSearchService from basic_memory.index.local_project import LocalProjectIndexRuntime, run_local_project_index +from basic_memory.index.local_runtime import LocalInlineStorageEventResultRecorder from basic_memory.indexing.change_planning import ChangeReport from basic_memory.indexing.directory_delete_runner import ( DirectoryDeleteRuntime, @@ -25,7 +27,9 @@ from basic_memory.indexing.project_index_maintenance import ( ProjectIndexDeleteRun, ProjectIndexMoveRun, + ProjectIndexMovedEntitySearchRefresher, ) +from basic_memory.indexing.relation_resolution import RelationResolutionRuntime from basic_memory.models import Project from basic_memory.models.knowledge import Entity from basic_memory.read_cache import ( @@ -52,6 +56,8 @@ StorageEventPayload, StorageObjectIdentity, StorageObjectVersion, + RuntimeStorageEventOperation, + RuntimeStorageEventOperationKind, ) from basic_memory.services.directory_deletes import DirectoryDeleteService from basic_memory.services.file_service import FileService @@ -290,6 +296,14 @@ def _move_event(event_name: str, path: str) -> StorageEventPayload: ) +def _index_operation(path: str) -> RuntimeStorageEventOperation: + return RuntimeStorageEventOperation( + kind=RuntimeStorageEventOperationKind.index_file, + storage_event=_move_event("OBJECT_CREATED_PUT", path), + relative_path=path, + ) + + @pytest.mark.asyncio async def test_watcher_move_completion_invalidates_real_redis( test_project: Project, @@ -332,6 +346,42 @@ async def test_watcher_move_completion_invalidates_real_redis( assert generation_after != generation_before +@pytest.mark.asyncio +async def test_watcher_index_failure_invalidates_real_redis( + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="watcher-index-failure", + ) + recorder = LocalInlineStorageEventResultRecorder( + project=ProjectRuntimeReference.from_project(test_project), + search_service=cast(LocalIndexSearchService, object()), + relation_cleanup_search_refresher=cast( + ProjectIndexMovedEntitySearchRefresher, + object(), + ), + relation_runtime=cast(RelationResolutionRuntime, object()), + index_embeddings=False, + read_cache=redis_cache.cache, + ) + + await recorder.event_failed( + _index_operation("notes/partial-index.md"), + RuntimeError("partial watcher index failure"), + ) + + generation_after = await _initialized_generation( + redis_cache, + project_external_id, + request="watcher-index-failure", + ) + assert generation_after != generation_before + + @pytest.mark.asyncio async def test_startup_materialization_recovery_invalidates_real_redis( engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], From 78e109d7502a45e6454fa296195f61175fcf4a9a Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 29 Jul 2026 06:21:18 -0500 Subject: [PATCH 06/28] fix(api): invalidate cache after imports Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 20 +++-- .../api/v2/routers/importer_router.py | 72 ++++++++++++++-- test-int/read_cache/test_api_read_cache.py | 85 ++++++++++++++++++- 3 files changed, 163 insertions(+), 14 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index 4ed3c733d..b436169c3 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -89,6 +89,9 @@ tenant: 1. Put direct single-file and watcher file-index invalidation in failure-safe boundaries. Entity transactions can commit before search refresh or note-content reconciliation raises, so a failed index attempt can still publish cache-relevant state. +1. Inject the namespace-bound cache into import endpoints and workers. Invalidate after every + import attempt that may have written files, including importers that return a failed result or + raise after partial progress, so cached file-first resources cannot survive overwritten bytes. 1. Invalidate directory deletion immediately after its acceptance transaction commits, then again after file cleanup and surviving-relation refresh. A slow or failed cleanup must not keep deleted entities reachable through the pre-acceptance generation. @@ -216,13 +219,14 @@ Primary integration points: Invalidation belongs at portable mutation and indexing completion boundaries, not only in FastAPI routes. It must cover accepted note writes, terminal deferred materialization and status publication, direct file indexing, filesystem watcher updates, project indexing, directory -mutations, watcher-detected paired moves, startup recovery or reconciliation, Cloud storage -events, and relation-resolution changes that affect cached responses. Each later phase +mutations, imports, watcher-detected paired moves, startup recovery or reconciliation, Cloud +storage events, and relation-resolution changes that affect cached responses. Each later phase invalidates again so a value filled after an earlier generation bump cannot outlive the state -that phase publishes. Directory deletion invalidates after acceptance and after cleanup. Project -indexing invalidates even after a partial failure. Direct single-file and watcher file indexing -invalidate even when a follow-up fails after the entity commit. Recovery invalidation runs before -the serving barrier is released and includes terminal conflict or failure publication. +that phase publishes. Directory deletion invalidates after acceptance and after cleanup. Imports +invalidate after every attempt that may have written files. Project indexing invalidates even +after a partial failure. Direct single-file and watcher file indexing invalidate even when a +follow-up fails after the entity commit. Recovery invalidation runs before the serving barrier is +released and includes terminal conflict or failure publication. ## Dependency And Lifecycle @@ -299,6 +303,7 @@ The real-Redis suite must prove: - project-index failures invalidate any earlier committed batches; - direct and watcher file-index failures invalidate any entity state committed before failed search or reconciliation follow-ups; +- import attempts invalidate cached file-first resources after success and partial failure; - directory deletion invalidates before cleanup starts and again after cleanup completes. Run route behavior against both SQLite and Postgres where persistence behavior differs. Redis @@ -331,8 +336,9 @@ semantics themselves are asserted only against the real Redis integration fixtur deletion both after acceptance commit and after cleanup/relation refresh. - Invalidate direct and watcher file indexing from failure-safe boundaries because entity commits precede some search and reconciliation follow-ups. +- Invalidate every import attempt that may write files, including partial failures. - Enable reads for a tenant only after every request, worker, partial-index, direct-index, - accepted-delete, move, and recovery boundary has namespace and invalidation parity. + import, accepted-delete, move, and recovery boundary has namespace and invalidation parity. - Start with shadow telemetry or a limited tenant cohort. - Compare hit rate, Redis latency, database query volume, and end-to-end tool latency. diff --git a/src/basic_memory/api/v2/routers/importer_router.py b/src/basic_memory/api/v2/routers/importer_router.py index 52291b8b1..42371f9da 100644 --- a/src/basic_memory/api/v2/routers/importer_router.py +++ b/src/basic_memory/api/v2/routers/importer_router.py @@ -7,7 +7,7 @@ import json import logging -from fastapi import APIRouter, Form, HTTPException, UploadFile, status, Path +from fastapi import APIRouter, Form, HTTPException, Path, UploadFile, status from basic_memory.deps import ( AppConfigDep, @@ -15,8 +15,10 @@ ClaudeConversationsImporterV2ExternalDep, ClaudeProjectsImporterV2ExternalDep, MemoryJsonImporterV2ExternalDep, + ReadCacheDep, ) from basic_memory.importers import Importer +from basic_memory.read_cache import ReadCache, invalidate_project_read_cache from basic_memory.schemas.importer import ( ChatImportResult, EntityImportResult, @@ -45,6 +47,7 @@ async def import_chatgpt( importer: ChatGPTImporterV2ExternalDep, config: AppConfigDep, file: UploadFile, + read_cache: ReadCacheDep, project_id: str = Path(..., description="Project external UUID"), directory: str = Form("conversations"), ) -> ChatImportResult: @@ -63,7 +66,14 @@ async def import_chatgpt( HTTPException: If import fails. """ logger.info(f"V2 Importing ChatGPT conversations for project {project_id}") - return await import_file(importer, file, directory, config.import_upload_max_bytes) + return await import_file( + importer, + file, + directory, + config.import_upload_max_bytes, + read_cache=read_cache, + project_external_id=project_id, + ) @router.post("/claude/conversations", response_model=ChatImportResult) @@ -71,6 +81,7 @@ async def import_claude_conversations( importer: ClaudeConversationsImporterV2ExternalDep, config: AppConfigDep, file: UploadFile, + read_cache: ReadCacheDep, project_id: str = Path(..., description="Project external UUID"), directory: str = Form("conversations"), ) -> ChatImportResult: @@ -89,7 +100,14 @@ async def import_claude_conversations( HTTPException: If import fails. """ logger.info(f"V2 Importing Claude conversations for project {project_id}") - return await import_file(importer, file, directory, config.import_upload_max_bytes) + return await import_file( + importer, + file, + directory, + config.import_upload_max_bytes, + read_cache=read_cache, + project_external_id=project_id, + ) @router.post("/claude/projects", response_model=ProjectImportResult) @@ -97,6 +115,7 @@ async def import_claude_projects( importer: ClaudeProjectsImporterV2ExternalDep, config: AppConfigDep, file: UploadFile, + read_cache: ReadCacheDep, project_id: str = Path(..., description="Project external UUID"), directory: str = Form("projects"), ) -> ProjectImportResult: @@ -115,7 +134,14 @@ async def import_claude_projects( HTTPException: If import fails. """ logger.info(f"V2 Importing Claude projects for project {project_id}") - return await import_file(importer, file, directory, config.import_upload_max_bytes) + return await import_file( + importer, + file, + directory, + config.import_upload_max_bytes, + read_cache=read_cache, + project_external_id=project_id, + ) @router.post("/memory-json", response_model=EntityImportResult) @@ -123,6 +149,7 @@ async def import_memory_json( importer: MemoryJsonImporterV2ExternalDep, config: AppConfigDep, file: UploadFile, + read_cache: ReadCacheDep, project_id: str = Path(..., description="Project external UUID"), directory: str = Form("conversations"), ) -> EntityImportResult: @@ -149,7 +176,13 @@ async def import_memory_json( json_data = json.loads(line) file_data.append(json_data) - result = await importer.import_data(file_data, directory) + result = await run_import_with_invalidation( + importer, + file_data, + directory, + read_cache=read_cache, + project_external_id=project_id, + ) if not result.success: # pragma: no cover raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -171,6 +204,9 @@ async def import_file[ImportResultT: ImportResult]( file: UploadFile, destination_directory: str, max_bytes: int, + *, + read_cache: ReadCache, + project_external_id: str, ) -> ImportResultT: """Helper function to import a file using an importer instance. @@ -190,7 +226,13 @@ async def import_file[ImportResultT: ImportResult]( # Process file upload_bytes = await read_import_upload(file, max_bytes) json_data = json.loads(upload_bytes) - result = await importer.import_data(json_data, destination_directory) + result = await run_import_with_invalidation( + importer, + json_data, + destination_directory, + read_cache=read_cache, + project_external_id=project_external_id, + ) if not result.success: # pragma: no cover raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -207,3 +249,21 @@ async def import_file[ImportResultT: ImportResult]( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Import failed: {str(e)}", ) + + +async def run_import_with_invalidation[ImportResultT: ImportResult]( + importer: Importer[ImportResultT], + source_data: object, + destination_directory: str, + *, + read_cache: ReadCache, + project_external_id: str, +) -> ImportResultT: + """Run one import attempt and invalidate files it may have written.""" + try: + return await importer.import_data(source_data, destination_directory) + finally: + # Importers write files one at a time and may return a failed result after + # earlier writes. Invalidate every attempted import so cached file-first + # resources cannot survive either success or partial failure. + await invalidate_project_read_cache(read_cache, project_external_id) diff --git a/test-int/read_cache/test_api_read_cache.py b/test-int/read_cache/test_api_read_cache.py index fe9f7c13b..46d372ca3 100644 --- a/test-int/read_cache/test_api_read_cache.py +++ b/test-int/read_cache/test_api_read_cache.py @@ -13,7 +13,11 @@ from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker from basic_memory import db -from basic_memory.deps import get_index_file_executor_v2_external, get_read_cache +from basic_memory.deps import ( + get_chatgpt_importer_v2_external, + get_index_file_executor_v2_external, + get_read_cache, +) from basic_memory.indexing.models import FileIndexResult from basic_memory.models import Project from basic_memory.models.knowledge import Entity @@ -57,6 +61,23 @@ async def index_file( raise RuntimeError("partial direct index failure") +class PartiallyFailingImporter: + """Overwrite one resource before reporting an import failure.""" + + def __init__(self, target: Path) -> None: + self.target = target + + async def import_data( + self, + source_data: object, + destination_folder: str, + **kwargs: object, + ) -> None: + del source_data, destination_folder, kwargs + self.target.write_text("# Imported before failure\n", encoding="utf-8") + raise RuntimeError("partial import failure") + + def _cache_key( *, project_id: str, @@ -301,3 +322,65 @@ async def test_direct_index_failure_invalidates_real_redis( request="direct-index-failure", ) assert generation_after != generation_before + + +@pytest.mark.asyncio +async def test_partial_import_failure_invalidates_cached_resource_in_real_redis( + app: FastAPI, + client: AsyncClient, + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """File writes from a failed import cannot retain cached pre-import bytes.""" + app.dependency_overrides[get_read_cache] = lambda: redis_cache.cache + project_external_id = str(test_project.external_id) + file_path = "cache/partial-import.md" + disk_path = Path(test_project.path) / file_path + disk_path.parent.mkdir(parents=True, exist_ok=True) + disk_path.write_text("# Before import\n", encoding="utf-8") + + repository = EntityRepository(project_id=test_project.id) + _, session_maker = engine_factory + async with db.scoped_session(session_maker) as session: + entity = await repository.add( + session, + Entity( + title="partial-import.md", + note_type="note", + content_type="text/markdown", + file_path=file_path, + checksum="partial-import-checksum", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + ) + + resource_url = f"/v2/projects/{project_external_id}/resource/{entity.external_id}" + cached_response = await client.get(resource_url) + assert cached_response.status_code == 200 + assert cached_response.text == "# Before import\n" + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="partial-import-failure", + ) + app.dependency_overrides[get_chatgpt_importer_v2_external] = lambda: PartiallyFailingImporter( + disk_path + ) + + response = await client.post( + f"/v2/projects/{project_external_id}/import/chatgpt", + files={"file": ("conversations.json", b"[]", "application/json")}, + ) + + assert response.status_code == 500 + generation_after = await _initialized_generation( + redis_cache, + project_external_id, + request="partial-import-failure", + ) + assert generation_after != generation_before + refreshed_response = await client.get(resource_url) + assert refreshed_response.status_code == 200 + assert refreshed_response.text == "# Imported before failure\n" From 5994aca43f4e9082f2b18ec9e4e78af29619e35d Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 29 Jul 2026 06:45:38 -0500 Subject: [PATCH 07/28] fix(api): close remaining cache freshness gaps Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 35 +++- .../api/v2/routers/knowledge_router.py | 4 +- .../api/v2/routers/resource_router.py | 1 + src/basic_memory/services/entity_service.py | 39 ++-- src/basic_memory/services/initialization.py | 45 +++-- .../services/note_content_reads.py | 12 ++ test-int/read_cache/test_api_read_cache.py | 182 +++++++++++++++++- .../read_cache/test_runtime_invalidation.py | 55 ++++++ 8 files changed, 332 insertions(+), 41 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index b436169c3..25ac42db9 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -89,6 +89,9 @@ tenant: 1. Put direct single-file and watcher file-index invalidation in failure-safe boundaries. Entity transactions can commit before search refresh or note-content reconciliation raises, so a failed index attempt can still publish cache-relevant state. +1. Pass the namespace-bound cache into hosted note-content read repair. A resource or entity read + can bootstrap a missing accepted-content row; invalidate immediately after that commit and + before the repaired response is offered to read-through storage. 1. Inject the namespace-bound cache into import endpoints and workers. Invalidate after every import attempt that may have written files, including importers that return a failed result or raise after partial progress, so cached file-first resources cannot survive overwritten bytes. @@ -97,10 +100,15 @@ tenant: keep deleted entities reachable through the pre-acceptance generation. 1. Invalidate watcher-detected moves at their own completion boundary. Paired delete/create events are consumed by move processing and therefore bypass the ordinary watcher callbacks. +1. Invalidate directory moves after each individual file/database move commits, then again after + the final search and relation follow-ups. Directory moves are incremental batches, so a long or + partially failed request must not keep earlier files under the pre-move generation. 1. If startup recovery or reconciliation attempts can publish materialization, vacate, index, or relation state, invalidate through the same namespace-bound cache before releasing the serving - barrier or resuming tenant traffic. Terminal conflict and failure publication count even when - the recovery did not produce a written file. + barrier or resuming tenant traffic. Treat materialization and move-vacate recovery as separate + freshness phases: invalidate a completed first phase before starting the second so a later + setup/query failure cannot skip the earlier generation bump. Terminal conflict and failure + publication count even when the recovery did not produce a written file. 1. Keep `bm:read:v1` separate from rate-limit and Cloud control-plane prefixes, metrics, timeouts, and failure policies. The clients may target one Redis deployment, but a read-cache timeout must bypass while a rate-limit decision keeps its Cloud-owned security behavior. @@ -222,11 +230,13 @@ publication, direct file indexing, filesystem watcher updates, project indexing, mutations, imports, watcher-detected paired moves, startup recovery or reconciliation, Cloud storage events, and relation-resolution changes that affect cached responses. Each later phase invalidates again so a value filled after an earlier generation bump cannot outlive the state -that phase publishes. Directory deletion invalidates after acceptance and after cleanup. Imports -invalidate after every attempt that may have written files. Project indexing invalidates even -after a partial failure. Direct single-file and watcher file indexing invalidate even when a -follow-up fails after the entity commit. Recovery invalidation runs before the serving barrier is -released and includes terminal conflict or failure publication. +that phase publishes. Hosted read repair invalidates after bootstrapping accepted content and +before a repaired entity or resource is stored. Directory moves invalidate after every committed +file plus the final reindex; directory deletion invalidates after acceptance and after cleanup. +Imports invalidate after every attempt that may have written files. Project indexing invalidates +even after a partial failure. Direct single-file and watcher file indexing invalidate even when a +follow-up fails after the entity commit. Recovery phases invalidate independently before the +serving barrier is released and include terminal conflict or failure publication. ## Dependency And Lifecycle @@ -303,8 +313,11 @@ The real-Redis suite must prove: - project-index failures invalidate any earlier committed batches; - direct and watcher file-index failures invalidate any entity state committed before failed search or reconciliation follow-ups; +- hosted read repair invalidates cached entity metadata before storing the repaired resource; - import attempts invalidate cached file-first resources after success and partial failure; +- directory moves invalidate after each committed file and again after final reindexing; - directory deletion invalidates before cleanup starts and again after cleanup completes. +- startup materialization recovery remains invalidated when a later move-vacate phase fails. Run route behavior against both SQLite and Postgres where persistence behavior differs. Redis semantics themselves are asserted only against the real Redis integration fixture. @@ -336,9 +349,15 @@ semantics themselves are asserted only against the real Redis integration fixtur deletion both after acceptance commit and after cleanup/relation refresh. - Invalidate direct and watcher file indexing from failure-safe boundaries because entity commits precede some search and reconciliation follow-ups. +- Invalidate hosted note-content read repair after it commits and before returning its repaired + entity or resource to the read-through helper. - Invalidate every import attempt that may write files, including partial failures. +- Invalidate directory moves after every committed file and again after search/relation + follow-ups. +- Invalidate each startup recovery phase before beginning the next phase. - Enable reads for a tenant only after every request, worker, partial-index, direct-index, - import, accepted-delete, move, and recovery boundary has namespace and invalidation parity. + read-repair, import, accepted-delete, move, and recovery boundary has namespace and invalidation + parity. - Start with shadow telemetry or a limited tenant cohort. - Compare hit rate, Redis latency, database query volume, and end-to-end tool latency. diff --git a/src/basic_memory/api/v2/routers/knowledge_router.py b/src/basic_memory/api/v2/routers/knowledge_router.py index 582131a20..b35ee8784 100644 --- a/src/basic_memory/api/v2/routers/knowledge_router.py +++ b/src/basic_memory/api/v2/routers/knowledge_router.py @@ -604,6 +604,7 @@ async def load() -> EntityResponseV2: project_external_id=project_external_id, entity_external_id=entity_id, session=session, + read_cache=read_cache, ) ) if note_payload is not None: @@ -1013,8 +1014,9 @@ async def move_directory( destination_directory=data.destination_directory, project_config=project_config, app_config=app_config, + project_external_id=project_external_id, + read_cache=read_cache, ) - await invalidate_project_read_cache(read_cache, project_external_id) try: # Reindex moved entities diff --git a/src/basic_memory/api/v2/routers/resource_router.py b/src/basic_memory/api/v2/routers/resource_router.py index 9f265f1da..f8b4379ed 100644 --- a/src/basic_memory/api/v2/routers/resource_router.py +++ b/src/basic_memory/api/v2/routers/resource_router.py @@ -96,6 +96,7 @@ async def load() -> CachedResourceResponse: project_external_id=project_id, entity_external_id=entity_id, session=session, + read_cache=read_cache, ) if note_resource is not None: return CachedResourceResponse( diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index 610d4647a..54e805e3e 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -22,6 +22,7 @@ from basic_memory.models.knowledge import Entity from basic_memory.repository import ObservationRepository, RelationRepository from basic_memory.repository.entity_repository import EntityRepository +from basic_memory.read_cache import ReadCache, invalidate_project_read_cache from basic_memory.runtime.note_move import normalize_note_move_destination_path from basic_memory.schemas import Entity as EntitySchema from basic_memory.schemas.base import Permalink @@ -1129,6 +1130,9 @@ async def move_directory( destination_directory: str, project_config: ProjectConfig, app_config: BasicMemoryConfig, + *, + project_external_id: str, + read_cache: ReadCache, ) -> DirectoryMoveResult: """Move all entities in a directory to a new location. @@ -1141,6 +1145,8 @@ async def move_directory( destination_directory: Destination directory path relative to project root project_config: Project configuration for file operations app_config: App configuration for permalink update settings + project_external_id: Canonical project UUID used for cache invalidation + read_cache: Namespace-bound semantic read cache Returns: DirectoryMoveResult with counts and details of moved files @@ -1177,18 +1183,16 @@ async def move_directory( # Process each entity for entity in entities: - try: - # Calculate new path by replacing source prefix with destination - old_path = entity.file_path - # Replace only the first occurrence of the source directory prefix - if old_path.startswith(f"{source_directory}/"): - new_path = old_path.replace( - f"{source_directory}/", f"{destination_directory}/", 1 - ) - else: # pragma: no cover - # Entity is directly in the source directory (shouldn't happen with prefix match) - new_path = f"{destination_directory}/{old_path}" + # Calculate new path by replacing source prefix with destination + old_path = entity.file_path + # Replace only the first occurrence of the source directory prefix + if old_path.startswith(f"{source_directory}/"): + new_path = old_path.replace(f"{source_directory}/", f"{destination_directory}/", 1) + else: # pragma: no cover + # Entity is directly in the source directory (shouldn't happen with prefix match) + new_path = f"{destination_directory}/{old_path}" + try: # Move the individual entity await self.move_entity( identifier=entity.file_path, @@ -1196,15 +1200,18 @@ async def move_directory( project_config=project_config, app_config=app_config, ) - - moved_files.append(new_path) - successful_moves += 1 - logger.debug(f"Moved entity: {old_path} -> {new_path}") - except Exception as e: # pragma: no cover failed_moves += 1 errors.append(DirectoryMoveError(path=entity.file_path, error=str(e))) logger.error(f"Failed to move entity {entity.file_path}: {e}") + continue + + # Each move commits independently. Invalidate before the next file so a + # long directory batch cannot serve early moves from the old generation. + await invalidate_project_read_cache(read_cache, project_external_id) + moved_files.append(new_path) + successful_moves += 1 + logger.debug(f"Moved entity: {old_path} -> {new_path}") logger.info( f"Directory move complete: {successful_moves} succeeded, {failed_moves} failed " diff --git a/src/basic_memory/services/initialization.py b/src/basic_memory/services/initialization.py index 6df2b326a..a4e3d18f6 100644 --- a/src/basic_memory/services/initialization.py +++ b/src/basic_memory/services/initialization.py @@ -68,44 +68,61 @@ async def recover_project_materializations( recover_move_vacates, recover_stuck_materializations, ) + from basic_memory.read_cache import NullReadCache, invalidate_project_read_cache from basic_memory.services.file_service import FileService + # FileService needs only base_path to write the accepted markdown bytes; + # the markdown_processor/app_config are unused on the materialization path. + file_service = FileService(Path(project.path)) + active_read_cache = read_cache if read_cache is not None else NullReadCache() + try: - # FileService needs only base_path to write the accepted markdown bytes; - # the markdown_processor/app_config are unused on the materialization path. - file_service = FileService(Path(project.path)) materialization_recovery = await recover_stuck_materializations( session_maker=session_maker, file_service=file_service, project_id=project.id, ) + except Exception as e: # pragma: no cover - defensive startup guard + logger.error(f"Error recovering stuck materializations for project {project.name}: {e}") + return + + if materialization_recovery.attempted: + logger.info( + "Recovered note materialization state on startup", + project=project.name, + attempted_materializations=materialization_recovery.attempted, + recovered_materializations=materialization_recovery.written, + ) + + # Redis can outlive the process that left this materialization unfinished. + # Invalidate this committed phase before move-vacate recovery begins; a + # later setup/query failure must not leave its published state cached. + await invalidate_project_read_cache( + active_read_cache, + str(project.external_id), + ) + + try: recovered_vacates = await recover_move_vacates( session_maker=session_maker, file_service=file_service, project_id=project.id, ) except Exception as e: # pragma: no cover - defensive startup guard - logger.error(f"Error recovering stuck materializations for project {project.name}: {e}") + logger.error(f"Error recovering move vacates for project {project.name}: {e}") return - if not materialization_recovery.attempted and not recovered_vacates: + if not recovered_vacates: return logger.info( - "Recovered note materialization state on startup", + "Recovered note move-vacate state on startup", project=project.name, - attempted_materializations=materialization_recovery.attempted, - recovered_materializations=materialization_recovery.written, recovered_move_vacates=recovered_vacates, ) - # Redis can outlive the process that left this materialization unfinished. - # Invalidate before releasing the startup barrier so no pre-crash pending or - # failed payload remains reachable while background indexing catches up. - from basic_memory.read_cache import NullReadCache, invalidate_project_read_cache - await invalidate_project_read_cache( - read_cache if read_cache is not None else NullReadCache(), + active_read_cache, str(project.external_id), ) diff --git a/src/basic_memory/services/note_content_reads.py b/src/basic_memory/services/note_content_reads.py index c409b66f6..2b5957617 100644 --- a/src/basic_memory/services/note_content_reads.py +++ b/src/basic_memory/services/note_content_reads.py @@ -15,6 +15,7 @@ run_note_content_read_repair_with_default_reconciler, ) from basic_memory.models import Entity, NoteContent, Project +from basic_memory.read_cache import ReadCache, invalidate_project_read_cache from basic_memory.runtime.note_content import ( RuntimeNoteContentResource, RuntimeNoteContentResponsePayload, @@ -91,6 +92,7 @@ async def get_note_entity_payload_with_read_repair( entity_external_id: str, session: AsyncSession | None = None, source: str = "read_repair", + read_cache: ReadCache | None = None, ) -> RuntimeNoteContentResponsePayload | None: """Return entity payload, repairing missing note_content when a reader exists.""" payload = await self.get_note_entity_payload( @@ -108,6 +110,11 @@ async def get_note_entity_payload_with_read_repair( ) if not repaired: return None + if read_cache is not None: + # Read repair commits note_content before this method reloads the response. + # Advance the generation first so the surrounding read-through cannot + # publish the repaired payload under the pre-repair generation. + await invalidate_project_read_cache(read_cache, project_external_id) # The repair commits through a separate scoped session, so reopen the read to # avoid stale snapshots in caller-owned transactions. return await self.get_note_entity_payload( @@ -147,6 +154,7 @@ async def get_note_resource_with_read_repair( entity_external_id: str, session: AsyncSession | None = None, source: str = "read_repair", + read_cache: ReadCache | None = None, ) -> RuntimeNoteContentResource | None: """Return markdown resource, repairing missing note_content when possible.""" resource = await self.get_note_resource( @@ -164,6 +172,10 @@ async def get_note_resource_with_read_repair( ) if not repaired: return None + if read_cache is not None: + # A resource read can repair the row used by cached entity responses. + # Invalidate before returning the repaired resource to read-through. + await invalidate_project_read_cache(read_cache, project_external_id) # The repair commits through a separate scoped session, so reopen the read to # avoid stale snapshots in caller-owned transactions. return await self.get_note_resource( diff --git a/test-int/read_cache/test_api_read_cache.py b/test-int/read_cache/test_api_read_cache.py index 46d372ca3..7e9247356 100644 --- a/test-int/read_cache/test_api_read_cache.py +++ b/test-int/read_cache/test_api_read_cache.py @@ -4,7 +4,7 @@ from datetime import datetime, timezone from pathlib import Path -from typing import Protocol +from typing import Protocol, override import pytest from fastapi import FastAPI @@ -16,12 +16,22 @@ from basic_memory.deps import ( get_chatgpt_importer_v2_external, get_index_file_executor_v2_external, + get_note_content_query_service, get_read_cache, ) +from basic_memory.indexing.note_content_read_repair_runner import ( + NoteContentReadRepairFile, + NoteContentReadRepairTarget, +) from basic_memory.indexing.models import FileIndexResult from basic_memory.models import Project from basic_memory.models.knowledge import Entity -from basic_memory.read_cache import ReadCacheKey, ReadCacheOperation, read_cache_request_digest +from basic_memory.read_cache import ( + ReadCacheInvalidationStatus, + ReadCacheKey, + ReadCacheOperation, + read_cache_request_digest, +) from basic_memory.read_cache.keys import ( redis_read_cache_generation_key, redis_read_cache_keys, @@ -30,6 +40,7 @@ from basic_memory.repository import EntityRepository from basic_memory.runtime.note_content import NOTE_CONTENT_BASE_CHECKSUM_HEADER from basic_memory.schemas.v2 import EntityResolveRequest +from basic_memory.services.note_content_reads import NoteContentQueryService from basic_memory.workspace_context import ( WORKSPACE_SLUG_HEADER, WORKSPACE_TYPE_HEADER, @@ -78,6 +89,47 @@ async def import_data( raise RuntimeError("partial import failure") +class LocalReadRepairFileReader: + """Read canonical project files through the hosted read-repair protocol.""" + + async def read_note_content_repair_file( + self, + target: NoteContentReadRepairTarget[Project, Entity], + ) -> NoteContentReadRepairFile | None: + path = Path(target.project.path) / target.entity.file_path + if not path.exists(): + return None + stat = path.stat() + return NoteContentReadRepairFile( + markdown_content=path.read_text(encoding="utf-8"), + observed_at=datetime.fromtimestamp(stat.st_mtime, timezone.utc), + ) + + +class DirectoryMoveObservingRedisReadCache(RedisReadCache): + """Record how much of a directory move is durable at each real invalidation.""" + + def __init__( + self, + *, + client: Redis, + namespace: str, + prefix: str, + destination_paths: tuple[Path, ...], + ) -> None: + super().__init__(client=client, namespace=namespace, prefix=prefix) + self.destination_paths = destination_paths + self.destination_counts: list[int] = [] + + @override + async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: + status = await super().invalidate_project(project_id) + self.destination_counts.append( + sum(destination.exists() for destination in self.destination_paths) + ) + return status + + def _cache_key( *, project_id: str, @@ -384,3 +436,129 @@ async def test_partial_import_failure_invalidates_cached_resource_in_real_redis( refreshed_response = await client.get(resource_url) assert refreshed_response.status_code == 200 assert refreshed_response.text == "# Imported before failure\n" + + +@pytest.mark.asyncio +async def test_resource_read_repair_invalidates_cached_entity_in_real_redis( + app: FastAPI, + client: AsyncClient, + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """A resource-triggered note_content repair supersedes cached entity metadata.""" + app.dependency_overrides[get_read_cache] = lambda: redis_cache.cache + project_external_id = str(test_project.external_id) + file_path = "cache/read-repair.md" + disk_path = Path(test_project.path) / file_path + disk_path.parent.mkdir(parents=True, exist_ok=True) + disk_path.write_text("# Read repair\n\nCanonical file content.\n", encoding="utf-8") + + repository = EntityRepository(project_id=test_project.id) + _, session_maker = engine_factory + async with db.scoped_session(session_maker) as session: + entity = await repository.add( + session, + Entity( + title="Read repair", + note_type="note", + content_type="text/markdown", + file_path=file_path, + checksum="read-repair-checksum", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + ) + + project_url = f"/v2/projects/{project_external_id}" + entity_url = f"{project_url}/knowledge/entities/{entity.external_id}" + resource_url = f"{project_url}/resource/{entity.external_id}" + cached_entity = await client.get(entity_url) + assert cached_entity.status_code == 200 + assert cached_entity.json().get("db_version") is None + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="resource-read-repair", + ) + + query_service = NoteContentQueryService( + session_maker=session_maker, + read_repair_file_reader=LocalReadRepairFileReader(), + ) + app.dependency_overrides[get_note_content_query_service] = lambda: query_service + + repaired_resource = await client.get(resource_url) + assert repaired_resource.status_code == 200 + assert repaired_resource.text == "# Read repair\n\nCanonical file content.\n" + generation_after = await _initialized_generation( + redis_cache, + project_external_id, + request="resource-read-repair", + ) + assert generation_after != generation_before + + refreshed_entity = await client.get(entity_url) + assert refreshed_entity.status_code == 200 + assert refreshed_entity.json()["db_version"] == 1 + assert "Canonical file content." in refreshed_entity.json()["content"] + + +@pytest.mark.asyncio +async def test_directory_move_invalidates_after_each_committed_file_in_real_redis( + app: FastAPI, + client: AsyncClient, + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """Each file move advances Redis before the next directory item is processed.""" + project_external_id = str(test_project.external_id) + project_url = f"/v2/projects/{project_external_id}" + created_paths: list[str] = [] + for title in ("First directory move", "Second directory move"): + response = await client.post( + f"{project_url}/knowledge/entities", + json={ + "title": title, + "directory": "move-source", + "content": f"# {title}\n", + }, + ) + assert response.status_code == 202 + created_paths.append(response.json()["file_path"]) + + destination_paths = tuple( + Path(test_project.path) / source_path.replace("move-source/", "move-destination/", 1) + for source_path in created_paths + ) + observing_cache = DirectoryMoveObservingRedisReadCache( + client=redis_cache.client, + namespace=redis_cache.namespace, + prefix=redis_cache.prefix, + destination_paths=destination_paths, + ) + app.dependency_overrides[get_read_cache] = lambda: observing_cache + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="directory-move", + ) + + response = await client.post( + f"{project_url}/knowledge/move-directory", + json={ + "source_directory": "move-source", + "destination_directory": "move-destination", + }, + ) + + assert response.status_code == 200 + assert response.json()["successful_moves"] == 2 + assert observing_cache.destination_counts[:2] == [1, 2] + assert all(destination.exists() for destination in destination_paths) + generation_after = await _initialized_generation( + redis_cache, + project_external_id, + request="directory-move", + ) + assert generation_after != generation_before diff --git a/test-int/read_cache/test_runtime_invalidation.py b/test-int/read_cache/test_runtime_invalidation.py index 5e2dfe05a..45fb3b0ca 100644 --- a/test-int/read_cache/test_runtime_invalidation.py +++ b/test-int/read_cache/test_runtime_invalidation.py @@ -12,6 +12,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker from basic_memory import db +from basic_memory.index import note_content_materialization from basic_memory.index.local_moves import ( LocalMoveEntityRepository, LocalWatchMoveProcessor, @@ -419,6 +420,60 @@ async def test_startup_materialization_recovery_invalidates_real_redis( assert generation_after != generation_before +@pytest.mark.asyncio +async def test_startup_recovery_invalidates_before_later_vacate_failure_in_real_redis( + monkeypatch: pytest.MonkeyPatch, + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """A later recovery phase failure cannot retain phase-one published state.""" + _, session_maker = engine_factory + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="startup-recovery-later-failure", + ) + entity = await _seed_recovery_note( + session_maker, + test_project, + title="Recovered Before Vacate Failure", + file_path="notes/recovered-before-vacate-failure.md", + markdown_content="# Recovered before later failure\n", + ) + + async def fail_move_vacate_recovery( + *, + session_maker: async_sessionmaker[AsyncSession], + file_service: FileService, + project_id: int, + ) -> int: + del session_maker, file_service, project_id + raise RuntimeError("move-vacate setup failure") + + monkeypatch.setattr( + note_content_materialization, + "recover_move_vacates", + fail_move_vacate_recovery, + ) + + await recover_project_materializations( + test_project, + session_maker, + read_cache=redis_cache.cache, + ) + + written = Path(test_project.path) / entity.file_path + assert written.read_text(encoding="utf-8") == "# Recovered before later failure\n" + generation_after = await _initialized_generation( + redis_cache, + project_external_id, + request="startup-recovery-later-failure", + ) + assert generation_after != generation_before + + @pytest.mark.asyncio async def test_startup_recovery_conflict_invalidates_published_failure_in_real_redis( engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], From 2b7eea3ffcfe3c130e04423f052a7322766f587c Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 29 Jul 2026 07:08:25 -0500 Subject: [PATCH 08/28] fix(api): preserve cache freshness on failure paths Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 18 ++++- src/basic_memory/read_cache/redis.py | 16 +++- .../services/note_content_writes.py | 74 ++++++++++++++----- test-int/read_cache/test_api_read_cache.py | 26 ++++++- test-int/read_cache/test_redis_read_cache.py | 39 ++++++++++ 5 files changed, 144 insertions(+), 29 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index 25ac42db9..940c1d0e5 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -92,6 +92,10 @@ tenant: 1. Pass the namespace-bound cache into hosted note-content read repair. A resource or entity read can bootstrap a missing accepted-content row; invalidate immediately after that commit and before the repaired response is offered to read-through storage. +1. Pass the namespace-bound cache into pre-mutation content freshening. Freshening can index an + externally edited file before the accepted mutation begins, so invalidate after every + freshening attempt that may have published state, including when the later mutation is + rejected or raises. 1. Inject the namespace-bound cache into import endpoints and workers. Invalidate after every import attempt that may have written files, including importers that return a failed result or raise after partial progress, so cached file-first resources cannot survive overwritten bytes. @@ -256,12 +260,14 @@ local ASGI transport does not run FastAPI lifespan. ## Failure Behavior -- Connection and timeout failures are represented explicitly as cache-unavailable outcomes. +- Operational Redis command failures, including connection, timeout, capacity, replica-read-only, + and server response failures, are represented explicitly as cache-unavailable outcomes. - Reads bypass Redis and use the authoritative path when the cache is unavailable. - Cache-store failures do not fail an otherwise successful read. - Cache-invalidation failures do not fail committed writes, but they emit prominent telemetry. - Short initial TTLs bound stale-data exposure after an invalidation failure and Redis recovery. -- Serialization and programming errors fail fast rather than masquerading as cache misses. +- Redis client-input errors plus local serialization, decoding, and programming errors fail fast + rather than masquerading as cache misses. Rate-limit failure behavior remains entirely Cloud-owned. @@ -306,7 +312,11 @@ The real-Redis suite must prove: - no invalidation operation touches keys outside the Basic Memory prefix; - payload size limits; - repeated API entity reads use the real cached representation; -- successful writes invalidate while rejected or rolled-back writes do not; +- successful writes invalidate; a rejected write also invalidates when pre-write freshening may + already have published external file state, while a rolled-back transaction without such a + publication does not; +- real Redis no-eviction capacity failures bypass cache storage and cannot fail committed-write + invalidation; - watcher-detected paired moves invalidate even though their events bypass ordinary callbacks; - startup recovery that publishes written, conflict, or failed materialization state invalidates before serving resumes; @@ -351,6 +361,8 @@ semantics themselves are asserted only against the real Redis integration fixtur precede some search and reconciliation follow-ups. - Invalidate hosted note-content read repair after it commits and before returning its repaired entity or resource to the read-through helper. +- Invalidate pre-mutation content freshening even when a later accepted mutation is rejected or + fails, because the freshening index may already have committed external file state. - Invalidate every import attempt that may write files, including partial failures. - Invalidate directory moves after every committed file and again after search/relation follow-ups. diff --git a/src/basic_memory/read_cache/redis.py b/src/basic_memory/read_cache/redis.py index b3f26e5da..97386df50 100644 --- a/src/basic_memory/read_cache/redis.py +++ b/src/basic_memory/read_cache/redis.py @@ -7,7 +7,10 @@ from uuid import uuid4 from redis.asyncio import Redis +from redis.exceptions import ClusterError as RedisClusterError from redis.exceptions import ConnectionError as RedisConnectionError +from redis.exceptions import InvalidResponse as RedisInvalidResponse +from redis.exceptions import ResponseError as RedisResponseError from redis.exceptions import TimeoutError as RedisTimeoutError from basic_memory.read_cache.contract import ( @@ -26,6 +29,13 @@ ) _ENVELOPE_SEPARATOR = b"\n" +_REDIS_OPERATIONAL_ERRORS = ( + RedisClusterError, + RedisConnectionError, + RedisInvalidResponse, + RedisResponseError, + RedisTimeoutError, +) _INITIALIZE_GENERATION_SCRIPT = """ local generation = redis.call("GET", KEYS[1]) if generation then @@ -132,7 +142,7 @@ async def lookup(self, key: ReadCacheKey) -> ReadCacheLookup: generation_value, cached_value = await self._client.mget([keys.generation, keys.data]) if generation_value is None: generation_value = await self._initialize_generation(keys.generation) - except (RedisConnectionError, RedisTimeoutError) as error: + except _REDIS_OPERATIONAL_ERRORS as error: raise ReadCacheUnavailable("Redis cache lookup failed") from error generation, generation_text = _decode_generation(generation_value) @@ -173,7 +183,7 @@ async def store( encoded, ttl_seconds, ) - except (RedisConnectionError, RedisTimeoutError) as error: + except _REDIS_OPERATIONAL_ERRORS as error: raise ReadCacheUnavailable("Redis cache store failed") from error return _store_status(stored) @@ -186,6 +196,6 @@ async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStat ) try: await self._client.set(generation_key, uuid4().hex.encode("ascii")) - except (RedisConnectionError, RedisTimeoutError) as error: + except _REDIS_OPERATIONAL_ERRORS as error: raise ReadCacheUnavailable("Redis project invalidation failed") from error return ReadCacheInvalidationStatus.invalidated diff --git a/src/basic_memory/services/note_content_writes.py b/src/basic_memory/services/note_content_writes.py index d7993792b..77e2f25fc 100644 --- a/src/basic_memory/services/note_content_writes.py +++ b/src/basic_memory/services/note_content_writes.py @@ -178,14 +178,23 @@ async def freshen_existing_note_content( *, project_external_id: str, entity_external_id: str, - ) -> None: - """Let the runtime converge observed file state before an existing-note mutation.""" + ) -> bool: + """Converge observed file state and report whether it may have published state.""" if self.content_freshener is None: - return - await self.content_freshener.freshen_note_content( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - ) + return False + + try: + await self.content_freshener.freshen_note_content( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ) + except BaseException: + # Freshening can commit entity and note-content state before a later + # indexing follow-up raises. Invalidate before propagating so those + # partial publications cannot retain the previous cache generation. + await self._invalidate_project(project_external_id) + raise + return True async def create_note( self, @@ -254,8 +263,9 @@ async def update_note( actor_kind=actor_kind, actor_name=actor_name, ) + freshening_may_have_published = False try: - await self.freshen_existing_note_content( + freshening_may_have_published = await self.freshen_existing_note_content( project_external_id=project_external_id, entity_external_id=entity_external_id, ) @@ -276,10 +286,18 @@ async def update_note( ), dependencies=self.mutation_dependencies, ) - await self._invalidate_project(project_external_id) - return accepted except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error + finally: + # A rejected or failed mutation can follow a successful freshening + # index commit. The freshening attempt therefore owns invalidation + # for every downstream outcome, not only accepted writes. + if freshening_may_have_published: + await self._invalidate_project(project_external_id) + + if not freshening_may_have_published: + await self._invalidate_project(project_external_id) + return accepted async def edit_note( self, @@ -300,8 +318,9 @@ async def edit_note( actor_kind=actor_kind, actor_name=actor_name, ) + freshening_may_have_published = False try: - await self.freshen_existing_note_content( + freshening_may_have_published = await self.freshen_existing_note_content( project_external_id=project_external_id, entity_external_id=entity_external_id, ) @@ -321,10 +340,15 @@ async def edit_note( ), dependencies=self.mutation_dependencies, ) - await self._invalidate_project(project_external_id) - return accepted except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error + finally: + if freshening_may_have_published: + await self._invalidate_project(project_external_id) + + if not freshening_may_have_published: + await self._invalidate_project(project_external_id) + return accepted async def move_note( self, @@ -345,8 +369,9 @@ async def move_note( actor_kind=actor_kind, actor_name=actor_name, ) + freshening_may_have_published = False try: - await self.freshen_existing_note_content( + freshening_may_have_published = await self.freshen_existing_note_content( project_external_id=project_external_id, entity_external_id=entity_external_id, ) @@ -366,10 +391,15 @@ async def move_note( ), dependencies=self.mutation_dependencies, ) - await self._invalidate_project(project_external_id) - return accepted except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error + finally: + if freshening_may_have_published: + await self._invalidate_project(project_external_id) + + if not freshening_may_have_published: + await self._invalidate_project(project_external_id) + return accepted async def delete_note( self, @@ -378,8 +408,9 @@ async def delete_note( entity_external_id: str, ) -> AcceptedNoteChange: """DELETE the DB note and return the runtime follow-up change.""" + freshening_may_have_published = False try: - await self.freshen_existing_note_content( + freshening_may_have_published = await self.freshen_existing_note_content( project_external_id=project_external_id, entity_external_id=entity_external_id, ) @@ -392,7 +423,12 @@ async def delete_note( ), dependencies=self.mutation_dependencies, ) - await self._invalidate_project(project_external_id) - return accepted except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error + finally: + if freshening_may_have_published: + await self._invalidate_project(project_external_id) + + if not freshening_may_have_published: + await self._invalidate_project(project_external_id) + return accepted diff --git a/test-int/read_cache/test_api_read_cache.py b/test-int/read_cache/test_api_read_cache.py index 7e9247356..ac5b48d4a 100644 --- a/test-int/read_cache/test_api_read_cache.py +++ b/test-int/read_cache/test_api_read_cache.py @@ -24,6 +24,7 @@ NoteContentReadRepairTarget, ) from basic_memory.indexing.models import FileIndexResult +from basic_memory.index.note_content_materialization import drain_pending_materializations from basic_memory.models import Project from basic_memory.models.knowledge import Entity from basic_memory.read_cache import ( @@ -169,13 +170,13 @@ async def _initialized_generation( @pytest.mark.asyncio -async def test_entity_resolve_and_markdown_reads_cache_then_write_invalidates( +async def test_entity_resolve_and_markdown_reads_cache_then_freshened_write_invalidates( app: FastAPI, client: AsyncClient, test_project: Project, redis_cache: RedisCacheHarness, ) -> None: - """Successful reads populate Redis; rejected writes preserve and accepted writes bump.""" + """Pre-write freshening invalidates even when the following write is rejected.""" app.dependency_overrides[get_read_cache] = lambda: redis_cache.cache project_external_id = str(test_project.external_id) project_url = f"/v2/projects/{project_external_id}" @@ -191,6 +192,7 @@ async def test_entity_resolve_and_markdown_reads_cache_then_write_invalidates( assert created_response.status_code == 202 created = created_response.json() entity_id = created["external_id"] + await drain_pending_materializations() entity_response = await client.get(f"{project_url}/knowledge/entities/{entity_id}") resolve_request = EntityResolveRequest(identifier=created["permalink"]) @@ -254,6 +256,11 @@ async def test_entity_resolve_and_markdown_reads_cache_then_write_invalidates( populated_generation = await redis_cache.client.get(generation_key) assert populated_generation is not None + disk_path = Path(test_project.path) / created["file_path"] + disk_path.write_text( + "# Redis Cached Note\n\nExternally edited before rejected update.", + encoding="utf-8", + ) rejected_response = await client.put( f"{project_url}/knowledge/entities/{entity_id}", headers={NOTE_CONTENT_BASE_CHECKSUM_HEADER: "stale-checksum"}, @@ -264,7 +271,16 @@ async def test_entity_resolve_and_markdown_reads_cache_then_write_invalidates( }, ) assert rejected_response.status_code == 409 - assert await redis_cache.client.get(generation_key) == populated_generation + freshened_generation = await redis_cache.client.get(generation_key) + assert freshened_generation is not None + assert freshened_generation != populated_generation + + freshened_entity = await client.get(f"{project_url}/knowledge/entities/{entity_id}") + freshened_resource = await client.get(f"{project_url}/resource/{entity_id}") + assert freshened_entity.status_code == 200 + assert "Externally edited" in freshened_entity.json()["content"] + assert freshened_resource.status_code == 200 + assert "Externally edited" in freshened_resource.text edited_response = await client.patch( f"{project_url}/knowledge/entities/{entity_id}", @@ -276,13 +292,15 @@ async def test_entity_resolve_and_markdown_reads_cache_then_write_invalidates( assert edited_response.status_code == 202 invalidated_generation = await redis_cache.client.get(generation_key) assert invalidated_generation is not None - assert invalidated_generation != populated_generation + assert invalidated_generation != freshened_generation refreshed_entity = await client.get(f"{project_url}/knowledge/entities/{entity_id}") refreshed_resource = await client.get(f"{project_url}/resource/{entity_id}") assert refreshed_entity.status_code == 200 + assert "Externally edited" in refreshed_entity.json()["content"] assert "Version two." in refreshed_entity.json()["content"] assert refreshed_resource.status_code == 200 + assert "Externally edited" in refreshed_resource.text assert "Version two." in refreshed_resource.text diff --git a/test-int/read_cache/test_redis_read_cache.py b/test-int/read_cache/test_redis_read_cache.py index c2cdb3b25..31cc443f5 100644 --- a/test-int/read_cache/test_redis_read_cache.py +++ b/test-int/read_cache/test_redis_read_cache.py @@ -265,6 +265,45 @@ async def test_unavailable_redis_is_explicit_for_every_operation() -> None: await client.aclose() +@pytest.mark.asyncio +async def test_real_redis_capacity_failures_are_cache_unavailable( + redis_cache: RedisCacheHarness, +) -> None: + """A no-eviction maxmemory error cannot fail reads or committed-write invalidation.""" + key = _key(request="capacity-limited-store") + original_maxmemory = str((await redis_cache.client.config_get("maxmemory"))["maxmemory"]) + original_policy = str( + (await redis_cache.client.config_get("maxmemory-policy"))["maxmemory-policy"] + ) + await redis_cache.cache.lookup(key) + + async def load() -> CachedEntity: + return CachedEntity(external_id="entity-1", title="Authoritative") + + try: + await redis_cache.client.config_set("maxmemory-policy", "noeviction") + await redis_cache.client.config_set("maxmemory", "1") + + result = await read_through_model( + cache=redis_cache.cache, + key=key, + model_type=CachedEntity, + load=load, + ttl_seconds=60, + max_payload_bytes=1_024, + ) + invalidation_status = await invalidate_project_read_cache( + redis_cache.cache, + PROJECT_ID, + ) + finally: + await redis_cache.client.config_set("maxmemory", original_maxmemory) + await redis_cache.client.config_set("maxmemory-policy", original_policy) + + assert result.title == "Authoritative" + assert invalidation_status is ReadCacheInvalidationStatus.unavailable + + @pytest.mark.asyncio async def test_null_cache_preserves_disabled_semantics() -> None: class StoreMustNotRun(NullReadCache): From 9f02ff39d8c78984cafdf922676ed0a8945d247d Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 29 Jul 2026 07:24:37 -0500 Subject: [PATCH 09/28] fix(api): invalidate cache after project root changes Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 7 ++ .../api/v2/routers/project_router.py | 12 ++- test-int/read_cache/test_api_read_cache.py | 83 +++++++++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index 940c1d0e5..42c62b5e0 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -107,6 +107,9 @@ tenant: 1. Invalidate directory moves after each individual file/database move commits, then again after the final search and relation follow-ups. Directory moves are incremental batches, so a long or partially failed request must not keep earlier files under the pre-move generation. +1. Inject the namespace-bound cache into project-root path mutations. A root change preserves the + project and entity UUIDs while changing the filesystem source behind resource reads, so + invalidate in a failure-safe boundary around the committed path update. 1. If startup recovery or reconciliation attempts can publish materialization, vacate, index, or relation state, invalidate through the same namespace-bound cache before releasing the serving barrier or resuming tenant traffic. Treat materialization and move-vacate recovery as separate @@ -326,6 +329,8 @@ The real-Redis suite must prove: - hosted read repair invalidates cached entity metadata before storing the repaired resource; - import attempts invalidate cached file-first resources after success and partial failure; - directory moves invalidate after each committed file and again after final reindexing; +- project-root path changes invalidate cached resources whose project/entity UUID keys remain + stable; - directory deletion invalidates before cleanup starts and again after cleanup completes. - startup materialization recovery remains invalidated when a later move-vacate phase fails. @@ -366,6 +371,8 @@ semantics themselves are asserted only against the real Redis integration fixtur - Invalidate every import attempt that may write files, including partial failures. - Invalidate directory moves after every committed file and again after search/relation follow-ups. +- Invalidate project-root path changes in a failure-safe boundary because the filesystem source + can change while every cache identity remains stable. - Invalidate each startup recovery phase before beginning the next phase. - Enable reads for a tenant only after every request, worker, partial-index, direct-index, read-repair, import, accepted-delete, move, and recovery boundary has namespace and invalidation diff --git a/src/basic_memory/api/v2/routers/project_router.py b/src/basic_memory/api/v2/routers/project_router.py index 3fd00fc9d..db57cb592 100644 --- a/src/basic_memory/api/v2/routers/project_router.py +++ b/src/basic_memory/api/v2/routers/project_router.py @@ -24,10 +24,12 @@ ProjectIndexCommandDep, ProjectIndexObserverDep, ProjectExternalIdPathDep, + ReadCacheDep, SessionDep, SessionMakerDep, ) from basic_memory.index.local_project import ProjectIndexRouteRequest +from basic_memory.read_cache import invalidate_project_read_cache from basic_memory.schemas import ProjectIndexStatusResponse from basic_memory.models import Project from basic_memory.repository.project_repository import ProjectRepository @@ -408,6 +410,7 @@ async def update_project_by_id( project_service: ProjectServiceDep, session_maker: SessionMakerDep, project_repository: ProjectRepositoryDep, + read_cache: ReadCacheDep, project_id: str = Path(..., description="Project external ID (UUID)"), path: Optional[str] = Body(None, description="New absolute path for the project"), is_active: Optional[bool] = Body(None, description="Status of the project (active/inactive)"), @@ -454,7 +457,14 @@ async def update_project_by_id( # Update using project name (service layer still uses names internally) if path: - await project_service.move_project(old_project.name, path) + try: + await project_service.move_project(old_project.name, path) + finally: + # A path update changes the filesystem source behind every + # resource key while project and entity UUIDs stay stable. The + # service can update config before its DB follow-up completes, + # so invalidate on every attempted move completion path. + await invalidate_project_read_cache(read_cache, project_id) elif is_active is not None: await project_service.update_project(old_project.name, is_active=is_active) diff --git a/test-int/read_cache/test_api_read_cache.py b/test-int/read_cache/test_api_read_cache.py index ac5b48d4a..30f93992b 100644 --- a/test-int/read_cache/test_api_read_cache.py +++ b/test-int/read_cache/test_api_read_cache.py @@ -28,6 +28,7 @@ from basic_memory.models import Project from basic_memory.models.knowledge import Entity from basic_memory.read_cache import ( + ReadCache, ReadCacheInvalidationStatus, ReadCacheKey, ReadCacheOperation, @@ -107,6 +108,21 @@ async def read_note_content_repair_file( ) +class NoAcceptedNoteContent: + """Keep this resource on the file-first path for project-root cache coverage.""" + + async def get_note_resource_with_read_repair( + self, + *, + project_external_id: str, + entity_external_id: str, + session: AsyncSession, + read_cache: ReadCache, + ) -> None: + del project_external_id, entity_external_id, session, read_cache + return None + + class DirectoryMoveObservingRedisReadCache(RedisReadCache): """Record how much of a directory move is durable at each real invalidation.""" @@ -356,6 +372,73 @@ async def test_non_markdown_resource_is_never_cached( assert await redis_cache.client.exists(redis_keys.data) == 0 +@pytest.mark.asyncio +async def test_project_root_change_invalidates_cached_markdown_resource( + app: FastAPI, + client: AsyncClient, + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], + test_project: Project, + redis_cache: RedisCacheHarness, + tmp_path: Path, +) -> None: + """A stable project/entity UUID cannot retain bytes from the previous root.""" + app.dependency_overrides[get_read_cache] = lambda: redis_cache.cache + app.dependency_overrides[get_note_content_query_service] = NoAcceptedNoteContent + project_external_id = str(test_project.external_id) + file_path = "cache/project-root.md" + old_disk_path = Path(test_project.path) / file_path + old_disk_path.parent.mkdir(parents=True, exist_ok=True) + old_disk_path.write_text("# Old project root\n", encoding="utf-8") + + new_root = tmp_path / "new-project-root" + new_disk_path = new_root / file_path + new_disk_path.parent.mkdir(parents=True, exist_ok=True) + new_disk_path.write_text("# New project root\n", encoding="utf-8") + + repository = EntityRepository(project_id=test_project.id) + _, session_maker = engine_factory + async with db.scoped_session(session_maker) as session: + entity = await repository.add( + session, + Entity( + title="project-root.md", + note_type="file", + content_type="text/markdown", + file_path=file_path, + checksum="project-root-checksum", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + ) + + project_url = f"/v2/projects/{project_external_id}" + resource_url = f"{project_url}/resource/{entity.external_id}" + cached_response = await client.get(resource_url) + assert cached_response.status_code == 200 + assert cached_response.text == "# Old project root\n" + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="project-root-change", + ) + + move_response = await client.patch( + project_url, + json={"path": str(new_root)}, + ) + + assert move_response.status_code == 200 + generation_after = await _initialized_generation( + redis_cache, + project_external_id, + request="project-root-change", + ) + assert generation_after != generation_before + refreshed_response = await client.get(resource_url) + assert refreshed_response.status_code == 200 + assert refreshed_response.text == "# New project root\n" + + @pytest.mark.asyncio async def test_direct_index_failure_invalidates_real_redis( app: FastAPI, From b9e736a677e1ba6a537247c9108b610c52ccf865 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 29 Jul 2026 20:41:26 -0500 Subject: [PATCH 10/28] fix(api): expire Redis cache generations Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 8 ++- src/basic_memory/read_cache/redis.py | 47 ++++++++----- test-int/read_cache/test_redis_read_cache.py | 69 ++++++++++++++++++++ 3 files changed, 105 insertions(+), 19 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index 42c62b5e0..99baf4c89 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -185,7 +185,11 @@ Each value records the random generation token under which it was created: 1. Read the generation and data key together. 1. Accept the cached value only when its embedded generation matches. 1. After a successful mutation commit, replace the generation with a new random token. -1. Let unreachable entries expire; never scan or bulk-delete keys. +1. Give generation metadata a bounded TTL, renew it to at least each stored response's remaining + lifetime, and reset that TTL on invalidation. A lookup also migrates a legacy persistent + generation key to a bounded TTL. +1. Let unreachable entries and inactive-project generation metadata expire; never scan or + bulk-delete keys. Random tokens prevent an evicted generation key from returning to an old integer generation and reviving stale data. A read that fills after concurrent invalidation also remains safe because its @@ -311,6 +315,8 @@ The real-Redis suite must prove: - project invalidation; - a fill that completes after invalidation is never served; - loss of the generation key cannot revive an older value; +- generation metadata expires for inactive projects, including legacy persistent keys, and stores + keep it alive at least as long as their response data; - Redis restart or unavailability produces explicit bypass behavior; - no invalidation operation touches keys outside the Basic Memory prefix; - payload size limits; diff --git a/src/basic_memory/read_cache/redis.py b/src/basic_memory/read_cache/redis.py index 97386df50..53fd79829 100644 --- a/src/basic_memory/read_cache/redis.py +++ b/src/basic_memory/read_cache/redis.py @@ -29,6 +29,7 @@ ) _ENVELOPE_SEPARATOR = b"\n" +_DEFAULT_GENERATION_TTL_SECONDS = 60 _REDIS_OPERATIONAL_ERRORS = ( RedisClusterError, RedisConnectionError, @@ -36,19 +37,26 @@ RedisResponseError, RedisTimeoutError, ) -_INITIALIZE_GENERATION_SCRIPT = """ +_LOOKUP_SCRIPT = """ local generation = redis.call("GET", KEYS[1]) if generation then - return generation + if redis.call("TTL", KEYS[1]) == -1 then + redis.call("EXPIRE", KEYS[1], ARGV[2]) + end +else + redis.call("SET", KEYS[1], ARGV[1], "EX", ARGV[2]) end -redis.call("SET", KEYS[1], ARGV[1]) -return ARGV[1] +return redis.call("MGET", KEYS[1], KEYS[2]) """ _STORE_IF_CURRENT_SCRIPT = """ if redis.call("GET", KEYS[1]) ~= ARGV[1] then return 0 end redis.call("SET", KEYS[2], ARGV[2], "EX", ARGV[3]) +local response_ttl_ms = tonumber(ARGV[3]) * 1000 +if redis.call("PTTL", KEYS[1]) < response_ttl_ms then + redis.call("PEXPIRE", KEYS[1], response_ttl_ms) +end return 1 """ @@ -111,13 +119,17 @@ def __init__( client: Redis, namespace: str, prefix: str = DEFAULT_READ_CACHE_PREFIX, + generation_ttl_seconds: int = _DEFAULT_GENERATION_TTL_SECONDS, ) -> None: namespace = namespace.strip() if not namespace: raise ValueError("read-cache namespace must not be empty") + if generation_ttl_seconds <= 0: + raise ValueError("read-cache generation_ttl_seconds must be positive") self._client = client self._namespace = namespace self._prefix = prefix + self._generation_ttl_seconds = generation_ttl_seconds def _keys(self, key: ReadCacheKey) -> RedisReadCacheKeys: return redis_read_cache_keys( @@ -126,22 +138,17 @@ def _keys(self, key: ReadCacheKey) -> RedisReadCacheKeys: key=key, ) - async def _initialize_generation(self, generation_key: str) -> bytes: - token = uuid4().hex.encode("ascii") - generation = await self._client.eval( - _INITIALIZE_GENERATION_SCRIPT, - 1, - generation_key, - token, - ) - return _required_bytes(generation, field="generation") - async def lookup(self, key: ReadCacheKey) -> ReadCacheLookup: keys = self._keys(key) try: - generation_value, cached_value = await self._client.mget([keys.generation, keys.data]) - if generation_value is None: - generation_value = await self._initialize_generation(keys.generation) + generation_value, cached_value = await self._client.eval( + _LOOKUP_SCRIPT, + 2, + keys.generation, + keys.data, + uuid4().hex.encode("ascii"), + self._generation_ttl_seconds, + ) except _REDIS_OPERATIONAL_ERRORS as error: raise ReadCacheUnavailable("Redis cache lookup failed") from error @@ -195,7 +202,11 @@ async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStat project_id=project_id, ) try: - await self._client.set(generation_key, uuid4().hex.encode("ascii")) + await self._client.set( + generation_key, + uuid4().hex.encode("ascii"), + ex=self._generation_ttl_seconds, + ) except _REDIS_OPERATIONAL_ERRORS as error: raise ReadCacheUnavailable("Redis project invalidation failed") from error return ReadCacheInvalidationStatus.invalidated diff --git a/test-int/read_cache/test_redis_read_cache.py b/test-int/read_cache/test_redis_read_cache.py index 31cc443f5..4871087c3 100644 --- a/test-int/read_cache/test_redis_read_cache.py +++ b/test-int/read_cache/test_redis_read_cache.py @@ -91,6 +91,69 @@ async def test_round_trip_and_ttl_expiry(redis_cache: RedisCacheHarness) -> None assert expired.generation == miss.generation +@pytest.mark.asyncio +async def test_generation_metadata_expires_and_tracks_response_lifetime( + redis_cache: RedisCacheHarness, +) -> None: + """Inactive projects cannot leave permanent generation metadata in Redis.""" + cache = RedisReadCache( + client=redis_cache.client, + namespace=redis_cache.namespace, + prefix=redis_cache.prefix, + generation_ttl_seconds=1, + ) + long_lived_key = _key(request="long-lived-generation") + redis_keys = redis_read_cache_keys( + prefix=redis_cache.prefix, + namespace=redis_cache.namespace, + key=long_lived_key, + ) + + # Existing deployments may already have persistent generation keys. A + # lookup migrates them to bounded metadata without changing the token. + legacy_generation = b"0" * 32 + await redis_cache.client.set(redis_keys.generation, legacy_generation) + lookup = await cache.lookup(long_lived_key) + assert lookup.generation == legacy_generation.decode("ascii") + assert await redis_cache.client.pttl(redis_keys.generation) > 0 + + stored = await cache.store( + long_lived_key, + lookup, + b"long-lived payload", + ttl_seconds=3, + ) + generation_ttl = await redis_cache.client.pttl(redis_keys.generation) + data_ttl = await redis_cache.client.pttl(redis_keys.data) + assert stored is ReadCacheStoreStatus.stored + assert generation_ttl >= data_ttl > 2_000 + + short_lived_key = _key(request="short-lived-generation") + short_lookup = await cache.lookup(short_lived_key) + await cache.store( + short_lived_key, + short_lookup, + b"short-lived payload", + ttl_seconds=1, + ) + assert await redis_cache.client.pttl(redis_keys.generation) > 2_000 + + generation_before_invalidation = await redis_cache.client.get(redis_keys.generation) + await cache.invalidate_project(PROJECT_ID) + generation_after_invalidation = await redis_cache.client.get(redis_keys.generation) + invalidated_ttl = await redis_cache.client.pttl(redis_keys.generation) + assert generation_before_invalidation is not None + assert isinstance(generation_after_invalidation, bytes) + assert generation_after_invalidation != generation_before_invalidation + assert 0 < invalidated_ttl <= 1_000 + + await asyncio.sleep(1.1) + assert not await redis_cache.client.exists(redis_keys.generation) + after_expiry = await cache.lookup(long_lived_key) + assert not after_expiry.is_hit + assert after_expiry.generation != generation_after_invalidation.decode("ascii") + + @pytest.mark.asyncio async def test_cache_identity_isolates_every_key_dimension( redis_cache: RedisCacheHarness, @@ -648,6 +711,12 @@ async def test_invalid_store_inputs_fail_before_redis( RedisReadCache(client=redis_cache.client, namespace="") with pytest.raises(ValueError, match="namespace"): RedisReadCache(client=redis_cache.client, namespace=" ") + with pytest.raises(ValueError, match="generation_ttl_seconds"): + RedisReadCache( + client=redis_cache.client, + namespace="tenant", + generation_ttl_seconds=0, + ) with pytest.raises(ValueError, match="project_id"): await redis_cache.cache.invalidate_project("") From 0f18c3729117b637d60cd71bc9e4d943e1b325cc Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 29 Jul 2026 21:03:20 -0500 Subject: [PATCH 11/28] fix(ci): skip Redis tests without Windows Docker Signed-off-by: phernandez --- test-int/read_cache/conftest.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test-int/read_cache/conftest.py b/test-int/read_cache/conftest.py index f9c6cc97c..7ceae4d20 100644 --- a/test-int/read_cache/conftest.py +++ b/test-int/read_cache/conftest.py @@ -11,6 +11,7 @@ import pytest import pytest_asyncio +from docker.errors import DockerException from redis import Redis as SyncRedis from redis.asyncio import Redis from redis.exceptions import RedisError @@ -54,7 +55,16 @@ def redis_url() -> Generator[str]: pytest.skip("Docker is required for real Redis integration tests") image = os.environ.get("BASIC_MEMORY_TEST_REDIS_IMAGE", "redis:8.8-alpine") - with DockerContainer(image).with_exposed_ports(6379) as container: + try: + container = DockerContainer(image).with_exposed_ports(6379) + except DockerException: + # GitHub's Windows image includes the Docker CLI without a running daemon. + # Keep real Redis coverage on Linux CI while allowing Windows coverage to proceed. + if os.name == "nt": + pytest.skip("A Docker daemon is required for real Redis integration tests") + raise + + with container: host = container.get_container_host_ip() port = container.get_exposed_port(6379) url = f"redis://{host}:{port}/0" From 10511e63ee0eb9ef5a12083ed560b1cf61d89b56 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 29 Jul 2026 21:32:29 -0500 Subject: [PATCH 12/28] fix(ci): skip Redis testcontainers on Windows Signed-off-by: phernandez --- test-int/read_cache/conftest.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/test-int/read_cache/conftest.py b/test-int/read_cache/conftest.py index 7ceae4d20..505855715 100644 --- a/test-int/read_cache/conftest.py +++ b/test-int/read_cache/conftest.py @@ -11,7 +11,6 @@ import pytest import pytest_asyncio -from docker.errors import DockerException from redis import Redis as SyncRedis from redis.asyncio import Redis from redis.exceptions import RedisError @@ -51,20 +50,16 @@ def redis_url() -> Generator[str]: yield configured_url return + # GitHub's Windows runners cannot start the Linux Redis image through testcontainers. + # A configured URL still runs the same real-server suite on Windows when one is available. + if os.name == "nt": + pytest.skip("Set BASIC_MEMORY_TEST_REDIS_URL to run real Redis tests on Windows") + if shutil.which("docker") is None: pytest.skip("Docker is required for real Redis integration tests") image = os.environ.get("BASIC_MEMORY_TEST_REDIS_IMAGE", "redis:8.8-alpine") - try: - container = DockerContainer(image).with_exposed_ports(6379) - except DockerException: - # GitHub's Windows image includes the Docker CLI without a running daemon. - # Keep real Redis coverage on Linux CI while allowing Windows coverage to proceed. - if os.name == "nt": - pytest.skip("A Docker daemon is required for real Redis integration tests") - raise - - with container: + with DockerContainer(image).with_exposed_ports(6379) as container: host = container.get_container_host_ip() port = container.get_exposed_port(6379) url = f"redis://{host}:{port}/0" From d4121acdf235c7b32db2ecd4a1cbdc26ca7afb57 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 29 Jul 2026 23:22:38 -0500 Subject: [PATCH 13/28] refactor(api): simplify read-cache control flow Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 51 ++- .../api/v2/routers/importer_router.py | 12 +- .../api/v2/routers/knowledge_router.py | 101 +++-- .../api/v2/routers/project_router.py | 14 +- .../api/v2/routers/resource_router.py | 63 +-- src/basic_memory/deps/__init__.py | 4 + src/basic_memory/deps/read_cache.py | 26 +- src/basic_memory/index/local_moves.py | 15 +- src/basic_memory/index/local_project.py | 32 +- src/basic_memory/index/local_runtime.py | 39 +- src/basic_memory/index/local_schedulers.py | 15 +- .../index/note_content_materialization.py | 15 +- src/basic_memory/read_cache/__init__.py | 12 +- src/basic_memory/read_cache/contract.py | 23 +- src/basic_memory/read_cache/invalidation.py | 19 +- src/basic_memory/read_cache/keys.py | 35 +- src/basic_memory/read_cache/read_through.py | 204 ++++++---- src/basic_memory/read_cache/redis.py | 8 +- .../services/note_content_reads.py | 6 +- test-int/read_cache/test_api_read_cache.py | 8 +- test-int/read_cache/test_redis_read_cache.py | 361 ++++++++++++------ tests/test_deps.py | 30 +- 22 files changed, 683 insertions(+), 410 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index 99baf4c89..5e64c7da1 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -69,9 +69,12 @@ tenant: boundary, even if current database constraints make it globally unique. 1. Complete authorization and tenant database/schema selection before cache lookup. The namespace prevents key collisions; it does not replace Cloud's access-control boundary. -1. Override `get_read_cache` at the Cloud composition root. Construct +1. Override the low-level `get_read_cache` dependency at the Cloud composition root. Construct `RedisReadCache(client=shared_basic_memory_client, namespace=trusted_namespace)` as a lightweight request-scoped adapter; reuse the long-lived client and connection pool. + FastAPI then resolves `ConfiguredReadCacheDep` from that backend and binds Basic Memory's TTL + and payload-size policy. Cloud should not duplicate those policy constants or override + `get_configured_read_cache`. 1. Pass the trusted tenant/workspace identity through internal queue payloads, or include enough trusted identifiers for workers to derive the exact same namespace. Never copy a namespace from a public request field. @@ -152,10 +155,11 @@ flowchart LR Introduce `src/basic_memory/read_cache/` with: - a narrow `ReadCache` protocol; +- a narrower `ReadCacheInvalidator` protocol for mutation and repair paths; - immutable request/key values; - a `NullReadCache` default; - canonical key construction; -- typed Pydantic read-through helpers; +- a configured Pydantic read-through dependency; - an optional `RedisReadCache` adapter. The cache is namespace-bound at construction. Its public operations are: @@ -223,8 +227,41 @@ workspace type in addition to the validated request body. ## Placement -Cache typed boundary values rather than SQLAlchemy models. Use an explicit read-through helper in -the API routes so hit, miss, serialization, and fallback behavior remain visible. +Cache typed boundary values rather than SQLAlchemy models. Use an explicit read-through scope in +the API routes so hit, miss, serialization, and fallback behavior remain visible. FastAPI injects +`ConfiguredReadCacheDep`; its provider binds the injected backend to Basic Memory's TTL and +payload-size policy at the dependency boundary. Routes keep the authoritative read inline inside +a Python async context manager instead of constructing loader callbacks: + +```python +cache_key = ReadCacheKey( + project_id=project_external_id, + operation=ReadCacheOperation.entity, + request_digest=read_cache_request_digest(entity_id), +) +async with read_cache.read(key=cache_key, model_type=EntityResponseV2) as cached: + if cached.value is not None: + return cached.value + + entity = await entity_repository.get_by_external_id(session, entity_id) + result = EntityResponseV2.model_validate(entity) + cached.value = result + return result +``` + +The context manager performs lookup before entering the body and stores an eligible miss when the +body exits normally. Exceptions and cancellation propagate without storing. A route that performs +read repair passes the configured dependency itself through the narrow `ReadCacheInvalidator` +capability; it never reaches through the facade to a Redis/backend attribute. + +Mutation and indexing code uses the same direct scope pattern when invalidation is unconditional: + +```python +async with invalidate_cache(read_cache, project_id): + await importer.import_data(...) +``` + +Conditional and multi-phase invalidation remains explicit so the freshness boundary is visible. Primary integration points: @@ -261,6 +298,11 @@ in-process ASGI routing, and the standalone API remain on `NullReadCache` in the A later standalone Redis setting can create and close a client in the FastAPI lifespan without changing the cache contract. +`get_read_cache` is the host override point. `get_configured_read_cache` is Core-owned FastAPI +composition: it receives that backend through dependency injection and returns the route-facing +cache with validated policy. Portable mutation and indexing runtimes continue to depend only on +the backend or the narrower invalidation capability; they do not depend on FastAPI. + The FastAPI Redis SDK is not the foundational dependency for this work. The cache contract must also participate in portable indexing and hosted storage-event invalidation, and Basic Memory's local ASGI transport does not run FastAPI lifespan. @@ -326,6 +368,7 @@ The real-Redis suite must prove: publication does not; - real Redis no-eviction capacity failures bypass cache storage and cannot fail committed-write invalidation; +- authoritative read exceptions propagate without populating the missed cache key; - watcher-detected paired moves invalidate even though their events bypass ordinary callbacks; - startup recovery that publishes written, conflict, or failed materialization state invalidates before serving resumes; diff --git a/src/basic_memory/api/v2/routers/importer_router.py b/src/basic_memory/api/v2/routers/importer_router.py index 42371f9da..35594b671 100644 --- a/src/basic_memory/api/v2/routers/importer_router.py +++ b/src/basic_memory/api/v2/routers/importer_router.py @@ -18,7 +18,7 @@ ReadCacheDep, ) from basic_memory.importers import Importer -from basic_memory.read_cache import ReadCache, invalidate_project_read_cache +from basic_memory.read_cache import ReadCache, invalidate_cache from basic_memory.schemas.importer import ( ChatImportResult, EntityImportResult, @@ -260,10 +260,8 @@ async def run_import_with_invalidation[ImportResultT: ImportResult]( project_external_id: str, ) -> ImportResultT: """Run one import attempt and invalidate files it may have written.""" - try: + # Importers write files one at a time and may return a failed result after + # earlier writes. Invalidate every attempted import so cached file-first + # resources cannot survive either success or partial failure. + async with invalidate_cache(read_cache, project_external_id): return await importer.import_data(source_data, destination_directory) - finally: - # Importers write files one at a time and may return a failed result after - # earlier writes. Invalidate every attempted import so cached file-first - # resources cannot survive either success or partial failure. - await invalidate_project_read_cache(read_cache, project_external_id) diff --git a/src/basic_memory/api/v2/routers/knowledge_router.py b/src/basic_memory/api/v2/routers/knowledge_router.py index b35ee8784..9625e87b7 100644 --- a/src/basic_memory/api/v2/routers/knowledge_router.py +++ b/src/basic_memory/api/v2/routers/knowledge_router.py @@ -30,6 +30,7 @@ should_ignore_path, ) from basic_memory.deps import ( + ConfiguredReadCacheDep, EntityServiceV2ExternalDep, FileServiceV2ExternalDep, SearchServiceV2ExternalDep, @@ -54,13 +55,8 @@ from basic_memory.read_cache import ( ReadCacheKey, ReadCacheOperation, - invalidate_project_read_cache, + invalidate_cache, read_cache_request_digest, - read_through_model, -) -from basic_memory.read_cache.policy import ( - READ_CACHE_MAX_PAYLOAD_BYTES, - READ_CACHE_TTL_SECONDS, ) from basic_memory.runtime.note_content import ( NOTE_CONTENT_BASE_CHECKSUM_HEADER, @@ -251,7 +247,7 @@ async def resolve_identifier( entity_repository: EntityRepositoryV2ExternalDep, project_repository: ProjectRepositoryDep, session: SessionDep, - read_cache: ReadCacheDep, + read_cache: ConfiguredReadCacheDep, ) -> EntityResolveResponse: """Resolve a string identifier (external_id, permalink, title, or path) to entity info. @@ -290,7 +286,23 @@ async def resolve_identifier( ): logger.info(f"API v2 request: resolve_identifier for '{data.identifier}'") - async def load() -> EntityResolveResponse: + workspace_context = current_workspace_permalink_context() + cache_key = ReadCacheKey( + project_id=project_external_id, + operation=ReadCacheOperation.resolve, + request_digest=read_cache_request_digest( + data.model_dump_json(), + workspace_context.workspace_slug if workspace_context else "", + workspace_context.workspace_type if workspace_context else "", + ), + ) + async with read_cache.read( + key=cache_key, + model_type=EntityResolveResponse, + ) as cached: + if cached.value is not None: + return cached.value + entity = await entity_repository.get_by_external_id(session, data.identifier) resolution_method = "external_id" if entity else "search" @@ -343,28 +355,11 @@ async def load() -> EntityResolveResponse: f"API v2 response: resolved '{data.identifier}' " f"to external_id={result.external_id} via {resolution_method}" ) - return result - - workspace_context = current_workspace_permalink_context() - return await read_through_model( - cache=read_cache, - key=ReadCacheKey( - project_id=project_external_id, - operation=ReadCacheOperation.resolve, - request_digest=read_cache_request_digest( - data.model_dump_json(), - workspace_context.workspace_slug if workspace_context else "", - workspace_context.workspace_type if workspace_context else "", - ), - ), - model_type=EntityResolveResponse, - load=load, - ttl_seconds=READ_CACHE_TTL_SECONDS, - max_payload_bytes=READ_CACHE_MAX_PAYLOAD_BYTES, # Cross-project references depend on two projects. Keep phase-one # generation invalidation exact by caching only local resolutions. - should_store=lambda resolved: resolved.project_external_id == project_external_id, - ) + cached.cacheable = result.project_external_id == project_external_id + cached.value = result + return result ## Single-file indexing endpoint @@ -530,13 +525,11 @@ async def index_file( detail=f"Only markdown files can be indexed: '{data.file_path}'", ) - try: + # The file indexer commits entity state before search and reconciliation + # follow-ups finish. Invalidate even when a later phase raises so those + # partial commits cannot remain reachable through the old generation. + async with invalidate_cache(read_cache, project_external_id): indexed = await file_indexer.index_file(file_path, source="api-index-file") - finally: - # The file indexer commits entity state before search and reconciliation - # follow-ups finish. Invalidate even when a later phase raises so those - # partial commits cannot remain reachable through the old generation. - await invalidate_project_read_cache(read_cache, project_external_id) async with db.scoped_session(session_maker) as session: entity = await entity_repository.get_by_id(session, indexed.entity_id) if entity is None: # pragma: no cover @@ -573,7 +566,7 @@ async def get_entity_by_id( entity_repository: EntityRepositoryV2ExternalDep, note_content_query_service: NoteContentQueryServiceDep, session: SessionDep, - read_cache: ReadCacheDep, + read_cache: ConfiguredReadCacheDep, entity_id: str = Path(..., description="Entity external ID (UUID)"), ) -> EntityResponseV2: """Get an entity by its external ID (UUID). @@ -598,7 +591,18 @@ async def get_entity_by_id( ): logger.info(f"API v2 request: get_entity_by_id entity_id={entity_id}") - async def load() -> EntityResponseV2: + cache_key = ReadCacheKey( + project_id=project_external_id, + operation=ReadCacheOperation.entity, + request_digest=read_cache_request_digest(entity_id), + ) + async with read_cache.read( + key=cache_key, + model_type=EntityResponseV2, + ) as cached: + if cached.value is not None: + return cached.value + note_payload = ( await note_content_query_service.get_note_entity_payload_with_read_repair( project_external_id=project_external_id, @@ -610,6 +614,7 @@ async def load() -> EntityResponseV2: if note_payload is not None: result = entity_response_from_note_content_payload(note_payload) logger.info(f"API v2 response: external_id={entity_id}, title='{result.title}'") + cached.value = result return result entity = await entity_repository.get_by_external_id(session, entity_id) @@ -621,21 +626,9 @@ async def load() -> EntityResponseV2: result = EntityResponseV2.model_validate(entity) logger.info(f"API v2 response: external_id={entity_id}, title='{result.title}'") + cached.value = result return result - return await read_through_model( - cache=read_cache, - key=ReadCacheKey( - project_id=project_external_id, - operation=ReadCacheOperation.entity, - request_digest=read_cache_request_digest(entity_id), - ), - model_type=EntityResponseV2, - load=load, - ttl_seconds=READ_CACHE_TTL_SECONDS, - max_payload_bytes=READ_CACHE_MAX_PAYLOAD_BYTES, - ) - ## Create endpoints @@ -1018,7 +1011,10 @@ async def move_directory( read_cache=read_cache, ) - try: + # Reindexing can alter entity responses after the move was first + # invalidated. Close that fill window even after partial + # follow-up failure. + async with invalidate_cache(read_cache, project_external_id): # Reindex moved entities for file_path in result.moved_files: async with db.scoped_session(session_maker) as session: @@ -1034,11 +1030,6 @@ async def move_directory( entity_id=entity.id, project_id=project_id, ) - finally: - # Reindexing can alter entity responses after the move was first - # invalidated. Close that fill window even after partial - # follow-up failure. - await invalidate_project_read_cache(read_cache, project_external_id) logger.info( f"API v2 response: move_directory " diff --git a/src/basic_memory/api/v2/routers/project_router.py b/src/basic_memory/api/v2/routers/project_router.py index db57cb592..7064c7952 100644 --- a/src/basic_memory/api/v2/routers/project_router.py +++ b/src/basic_memory/api/v2/routers/project_router.py @@ -29,7 +29,7 @@ SessionMakerDep, ) from basic_memory.index.local_project import ProjectIndexRouteRequest -from basic_memory.read_cache import invalidate_project_read_cache +from basic_memory.read_cache import invalidate_cache from basic_memory.schemas import ProjectIndexStatusResponse from basic_memory.models import Project from basic_memory.repository.project_repository import ProjectRepository @@ -457,14 +457,12 @@ async def update_project_by_id( # Update using project name (service layer still uses names internally) if path: - try: + # A path update changes the filesystem source behind every + # resource key while project and entity UUIDs stay stable. The + # service can update config before its DB follow-up completes, + # so invalidate on every attempted move completion path. + async with invalidate_cache(read_cache, project_id): await project_service.move_project(old_project.name, path) - finally: - # A path update changes the filesystem source behind every - # resource key while project and entity UUIDs stay stable. The - # service can update config before its DB follow-up completes, - # so invalidate on every attempted move completion path. - await invalidate_project_read_cache(read_cache, project_id) elif is_active is not None: await project_service.update_project(old_project.name, is_active=is_active) diff --git a/src/basic_memory/api/v2/routers/resource_router.py b/src/basic_memory/api/v2/routers/resource_router.py index f8b4379ed..7da151856 100644 --- a/src/basic_memory/api/v2/routers/resource_router.py +++ b/src/basic_memory/api/v2/routers/resource_router.py @@ -18,22 +18,17 @@ import logfire from basic_memory import db from basic_memory.deps import ( + ConfiguredReadCacheDep, ProjectConfigV2ExternalDep, FileServiceV2ExternalDep, EntityRepositoryV2ExternalDep, NoteContentQueryServiceDep, - ReadCacheDep, SessionMakerDep, ) from basic_memory.read_cache import ( ReadCacheKey, ReadCacheOperation, read_cache_request_digest, - read_through_model, -) -from basic_memory.read_cache.policy import ( - READ_CACHE_MAX_PAYLOAD_BYTES, - READ_CACHE_TTL_SECONDS, ) from basic_memory.utils import validate_project_path @@ -59,7 +54,7 @@ async def get_resource_content( entity_repository: EntityRepositoryV2ExternalDep, file_service: FileServiceV2ExternalDep, note_content_query_service: NoteContentQueryServiceDep, - read_cache: ReadCacheDep, + read_cache: ConfiguredReadCacheDep, session_maker: SessionMakerDep, project_id: str = Path(..., description="Project external UUID"), entity_id: str = Path(..., description="Entity external UUID"), @@ -87,7 +82,21 @@ async def get_resource_content( ): logger.debug(f"V2 Getting content for project {project_id}, entity_id: {entity_id}") - async def load() -> CachedResourceResponse: + cache_key = ReadCacheKey( + project_id=project_id, + operation=ReadCacheOperation.resource, + request_digest=read_cache_request_digest(entity_id), + ) + async with read_cache.read( + key=cache_key, + model_type=CachedResourceResponse, + ) as cached: + if cached.value is not None: + return Response( + content=cached.value.content, + media_type=cached.value.media_type, + ) + # Keep the DB session open only for the lookups; close it before the # filesystem I/O below so large/slow resource reads don't pin a pooled # connection (and an open read transaction on Postgres) for their duration. @@ -99,10 +108,15 @@ async def load() -> CachedResourceResponse: read_cache=read_cache, ) if note_resource is not None: - return CachedResourceResponse( + resource = CachedResourceResponse( content=note_resource.content.encode("utf-8"), media_type=note_resource.content_type, ) + cached.value = resource + return Response( + content=resource.content, + media_type=resource.media_type, + ) with logfire.span( "api.resource.get_content.load_entity", @@ -112,7 +126,10 @@ async def load() -> CachedResourceResponse: ): entity = await entity_repository.get_by_external_id(session, entity_id) if not entity: - raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found") + raise HTTPException( + status_code=404, + detail=f"Entity {entity_id} not found", + ) # Copy the scalar columns needed for file I/O so the session can close. entity_file_path = entity.file_path entity_db_id = entity.id @@ -154,25 +171,13 @@ async def load() -> CachedResourceResponse: content = await file_service.read_file_bytes(entity_file_path) content_type = file_service.content_type(entity_file_path) - return CachedResourceResponse( + resource = CachedResourceResponse( content=content, media_type=content_type, ) - - resource = await read_through_model( - cache=read_cache, - key=ReadCacheKey( - project_id=project_id, - operation=ReadCacheOperation.resource, - request_digest=read_cache_request_digest(entity_id), - ), - model_type=CachedResourceResponse, - load=load, - ttl_seconds=READ_CACHE_TTL_SECONDS, - max_payload_bytes=READ_CACHE_MAX_PAYLOAD_BYTES, - should_store=_is_markdown_resource, - ) - return Response( - content=resource.content, - media_type=resource.media_type, - ) + cached.cacheable = _is_markdown_resource(resource) + cached.value = resource + return Response( + content=resource.content, + media_type=resource.media_type, + ) diff --git a/src/basic_memory/deps/__init__.py b/src/basic_memory/deps/__init__.py index f2dca6f7a..4cc77e7f4 100644 --- a/src/basic_memory/deps/__init__.py +++ b/src/basic_memory/deps/__init__.py @@ -39,7 +39,9 @@ ) from basic_memory.deps.read_cache import ( + get_configured_read_cache, get_read_cache, + ConfiguredReadCacheDep, ReadCacheDep, ) @@ -129,7 +131,9 @@ "get_project_config_v2_external", "ProjectConfigV2ExternalDep", # Read cache + "get_configured_read_cache", "get_read_cache", + "ConfiguredReadCacheDep", "ReadCacheDep", # Repositories "get_entity_repository_v2_external", diff --git a/src/basic_memory/deps/read_cache.py b/src/basic_memory/deps/read_cache.py index 6cf95c1e3..47c4edb84 100644 --- a/src/basic_memory/deps/read_cache.py +++ b/src/basic_memory/deps/read_cache.py @@ -4,14 +4,22 @@ from fastapi import Depends, Request -from basic_memory.read_cache import ReadCache +from basic_memory.read_cache import ConfiguredReadCache, ReadCache +from basic_memory.read_cache.policy import ( + READ_CACHE_MAX_PAYLOAD_BYTES, + READ_CACHE_TTL_SECONDS, +) def get_read_cache(request: Request) -> ReadCache: """Return the host-injected cache or the container's no-op default.""" - container = getattr(request.app.state, "container", None) - if container is not None: + try: + container = request.app.state.container + except AttributeError: + pass + else: return container.read_cache + # Deferred import avoids api.app -> routers -> deps circular initialization. from basic_memory.api.container import resolve_container @@ -19,3 +27,15 @@ def get_read_cache(request: Request) -> ReadCache: ReadCacheDep = Annotated[ReadCache, Depends(get_read_cache)] + + +def get_configured_read_cache(read_cache: ReadCacheDep) -> ConfiguredReadCache: + """Bind the host cache to Basic Memory's API read policy.""" + return ConfiguredReadCache( + backend=read_cache, + ttl_seconds=READ_CACHE_TTL_SECONDS, + max_payload_bytes=READ_CACHE_MAX_PAYLOAD_BYTES, + ) + + +ConfiguredReadCacheDep = Annotated[ConfiguredReadCache, Depends(get_configured_read_cache)] diff --git a/src/basic_memory/index/local_moves.py b/src/basic_memory/index/local_moves.py index 91af49dd3..f23d5a91b 100644 --- a/src/basic_memory/index/local_moves.py +++ b/src/basic_memory/index/local_moves.py @@ -35,7 +35,7 @@ STORAGE_OBJECT_CREATED_EVENTS, STORAGE_OBJECT_DELETED_EVENT, ) -from basic_memory.read_cache import ReadCache, invalidate_project_read_cache +from basic_memory.read_cache import ReadCache, invalidate_cache from basic_memory.services import FileService @@ -185,7 +185,10 @@ async def process_moves( removed_event_indexes: set[int] = set() if moved_files: - try: + # Move pairs bypass ordinary file/delete completion callbacks. + # Invalidate after maintenance and search refresh so cached paths, + # permalinks, and relations cannot retain the pre-move state. + async with invalidate_cache(self.read_cache, self.project_external_id): move_run = await self.maintenance_runner.run_move_batches( moved_files=moved_files, batch_size=self.batch_size, @@ -197,14 +200,6 @@ async def process_moves( await self.moved_entity_search_refresher.refresh_moved_entities( sorted(refresh_entity_ids) ) - finally: - # Move pairs bypass ordinary file/delete completion callbacks. - # Invalidate after maintenance and search refresh so cached paths, - # permalinks, and relations cannot retain the pre-move state. - await invalidate_project_read_cache( - self.read_cache, - self.project_external_id, - ) moved_old_paths = set(moved_files) - set(move_run.missing_paths) moved_new_paths = {moved_files[old_path] for old_path in moved_old_paths} diff --git a/src/basic_memory/index/local_project.py b/src/basic_memory/index/local_project.py index a3919efb4..35367ff04 100644 --- a/src/basic_memory/index/local_project.py +++ b/src/basic_memory/index/local_project.py @@ -79,7 +79,7 @@ resolve_project_index_completion_relations, ) from basic_memory.models import Entity, Project -from basic_memory.read_cache import NullReadCache, ReadCache, invalidate_project_read_cache +from basic_memory.read_cache import NullReadCache, ReadCache, invalidate_cache from basic_memory.repository import NoteContentRepository from basic_memory.runtime.jobs import ( RuntimeIndexFileBatchJobRequest, @@ -761,7 +761,13 @@ async def run_local_project_index( runtime: LocalProjectIndexRuntime, ) -> ProjectIndexCoordinatorResult: """Run project-wide local indexing through the storage-neutral coordinator.""" - try: + # The coordinator commits moves, deletes, and file batches incrementally. + # Invalidate even when a later batch or vector sync raises so already + # published changes cannot remain behind the previous generation. + async with invalidate_cache( + runtime.read_cache, + request.project.project_external_id, + ): result = await run_project_index_coordinator( request, coordinator_job_id=runtime.coordinator_job_id, @@ -775,16 +781,14 @@ async def run_local_project_index( batch_size=runtime.batch_size, embedding_vector_sync=runtime.embedding_vector_sync, ) - finally: - # The coordinator commits moves, deletes, and file batches incrementally. - # Invalidate even when a later batch or vector sync raises so already - # published changes cannot remain behind the previous generation. - await invalidate_project_read_cache( + if runtime.completion_relation_runtime is not None: + # Relation repair can mutate cached entity responses after the first + # invalidation. Clear any value filled during that window, including + # when repair commits partial progress before raising. + async with invalidate_cache( runtime.read_cache, request.project.project_external_id, - ) - if runtime.completion_relation_runtime is not None: - try: + ): await resolve_project_index_completion_relations( ProjectIndexRelationResolutionContext( project_id=request.project.project_id, @@ -792,12 +796,4 @@ async def run_local_project_index( ), runtime.completion_relation_runtime, ) - finally: - # Relation repair can mutate cached entity responses after the first - # invalidation. Clear any value filled during that window, including - # when repair commits partial progress before raising. - await invalidate_project_read_cache( - runtime.read_cache, - request.project.project_external_id, - ) return result diff --git a/src/basic_memory/index/local_runtime.py b/src/basic_memory/index/local_runtime.py index 8906beedf..b49c80dcb 100644 --- a/src/basic_memory/index/local_runtime.py +++ b/src/basic_memory/index/local_runtime.py @@ -57,7 +57,12 @@ RepositoryExternalFileDeleteEntities, ) from basic_memory.models import Entity, Project -from basic_memory.read_cache import NullReadCache, ReadCache, invalidate_project_read_cache +from basic_memory.read_cache import ( + NullReadCache, + ReadCache, + invalidate_cache, + invalidate_project_read_cache, +) from basic_memory.repository import NoteContentRepository from basic_memory.runtime.projects import ProjectRuntimeReference from basic_memory.runtime.storage import ( @@ -177,7 +182,13 @@ async def index_file_completed( ) ) if relation_request is not None: - try: + # Relation repair changes cached entity payloads after indexing. + # A second generation bump closes the fill window opened by the + # first post-index invalidation, even after partial failure. + async with invalidate_cache( + self.read_cache, + self.project.project_external_id, + ): relation_result = await resolve_project_relations(self.relation_runtime) logger.info( "Local event-index relation repair completed", @@ -187,14 +198,6 @@ async def index_file_completed( remaining=relation_result.remaining, passes=relation_result.passes, ) - finally: - # Relation repair changes cached entity payloads after indexing. - # A second generation bump closes the fill window opened by the - # first post-index invalidation, even after partial failure. - await invalidate_project_read_cache( - self.read_cache, - self.project.project_external_id, - ) # --- Semantic embedding --- # Trigger: a file was (re)indexed and semantic embeddings are enabled. @@ -231,7 +234,13 @@ async def delete_file_completed( self.read_cache, self.project.project_external_id, ) - try: + # Cleanup may rewrite relations on surviving entities. Invalidate + # values filled after the delete became visible, including partial + # cleanup progress followed by an error. + async with invalidate_cache( + self.read_cache, + self.project.project_external_id, + ): if not isinstance(result.deleted_entity, Entity): raise RuntimeError( "Local external file delete returned an incomplete entity result" @@ -240,14 +249,6 @@ async def delete_file_completed( await self.relation_cleanup_search_refresher.refresh_moved_entities( tuple(sorted(result.relation_cleanup_entity_ids)), ) - finally: - # Cleanup may rewrite relations on surviving entities. Invalidate - # values filled after the delete became visible, including partial - # cleanup progress followed by an error. - await invalidate_project_read_cache( - self.read_cache, - self.project.project_external_id, - ) async def skip_event(self, operation: RuntimeStorageEventOperation) -> None: logger.debug( diff --git a/src/basic_memory/index/local_schedulers.py b/src/basic_memory/index/local_schedulers.py index 5e32cd382..75ff25850 100644 --- a/src/basic_memory/index/local_schedulers.py +++ b/src/basic_memory/index/local_schedulers.py @@ -20,7 +20,7 @@ RelationResolutionRuntime, resolve_project_relations, ) -from basic_memory.read_cache import ReadCache, invalidate_project_read_cache +from basic_memory.read_cache import ReadCache, invalidate_cache from basic_memory.runtime.vector_sync import EntityVectorSync # --- Background Task Machinery --- @@ -223,16 +223,11 @@ async def _resolve_after_debounce(self, project_id: int) -> None: # Writes up to here are covered by the scan we are about to run, so only # writes that land DURING the scan should force a re-run. _dirty_relation_resolution.discard(project_id) - try: + # Relation resolution commits entity changes after the index pass. + # A second bump closes the window in which an intermediate entity + # response could have populated the current generation. + async with invalidate_cache(self.read_cache, self.project_external_id): await resolve_project_relations(self.relation_runtime) - finally: - # Relation resolution commits entity changes after the index pass. - # A second bump closes the window in which an intermediate entity - # response could have populated the current generation. - await invalidate_project_read_cache( - self.read_cache, - self.project_external_id, - ) finally: rerun = project_id in _dirty_relation_resolution _dirty_relation_resolution.discard(project_id) diff --git a/src/basic_memory/index/note_content_materialization.py b/src/basic_memory/index/note_content_materialization.py index e4e7ef73f..0de4e9dde 100644 --- a/src/basic_memory/index/note_content_materialization.py +++ b/src/basic_memory/index/note_content_materialization.py @@ -51,7 +51,7 @@ NoteFileVacateRepository, RecoverableVacate, ) -from basic_memory.read_cache import ReadCache, invalidate_project_read_cache +from basic_memory.read_cache import ReadCache, invalidate_cache from basic_memory.schemas.response import ObservationResponse, RelationResponse from basic_memory.services.file_service import FileService @@ -640,7 +640,10 @@ async def _materialize_write_now( ) -> RuntimeAcceptedNoteChange[RuntimeNoteContentResponsePayload]: if accepted.materialization is None: # pragma: no cover - guarded by caller return accepted - try: + # The accepted-write invalidation runs before deferred materialization. + # Invalidate again after status publication and indexing so a read + # filled during that window cannot survive the terminal state. + async with invalidate_cache(self.read_cache, self.project_external_id): storage = LocalNoteContentStorage(self.file_service) cleanup_enqueuer = InlineNoteFileDeleteEnqueuer( storage, @@ -695,14 +698,6 @@ async def _materialize_write_now( ), ) return accepted - finally: - # The accepted-write invalidation runs before deferred materialization. - # Invalidate again after status publication and indexing so a read - # filled during that window cannot survive the terminal state. - await invalidate_project_read_cache( - self.read_cache, - self.project_external_id, - ) async def materialize_delete_change( self, diff --git a/src/basic_memory/read_cache/__init__.py b/src/basic_memory/read_cache/__init__.py index a5bc41832..3a54f59e5 100644 --- a/src/basic_memory/read_cache/__init__.py +++ b/src/basic_memory/read_cache/__init__.py @@ -3,6 +3,7 @@ from basic_memory.read_cache.contract import ( ReadCache, ReadCacheDataError, + ReadCacheInvalidator, ReadCacheInvalidationStatus, ReadCacheKey, ReadCacheLookup, @@ -10,22 +11,27 @@ ReadCacheStoreStatus, ReadCacheUnavailable, ) -from basic_memory.read_cache.invalidation import invalidate_project_read_cache +from basic_memory.read_cache.invalidation import ( + invalidate_cache, + invalidate_project_read_cache, +) from basic_memory.read_cache.keys import read_cache_request_digest from basic_memory.read_cache.null import NullReadCache -from basic_memory.read_cache.read_through import read_through_model +from basic_memory.read_cache.read_through import ConfiguredReadCache __all__ = [ + "ConfiguredReadCache", "NullReadCache", "ReadCache", "ReadCacheDataError", + "ReadCacheInvalidator", "ReadCacheInvalidationStatus", "ReadCacheKey", "ReadCacheLookup", "ReadCacheOperation", "ReadCacheStoreStatus", "ReadCacheUnavailable", + "invalidate_cache", "invalidate_project_read_cache", "read_cache_request_digest", - "read_through_model", ] diff --git a/src/basic_memory/read_cache/contract.py b/src/basic_memory/read_cache/contract.py index 4f855dd1c..2e96c238b 100644 --- a/src/basic_memory/read_cache/contract.py +++ b/src/basic_memory/read_cache/contract.py @@ -57,9 +57,12 @@ def __post_init__(self) -> None: if len(self.request_digest) != 64: raise ValueError("read-cache request_digest must be a SHA-256 hex digest") try: - bytes.fromhex(self.request_digest) + decoded_digest = bytes.fromhex(self.request_digest) except ValueError as error: raise ValueError("read-cache request_digest must be a SHA-256 hex digest") from error + if len(decoded_digest) != 32: + raise ValueError("read-cache request_digest must be a SHA-256 hex digest") + object.__setattr__(self, "request_digest", self.request_digest.lower()) @dataclass(frozen=True, slots=True) @@ -73,6 +76,10 @@ class ReadCacheLookup: generation: str | None payload: bytes | None = None + def __post_init__(self) -> None: + if self.generation is None and self.payload is not None: + raise ValueError("read-cache payload requires a lookup generation") + @property def is_hit(self) -> bool: return self.payload is not None @@ -86,8 +93,15 @@ class ReadCacheDataError(RuntimeError): """A cached value violated the Basic Memory cache encoding contract.""" -class ReadCache(Protocol): - """Best-effort read cache with project-generation invalidation.""" +class ReadCacheInvalidator(Protocol): + """Capability for making one project's cached reads unreachable.""" + + async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: + """Make every existing cached value for one project unreachable.""" + + +class ReadCache(ReadCacheInvalidator, Protocol): + """Best-effort read-through backend with project-generation invalidation.""" async def lookup(self, key: ReadCacheKey) -> ReadCacheLookup: """Return a cached payload and the generation observed by this lookup.""" @@ -101,6 +115,3 @@ async def store( ttl_seconds: int, ) -> ReadCacheStoreStatus: """Store a payload under the generation observed by ``lookup``.""" - - async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: - """Make every existing cached value for one project unreachable.""" diff --git a/src/basic_memory/read_cache/invalidation.py b/src/basic_memory/read_cache/invalidation.py index 3d6c07c6a..a55d4520e 100644 --- a/src/basic_memory/read_cache/invalidation.py +++ b/src/basic_memory/read_cache/invalidation.py @@ -1,10 +1,13 @@ """Best-effort project invalidation shared by mutation and indexing runtimes.""" +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + from loguru import logger import logfire from basic_memory.read_cache.contract import ( - ReadCache, + ReadCacheInvalidator, ReadCacheInvalidationStatus, ReadCacheUnavailable, ) @@ -21,7 +24,7 @@ def _record_invalidation_event(event: ReadCacheInvalidationStatus) -> None: async def invalidate_project_read_cache( - cache: ReadCache, + cache: ReadCacheInvalidator, project_id: str, ) -> ReadCacheInvalidationStatus: """Invalidate one project without failing an already-committed mutation.""" @@ -43,3 +46,15 @@ async def invalidate_project_read_cache( _record_invalidation_event(status) span.set_attribute("cache.outcome", status.value) return status + + +@asynccontextmanager +async def invalidate_cache( + cache: ReadCacheInvalidator, + project_id: str, +) -> AsyncIterator[None]: + """Invalidate one project's read cache when the enclosed operation exits.""" + try: + yield + finally: + await invalidate_project_read_cache(cache, project_id) diff --git a/src/basic_memory/read_cache/keys.py b/src/basic_memory/read_cache/keys.py index 66ac1f080..cd4c1f21f 100644 --- a/src/basic_memory/read_cache/keys.py +++ b/src/basic_memory/read_cache/keys.py @@ -25,11 +25,12 @@ def read_cache_request_digest(*parts: str) -> str: class RedisReadCacheKeys: """Redis keys for one project generation and canonical request.""" - generation: str - data: str + generation_key: str + data_key: str def _redis_read_cache_cluster_scope(*, namespace: str, project_id: str) -> str: + namespace = namespace.strip() if not namespace: raise ValueError("read-cache namespace must not be empty") @@ -40,13 +41,12 @@ def _redis_read_cache_cluster_scope(*, namespace: str, project_id: str) -> str: return f"{{{scope_digest}}}" -def redis_read_cache_generation_key( +def _redis_read_cache_key_base( *, prefix: str, namespace: str, project_id: str, ) -> str: - """Build the generation key shared by every cached read in one project.""" if not prefix or any(character.isspace() for character in prefix): raise ValueError("read-cache prefix must be non-empty and contain no whitespace") @@ -54,7 +54,22 @@ def redis_read_cache_generation_key( namespace=namespace, project_id=project_id, ) - return f"{prefix}:{cluster_scope}:generation" + return f"{prefix}:{cluster_scope}" + + +def redis_read_cache_generation_key( + *, + prefix: str, + namespace: str, + project_id: str, +) -> str: + """Build the generation key shared by every cached read in one project.""" + key_base = _redis_read_cache_key_base( + prefix=prefix, + namespace=namespace, + project_id=project_id, + ) + return f"{key_base}:generation" def redis_read_cache_keys( @@ -64,16 +79,12 @@ def redis_read_cache_keys( key: ReadCacheKey, ) -> RedisReadCacheKeys: """Build versioned Redis keys without exposing namespace values.""" - cluster_scope = _redis_read_cache_cluster_scope( - namespace=namespace, - project_id=key.project_id, - ) - generation = redis_read_cache_generation_key( + key_base = _redis_read_cache_key_base( prefix=prefix, namespace=namespace, project_id=key.project_id, ) return RedisReadCacheKeys( - generation=generation, - data=f"{prefix}:{cluster_scope}:{key.operation.value}:{key.request_digest}", + generation_key=f"{key_base}:generation", + data_key=f"{key_base}:{key.operation.value}:{key.request_digest}", ) diff --git a/src/basic_memory/read_cache/read_through.py b/src/basic_memory/read_cache/read_through.py index 7549a7a33..973ea0174 100644 --- a/src/basic_memory/read_cache/read_through.py +++ b/src/basic_memory/read_cache/read_through.py @@ -1,12 +1,15 @@ """Typed read-through behavior shared by cacheable API boundaries.""" -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass import logfire from pydantic import BaseModel from basic_memory.read_cache.contract import ( ReadCache, + ReadCacheInvalidationStatus, ReadCacheKey, ReadCacheUnavailable, ) @@ -22,90 +25,125 @@ def _record_event(key: ReadCacheKey, event: str) -> None: ) -async def read_through_model[ModelT: BaseModel]( - *, - cache: ReadCache, - key: ReadCacheKey, - model_type: type[ModelT], - load: Callable[[], Awaitable[ModelT]], - ttl_seconds: int, - max_payload_bytes: int, - should_store: Callable[[ModelT], bool] | None = None, -) -> ModelT: - """Return a validated cached model or load and best-effort cache it.""" - if ttl_seconds <= 0: - raise ValueError("read-cache ttl_seconds must be positive") - if max_payload_bytes <= 0: - raise ValueError("read-cache max_payload_bytes must be positive") - - with logfire.span( - "read_cache.read_through", - operation=key.operation.value, - ) as span: - try: - lookup = await cache.lookup(key) - except ReadCacheUnavailable: - # Trigger: Redis is unreachable or timed out. - # Why: the database or storage path remains authoritative. - # Outcome: return fresh data without attempting another cache operation. - _record_event(key, "bypass") - span.set_attribute("cache.outcome", "bypass") - return await load() - - if lookup.generation is None: - # Trigger: the host selected the no-op cache implementation. - # Why: an optional cache must not serialize every response merely to - # discover that storage is disabled. - # Outcome: execute only the authoritative read path. - _record_event(key, "disabled") - span.set_attribute("cache.outcome", "disabled") - return await load() - - if lookup.payload is not None: - _record_event(key, "hit") - span.set_attributes( - { - "cache.outcome": "hit", - "cache.payload_bytes": len(lookup.payload), - } - ) - return model_type.model_validate_json(lookup.payload) - - _record_event(key, "miss") - value = await load() - if should_store is not None and not should_store(value): - _record_event(key, "ineligible") - span.set_attribute("cache.outcome", "ineligible") - return value - - payload = value.model_dump_json().encode("utf-8") - if len(payload) > max_payload_bytes: - _record_event(key, "oversize") +@dataclass(slots=True) +class ReadCacheScope[ModelT: BaseModel]: + """Mutable state exchanged with one configured read-cache scope.""" + + value: ModelT | None = None + cacheable: bool = True + + def require_value(self) -> ModelT: + """Return the authoritative value supplied by the route.""" + if self.value is None: + raise RuntimeError("read-through cache scope exited without a result") + return self.value + + +@dataclass(frozen=True, slots=True) +class ConfiguredReadCache: + """Request dependency that binds one cache backend to read policy.""" + + backend: ReadCache + ttl_seconds: int + max_payload_bytes: int + + def __post_init__(self) -> None: + if self.ttl_seconds <= 0: + raise ValueError("read-cache ttl_seconds must be positive") + if self.max_payload_bytes <= 0: + raise ValueError("read-cache max_payload_bytes must be positive") + + async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: + """Delegate invalidation without exposing the backend to API routes.""" + return await self.backend.invalidate_project(project_id) + + @asynccontextmanager + async def read[ModelT: BaseModel]( + self, + *, + key: ReadCacheKey, + model_type: type[ModelT], + ) -> AsyncIterator[ReadCacheScope[ModelT]]: + """Yield a cached model or store the authoritative value supplied by the route.""" + with logfire.span( + "read_cache.read_through", + operation=key.operation.value, + ) as span: + try: + lookup = await self.backend.lookup(key) + except ReadCacheUnavailable: + # Trigger: Redis is unreachable or timed out. + # Why: the database or storage path remains authoritative. + # Outcome: return fresh data without attempting another cache operation. + _record_event(key, "bypass") + span.set_attribute("cache.outcome", "bypass") + result = ReadCacheScope[ModelT]() + yield result + result.require_value() + return + + if lookup.generation is None: + # Trigger: the host selected the no-op cache implementation. + # Why: an optional cache must not serialize every response merely to + # discover that storage is disabled. + # Outcome: execute only the authoritative read path. + _record_event(key, "disabled") + span.set_attribute("cache.outcome", "disabled") + result = ReadCacheScope[ModelT]() + yield result + result.require_value() + return + + if lookup.payload is not None: + _record_event(key, "hit") + span.set_attributes( + { + "cache.outcome": "hit", + "cache.payload_bytes": len(lookup.payload), + } + ) + yield ReadCacheScope( + value=model_type.model_validate_json(lookup.payload), + cacheable=False, + ) + return + + _record_event(key, "miss") + result = ReadCacheScope[ModelT]() + yield result + value = result.require_value() + if not result.cacheable: + _record_event(key, "ineligible") + span.set_attribute("cache.outcome", "ineligible") + return + + payload = value.model_dump_json().encode("utf-8") + if len(payload) > self.max_payload_bytes: + _record_event(key, "oversize") + span.set_attributes( + { + "cache.outcome": "oversize", + "cache.payload_bytes": len(payload), + } + ) + return + + try: + store_status = await self.backend.store( + key, + lookup, + payload, + ttl_seconds=self.ttl_seconds, + ) + except ReadCacheUnavailable: + _record_event(key, "store_unavailable") + span.set_attribute("cache.outcome", "store_unavailable") + return + + _record_event(key, store_status.value) span.set_attributes( { - "cache.outcome": "oversize", + "cache.outcome": store_status.value, "cache.payload_bytes": len(payload), } ) - return value - - try: - store_status = await cache.store( - key, - lookup, - payload, - ttl_seconds=ttl_seconds, - ) - except ReadCacheUnavailable: - _record_event(key, "store_unavailable") - span.set_attribute("cache.outcome", "store_unavailable") - return value - - _record_event(key, store_status.value) - span.set_attributes( - { - "cache.outcome": store_status.value, - "cache.payload_bytes": len(payload), - } - ) - return value diff --git a/src/basic_memory/read_cache/redis.py b/src/basic_memory/read_cache/redis.py index 53fd79829..9ba7e0046 100644 --- a/src/basic_memory/read_cache/redis.py +++ b/src/basic_memory/read_cache/redis.py @@ -144,8 +144,8 @@ async def lookup(self, key: ReadCacheKey) -> ReadCacheLookup: generation_value, cached_value = await self._client.eval( _LOOKUP_SCRIPT, 2, - keys.generation, - keys.data, + keys.generation_key, + keys.data_key, uuid4().hex.encode("ascii"), self._generation_ttl_seconds, ) @@ -184,8 +184,8 @@ async def store( stored = await self._client.eval( _STORE_IF_CURRENT_SCRIPT, 2, - keys.generation, - keys.data, + keys.generation_key, + keys.data_key, generation, encoded, ttl_seconds, diff --git a/src/basic_memory/services/note_content_reads.py b/src/basic_memory/services/note_content_reads.py index 2b5957617..0225d50d0 100644 --- a/src/basic_memory/services/note_content_reads.py +++ b/src/basic_memory/services/note_content_reads.py @@ -15,7 +15,7 @@ run_note_content_read_repair_with_default_reconciler, ) from basic_memory.models import Entity, NoteContent, Project -from basic_memory.read_cache import ReadCache, invalidate_project_read_cache +from basic_memory.read_cache import ReadCacheInvalidator, invalidate_project_read_cache from basic_memory.runtime.note_content import ( RuntimeNoteContentResource, RuntimeNoteContentResponsePayload, @@ -92,7 +92,7 @@ async def get_note_entity_payload_with_read_repair( entity_external_id: str, session: AsyncSession | None = None, source: str = "read_repair", - read_cache: ReadCache | None = None, + read_cache: ReadCacheInvalidator | None = None, ) -> RuntimeNoteContentResponsePayload | None: """Return entity payload, repairing missing note_content when a reader exists.""" payload = await self.get_note_entity_payload( @@ -154,7 +154,7 @@ async def get_note_resource_with_read_repair( entity_external_id: str, session: AsyncSession | None = None, source: str = "read_repair", - read_cache: ReadCache | None = None, + read_cache: ReadCacheInvalidator | None = None, ) -> RuntimeNoteContentResource | None: """Return markdown resource, repairing missing note_content when possible.""" resource = await self.get_note_resource( diff --git a/test-int/read_cache/test_api_read_cache.py b/test-int/read_cache/test_api_read_cache.py index 30f93992b..090b4d453 100644 --- a/test-int/read_cache/test_api_read_cache.py +++ b/test-int/read_cache/test_api_read_cache.py @@ -28,7 +28,7 @@ from basic_memory.models import Project from basic_memory.models.knowledge import Entity from basic_memory.read_cache import ( - ReadCache, + ReadCacheInvalidator, ReadCacheInvalidationStatus, ReadCacheKey, ReadCacheOperation, @@ -117,7 +117,7 @@ async def get_note_resource_with_read_repair( project_external_id: str, entity_external_id: str, session: AsyncSession, - read_cache: ReadCache, + read_cache: ReadCacheInvalidator, ) -> None: del project_external_id, entity_external_id, session, read_cache return None @@ -262,7 +262,7 @@ async def test_entity_resolve_and_markdown_reads_cache_then_freshened_write_inva namespace=redis_cache.namespace, key=key, ) - assert await redis_cache.client.exists(redis_keys.data) == 1 + assert await redis_cache.client.exists(redis_keys.data_key) == 1 generation_key = redis_read_cache_generation_key( prefix=redis_cache.prefix, @@ -369,7 +369,7 @@ async def test_non_markdown_resource_is_never_cached( namespace=redis_cache.namespace, key=key, ) - assert await redis_cache.client.exists(redis_keys.data) == 0 + assert await redis_cache.client.exists(redis_keys.data_key) == 0 @pytest.mark.asyncio diff --git a/test-int/read_cache/test_redis_read_cache.py b/test-int/read_cache/test_redis_read_cache.py index 4871087c3..0fda9e571 100644 --- a/test-int/read_cache/test_redis_read_cache.py +++ b/test-int/read_cache/test_redis_read_cache.py @@ -12,6 +12,7 @@ from redis.asyncio import Redis from basic_memory.read_cache import ( + ConfiguredReadCache, NullReadCache, ReadCacheDataError, ReadCacheInvalidationStatus, @@ -20,9 +21,9 @@ ReadCacheOperation, ReadCacheStoreStatus, ReadCacheUnavailable, + invalidate_cache, invalidate_project_read_cache, read_cache_request_digest, - read_through_model, ) from basic_memory.read_cache.keys import ( redis_read_cache_generation_key, @@ -112,10 +113,10 @@ async def test_generation_metadata_expires_and_tracks_response_lifetime( # Existing deployments may already have persistent generation keys. A # lookup migrates them to bounded metadata without changing the token. legacy_generation = b"0" * 32 - await redis_cache.client.set(redis_keys.generation, legacy_generation) + await redis_cache.client.set(redis_keys.generation_key, legacy_generation) lookup = await cache.lookup(long_lived_key) assert lookup.generation == legacy_generation.decode("ascii") - assert await redis_cache.client.pttl(redis_keys.generation) > 0 + assert await redis_cache.client.pttl(redis_keys.generation_key) > 0 stored = await cache.store( long_lived_key, @@ -123,8 +124,8 @@ async def test_generation_metadata_expires_and_tracks_response_lifetime( b"long-lived payload", ttl_seconds=3, ) - generation_ttl = await redis_cache.client.pttl(redis_keys.generation) - data_ttl = await redis_cache.client.pttl(redis_keys.data) + generation_ttl = await redis_cache.client.pttl(redis_keys.generation_key) + data_ttl = await redis_cache.client.pttl(redis_keys.data_key) assert stored is ReadCacheStoreStatus.stored assert generation_ttl >= data_ttl > 2_000 @@ -136,19 +137,19 @@ async def test_generation_metadata_expires_and_tracks_response_lifetime( b"short-lived payload", ttl_seconds=1, ) - assert await redis_cache.client.pttl(redis_keys.generation) > 2_000 + assert await redis_cache.client.pttl(redis_keys.generation_key) > 2_000 - generation_before_invalidation = await redis_cache.client.get(redis_keys.generation) + generation_before_invalidation = await redis_cache.client.get(redis_keys.generation_key) await cache.invalidate_project(PROJECT_ID) - generation_after_invalidation = await redis_cache.client.get(redis_keys.generation) - invalidated_ttl = await redis_cache.client.pttl(redis_keys.generation) + generation_after_invalidation = await redis_cache.client.get(redis_keys.generation_key) + invalidated_ttl = await redis_cache.client.pttl(redis_keys.generation_key) assert generation_before_invalidation is not None assert isinstance(generation_after_invalidation, bytes) assert generation_after_invalidation != generation_before_invalidation assert 0 < invalidated_ttl <= 1_000 await asyncio.sleep(1.1) - assert not await redis_cache.client.exists(redis_keys.generation) + assert not await redis_cache.client.exists(redis_keys.generation_key) after_expiry = await cache.lookup(long_lived_key) assert not after_expiry.is_hit assert after_expiry.generation != generation_after_invalidation.decode("ascii") @@ -219,6 +220,51 @@ async def test_project_invalidation_rejects_a_concurrent_stale_fill( assert (await redis_cache.cache.lookup(other_project_key)).payload == b"other project" +@pytest.mark.asyncio +async def test_invalidation_context_invalidates_real_generation_on_exit( + redis_cache: RedisCacheHarness, +) -> None: + key = _key(request="context-normal-exit") + await redis_cache.cache.lookup(key) + generation_key = redis_read_cache_generation_key( + prefix=redis_cache.prefix, + namespace=redis_cache.namespace, + project_id=PROJECT_ID, + ) + generation_before = await redis_cache.client.get(generation_key) + + async with invalidate_cache(redis_cache.cache, PROJECT_ID): + assert await redis_cache.client.get(generation_key) == generation_before + + generation_after = await redis_cache.client.get(generation_key) + assert generation_before is not None + assert generation_after is not None + assert generation_after != generation_before + + +@pytest.mark.asyncio +async def test_invalidation_context_invalidates_real_generation_after_body_error( + redis_cache: RedisCacheHarness, +) -> None: + key = _key(request="context-error-exit") + await redis_cache.cache.lookup(key) + generation_key = redis_read_cache_generation_key( + prefix=redis_cache.prefix, + namespace=redis_cache.namespace, + project_id=PROJECT_ID, + ) + generation_before = await redis_cache.client.get(generation_key) + + with pytest.raises(RuntimeError, match="operation failed"): + async with invalidate_cache(redis_cache.cache, PROJECT_ID): + raise RuntimeError("operation failed") + + generation_after = await redis_cache.client.get(generation_key) + assert generation_before is not None + assert generation_after is not None + assert generation_after != generation_before + + @pytest.mark.asyncio async def test_lost_generation_key_cannot_revive_old_data( redis_cache: RedisCacheHarness, @@ -232,7 +278,7 @@ async def test_lost_generation_key_cannot_revive_old_data( key=key, ) - await redis_cache.client.delete(redis_keys.generation) + await redis_cache.client.delete(redis_keys.generation_key) after_eviction = await redis_cache.cache.lookup(key) assert not after_eviction.is_hit @@ -262,19 +308,19 @@ async def test_corrupt_redis_values_fail_fast(redis_cache: RedisCacheHarness) -> key=key, ) - await redis_cache.client.set(redis_keys.data, b"missing envelope") + await redis_cache.client.set(redis_keys.data_key, b"missing envelope") with pytest.raises(ReadCacheDataError, match="invalid generation envelope"): await redis_cache.cache.lookup(key) - await redis_cache.client.set(redis_keys.data, b"\npayload") + await redis_cache.client.set(redis_keys.data_key, b"\npayload") with pytest.raises(ReadCacheDataError, match="invalid generation envelope"): await redis_cache.cache.lookup(key) - await redis_cache.client.set(redis_keys.generation, b"\xff") + await redis_cache.client.set(redis_keys.generation_key, b"\xff") with pytest.raises(ReadCacheDataError, match="invalid generation token"): await redis_cache.cache.lookup(key) - await redis_cache.client.set(redis_keys.generation, b"abcd") + await redis_cache.client.set(redis_keys.generation_key, b"abcd") with pytest.raises(ReadCacheDataError, match="invalid generation token"): await redis_cache.cache.lookup(key) @@ -339,22 +385,26 @@ async def test_real_redis_capacity_failures_are_cache_unavailable( (await redis_cache.client.config_get("maxmemory-policy"))["maxmemory-policy"] ) await redis_cache.cache.lookup(key) - - async def load() -> CachedEntity: - return CachedEntity(external_id="entity-1", title="Authoritative") + read_cache = ConfiguredReadCache( + backend=redis_cache.cache, + ttl_seconds=60, + max_payload_bytes=1_024, + ) try: await redis_cache.client.config_set("maxmemory-policy", "noeviction") await redis_cache.client.config_set("maxmemory", "1") - result = await read_through_model( - cache=redis_cache.cache, + async with read_cache.read( key=key, model_type=CachedEntity, - load=load, - ttl_seconds=60, - max_payload_bytes=1_024, - ) + ) as cached: + assert cached.value is None + cached.value = CachedEntity( + external_id="entity-1", + title="Authoritative", + ) + result = cached.value invalidation_status = await invalidate_project_read_cache( redis_cache.cache, PROJECT_ID, @@ -390,18 +440,22 @@ async def store( assert store_status is ReadCacheStoreStatus.disabled status = await cache.invalidate_project(key.project_id) assert status is ReadCacheInvalidationStatus.disabled - - async def load() -> CachedEntity: - return CachedEntity(external_id="entity-1", title="Authoritative") - - result = await read_through_model( - cache=cache, - key=key, - model_type=CachedEntity, - load=load, + read_cache = ConfiguredReadCache( + backend=cache, ttl_seconds=60, max_payload_bytes=1_024, ) + + async with read_cache.read( + key=key, + model_type=CachedEntity, + ) as cached: + assert cached.value is None + cached.value = CachedEntity( + external_id="entity-1", + title="Authoritative", + ) + result = cached.value assert result.title == "Authoritative" @@ -410,29 +464,27 @@ async def test_typed_read_through_uses_real_cached_representation( redis_cache: RedisCacheHarness, ) -> None: loads = 0 - - async def load() -> CachedEntity: - nonlocal loads - loads += 1 - return CachedEntity(external_id="entity-1", title="First") - - first = await read_through_model( - cache=redis_cache.cache, - key=_key(), - model_type=CachedEntity, - load=load, - ttl_seconds=60, - max_payload_bytes=1_024, - ) - second = await read_through_model( - cache=redis_cache.cache, - key=_key(), - model_type=CachedEntity, - load=load, + read_cache = ConfiguredReadCache( + backend=redis_cache.cache, ttl_seconds=60, max_payload_bytes=1_024, ) + results: list[CachedEntity] = [] + for _ in range(2): + async with read_cache.read( + key=_key(), + model_type=CachedEntity, + ) as cached: + if cached.value is None: + loads += 1 + cached.value = CachedEntity( + external_id="entity-1", + title="First", + ) + results.append(cached.value) + + first, second = results assert first == CachedEntity(external_id="entity-1", title="First") assert second == first assert loads == 1 @@ -443,21 +495,23 @@ async def test_typed_read_through_does_not_cache_oversize_models( redis_cache: RedisCacheHarness, ) -> None: loads = 0 - - async def load() -> CachedEntity: - nonlocal loads - loads += 1 - return CachedEntity(external_id="entity-1", title="Too large") + read_cache = ConfiguredReadCache( + backend=redis_cache.cache, + ttl_seconds=60, + max_payload_bytes=1, + ) for _ in range(2): - await read_through_model( - cache=redis_cache.cache, + async with read_cache.read( key=_key(), model_type=CachedEntity, - load=load, - ttl_seconds=60, - max_payload_bytes=1, - ) + ) as cached: + assert cached.value is None + loads += 1 + cached.value = CachedEntity( + external_id="entity-1", + title="Too large", + ) assert loads == 2 @@ -467,26 +521,53 @@ async def test_typed_read_through_does_not_cache_ineligible_models( redis_cache: RedisCacheHarness, ) -> None: loads = 0 - - async def load() -> CachedEntity: - nonlocal loads - loads += 1 - return CachedEntity(external_id="cross-project", title="Other tenant") + read_cache = ConfiguredReadCache( + backend=redis_cache.cache, + ttl_seconds=60, + max_payload_bytes=1_024, + ) for _ in range(2): - await read_through_model( - cache=redis_cache.cache, + async with read_cache.read( key=_key(operation=ReadCacheOperation.resolve), model_type=CachedEntity, - load=load, - ttl_seconds=60, - max_payload_bytes=1_024, - should_store=lambda entity: entity.external_id != "cross-project", - ) + ) as cached: + assert cached.value is None + loads += 1 + cached.cacheable = False + cached.value = CachedEntity( + external_id="cross-project", + title="Other tenant", + ) assert loads == 2 +@pytest.mark.asyncio +async def test_typed_read_through_does_not_store_after_body_error( + redis_cache: RedisCacheHarness, +) -> None: + key = _key(request="failed-authoritative-read") + read_cache = ConfiguredReadCache( + backend=redis_cache.cache, + ttl_seconds=60, + max_payload_bytes=1_024, + ) + + with pytest.raises(RuntimeError, match="authoritative read failed"): + async with read_cache.read( + key=key, + model_type=CachedEntity, + ) as cached: + cached.value = CachedEntity( + external_id="entity-1", + title="Must not be stored", + ) + raise RuntimeError("authoritative read failed") + + assert not (await redis_cache.cache.lookup(key)).is_hit + + @pytest.mark.asyncio async def test_typed_read_through_rejects_invalid_cached_models( redis_cache: RedisCacheHarness, @@ -494,19 +575,18 @@ async def test_typed_read_through_rejects_invalid_cached_models( key = _key() miss = await redis_cache.cache.lookup(key) await redis_cache.cache.store(key, miss, b'{"wrong":"shape"}', ttl_seconds=60) - - async def load() -> CachedEntity: - raise AssertionError("invalid cache data must not fall through to the loader") + read_cache = ConfiguredReadCache( + backend=redis_cache.cache, + ttl_seconds=60, + max_payload_bytes=1_024, + ) with pytest.raises(ValidationError): - await read_through_model( - cache=redis_cache.cache, + async with read_cache.read( key=key, model_type=CachedEntity, - load=load, - ttl_seconds=60, - max_payload_bytes=1_024, - ) + ): + raise AssertionError("invalid cache data must not enter the read scope") @pytest.mark.asyncio @@ -520,19 +600,23 @@ async def test_typed_read_through_bypasses_unavailable_real_redis() -> None: socket_timeout=0.05, ) cache = RedisReadCache(client=client, namespace="unavailable") - - async def load() -> CachedEntity: - return CachedEntity(external_id="entity-1", title="Authoritative") + read_cache = ConfiguredReadCache( + backend=cache, + ttl_seconds=60, + max_payload_bytes=1_024, + ) try: - result = await read_through_model( - cache=cache, + async with read_cache.read( key=_key(), model_type=CachedEntity, - load=load, - ttl_seconds=60, - max_payload_bytes=1_024, - ) + ) as cached: + assert cached.value is None + cached.value = CachedEntity( + external_id="entity-1", + title="Authoritative", + ) + result = cached.value finally: await client.aclose() @@ -570,20 +654,24 @@ async def test_typed_read_through_returns_data_when_real_redis_store_times_out( namespace="paused-store", prefix=prefix, ) - - async def load() -> CachedEntity: - await redis_cache.client.execute_command("CLIENT", "PAUSE", 200, "WRITE") - return CachedEntity(external_id="entity-1", title="Authoritative") + read_cache = ConfiguredReadCache( + backend=cache, + ttl_seconds=60, + max_payload_bytes=1_024, + ) try: - result = await read_through_model( - cache=cache, + async with read_cache.read( key=_key(), model_type=CachedEntity, - load=load, - ttl_seconds=60, - max_payload_bytes=1_024, - ) + ) as cached: + assert cached.value is None + await redis_cache.client.execute_command("CLIENT", "PAUSE", 200, "WRITE") + cached.value = CachedEntity( + external_id="entity-1", + title="Authoritative", + ) + result = cached.value assert result.title == "Authoritative" finally: await asyncio.sleep(0.25) @@ -595,27 +683,40 @@ async def load() -> CachedEntity: @pytest.mark.asyncio async def test_typed_read_through_validates_policy_before_loading() -> None: - async def load() -> CachedEntity: - raise AssertionError("invalid policy must fail before loading") - with pytest.raises(ValueError, match="ttl_seconds"): - await read_through_model( - cache=NullReadCache(), - key=_key(), - model_type=CachedEntity, - load=load, + async with ConfiguredReadCache( + backend=NullReadCache(), ttl_seconds=0, max_payload_bytes=1, - ) - with pytest.raises(ValueError, match="max_payload_bytes"): - await read_through_model( - cache=NullReadCache(), + ).read( key=_key(), model_type=CachedEntity, - load=load, + ): + raise AssertionError("invalid policy must fail before entering the read scope") + with pytest.raises(ValueError, match="max_payload_bytes"): + async with ConfiguredReadCache( + backend=NullReadCache(), ttl_seconds=1, max_payload_bytes=0, - ) + ).read( + key=_key(), + model_type=CachedEntity, + ): + raise AssertionError("invalid policy must fail before entering the read scope") + + +@pytest.mark.asyncio +async def test_typed_read_through_requires_an_authoritative_result() -> None: + with pytest.raises(RuntimeError, match="exited without a result"): + async with ConfiguredReadCache( + backend=NullReadCache(), + ttl_seconds=1, + max_payload_bytes=1, + ).read( + key=_key(), + model_type=CachedEntity, + ): + pass def test_key_validation_and_canonicalization() -> None: @@ -634,13 +735,18 @@ def test_key_validation_and_canonicalization() -> None: namespace="tenant", key=key, ) - assert redis_keys.generation == generation_key - assert f"{{{read_cache_request_digest('tenant', key.project_id)}}}" in redis_keys.data + assert redis_keys.generation_key == generation_key + assert f"{{{read_cache_request_digest('tenant', key.project_id)}}}" in redis_keys.data_key assert generation_key == redis_read_cache_generation_key( prefix="bm:read:v1", namespace="tenant", project_id=PROJECT_ID.upper(), ) + assert generation_key == redis_read_cache_generation_key( + prefix="bm:read:v1", + namespace=" tenant ", + project_id=PROJECT_ID, + ) with pytest.raises(ValueError, match="project_id"): _key(project_id="") @@ -658,6 +764,23 @@ def test_key_validation_and_canonicalization() -> None: operation=ReadCacheOperation.entity, request_digest="z" * 64, ) + with pytest.raises(ValueError, match="SHA-256"): + ReadCacheKey( + project_id=PROJECT_ID, + operation=ReadCacheOperation.entity, + request_digest=("0" * 62) + " ", + ) + uppercase_digest = read_cache_request_digest("uppercase").upper() + assert ( + ReadCacheKey( + project_id=PROJECT_ID, + operation=ReadCacheOperation.entity, + request_digest=uppercase_digest, + ).request_digest + == uppercase_digest.lower() + ) + with pytest.raises(ValueError, match="payload requires"): + ReadCacheLookup(generation=None, payload=b"orphaned") with pytest.raises(ValueError, match="prefix"): redis_read_cache_generation_key( prefix="", diff --git a/tests/test_deps.py b/tests/test_deps.py index 252da8512..2c1af30a6 100644 --- a/tests/test_deps.py +++ b/tests/test_deps.py @@ -5,8 +5,9 @@ from basic_memory.api import container as container_module from basic_memory.api.container import ApiContainer, resolve_container -from basic_memory.deps import get_app_config, validate_project_external_id +from basic_memory.deps import get_app_config, get_read_cache, validate_project_external_id from basic_memory.models.project import Project +from basic_memory.read_cache import NullReadCache from basic_memory.repository.project_repository import ProjectRepository from basic_memory.runtime.mode import resolve_runtime_mode @@ -44,6 +45,33 @@ def test_resolve_container_prefers_installed_container(app_config, monkeypatch): assert resolve_container() is installed +def test_get_read_cache_reads_lifespan_container(app_config): + """API requests get the cache the lifespan stored on app.state.""" + app = FastAPI() + cache = NullReadCache() + app.state.container = ApiContainer( + config=app_config, + mode=resolve_runtime_mode(is_test_env=True), + read_cache=cache, + ) + + assert get_read_cache(_request_for(app)) is cache + + +def test_get_read_cache_falls_back_to_composition_root(app_config, monkeypatch): + """Off-lifespan requests resolve the cache from the API composition root.""" + app = FastAPI() + cache = NullReadCache() + installed = ApiContainer( + config=app_config, + mode=resolve_runtime_mode(is_test_env=True), + read_cache=cache, + ) + monkeypatch.setattr(container_module, "_container", installed) + + assert get_read_cache(_request_for(app)) is cache + + @pytest.mark.asyncio async def test_validate_project_external_id_success( project_repository: ProjectRepository, test_project: Project, session_maker From 007b54ca4325522c380cda88f96100c625d63e24 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 29 Jul 2026 23:41:44 -0500 Subject: [PATCH 14/28] fix(api): finish cache invalidation after cancellation Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 6 + .../services/note_content_writes.py | 240 ++++++++++-------- test-int/read_cache/test_api_read_cache.py | 110 ++++++++ 3 files changed, 257 insertions(+), 99 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index 5e64c7da1..74c487562 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -87,6 +87,10 @@ tenant: the accepted-note transaction commits, again after terminal materialization/status publication and indexing, and again after relation resolution completes. This prevents a read filled between phases from surviving the later worker commit. +1. Make accepted-mutation invalidation cancellation-safe. A request or worker can be cancelled + after its database commit succeeds but before the transaction context returns. Finish the + namespace-bound generation bump before re-propagating cancellation so committed state cannot + remain hidden behind the previous generation. 1. Put project-index invalidation in a failure-safe completion boundary. Move, delete, file-index, and vector batches can commit incrementally before a later batch raises. 1. Put direct single-file and watcher file-index invalidation in failure-safe boundaries. Entity @@ -366,6 +370,8 @@ The real-Redis suite must prove: - successful writes invalidate; a rejected write also invalidates when pre-write freshening may already have published external file state, while a rolled-back transaction without such a publication does not; +- cancellation after a real accepted-note transaction commits cannot interrupt the real Redis + generation bump, including repeated cancellation while invalidation is in progress; - real Redis no-eviction capacity failures bypass cache storage and cannot fail committed-write invalidation; - authoritative read exceptions propagate without populating the missed cache key; diff --git a/src/basic_memory/services/note_content_writes.py b/src/basic_memory/services/note_content_writes.py index 77e2f25fc..7a8c7013f 100644 --- a/src/basic_memory/services/note_content_writes.py +++ b/src/basic_memory/services/note_content_writes.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio from collections.abc import AsyncIterator from contextlib import asynccontextmanager from dataclasses import dataclass @@ -148,8 +149,60 @@ def __init__( self.read_cache = read_cache if read_cache is not None else NullReadCache() async def _invalidate_project(self, project_external_id: str) -> None: - """Invalidate semantic reads after the accepted transaction commits.""" - await invalidate_project_read_cache(self.read_cache, project_external_id) + """Finish invalidation before propagating request cancellation.""" + invalidation = asyncio.create_task( + invalidate_project_read_cache(self.read_cache, project_external_id) + ) + try: + await asyncio.shield(invalidation) + except asyncio.CancelledError as cancellation: + # Trigger: request cancellation landed after a mutation may have committed. + # Why: abandoning this Redis write can leave the committed DB state hidden + # behind the previous generation until its TTL expires. + # Outcome: finish invalidation, then preserve the caller's cancellation. + cleanup_error: BaseException | None = None + while not invalidation.done(): + try: + await asyncio.shield(invalidation) + except asyncio.CancelledError: + continue + except BaseException as error: + cleanup_error = error + if cleanup_error is None: + if invalidation.cancelled(): + cleanup_error = asyncio.CancelledError( + "read-cache invalidation task was cancelled" + ) + else: + cleanup_error = invalidation.exception() + if cleanup_error is not None: + cancellation.add_note( + f"Read-cache invalidation failed during cancellation: {cleanup_error!r}" + ) + raise + + @asynccontextmanager + async def _mutation_cache_scope( + self, + project_external_id: str, + *, + invalidate_on_rejection: bool = False, + ) -> AsyncIterator[None]: + """Invalidate after a mutation can publish authoritative state.""" + try: + yield + except AcceptedNoteMutationRejected: + if invalidate_on_rejection: + await self._invalidate_project(project_external_id) + raise + except BaseException: + # Transaction exit can raise after its commit reached the database. + # Invalidation is safe after an earlier rollback and required after + # any commit whose response was interrupted. + await self._invalidate_project(project_external_id) + raise + else: + await self._invalidate_project(project_external_id) def _resolve_actor( self, @@ -215,22 +268,22 @@ async def create_note( actor_name=actor_name, ) try: - async with accepted_note_transaction(self.session_maker) as session: - accepted = await run_accepted_note_create( - session, - request=AcceptedNoteCreateMutation( - project_external_id=project_external_id, - data=data, - actor=accepted_note_mutation_actor( - user_profile_id=actor_context.user_profile_id, - actor_kind=actor_context.actor_kind, - actor_name=actor_context.actor_name, + async with self._mutation_cache_scope(project_external_id): + async with accepted_note_transaction(self.session_maker) as session: + accepted = await run_accepted_note_create( + session, + request=AcceptedNoteCreateMutation( + project_external_id=project_external_id, + data=data, + actor=accepted_note_mutation_actor( + user_profile_id=actor_context.user_profile_id, + actor_kind=actor_context.actor_kind, + actor_name=actor_context.actor_name, + ), + source=actor_context.source, ), - source=actor_context.source, - ), - dependencies=self.mutation_dependencies, - ) - await self._invalidate_project(project_external_id) + dependencies=self.mutation_dependencies, + ) return accepted except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error @@ -269,34 +322,29 @@ async def update_note( project_external_id=project_external_id, entity_external_id=entity_external_id, ) - async with accepted_note_transaction(self.session_maker) as session: - accepted = await run_accepted_note_update( - session, - request=AcceptedNoteUpdateMutation( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - data=data, - actor=accepted_note_mutation_actor( - user_profile_id=actor_context.user_profile_id, - actor_kind=actor_context.actor_kind, - actor_name=actor_context.actor_name, + async with self._mutation_cache_scope( + project_external_id, + invalidate_on_rejection=freshening_may_have_published, + ): + async with accepted_note_transaction(self.session_maker) as session: + accepted = await run_accepted_note_update( + session, + request=AcceptedNoteUpdateMutation( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + data=data, + actor=accepted_note_mutation_actor( + user_profile_id=actor_context.user_profile_id, + actor_kind=actor_context.actor_kind, + actor_name=actor_context.actor_name, + ), + source=actor_context.source, + base_checksum=base_checksum, ), - source=actor_context.source, - base_checksum=base_checksum, - ), - dependencies=self.mutation_dependencies, - ) + dependencies=self.mutation_dependencies, + ) except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error - finally: - # A rejected or failed mutation can follow a successful freshening - # index commit. The freshening attempt therefore owns invalidation - # for every downstream outcome, not only accepted writes. - if freshening_may_have_published: - await self._invalidate_project(project_external_id) - - if not freshening_may_have_published: - await self._invalidate_project(project_external_id) return accepted async def edit_note( @@ -324,30 +372,28 @@ async def edit_note( project_external_id=project_external_id, entity_external_id=entity_external_id, ) - async with accepted_note_transaction(self.session_maker) as session: - accepted = await run_accepted_note_edit( - session, - request=AcceptedNoteEditMutation( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - data=data, - actor=accepted_note_mutation_actor( - user_profile_id=actor_context.user_profile_id, - actor_kind=actor_context.actor_kind, - actor_name=actor_context.actor_name, + async with self._mutation_cache_scope( + project_external_id, + invalidate_on_rejection=freshening_may_have_published, + ): + async with accepted_note_transaction(self.session_maker) as session: + accepted = await run_accepted_note_edit( + session, + request=AcceptedNoteEditMutation( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + data=data, + actor=accepted_note_mutation_actor( + user_profile_id=actor_context.user_profile_id, + actor_kind=actor_context.actor_kind, + actor_name=actor_context.actor_name, + ), + source=actor_context.source, ), - source=actor_context.source, - ), - dependencies=self.mutation_dependencies, - ) + dependencies=self.mutation_dependencies, + ) except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error - finally: - if freshening_may_have_published: - await self._invalidate_project(project_external_id) - - if not freshening_may_have_published: - await self._invalidate_project(project_external_id) return accepted async def move_note( @@ -375,30 +421,28 @@ async def move_note( project_external_id=project_external_id, entity_external_id=entity_external_id, ) - async with accepted_note_transaction(self.session_maker) as session: - accepted = await run_accepted_note_move( - session, - request=AcceptedNoteMoveMutation( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - destination_path=destination_path, - actor=accepted_note_mutation_actor( - user_profile_id=actor_context.user_profile_id, - actor_kind=actor_context.actor_kind, - actor_name=actor_context.actor_name, + async with self._mutation_cache_scope( + project_external_id, + invalidate_on_rejection=freshening_may_have_published, + ): + async with accepted_note_transaction(self.session_maker) as session: + accepted = await run_accepted_note_move( + session, + request=AcceptedNoteMoveMutation( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + destination_path=destination_path, + actor=accepted_note_mutation_actor( + user_profile_id=actor_context.user_profile_id, + actor_kind=actor_context.actor_kind, + actor_name=actor_context.actor_name, + ), + source=actor_context.source, ), - source=actor_context.source, - ), - dependencies=self.mutation_dependencies, - ) + dependencies=self.mutation_dependencies, + ) except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error - finally: - if freshening_may_have_published: - await self._invalidate_project(project_external_id) - - if not freshening_may_have_published: - await self._invalidate_project(project_external_id) return accepted async def delete_note( @@ -414,21 +458,19 @@ async def delete_note( project_external_id=project_external_id, entity_external_id=entity_external_id, ) - async with accepted_note_transaction(self.session_maker) as session: - accepted = await run_accepted_note_delete( - session, - request=AcceptedNoteDeleteMutation( - project_external_id=project_external_id, - entity_external_id=entity_external_id, - ), - dependencies=self.mutation_dependencies, - ) + async with self._mutation_cache_scope( + project_external_id, + invalidate_on_rejection=freshening_may_have_published, + ): + async with accepted_note_transaction(self.session_maker) as session: + accepted = await run_accepted_note_delete( + session, + request=AcceptedNoteDeleteMutation( + project_external_id=project_external_id, + entity_external_id=entity_external_id, + ), + dependencies=self.mutation_dependencies, + ) except AcceptedNoteMutationRejected as error: raise note_content_mutation_error_from_rejection(error.rejection) from error - finally: - if freshening_may_have_published: - await self._invalidate_project(project_external_id) - - if not freshening_may_have_published: - await self._invalidate_project(project_external_id) return accepted diff --git a/test-int/read_cache/test_api_read_cache.py b/test-int/read_cache/test_api_read_cache.py index 090b4d453..331de18ac 100644 --- a/test-int/read_cache/test_api_read_cache.py +++ b/test-int/read_cache/test_api_read_cache.py @@ -2,9 +2,13 @@ from __future__ import annotations +import asyncio +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager from datetime import datetime, timezone from pathlib import Path from typing import Protocol, override +from uuid import uuid4 import pytest from fastapi import FastAPI @@ -12,6 +16,7 @@ from redis.asyncio import Redis from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker +import basic_memory.services.note_content_writes as note_content_writes from basic_memory import db from basic_memory.deps import ( get_chatgpt_importer_v2_external, @@ -147,6 +152,27 @@ async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStat return status +class BlockingInvalidationRedisReadCache(RedisReadCache): + """Hold one real invalidation so the request can be cancelled repeatedly.""" + + def __init__( + self, + *, + client: Redis, + namespace: str, + prefix: str, + ) -> None: + super().__init__(client=client, namespace=namespace, prefix=prefix) + self.invalidation_started = asyncio.Event() + self.release_invalidation = asyncio.Event() + + @override + async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: + self.invalidation_started.set() + await self.release_invalidation.wait() + return await super().invalidate_project(project_id) + + def _cache_key( *, project_id: str, @@ -185,6 +211,90 @@ async def _initialized_generation( return generation +@pytest.mark.asyncio +async def test_cancelled_committed_create_finishes_real_redis_invalidation( + app: FastAPI, + client: AsyncClient, + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], + test_project: Project, + redis_cache: RedisCacheHarness, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Cancellation after commit waits for invalidation before propagating.""" + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="cancelled-committed-create", + ) + blocking_cache = BlockingInvalidationRedisReadCache( + client=redis_cache.client, + namespace=redis_cache.namespace, + prefix=redis_cache.prefix, + ) + app.dependency_overrides[get_read_cache] = lambda: blocking_cache + + transaction_committed = asyncio.Event() + hold_after_commit = asyncio.Event() + original_transaction = note_content_writes.accepted_note_transaction + + @asynccontextmanager + async def pause_after_commit( + session_maker: async_sessionmaker[AsyncSession], + ) -> AsyncIterator[AsyncSession]: + async with original_transaction(session_maker) as session: + yield session + transaction_committed.set() + await hold_after_commit.wait() + + monkeypatch.setattr( + note_content_writes, + "accepted_note_transaction", + pause_after_commit, + ) + title = f"Cancelled committed create {uuid4()}" + request_task = asyncio.create_task( + client.post( + f"/v2/projects/{project_external_id}/knowledge/entities", + json={ + "title": title, + "directory": "cache", + "content": f"# {title}\n", + }, + ) + ) + + async with asyncio.timeout(5): + await transaction_committed.wait() + request_task.cancel() + async with asyncio.timeout(5): + await blocking_cache.invalidation_started.wait() + + # A second cancellation must not abandon the already-running cleanup. + request_task.cancel() + await asyncio.sleep(0) + assert not request_task.done() + + blocking_cache.release_invalidation.set() + with pytest.raises(asyncio.CancelledError): + await request_task + + generation_after = await _initialized_generation( + redis_cache, + project_external_id, + request="cancelled-committed-create", + ) + assert generation_after != generation_before + + _, session_maker = engine_factory + async with db.scoped_session(session_maker) as session: + committed_entities = await EntityRepository(project_id=test_project.id).get_by_title( + session, + title, + ) + assert len(committed_entities) == 1 + + @pytest.mark.asyncio async def test_entity_resolve_and_markdown_reads_cache_then_freshened_write_invalidates( app: FastAPI, From b8c5f72470be487ff6be1c830a3b068804437661 Mon Sep 17 00:00:00 2001 From: phernandez Date: Thu, 30 Jul 2026 00:14:49 -0500 Subject: [PATCH 15/28] fix(index): invalidate each committed project batch Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 8 +- src/basic_memory/index/local_project.py | 54 ++++-- .../indexing/project_index_maintenance.py | 58 ++++++- .../read_cache/test_runtime_invalidation.py | 159 ++++++++++++++++-- ...est_local_project_vector_cleaner_wiring.py | 12 +- 5 files changed, 258 insertions(+), 33 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index 74c487562..b441a6725 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -91,8 +91,10 @@ tenant: after its database commit succeeds but before the transaction context returns. Finish the namespace-bound generation bump before re-propagating cancellation so committed state cannot remain hidden behind the previous generation. -1. Put project-index invalidation in a failure-safe completion boundary. Move, delete, file-index, - and vector batches can commit incrementally before a later batch raises. +1. Invalidate after each committed project-index move, delete, and file-index batch, while + retaining the final failure-safe completion boundary for later-phase errors. Local inline file + batches invalidate when their runner returns; queued Cloud file batches invalidate in the child + worker after its durable commit, not merely when the coordinator enqueues the job. 1. Put direct single-file and watcher file-index invalidation in failure-safe boundaries. Entity transactions can commit before search refresh or note-content reconciliation raises, so a failed index attempt can still publish cache-relevant state. @@ -379,6 +381,8 @@ The real-Redis suite must prove: - startup recovery that publishes written, conflict, or failed materialization state invalidates before serving resumes; - project-index failures invalidate any earlier committed batches; +- consecutive project-index move, delete, and inline file batches each advance the real Redis + generation before the next batch begins; - direct and watcher file-index failures invalidate any entity state committed before failed search or reconciliation follow-ups; - hosted read repair invalidates cached entity metadata before storing the repaired resource; diff --git a/src/basic_memory/index/local_project.py b/src/basic_memory/index/local_project.py index 35367ff04..a19899519 100644 --- a/src/basic_memory/index/local_project.py +++ b/src/basic_memory/index/local_project.py @@ -65,6 +65,7 @@ run_project_index_coordinator, ) from basic_memory.indexing.project_index_maintenance import ( + InvalidatingProjectIndexBatchStore, ProjectIndexDeletePathVerifier, ProjectIndexMaintenanceRunner, ProjectIndexMovedEntitySearchRefresher, @@ -79,7 +80,12 @@ resolve_project_index_completion_relations, ) from basic_memory.models import Entity, Project -from basic_memory.read_cache import NullReadCache, ReadCache, invalidate_cache +from basic_memory.read_cache import ( + NullReadCache, + ReadCache, + ReadCacheInvalidator, + invalidate_cache, +) from basic_memory.repository import NoteContentRepository from basic_memory.runtime.jobs import ( RuntimeIndexFileBatchJobRequest, @@ -546,6 +552,7 @@ class LocalProjectIndexBatchEnqueuer(ProjectIndexBatchEnqueuer): reader: IndexFileBatchReader[IndexInputFile] indexer: IndexFileBatchIndexer[IndexInputFile] content_classifier: IndexFileBatchContentClassifier + read_cache: ReadCacheInvalidator = field(default_factory=NullReadCache) read_max_concurrent: int = 8 index_max_concurrent: int = 8 @@ -554,15 +561,19 @@ async def enqueue_index_file_batch( self, request: RuntimeIndexFileBatchJobRequest, ) -> IndexFileBatchJobResult: - return await run_index_file_batch( - request, - checker=self.checker, - reader=self.reader, - indexer=self.indexer, - content_classifier=self.content_classifier, - read_max_concurrent=self.read_max_concurrent, - index_max_concurrent=self.index_max_concurrent, - ) + async with invalidate_cache( + self.read_cache, + request.project.project_external_id, + ): + return await run_index_file_batch( + request, + checker=self.checker, + reader=self.reader, + indexer=self.indexer, + content_classifier=self.content_classifier, + read_max_concurrent=self.read_max_concurrent, + index_max_concurrent=self.index_max_concurrent, + ) @dataclass(frozen=True, slots=True) @@ -583,6 +594,8 @@ async def dependencies_for_project(self, project: Project) -> LocalIndexProjectD def runtime_from_dependencies( self, dependencies: LocalIndexProjectDependencies, + *, + project_external_id: str, ) -> LocalProjectIndexRuntime: metadata_source = LocalStorageFileMetadataSource(dependencies.file_service) checker = FileIndexChecker( @@ -616,6 +629,12 @@ def runtime_from_dependencies( # concurrent creation that must be checksum-verified before deletion. verify_replaced_move_targets=True, ) + invalidating_maintenance_store = InvalidatingProjectIndexBatchStore( + move_store=maintenance_store, + delete_store=maintenance_store, + read_cache=self.read_cache, + project_external_id=project_external_id, + ) return LocalProjectIndexRuntime( observed_file_source=LocalProjectIndexObservedFileSource( dependencies.file_service, @@ -629,8 +648,8 @@ def runtime_from_dependencies( entity_repository=dependencies.entity_repository, ), maintenance_runner=StoreProjectIndexMaintenanceRunner( - move_store=maintenance_store, - delete_store=maintenance_store, + move_store=invalidating_maintenance_store, + delete_store=invalidating_maintenance_store, ), moved_entity_search_refresher=RepositoryProjectIndexMovedEntitySearchRefresher( session_maker=dependencies.session_maker, @@ -642,6 +661,7 @@ def runtime_from_dependencies( reader=LocalIndexFileBatchReader(dependencies.file_service), indexer=dependencies.file_batch_indexer, content_classifier=dependencies.file_service, + read_cache=self.read_cache, read_max_concurrent=self.read_max_concurrent, index_max_concurrent=self.index_max_concurrent, ), @@ -659,7 +679,10 @@ def runtime_from_dependencies( ) async def runtime_for_project(self, project: Project) -> LocalProjectIndexRuntime: - return self.runtime_from_dependencies(await self.dependencies_for_project(project)) + return self.runtime_from_dependencies( + await self.dependencies_for_project(project), + project_external_id=str(project.external_id), + ) async def run_local_project_index_for_project( @@ -703,7 +726,10 @@ async def _get_project(self, project_id: int) -> Project: async def observe_project(self, project_id: int) -> LocalProjectIndexObservation: project = await self._get_project(project_id) dependencies = await self.runtime_factory.dependencies_for_project(project) - runtime = self.runtime_factory.runtime_from_dependencies(dependencies) + runtime = self.runtime_factory.runtime_from_dependencies( + dependencies, + project_external_id=str(project.external_id), + ) observed_files = await runtime.observed_file_source.list_observed_index_files() return LocalProjectIndexObservation(observed_files=observed_files) diff --git a/src/basic_memory/indexing/project_index_maintenance.py b/src/basic_memory/indexing/project_index_maintenance.py index 57f84e451..e5c811cdb 100644 --- a/src/basic_memory/indexing/project_index_maintenance.py +++ b/src/basic_memory/indexing/project_index_maintenance.py @@ -16,7 +16,8 @@ ProjectIndexExternalVectorCleaner, delete_project_index_vector_rows, ) -from basic_memory.runtime.storage import ProjectId +from basic_memory.read_cache import ReadCacheInvalidator, invalidate_cache +from basic_memory.runtime.storage import ProjectExternalId, ProjectId class ProjectIndexMaintenanceRunner(Protocol): @@ -37,6 +38,24 @@ async def run_delete_batches( ) -> ProjectIndexDeleteRun: ... +class ProjectIndexMoveBatchStore(Protocol): + """Capability that commits one project-index move batch.""" + + async def apply_project_index_move_batch( + self, + move_batch: ProjectIndexMoveBatch, + ) -> ProjectIndexMoveBatchResult: ... + + +class ProjectIndexDeleteBatchStore(Protocol): + """Capability that commits one project-index delete batch.""" + + async def apply_project_index_delete_batch( + self, + delete_batch: ProjectIndexDeleteBatch, + ) -> ProjectIndexDeleteBatchResult: ... + + class ProjectIndexMovedEntityRepository(Protocol): """Repository capability for loading moved entities after path maintenance.""" @@ -882,12 +901,41 @@ async def apply_project_index_delete_batch( ) +@dataclass(frozen=True, slots=True) +class InvalidatingProjectIndexBatchStore( + ProjectIndexMoveBatchStore, + ProjectIndexDeleteBatchStore, +): + """Invalidate semantic reads after each durable move or delete batch.""" + + move_store: ProjectIndexMoveBatchStore + delete_store: ProjectIndexDeleteBatchStore + read_cache: ReadCacheInvalidator + project_external_id: ProjectExternalId + + @override + async def apply_project_index_move_batch( + self, + move_batch: ProjectIndexMoveBatch, + ) -> ProjectIndexMoveBatchResult: + async with invalidate_cache(self.read_cache, self.project_external_id): + return await self.move_store.apply_project_index_move_batch(move_batch) + + @override + async def apply_project_index_delete_batch( + self, + delete_batch: ProjectIndexDeleteBatch, + ) -> ProjectIndexDeleteBatchResult: + async with invalidate_cache(self.read_cache, self.project_external_id): + return await self.delete_store.apply_project_index_delete_batch(delete_batch) + + @dataclass(frozen=True, slots=True) class StoreProjectIndexMaintenanceRunner(ProjectIndexMaintenanceRunner): """Run project-index maintenance through explicit move/delete batch stores.""" - move_store: RepositoryProjectIndexMaintenanceStore - delete_store: RepositoryProjectIndexMaintenanceStore + move_store: ProjectIndexMoveBatchStore + delete_store: ProjectIndexDeleteBatchStore @override async def run_move_batches( @@ -1011,7 +1059,7 @@ async def run_project_index_move_batches( *, moved_files: Mapping[str, str], batch_size: int, - move_store: RepositoryProjectIndexMaintenanceStore, + move_store: ProjectIndexMoveBatchStore, ) -> ProjectIndexMoveRun: """Apply project-index move maintenance through a storage adapter.""" move_plan = build_project_index_move_batch_plan( @@ -1064,7 +1112,7 @@ async def run_project_index_delete_batches( *, deleted_paths: Sequence[str], batch_size: int, - delete_store: RepositoryProjectIndexMaintenanceStore, + delete_store: ProjectIndexDeleteBatchStore, ) -> ProjectIndexDeleteRun: """Apply project-index delete maintenance through a storage adapter.""" delete_plan = build_project_index_delete_batch_plan( diff --git a/test-int/read_cache/test_runtime_invalidation.py b/test-int/read_cache/test_runtime_invalidation.py index 45fb3b0ca..69781efd2 100644 --- a/test-int/read_cache/test_runtime_invalidation.py +++ b/test-int/read_cache/test_runtime_invalidation.py @@ -18,17 +18,34 @@ LocalWatchMoveProcessor, ) from basic_memory.index.local_dependencies import LocalIndexSearchService -from basic_memory.index.local_project import LocalProjectIndexRuntime, run_local_project_index +from basic_memory.index.local_project import ( + LocalProjectIndexBatchEnqueuer, + LocalProjectIndexRuntime, + run_local_project_index, +) from basic_memory.index.local_runtime import LocalInlineStorageEventResultRecorder from basic_memory.indexing.change_planning import ChangeReport from basic_memory.indexing.directory_delete_runner import ( DirectoryDeleteRuntime, RepositoryDirectoryDeleteAcceptanceStore, ) +from basic_memory.indexing.file_batch_runner import ( + IndexFileBatchChecker, + IndexFileBatchContentClassifier, + IndexFileBatchIndexer, + IndexFileBatchReader, +) +from basic_memory.indexing.models import IndexInputFile from basic_memory.indexing.project_index_maintenance import ( + InvalidatingProjectIndexBatchStore, + ProjectIndexDeleteBatch, + ProjectIndexDeleteBatchResult, ProjectIndexDeleteRun, + ProjectIndexMoveBatch, + ProjectIndexMoveBatchResult, ProjectIndexMoveRun, ProjectIndexMovedEntitySearchRefresher, + StoreProjectIndexMaintenanceRunner, ) from basic_memory.indexing.relation_resolution import RelationResolutionRuntime from basic_memory.models import Project @@ -74,6 +91,21 @@ class RedisCacheHarness(Protocol): prefix: str +async def _current_generation( + redis_cache: RedisCacheHarness, + project_external_id: str, +) -> bytes | str: + generation = await redis_cache.client.get( + redis_read_cache_generation_key( + prefix=redis_cache.prefix, + namespace=redis_cache.namespace, + project_id=project_external_id, + ) + ) + assert generation is not None + return generation + + class DetectedMoveProcessor(LocalWatchMoveProcessor): """Exercise move completion without coupling the test to detection I/O.""" @@ -189,6 +221,38 @@ async def enqueue_index_file_batch( raise AssertionError("failing maintenance must stop before file batches") +class GenerationObservingProjectIndexBatchStore: + """Record the real Redis generation seen before each durable batch.""" + + def __init__( + self, + redis_cache: RedisCacheHarness, + project_external_id: str, + ) -> None: + self.redis_cache = redis_cache + self.project_external_id = project_external_id + self.move_generations: list[bytes | str] = [] + self.delete_generations: list[bytes | str] = [] + + async def apply_project_index_move_batch( + self, + move_batch: ProjectIndexMoveBatch, + ) -> ProjectIndexMoveBatchResult: + self.move_generations.append( + await _current_generation(self.redis_cache, self.project_external_id) + ) + return ProjectIndexMoveBatchResult(updated_files=len(move_batch.targets)) + + async def apply_project_index_delete_batch( + self, + delete_batch: ProjectIndexDeleteBatch, + ) -> ProjectIndexDeleteBatchResult: + self.delete_generations.append( + await _current_generation(self.redis_cache, self.project_external_id) + ) + return ProjectIndexDeleteBatchResult(deleted_entities=len(delete_batch.paths)) + + class GenerationObservingDirectoryDeleteEnqueuer: def __init__( self, @@ -231,15 +295,7 @@ async def _initialized_generation( request_digest=read_cache_request_digest(request), ) ) - generation = await redis_cache.client.get( - redis_read_cache_generation_key( - prefix=redis_cache.prefix, - namespace=redis_cache.namespace, - project_id=project_external_id, - ) - ) - assert generation is not None - return generation + return await _current_generation(redis_cache, project_external_id) async def _seed_recovery_note( @@ -517,6 +573,89 @@ async def test_startup_recovery_conflict_invalidates_published_failure_in_real_r assert generation_after != generation_before +@pytest.mark.asyncio +async def test_project_index_batches_invalidate_real_redis( + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """Every inline move, delete, and file batch advances the generation.""" + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="project-index-batches", + ) + observed_store = GenerationObservingProjectIndexBatchStore( + redis_cache, + project_external_id, + ) + invalidating_store = InvalidatingProjectIndexBatchStore( + move_store=observed_store, + delete_store=observed_store, + read_cache=redis_cache.cache, + project_external_id=project_external_id, + ) + maintenance = StoreProjectIndexMaintenanceRunner( + move_store=invalidating_store, + delete_store=invalidating_store, + ) + + await maintenance.run_move_batches( + moved_files={ + "notes/old-1.md": "notes/new-1.md", + "notes/old-2.md": "notes/new-2.md", + }, + batch_size=1, + ) + assert observed_store.move_generations[0] == generation_before + assert observed_store.move_generations[1] != observed_store.move_generations[0] + generation_after_moves = await _current_generation(redis_cache, project_external_id) + assert generation_after_moves != observed_store.move_generations[1] + + await maintenance.run_delete_batches( + deleted_paths=("notes/deleted-1.md", "notes/deleted-2.md"), + batch_size=1, + ) + assert observed_store.delete_generations[0] == generation_after_moves + assert observed_store.delete_generations[1] != observed_store.delete_generations[0] + generation_after_deletes = await _current_generation(redis_cache, project_external_id) + assert generation_after_deletes != observed_store.delete_generations[1] + + batch_enqueuer = LocalProjectIndexBatchEnqueuer( + checker=cast(IndexFileBatchChecker, object()), + reader=cast(IndexFileBatchReader[IndexInputFile], object()), + indexer=cast(IndexFileBatchIndexer[IndexInputFile], object()), + content_classifier=cast(IndexFileBatchContentClassifier, object()), + read_cache=redis_cache.cache, + ) + project = ProjectRuntimeReference.from_project(test_project) + await batch_enqueuer.enqueue_index_file_batch( + RuntimeIndexFileBatchJobRequest( + project=project, + batch_index=0, + batch_count=2, + ) + ) + generation_after_first_file_batch = await _current_generation( + redis_cache, + project_external_id, + ) + assert generation_after_first_file_batch != generation_after_deletes + + await batch_enqueuer.enqueue_index_file_batch( + RuntimeIndexFileBatchJobRequest( + project=project, + batch_index=1, + batch_count=2, + ) + ) + generation_after_second_file_batch = await _current_generation( + redis_cache, + project_external_id, + ) + assert generation_after_second_file_batch != generation_after_first_file_batch + + @pytest.mark.asyncio async def test_project_index_failure_invalidates_real_redis( test_project: Project, diff --git a/tests/index/test_local_project_vector_cleaner_wiring.py b/tests/index/test_local_project_vector_cleaner_wiring.py index fcc7c9571..0480fa891 100644 --- a/tests/index/test_local_project_vector_cleaner_wiring.py +++ b/tests/index/test_local_project_vector_cleaner_wiring.py @@ -5,6 +5,7 @@ from basic_memory.index.local_dependencies import LocalIndexProjectDependencies from basic_memory.index.local_project import LocalProjectIndexRuntimeFactory from basic_memory.indexing.project_index_maintenance import ( + InvalidatingProjectIndexBatchStore, RepositoryProjectIndexMaintenanceStore, StoreProjectIndexMaintenanceRunner, ) @@ -33,11 +34,18 @@ def test_full_project_runtime_forwards_external_vector_cleaner() -> None: external_vector_cleaner=cleaner, ) - runtime = LocalProjectIndexRuntimeFactory().runtime_from_dependencies(dependencies) + runtime = LocalProjectIndexRuntimeFactory().runtime_from_dependencies( + dependencies, + project_external_id="project-external-id", + ) assert isinstance(runtime.maintenance_runner, StoreProjectIndexMaintenanceRunner) assert isinstance( runtime.maintenance_runner.delete_store, + InvalidatingProjectIndexBatchStore, + ) + assert isinstance( + runtime.maintenance_runner.delete_store.delete_store, RepositoryProjectIndexMaintenanceStore, ) - assert runtime.maintenance_runner.delete_store.external_vector_cleaner is cleaner + assert runtime.maintenance_runner.delete_store.delete_store.external_vector_cleaner is cleaner From bf006be9d0bd2fbec949ccc08402a37e74017416 Mon Sep 17 00:00:00 2001 From: phernandez Date: Thu, 30 Jul 2026 00:46:46 -0500 Subject: [PATCH 16/28] fix(api): finish cache invalidation after directory commits Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 15 ++- src/basic_memory/read_cache/__init__.py | 2 + src/basic_memory/read_cache/invalidation.py | 34 +++++ .../services/directory_deletes.py | 30 +++-- .../services/note_content_writes.py | 60 +++------ test-int/read_cache/test_redis_read_cache.py | 93 +++++++++++++ .../read_cache/test_runtime_invalidation.py | 122 +++++++++++++++++- 7 files changed, 298 insertions(+), 58 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index b441a6725..9f4274c1a 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -87,10 +87,10 @@ tenant: the accepted-note transaction commits, again after terminal materialization/status publication and indexing, and again after relation resolution completes. This prevents a read filled between phases from surviving the later worker commit. -1. Make accepted-mutation invalidation cancellation-safe. A request or worker can be cancelled - after its database commit succeeds but before the transaction context returns. Finish the - namespace-bound generation bump before re-propagating cancellation so committed state cannot - remain hidden behind the previous generation. +1. Make committed-mutation invalidation cancellation-safe. Accepted-note writes and directory + deletion can be cancelled after their database commit succeeds but before the transaction + context returns. Finish the namespace-bound generation bump before re-propagating cancellation + so committed state cannot remain hidden behind the previous generation. 1. Invalidate after each committed project-index move, delete, and file-index batch, while retaining the final failure-safe completion boundary for later-phase errors. Local inline file batches invalidate when their runner returns; queued Cloud file batches invalidate in the child @@ -372,8 +372,9 @@ The real-Redis suite must prove: - successful writes invalidate; a rejected write also invalidates when pre-write freshening may already have published external file state, while a rolled-back transaction without such a publication does not; -- cancellation after a real accepted-note transaction commits cannot interrupt the real Redis - generation bump, including repeated cancellation while invalidation is in progress; +- cancellation after a real accepted-note or directory-delete transaction commits cannot + interrupt the real Redis generation bump, including repeated cancellation while invalidation is + in progress; - real Redis no-eviction capacity failures bypass cache storage and cannot fail committed-write invalidation; - authoritative read exceptions propagate without populating the missed cache key; @@ -428,6 +429,8 @@ semantics themselves are asserted only against the real Redis integration fixtur - Invalidate pre-mutation content freshening even when a later accepted mutation is rejected or fails, because the freshening index may already have committed external file state. - Invalidate every import attempt that may write files, including partial failures. +- Finish directory-delete acceptance invalidation before re-propagating cancellation that lands + after the delete transaction may have committed. - Invalidate directory moves after every committed file and again after search/relation follow-ups. - Invalidate project-root path changes in a failure-safe boundary because the filesystem source diff --git a/src/basic_memory/read_cache/__init__.py b/src/basic_memory/read_cache/__init__.py index 3a54f59e5..c0e005fcc 100644 --- a/src/basic_memory/read_cache/__init__.py +++ b/src/basic_memory/read_cache/__init__.py @@ -12,6 +12,7 @@ ReadCacheUnavailable, ) from basic_memory.read_cache.invalidation import ( + finish_project_read_cache_invalidation, invalidate_cache, invalidate_project_read_cache, ) @@ -31,6 +32,7 @@ "ReadCacheOperation", "ReadCacheStoreStatus", "ReadCacheUnavailable", + "finish_project_read_cache_invalidation", "invalidate_cache", "invalidate_project_read_cache", "read_cache_request_digest", diff --git a/src/basic_memory/read_cache/invalidation.py b/src/basic_memory/read_cache/invalidation.py index a55d4520e..776d88e65 100644 --- a/src/basic_memory/read_cache/invalidation.py +++ b/src/basic_memory/read_cache/invalidation.py @@ -1,5 +1,6 @@ """Best-effort project invalidation shared by mutation and indexing runtimes.""" +import asyncio from collections.abc import AsyncIterator from contextlib import asynccontextmanager @@ -48,6 +49,39 @@ async def invalidate_project_read_cache( return status +async def finish_project_read_cache_invalidation( + cache: ReadCacheInvalidator, + project_id: str, +) -> ReadCacheInvalidationStatus: + """Finish invalidation before propagating caller cancellation.""" + invalidation = asyncio.create_task(invalidate_project_read_cache(cache, project_id)) + try: + return await asyncio.shield(invalidation) + except asyncio.CancelledError as cancellation: + # Trigger: cancellation landed after authoritative state may have committed. + # Why: abandoning this Redis write can leave that state hidden behind the + # previous generation until its TTL expires. + # Outcome: finish invalidation, then preserve the caller's cancellation. + cleanup_error: BaseException | None = None + while not invalidation.done(): + try: + await asyncio.shield(invalidation) + except asyncio.CancelledError: + continue + except BaseException as error: + cleanup_error = error + if cleanup_error is None: + if invalidation.cancelled(): + cleanup_error = asyncio.CancelledError("read-cache invalidation task was cancelled") + else: + cleanup_error = invalidation.exception() + if cleanup_error is not None: + cancellation.add_note( + f"Read-cache invalidation failed during cancellation: {cleanup_error!r}" + ) + raise + + @asynccontextmanager async def invalidate_cache( cache: ReadCacheInvalidator, diff --git a/src/basic_memory/services/directory_deletes.py b/src/basic_memory/services/directory_deletes.py index 4248640ce..e1869b35e 100644 --- a/src/basic_memory/services/directory_deletes.py +++ b/src/basic_memory/services/directory_deletes.py @@ -19,7 +19,11 @@ finish_directory_delete_acceptance, normalize_directory_delete_path, ) -from basic_memory.read_cache import NullReadCache, ReadCache, invalidate_project_read_cache +from basic_memory.read_cache import ( + NullReadCache, + ReadCache, + finish_project_read_cache_invalidation, +) class DirectoryDeleteServiceError(Exception): @@ -69,6 +73,8 @@ async def delete_directory( project_external_id=project_external_id, directory=directory, ) + active_read_cache = read_cache if read_cache is not None else NullReadCache() + delete_may_have_committed = False try: # scoped_session enables `PRAGMA foreign_keys=ON` for SQLite; this bulk # delete issues a Core DELETE on entity and relies on ON DELETE CASCADE @@ -80,18 +86,18 @@ async def delete_directory( request=request, store=self.runtime.store, ) + delete_may_have_committed = bool(accepted.files) except DirectoryDeleteRejected as error: raise directory_delete_service_error_from_rejection(error.rejection) from error - - active_read_cache = read_cache if read_cache is not None else NullReadCache() - if accepted.files: - # Acceptance commits entity and search deletion before storage cleanup. - # Invalidate now so deleted reads cannot survive a slow or failed - # follow-up phase. - await invalidate_project_read_cache( - active_read_cache, - project_external_id, - ) + finally: + if delete_may_have_committed: + # Acceptance commits entity and search deletion before storage cleanup. + # Finish the generation bump even when cancellation interrupts the + # transaction exit, then preserve that cancellation. + await finish_project_read_cache_invalidation( + active_read_cache, + project_external_id, + ) try: result = await finish_directory_delete_acceptance( @@ -116,7 +122,7 @@ async def delete_directory( if accepted.files: # Cleanup and relation refresh can publish additional state or fail # after partial progress. Close the fill window in either case. - await invalidate_project_read_cache( + await finish_project_read_cache_invalidation( active_read_cache, project_external_id, ) diff --git a/src/basic_memory/services/note_content_writes.py b/src/basic_memory/services/note_content_writes.py index 7a8c7013f..772d98e34 100644 --- a/src/basic_memory/services/note_content_writes.py +++ b/src/basic_memory/services/note_content_writes.py @@ -2,7 +2,6 @@ from __future__ import annotations -import asyncio from collections.abc import AsyncIterator from contextlib import asynccontextmanager from dataclasses import dataclass @@ -31,7 +30,11 @@ RuntimeAcceptedNoteChange, RuntimeNoteContentResponsePayload, ) -from basic_memory.read_cache import NullReadCache, ReadCache, invalidate_project_read_cache +from basic_memory.read_cache import ( + NullReadCache, + ReadCache, + finish_project_read_cache_invalidation, +) from basic_memory.schemas.base import Entity as EntitySchema from basic_memory.schemas.request import EditEntityRequest @@ -148,39 +151,6 @@ def __init__( self.actor_resolver = actor_resolver self.read_cache = read_cache if read_cache is not None else NullReadCache() - async def _invalidate_project(self, project_external_id: str) -> None: - """Finish invalidation before propagating request cancellation.""" - invalidation = asyncio.create_task( - invalidate_project_read_cache(self.read_cache, project_external_id) - ) - try: - await asyncio.shield(invalidation) - except asyncio.CancelledError as cancellation: - # Trigger: request cancellation landed after a mutation may have committed. - # Why: abandoning this Redis write can leave the committed DB state hidden - # behind the previous generation until its TTL expires. - # Outcome: finish invalidation, then preserve the caller's cancellation. - cleanup_error: BaseException | None = None - while not invalidation.done(): - try: - await asyncio.shield(invalidation) - except asyncio.CancelledError: - continue - except BaseException as error: - cleanup_error = error - if cleanup_error is None: - if invalidation.cancelled(): - cleanup_error = asyncio.CancelledError( - "read-cache invalidation task was cancelled" - ) - else: - cleanup_error = invalidation.exception() - if cleanup_error is not None: - cancellation.add_note( - f"Read-cache invalidation failed during cancellation: {cleanup_error!r}" - ) - raise - @asynccontextmanager async def _mutation_cache_scope( self, @@ -193,16 +163,25 @@ async def _mutation_cache_scope( yield except AcceptedNoteMutationRejected: if invalidate_on_rejection: - await self._invalidate_project(project_external_id) + await finish_project_read_cache_invalidation( + self.read_cache, + project_external_id, + ) raise except BaseException: # Transaction exit can raise after its commit reached the database. # Invalidation is safe after an earlier rollback and required after # any commit whose response was interrupted. - await self._invalidate_project(project_external_id) + await finish_project_read_cache_invalidation( + self.read_cache, + project_external_id, + ) raise else: - await self._invalidate_project(project_external_id) + await finish_project_read_cache_invalidation( + self.read_cache, + project_external_id, + ) def _resolve_actor( self, @@ -245,7 +224,10 @@ async def freshen_existing_note_content( # Freshening can commit entity and note-content state before a later # indexing follow-up raises. Invalidate before propagating so those # partial publications cannot retain the previous cache generation. - await self._invalidate_project(project_external_id) + await finish_project_read_cache_invalidation( + self.read_cache, + project_external_id, + ) raise return True diff --git a/test-int/read_cache/test_redis_read_cache.py b/test-int/read_cache/test_redis_read_cache.py index 0fda9e571..01347ac55 100644 --- a/test-int/read_cache/test_redis_read_cache.py +++ b/test-int/read_cache/test_redis_read_cache.py @@ -21,6 +21,7 @@ ReadCacheOperation, ReadCacheStoreStatus, ReadCacheUnavailable, + finish_project_read_cache_invalidation, invalidate_cache, invalidate_project_read_cache, read_cache_request_digest, @@ -265,6 +266,98 @@ async def test_invalidation_context_invalidates_real_generation_after_body_error assert generation_after != generation_before +@pytest.mark.asyncio +async def test_cancelled_invalidation_records_real_redis_cleanup_failure( + redis_cache: RedisCacheHarness, +) -> None: + """Preserve cancellation while attaching a failure from the shielded cleanup.""" + + class FailingAfterRealInvalidation(RedisReadCache): + def __init__(self) -> None: + super().__init__( + client=redis_cache.client, + namespace=redis_cache.namespace, + prefix=redis_cache.prefix, + ) + self.invalidation_started = asyncio.Event() + self.release_invalidation = asyncio.Event() + + @override + async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: + self.invalidation_started.set() + await self.release_invalidation.wait() + await super().invalidate_project(project_id) + raise ReadCacheDataError("cleanup failed after real invalidation") + + key = _key(request="cancelled-cleanup-failure") + await redis_cache.cache.lookup(key) + generation_key = redis_read_cache_generation_key( + prefix=redis_cache.prefix, + namespace=redis_cache.namespace, + project_id=PROJECT_ID, + ) + generation_before = await redis_cache.client.get(generation_key) + cache = FailingAfterRealInvalidation() + invalidation = asyncio.create_task(finish_project_read_cache_invalidation(cache, PROJECT_ID)) + + await cache.invalidation_started.wait() + invalidation.cancel() + await asyncio.sleep(0) + assert not invalidation.done() + cache.release_invalidation.set() + + with pytest.raises(asyncio.CancelledError) as exc_info: + await invalidation + + generation_after = await redis_cache.client.get(generation_key) + assert generation_before is not None + assert generation_after is not None + assert generation_after != generation_before + assert any( + "cleanup failed after real invalidation" in note + for note in getattr(exc_info.value, "__notes__", ()) + ) + + +@pytest.mark.asyncio +async def test_child_cancelled_after_real_invalidation_is_recorded( + redis_cache: RedisCacheHarness, +) -> None: + """A cancelled cleanup task remains visible on the propagated cancellation.""" + + class CancellingAfterRealInvalidation(RedisReadCache): + @override + async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: + await super().invalidate_project(project_id) + raise asyncio.CancelledError("cleanup task cancelled") + + key = _key(request="child-cancelled-after-invalidation") + await redis_cache.cache.lookup(key) + generation_key = redis_read_cache_generation_key( + prefix=redis_cache.prefix, + namespace=redis_cache.namespace, + project_id=PROJECT_ID, + ) + generation_before = await redis_cache.client.get(generation_key) + cache = CancellingAfterRealInvalidation( + client=redis_cache.client, + namespace=redis_cache.namespace, + prefix=redis_cache.prefix, + ) + + with pytest.raises(asyncio.CancelledError) as exc_info: + await finish_project_read_cache_invalidation(cache, PROJECT_ID) + + generation_after = await redis_cache.client.get(generation_key) + assert generation_before is not None + assert generation_after is not None + assert generation_after != generation_before + assert any( + "read-cache invalidation task was cancelled" in note + for note in getattr(exc_info.value, "__notes__", ()) + ) + + @pytest.mark.asyncio async def test_lost_generation_key_cannot_revive_old_data( redis_cache: RedisCacheHarness, diff --git a/test-int/read_cache/test_runtime_invalidation.py b/test-int/read_cache/test_runtime_invalidation.py index 69781efd2..c258b95b7 100644 --- a/test-int/read_cache/test_runtime_invalidation.py +++ b/test-int/read_cache/test_runtime_invalidation.py @@ -2,7 +2,9 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +import asyncio +from collections.abc import AsyncIterator, Mapping, Sequence +from contextlib import asynccontextmanager from datetime import UTC, datetime from pathlib import Path from typing import Protocol, cast, override @@ -11,6 +13,7 @@ from redis.asyncio import Redis from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker +import basic_memory.services.directory_deletes as directory_deletes from basic_memory import db from basic_memory.index import note_content_materialization from basic_memory.index.local_moves import ( @@ -51,6 +54,7 @@ from basic_memory.models import Project from basic_memory.models.knowledge import Entity from basic_memory.read_cache import ( + ReadCacheInvalidationStatus, ReadCacheKey, ReadCacheOperation, read_cache_request_digest, @@ -282,6 +286,27 @@ async def enqueue_directory_file_delete( ) +class BlockingInvalidationRedisReadCache(RedisReadCache): + """Hold one real invalidation so cancellation can repeat during cleanup.""" + + def __init__( + self, + *, + client: Redis, + namespace: str, + prefix: str, + ) -> None: + super().__init__(client=client, namespace=namespace, prefix=prefix) + self.invalidation_started = asyncio.Event() + self.release_invalidation = asyncio.Event() + + @override + async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: + self.invalidation_started.set() + await self.release_invalidation.wait() + return await super().invalidate_project(project_id) + + async def _initialized_generation( redis_cache: RedisCacheHarness, project_external_id: str, @@ -748,3 +773,98 @@ async def test_directory_delete_invalidates_before_and_after_cleanup_in_real_red request="directory-delete", ) assert generation_after != generation_during_cleanup + + +@pytest.mark.asyncio +async def test_cancelled_committed_directory_delete_finishes_real_redis_invalidation( + monkeypatch: pytest.MonkeyPatch, + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """Cancellation during transaction exit cannot skip delete invalidation.""" + _, session_maker = engine_factory + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="cancelled-committed-directory-delete", + ) + entity_repository = EntityRepository(project_id=test_project.id) + title = "Cancelled Committed Directory Delete" + async with db.scoped_session(session_maker) as session: + await entity_repository.add( + session, + Entity( + title=title, + note_type="note", + content_type="text/markdown", + file_path="cancelled-delete/note.md", + checksum="cancelled-directory-delete-checksum", + created_at=datetime.now(UTC), + updated_at=datetime.now(UTC), + ), + ) + + blocking_cache = BlockingInvalidationRedisReadCache( + client=redis_cache.client, + namespace=redis_cache.namespace, + prefix=redis_cache.prefix, + ) + transaction_committed = asyncio.Event() + hold_after_commit = asyncio.Event() + original_scoped_session = directory_deletes.db.scoped_session + + @asynccontextmanager + async def pause_after_commit( + scoped_session_maker: async_sessionmaker[AsyncSession], + ) -> AsyncIterator[AsyncSession]: + async with original_scoped_session(scoped_session_maker) as session: + yield session + transaction_committed.set() + await hold_after_commit.wait() + + monkeypatch.setattr(directory_deletes.db, "scoped_session", pause_after_commit) + service = DirectoryDeleteService( + session_maker=session_maker, + runtime=DirectoryDeleteRuntime( + store=RepositoryDirectoryDeleteAcceptanceStore(), + file_delete_enqueuer=GenerationObservingDirectoryDeleteEnqueuer( + redis_cache, + project_external_id, + ), + ), + ) + delete_task = asyncio.create_task( + service.delete_directory( + project_external_id=project_external_id, + directory="cancelled-delete", + read_cache=blocking_cache, + ) + ) + + async with asyncio.timeout(5): + await transaction_committed.wait() + delete_task.cancel() + async with asyncio.timeout(5): + await blocking_cache.invalidation_started.wait() + + # A second cancellation must not abandon the already-running generation bump. + delete_task.cancel() + await asyncio.sleep(0) + assert not delete_task.done() + + blocking_cache.release_invalidation.set() + with pytest.raises(asyncio.CancelledError): + await delete_task + + generation_after = await _initialized_generation( + redis_cache, + project_external_id, + request="cancelled-committed-directory-delete", + ) + assert generation_after != generation_before + + async with original_scoped_session(session_maker) as session: + deleted_entities = await entity_repository.get_by_title(session, title) + assert deleted_entities == [] From 1badd6fbf4db718cd20813d30e727d4bbca36ed0 Mon Sep 17 00:00:00 2001 From: phernandez Date: Thu, 30 Jul 2026 10:06:01 -0500 Subject: [PATCH 17/28] refactor(api): simplify optional read caching Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 83 ++++--- src/basic_memory/api/container.py | 6 +- .../api/v2/routers/importer_router.py | 12 +- .../api/v2/routers/knowledge_router.py | 71 ++++-- .../api/v2/routers/project_router.py | 8 +- .../api/v2/routers/resource_router.py | 34 ++- src/basic_memory/deps/__init__.py | 6 +- src/basic_memory/deps/read_cache.py | 24 +- src/basic_memory/index/local_moves.py | 10 +- src/basic_memory/index/local_project.py | 67 +++-- src/basic_memory/index/local_runtime.py | 50 ++-- src/basic_memory/index/local_schedulers.py | 12 +- .../index/note_content_materialization.py | 13 +- src/basic_memory/index/watch_coordinator.py | 4 +- src/basic_memory/read_cache/__init__.py | 7 +- src/basic_memory/read_cache/contract.py | 14 +- src/basic_memory/read_cache/null.py | 28 --- src/basic_memory/read_cache/read_through.py | 22 +- src/basic_memory/read_cache/redis.py | 2 - .../services/directory_deletes.py | 14 +- src/basic_memory/services/entity_service.py | 5 +- src/basic_memory/services/initialization.py | 35 ++- .../services/note_content_writes.py | 27 +- .../read_cache/test_read_cache_benchmark.py | 3 +- test-int/read_cache/test_redis_read_cache.py | 235 +++++++----------- .../test_note_content_materialization.py | 12 +- tests/index/test_local_schedulers.py | 14 +- tests/test_deps.py | 13 +- 28 files changed, 426 insertions(+), 405 deletions(-) delete mode 100644 src/basic_memory/read_cache/null.py diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index 9f4274c1a..74b211b33 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -25,7 +25,7 @@ Basic Memory owns: - typed serialization; - TTL and payload-size policy; - project-scoped invalidation; -- the no-op and Redis read-cache implementations. +- the model-bound read-through facade and Redis read-cache implementation. Cloud owns: @@ -72,9 +72,9 @@ tenant: 1. Override the low-level `get_read_cache` dependency at the Cloud composition root. Construct `RedisReadCache(client=shared_basic_memory_client, namespace=trusted_namespace)` as a lightweight request-scoped adapter; reuse the long-lived client and connection pool. - FastAPI then resolves `ConfiguredReadCacheDep` from that backend and binds Basic Memory's TTL - and payload-size policy. Cloud should not duplicate those policy constants or override - `get_configured_read_cache`. + FastAPI route dependencies then create lightweight `ModelReadCache` facades from that shared + backend and bind Basic Memory's response type, TTL, and payload-size policy. Cloud should not + duplicate those policy constants or construct Redis clients per response model. 1. Pass the trusted tenant/workspace identity through internal queue payloads, or include enough trusted identifiers for workers to derive the exact same namespace. Never copy a namespace from a public request field. @@ -143,15 +143,16 @@ and every worker must switch atomically enough to avoid missing invalidations. ```mermaid flowchart LR - H["Cloud or standalone API host"] -->|"client plus opaque namespace"| RC["Basic Memory ReadCache"] - API["Basic Memory read routes"] --> RC - RC -->|"hit"| API - RC -->|"miss"| DB["Services, repositories, and storage"] - DB -->|"successful result"| RC - W["Writes, indexing, recovery, and storage events"] -->|"invalidate after commit"| RC + H["Cloud or standalone API host"] -->|"optional client plus opaque namespace"| B["Raw ReadCache backend"] + B --> F["Typed ModelReadCache facades"] + API["Basic Memory read routes"] --> F + F -->|"hit"| API + F -->|"miss"| DB["Services, repositories, and storage"] + DB -->|"successful result"| F + W["Writes, indexing, recovery, and storage events"] -->|"invalidate after commit"| B RL["Cloud tenant rate limiter"] --> RLD["Cloud rate-limit keyspace"] - RC --> BMD["Basic Memory read-cache keyspace"] + B --> BMD["Basic Memory read-cache keyspace"] RLD -. "same or separate instance" .-> R["Redis"] BMD -. "same or separate instance" .-> R ``` @@ -163,20 +164,20 @@ Introduce `src/basic_memory/read_cache/` with: - a narrow `ReadCache` protocol; - a narrower `ReadCacheInvalidator` protocol for mutation and repair paths; - immutable request/key values; -- a `NullReadCache` default; - canonical key construction; -- a configured Pydantic read-through dependency; +- a generic `ModelReadCache[ModelT]` facade that owns one Pydantic response type and policy; - an optional `RedisReadCache` adapter. -The cache is namespace-bound at construction. Its public operations are: +The raw backend is namespace-bound at construction. Its public operations are: - `lookup(key)`, which returns the generation observed with a hit or miss; -- `store(key, lookup, payload, ttl)`, which reports stored, superseded, or disabled; +- `store(key, lookup, payload, ttl)`, which reports stored or superseded; - `invalidate_project(project_id)`. Cloud can create a lightweight namespace-bound adapter around a long-lived, Basic -Memory-specific async Redis client. Basic Memory does not receive tenant, subscription, or -rate-limit concepts. +Memory-specific async Redis client. Basic Memory then creates separate model-bound facades for +entity, resolution, and resource responses over that same adapter. Facades do not own clients or +connections. Basic Memory does not receive tenant, subscription, or rate-limit concepts. ## Keys And Invalidation @@ -235,9 +236,10 @@ workspace type in addition to the validated request body. Cache typed boundary values rather than SQLAlchemy models. Use an explicit read-through scope in the API routes so hit, miss, serialization, and fallback behavior remain visible. FastAPI injects -`ConfiguredReadCacheDep`; its provider binds the injected backend to Basic Memory's TTL and -payload-size policy at the dependency boundary. Routes keep the authoritative read inline inside -a Python async context manager instead of constructing loader callbacks: +a route-specific `ModelReadCache[ResponseType] | None`; its provider binds the optional backend +to Basic Memory's response type, TTL, and payload-size policy at the dependency boundary. Routes +keep the authoritative read inline inside a Python async context manager instead of constructing +loader callbacks: ```python cache_key = ReadCacheKey( @@ -245,7 +247,12 @@ cache_key = ReadCacheKey( operation=ReadCacheOperation.entity, request_digest=read_cache_request_digest(entity_id), ) -async with read_cache.read(key=cache_key, model_type=EntityResponseV2) as cached: +cache_scope = ( + read_cache.read(key=cache_key) + if read_cache is not None + else nullcontext(ReadCacheScope[EntityResponseV2]()) +) +async with cache_scope as cached: if cached.value is not None: return cached.value @@ -256,9 +263,11 @@ async with read_cache.read(key=cache_key, model_type=EntityResponseV2) as cached ``` The context manager performs lookup before entering the body and stores an eligible miss when the -body exits normally. Exceptions and cancellation propagate without storing. A route that performs -read repair passes the configured dependency itself through the narrow `ReadCacheInvalidator` -capability; it never reaches through the facade to a Redis/backend attribute. +body exits normally. Exceptions and cancellation propagate without storing. When no backend is +configured, callers do not invoke cache lookup, store, or invalidation; there is no disabled cache +result or no-op implementation. A route that performs read repair passes the model-bound facade +itself through the narrow `ReadCacheInvalidator` capability when present; it never reaches through +the facade to a Redis/backend attribute. Mutation and indexing code uses the same direct scope pattern when invalidation is unconditional: @@ -267,7 +276,8 @@ async with invalidate_cache(read_cache, project_id): await importer.import_data(...) ``` -Conditional and multi-phase invalidation remains explicit so the freshness boundary is visible. +Callers enter this scope only when a backend is present. Conditional and multi-phase invalidation +remains explicit so the freshness boundary is visible. Primary integration points: @@ -297,17 +307,17 @@ serving barrier is released and include terminal conflict or failure publication Use the official asynchronous `redis-py` client behind the Basic Memory protocol. Add it only as an optional package extra. A host may instead supply a compatible, already-owned client. -The Core `ApiContainer` carries `NullReadCache` by default. A managed host activates caching by -injecting or dependency-overriding a namespace-bound implementation and owns that client's -lifecycle; Cloud therefore reuses its long-lived Basic Memory cache client. Local CLI, MCP -in-process ASGI routing, and the standalone API remain on `NullReadCache` in the first rollout. -A later standalone Redis setting can create and close a client in the FastAPI lifespan without -changing the cache contract. +The Core `ApiContainer` carries `ReadCache | None` and defaults to `None`. A managed host +activates caching by injecting or dependency-overriding a namespace-bound implementation and owns +that client's lifecycle; Cloud therefore reuses its long-lived Basic Memory cache client. Local +CLI, MCP in-process ASGI routing, and the standalone API simply skip cache work in the first +rollout. A later standalone Redis setting can create and close a client in the FastAPI lifespan +without changing the cache contract. -`get_read_cache` is the host override point. `get_configured_read_cache` is Core-owned FastAPI -composition: it receives that backend through dependency injection and returns the route-facing -cache with validated policy. Portable mutation and indexing runtimes continue to depend only on -the backend or the narrower invalidation capability; they do not depend on FastAPI. +`get_read_cache` is the host override point and returns `ReadCache | None`. Core-owned, +route-specific FastAPI providers call `create_model_read_cache` to return a correctly typed facade +when that backend exists. Portable mutation and indexing runtimes continue to depend only on the +optional backend or narrower invalidation capability; they do not depend on FastAPI. The FastAPI Redis SDK is not the foundational dependency for this work. The cache contract must also participate in portable indexing and hosted storage-event invalidation, and Basic Memory's @@ -358,6 +368,7 @@ changing the test contract. The real-Redis suite must prove: - namespace, project, operation, and request isolation; +- distinct model-bound facades share one raw backend while retaining their own response type; - deterministic canonical keys; - cache hit and TTL expiry behavior; - project invalidation; @@ -401,7 +412,7 @@ semantics themselves are asserted only against the real Redis integration fixtur ### 1. Cache infrastructure -- Add the protocol, key values, no-op backend, Redis adapter, typed helper, optional dependency, +- Add the protocol, key values, Redis adapter, typed facade, optional dependency, telemetry, and real Redis integration tests. - Do not cache production routes yet. diff --git a/src/basic_memory/api/container.py b/src/basic_memory/api/container.py index 18bdb17ea..c0c585835 100644 --- a/src/basic_memory/api/container.py +++ b/src/basic_memory/api/container.py @@ -10,14 +10,14 @@ - Factories for services are provided, not singletons """ -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import TYPE_CHECKING from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker, AsyncSession from basic_memory import db from basic_memory.config import BasicMemoryConfig, ConfigManager -from basic_memory.read_cache import NullReadCache, ReadCache +from basic_memory.read_cache import ReadCache from basic_memory.runtime.mode import RuntimeMode, resolve_runtime_mode if TYPE_CHECKING: # pragma: no cover @@ -43,7 +43,7 @@ class ApiContainer: # --- Optional semantic read cache --- # Hosts inject a namespace-bound implementation; local and off-lifespan # ASGI requests deliberately stay dependency-free by default. - read_cache: ReadCache = field(default_factory=NullReadCache) + read_cache: ReadCache | None = None @classmethod def create(cls) -> "ApiContainer": # pragma: no cover diff --git a/src/basic_memory/api/v2/routers/importer_router.py b/src/basic_memory/api/v2/routers/importer_router.py index 35594b671..aa5452b73 100644 --- a/src/basic_memory/api/v2/routers/importer_router.py +++ b/src/basic_memory/api/v2/routers/importer_router.py @@ -6,6 +6,7 @@ import json import logging +from contextlib import nullcontext from fastapi import APIRouter, Form, HTTPException, Path, UploadFile, status @@ -205,7 +206,7 @@ async def import_file[ImportResultT: ImportResult]( destination_directory: str, max_bytes: int, *, - read_cache: ReadCache, + read_cache: ReadCache | None, project_external_id: str, ) -> ImportResultT: """Helper function to import a file using an importer instance. @@ -256,12 +257,17 @@ async def run_import_with_invalidation[ImportResultT: ImportResult]( source_data: object, destination_directory: str, *, - read_cache: ReadCache, + read_cache: ReadCache | None, project_external_id: str, ) -> ImportResultT: """Run one import attempt and invalidate files it may have written.""" # Importers write files one at a time and may return a failed result after # earlier writes. Invalidate every attempted import so cached file-first # resources cannot survive either success or partial failure. - async with invalidate_cache(read_cache, project_external_id): + invalidation_scope = ( + invalidate_cache(read_cache, project_external_id) + if read_cache is not None + else nullcontext() + ) + async with invalidation_scope: return await importer.import_data(source_data, destination_directory) diff --git a/src/basic_memory/api/v2/routers/knowledge_router.py b/src/basic_memory/api/v2/routers/knowledge_router.py index 9625e87b7..05fcffd31 100644 --- a/src/basic_memory/api/v2/routers/knowledge_router.py +++ b/src/basic_memory/api/v2/routers/knowledge_router.py @@ -11,12 +11,13 @@ """ from collections.abc import Mapping +from contextlib import nullcontext from hashlib import sha256 import os import pathlib from typing import Annotated -from fastapi import APIRouter, Header, HTTPException, Response, Path, status +from fastapi import APIRouter, Depends, Header, HTTPException, Response, Path, status from loguru import logger import logfire @@ -30,7 +31,7 @@ should_ignore_path, ) from basic_memory.deps import ( - ConfiguredReadCacheDep, + create_model_read_cache, EntityServiceV2ExternalDep, FileServiceV2ExternalDep, SearchServiceV2ExternalDep, @@ -53,8 +54,10 @@ SessionMakerDep, ) from basic_memory.read_cache import ( + ModelReadCache, ReadCacheKey, ReadCacheOperation, + ReadCacheScope, invalidate_cache, read_cache_request_digest, ) @@ -86,6 +89,32 @@ router = APIRouter(prefix="/knowledge", tags=["knowledge-v2"]) +def get_resolve_read_cache( + read_cache: ReadCacheDep, +) -> ModelReadCache[EntityResolveResponse] | None: + """Bind identifier-resolution responses to the optional cache backend.""" + return create_model_read_cache(read_cache, EntityResolveResponse) + + +ResolveReadCacheDep = Annotated[ + ModelReadCache[EntityResolveResponse] | None, + Depends(get_resolve_read_cache), +] + + +def get_entity_read_cache( + read_cache: ReadCacheDep, +) -> ModelReadCache[EntityResponseV2] | None: + """Bind entity responses to the optional cache backend.""" + return create_model_read_cache(read_cache, EntityResponseV2) + + +EntityReadCacheDep = Annotated[ + ModelReadCache[EntityResponseV2] | None, + Depends(get_entity_read_cache), +] + + def _schedule_post_write_followups( *, vector_sync_scheduler, @@ -247,7 +276,7 @@ async def resolve_identifier( entity_repository: EntityRepositoryV2ExternalDep, project_repository: ProjectRepositoryDep, session: SessionDep, - read_cache: ConfiguredReadCacheDep, + read_cache: ResolveReadCacheDep, ) -> EntityResolveResponse: """Resolve a string identifier (external_id, permalink, title, or path) to entity info. @@ -296,10 +325,12 @@ async def resolve_identifier( workspace_context.workspace_type if workspace_context else "", ), ) - async with read_cache.read( - key=cache_key, - model_type=EntityResolveResponse, - ) as cached: + cache_scope = ( + read_cache.read(key=cache_key) + if read_cache is not None + else nullcontext(ReadCacheScope[EntityResolveResponse]()) + ) + async with cache_scope as cached: if cached.value is not None: return cached.value @@ -528,7 +559,12 @@ async def index_file( # The file indexer commits entity state before search and reconciliation # follow-ups finish. Invalidate even when a later phase raises so those # partial commits cannot remain reachable through the old generation. - async with invalidate_cache(read_cache, project_external_id): + invalidation_scope = ( + invalidate_cache(read_cache, project_external_id) + if read_cache is not None + else nullcontext() + ) + async with invalidation_scope: indexed = await file_indexer.index_file(file_path, source="api-index-file") async with db.scoped_session(session_maker) as session: entity = await entity_repository.get_by_id(session, indexed.entity_id) @@ -566,7 +602,7 @@ async def get_entity_by_id( entity_repository: EntityRepositoryV2ExternalDep, note_content_query_service: NoteContentQueryServiceDep, session: SessionDep, - read_cache: ConfiguredReadCacheDep, + read_cache: EntityReadCacheDep, entity_id: str = Path(..., description="Entity external ID (UUID)"), ) -> EntityResponseV2: """Get an entity by its external ID (UUID). @@ -596,10 +632,12 @@ async def get_entity_by_id( operation=ReadCacheOperation.entity, request_digest=read_cache_request_digest(entity_id), ) - async with read_cache.read( - key=cache_key, - model_type=EntityResponseV2, - ) as cached: + cache_scope = ( + read_cache.read(key=cache_key) + if read_cache is not None + else nullcontext(ReadCacheScope[EntityResponseV2]()) + ) + async with cache_scope as cached: if cached.value is not None: return cached.value @@ -1014,7 +1052,12 @@ async def move_directory( # Reindexing can alter entity responses after the move was first # invalidated. Close that fill window even after partial # follow-up failure. - async with invalidate_cache(read_cache, project_external_id): + invalidation_scope = ( + invalidate_cache(read_cache, project_external_id) + if read_cache is not None + else nullcontext() + ) + async with invalidation_scope: # Reindex moved entities for file_path in result.moved_files: async with db.scoped_session(session_maker) as session: diff --git a/src/basic_memory/api/v2/routers/project_router.py b/src/basic_memory/api/v2/routers/project_router.py index 7064c7952..2ade0c1d2 100644 --- a/src/basic_memory/api/v2/routers/project_router.py +++ b/src/basic_memory/api/v2/routers/project_router.py @@ -10,6 +10,7 @@ - Consistent with v2 entity operations """ +from contextlib import nullcontext import os from typing import Literal, Optional @@ -461,7 +462,12 @@ async def update_project_by_id( # resource key while project and entity UUIDs stay stable. The # service can update config before its DB follow-up completes, # so invalidate on every attempted move completion path. - async with invalidate_cache(read_cache, project_id): + invalidation_scope = ( + invalidate_cache(read_cache, project_id) + if read_cache is not None + else nullcontext() + ) + async with invalidation_scope: await project_service.move_project(old_project.name, path) elif is_active is not None: await project_service.update_project(old_project.name, is_active=is_active) diff --git a/src/basic_memory/api/v2/routers/resource_router.py b/src/basic_memory/api/v2/routers/resource_router.py index 7da151856..4ab0b19aa 100644 --- a/src/basic_memory/api/v2/routers/resource_router.py +++ b/src/basic_memory/api/v2/routers/resource_router.py @@ -9,25 +9,30 @@ storage-event indexing pipeline. No API endpoint writes resource files inline. """ +from contextlib import nullcontext from pathlib import Path as PathLib +from typing import Annotated -from fastapi import APIRouter, HTTPException, Response, Path +from fastapi import APIRouter, Depends, HTTPException, Response, Path from loguru import logger from pydantic import BaseModel, ConfigDict import logfire from basic_memory import db from basic_memory.deps import ( - ConfiguredReadCacheDep, + create_model_read_cache, ProjectConfigV2ExternalDep, FileServiceV2ExternalDep, EntityRepositoryV2ExternalDep, NoteContentQueryServiceDep, + ReadCacheDep, SessionMakerDep, ) from basic_memory.read_cache import ( + ModelReadCache, ReadCacheKey, ReadCacheOperation, + ReadCacheScope, read_cache_request_digest, ) from basic_memory.utils import validate_project_path @@ -44,6 +49,19 @@ class CachedResourceResponse(BaseModel): model_config = ConfigDict(ser_json_bytes="base64", val_json_bytes="base64") +def get_resource_read_cache( + read_cache: ReadCacheDep, +) -> ModelReadCache[CachedResourceResponse] | None: + """Bind resource responses to the optional cache backend.""" + return create_model_read_cache(read_cache, CachedResourceResponse) + + +ResourceReadCacheDep = Annotated[ + ModelReadCache[CachedResourceResponse] | None, + Depends(get_resource_read_cache), +] + + def _is_markdown_resource(resource: CachedResourceResponse) -> bool: return resource.media_type.partition(";")[0].strip().lower() == "text/markdown" @@ -54,7 +72,7 @@ async def get_resource_content( entity_repository: EntityRepositoryV2ExternalDep, file_service: FileServiceV2ExternalDep, note_content_query_service: NoteContentQueryServiceDep, - read_cache: ConfiguredReadCacheDep, + read_cache: ResourceReadCacheDep, session_maker: SessionMakerDep, project_id: str = Path(..., description="Project external UUID"), entity_id: str = Path(..., description="Entity external UUID"), @@ -87,10 +105,12 @@ async def get_resource_content( operation=ReadCacheOperation.resource, request_digest=read_cache_request_digest(entity_id), ) - async with read_cache.read( - key=cache_key, - model_type=CachedResourceResponse, - ) as cached: + cache_scope = ( + read_cache.read(key=cache_key) + if read_cache is not None + else nullcontext(ReadCacheScope[CachedResourceResponse]()) + ) + async with cache_scope as cached: if cached.value is not None: return Response( content=cached.value.content, diff --git a/src/basic_memory/deps/__init__.py b/src/basic_memory/deps/__init__.py index 4cc77e7f4..d45c6e18f 100644 --- a/src/basic_memory/deps/__init__.py +++ b/src/basic_memory/deps/__init__.py @@ -39,9 +39,8 @@ ) from basic_memory.deps.read_cache import ( - get_configured_read_cache, + create_model_read_cache, get_read_cache, - ConfiguredReadCacheDep, ReadCacheDep, ) @@ -131,9 +130,8 @@ "get_project_config_v2_external", "ProjectConfigV2ExternalDep", # Read cache - "get_configured_read_cache", + "create_model_read_cache", "get_read_cache", - "ConfiguredReadCacheDep", "ReadCacheDep", # Repositories "get_entity_repository_v2_external", diff --git a/src/basic_memory/deps/read_cache.py b/src/basic_memory/deps/read_cache.py index 47c4edb84..d786a6338 100644 --- a/src/basic_memory/deps/read_cache.py +++ b/src/basic_memory/deps/read_cache.py @@ -3,16 +3,17 @@ from typing import Annotated from fastapi import Depends, Request +from pydantic import BaseModel -from basic_memory.read_cache import ConfiguredReadCache, ReadCache +from basic_memory.read_cache import ModelReadCache, ReadCache from basic_memory.read_cache.policy import ( READ_CACHE_MAX_PAYLOAD_BYTES, READ_CACHE_TTL_SECONDS, ) -def get_read_cache(request: Request) -> ReadCache: - """Return the host-injected cache or the container's no-op default.""" +def get_read_cache(request: Request) -> ReadCache | None: + """Return the optional host-injected cache backend.""" try: container = request.app.state.container except AttributeError: @@ -26,16 +27,19 @@ def get_read_cache(request: Request) -> ReadCache: return resolve_container().read_cache -ReadCacheDep = Annotated[ReadCache, Depends(get_read_cache)] +ReadCacheDep = Annotated[ReadCache | None, Depends(get_read_cache)] -def get_configured_read_cache(read_cache: ReadCacheDep) -> ConfiguredReadCache: - """Bind the host cache to Basic Memory's API read policy.""" - return ConfiguredReadCache( +def create_model_read_cache[ModelT: BaseModel]( + read_cache: ReadCache | None, + model_type: type[ModelT], +) -> ModelReadCache[ModelT] | None: + """Bind one response model to the host cache and Basic Memory's read policy.""" + if read_cache is None: + return None + return ModelReadCache( backend=read_cache, + model_type=model_type, ttl_seconds=READ_CACHE_TTL_SECONDS, max_payload_bytes=READ_CACHE_MAX_PAYLOAD_BYTES, ) - - -ConfiguredReadCacheDep = Annotated[ConfiguredReadCache, Depends(get_configured_read_cache)] diff --git a/src/basic_memory/index/local_moves.py b/src/basic_memory/index/local_moves.py index f23d5a91b..da6c332ba 100644 --- a/src/basic_memory/index/local_moves.py +++ b/src/basic_memory/index/local_moves.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import Mapping, Sequence +from contextlib import nullcontext from dataclasses import dataclass from pathlib import Path from typing import override, Protocol @@ -173,7 +174,7 @@ class LocalWatchMoveProcessor: maintenance_runner: ProjectIndexMaintenanceRunner moved_entity_search_refresher: ProjectIndexMovedEntitySearchRefresher project_external_id: str - read_cache: ReadCache + read_cache: ReadCache | None batch_size: int = 100 async def process_moves( @@ -188,7 +189,12 @@ async def process_moves( # Move pairs bypass ordinary file/delete completion callbacks. # Invalidate after maintenance and search refresh so cached paths, # permalinks, and relations cannot retain the pre-move state. - async with invalidate_cache(self.read_cache, self.project_external_id): + invalidation_scope = ( + invalidate_cache(self.read_cache, self.project_external_id) + if self.read_cache is not None + else nullcontext() + ) + async with invalidation_scope: move_run = await self.maintenance_runner.run_move_batches( moved_files=moved_files, batch_size=self.batch_size, diff --git a/src/basic_memory/index/local_project.py b/src/basic_memory/index/local_project.py index a19899519..1c56aa3b4 100644 --- a/src/basic_memory/index/local_project.py +++ b/src/basic_memory/index/local_project.py @@ -5,7 +5,8 @@ import asyncio import os from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field +from contextlib import nullcontext +from dataclasses import dataclass from pathlib import Path from typing import Any, override, Protocol @@ -81,7 +82,6 @@ ) from basic_memory.models import Entity, Project from basic_memory.read_cache import ( - NullReadCache, ReadCache, ReadCacheInvalidator, invalidate_cache, @@ -466,7 +466,7 @@ class LocalProjectIndexRuntime: embedding_vector_sync: EmbeddingBatchVectorSync | None = None batch_size: int = 100 coordinator_job_id: RuntimeJobId | None = None - read_cache: ReadCache = field(default_factory=NullReadCache) + read_cache: ReadCache | None = None LocalProjectIndexObservation = ProjectIndexObservation @@ -552,7 +552,7 @@ class LocalProjectIndexBatchEnqueuer(ProjectIndexBatchEnqueuer): reader: IndexFileBatchReader[IndexInputFile] indexer: IndexFileBatchIndexer[IndexInputFile] content_classifier: IndexFileBatchContentClassifier - read_cache: ReadCacheInvalidator = field(default_factory=NullReadCache) + read_cache: ReadCacheInvalidator | None = None read_max_concurrent: int = 8 index_max_concurrent: int = 8 @@ -561,10 +561,15 @@ async def enqueue_index_file_batch( self, request: RuntimeIndexFileBatchJobRequest, ) -> IndexFileBatchJobResult: - async with invalidate_cache( - self.read_cache, - request.project.project_external_id, - ): + invalidation_scope = ( + invalidate_cache( + self.read_cache, + request.project.project_external_id, + ) + if self.read_cache is not None + else nullcontext() + ) + async with invalidation_scope: return await run_index_file_batch( request, checker=self.checker, @@ -586,7 +591,7 @@ class LocalProjectIndexRuntimeFactory: batch_size: int = 100 read_max_concurrent: int = 8 index_max_concurrent: int = 8 - read_cache: ReadCache = field(default_factory=NullReadCache) + read_cache: ReadCache | None = None async def dependencies_for_project(self, project: Project) -> LocalIndexProjectDependencies: return await self.dependency_provider.dependencies_for_project(project) @@ -629,11 +634,15 @@ def runtime_from_dependencies( # concurrent creation that must be checksum-verified before deletion. verify_replaced_move_targets=True, ) - invalidating_maintenance_store = InvalidatingProjectIndexBatchStore( - move_store=maintenance_store, - delete_store=maintenance_store, - read_cache=self.read_cache, - project_external_id=project_external_id, + active_maintenance_store = ( + InvalidatingProjectIndexBatchStore( + move_store=maintenance_store, + delete_store=maintenance_store, + read_cache=self.read_cache, + project_external_id=project_external_id, + ) + if self.read_cache is not None + else maintenance_store ) return LocalProjectIndexRuntime( observed_file_source=LocalProjectIndexObservedFileSource( @@ -648,8 +657,8 @@ def runtime_from_dependencies( entity_repository=dependencies.entity_repository, ), maintenance_runner=StoreProjectIndexMaintenanceRunner( - move_store=invalidating_maintenance_store, - delete_store=invalidating_maintenance_store, + move_store=active_maintenance_store, + delete_store=active_maintenance_store, ), moved_entity_search_refresher=RepositoryProjectIndexMovedEntitySearchRefresher( session_maker=dependencies.session_maker, @@ -790,10 +799,15 @@ async def run_local_project_index( # The coordinator commits moves, deletes, and file batches incrementally. # Invalidate even when a later batch or vector sync raises so already # published changes cannot remain behind the previous generation. - async with invalidate_cache( - runtime.read_cache, - request.project.project_external_id, - ): + invalidation_scope = ( + invalidate_cache( + runtime.read_cache, + request.project.project_external_id, + ) + if runtime.read_cache is not None + else nullcontext() + ) + async with invalidation_scope: result = await run_project_index_coordinator( request, coordinator_job_id=runtime.coordinator_job_id, @@ -811,10 +825,15 @@ async def run_local_project_index( # Relation repair can mutate cached entity responses after the first # invalidation. Clear any value filled during that window, including # when repair commits partial progress before raising. - async with invalidate_cache( - runtime.read_cache, - request.project.project_external_id, - ): + relation_invalidation_scope = ( + invalidate_cache( + runtime.read_cache, + request.project.project_external_id, + ) + if runtime.read_cache is not None + else nullcontext() + ) + async with relation_invalidation_scope: await resolve_project_index_completion_relations( ProjectIndexRelationResolutionContext( project_id=request.project.project_id, diff --git a/src/basic_memory/index/local_runtime.py b/src/basic_memory/index/local_runtime.py index b49c80dcb..7e1df799c 100644 --- a/src/basic_memory/index/local_runtime.py +++ b/src/basic_memory/index/local_runtime.py @@ -2,7 +2,8 @@ from __future__ import annotations -from dataclasses import dataclass, field +from contextlib import nullcontext +from dataclasses import dataclass from loguru import logger @@ -58,7 +59,6 @@ ) from basic_memory.models import Entity, Project from basic_memory.read_cache import ( - NullReadCache, ReadCache, invalidate_cache, invalidate_project_read_cache, @@ -151,7 +151,7 @@ class LocalInlineStorageEventResultRecorder: relation_cleanup_search_refresher: ProjectIndexMovedEntitySearchRefresher relation_runtime: RelationResolutionRuntime index_embeddings: bool - read_cache: ReadCache = field(default_factory=NullReadCache) + read_cache: ReadCache | None = None async def index_file_completed( self, @@ -166,7 +166,7 @@ async def index_file_completed( entity_id=result.entity_id, ) - if result.status == IndexFileJobStatus.processed: + if result.status == IndexFileJobStatus.processed and self.read_cache is not None: await invalidate_project_read_cache( self.read_cache, self.project.project_external_id, @@ -185,10 +185,15 @@ async def index_file_completed( # Relation repair changes cached entity payloads after indexing. # A second generation bump closes the fill window opened by the # first post-index invalidation, even after partial failure. - async with invalidate_cache( - self.read_cache, - self.project.project_external_id, - ): + invalidation_scope = ( + invalidate_cache( + self.read_cache, + self.project.project_external_id, + ) + if self.read_cache is not None + else nullcontext() + ) + async with invalidation_scope: relation_result = await resolve_project_relations(self.relation_runtime) logger.info( "Local event-index relation repair completed", @@ -230,17 +235,23 @@ async def delete_file_completed( ) if not result.entity_deleted: return - await invalidate_project_read_cache( - self.read_cache, - self.project.project_external_id, - ) + if self.read_cache is not None: + await invalidate_project_read_cache( + self.read_cache, + self.project.project_external_id, + ) # Cleanup may rewrite relations on surviving entities. Invalidate # values filled after the delete became visible, including partial # cleanup progress followed by an error. - async with invalidate_cache( - self.read_cache, - self.project.project_external_id, - ): + invalidation_scope = ( + invalidate_cache( + self.read_cache, + self.project.project_external_id, + ) + if self.read_cache is not None + else nullcontext() + ) + async with invalidation_scope: if not isinstance(result.deleted_entity, Entity): raise RuntimeError( "Local external file delete returned an incomplete entity result" @@ -267,7 +278,10 @@ async def event_failed( file_path=operation.relative_path, error=str(exc), ) - if operation.kind == RuntimeStorageEventOperationKind.index_file: + if ( + operation.kind == RuntimeStorageEventOperationKind.index_file + and self.read_cache is not None + ): # LocalMarkdownFileIndexer commits the entity before all search and # reconciliation follow-ups complete. A watcher failure can therefore # publish partial state even though the success callback never runs. @@ -306,7 +320,7 @@ class LocalWatchEventIndexRuntimeFactory: # let runtime construction opt in via semantic_search_enabled (#1016). index_embeddings: bool = False move_batch_size: int = 100 - read_cache: ReadCache = field(default_factory=NullReadCache) + read_cache: ReadCache | None = None async def runtime_for_project(self, project: Project) -> StorageEventIndexRuntime: dependencies = await self.dependency_provider.dependencies_for_project(project) diff --git a/src/basic_memory/index/local_schedulers.py b/src/basic_memory/index/local_schedulers.py index 75ff25850..2ade6a102 100644 --- a/src/basic_memory/index/local_schedulers.py +++ b/src/basic_memory/index/local_schedulers.py @@ -9,6 +9,7 @@ """ import asyncio +from contextlib import nullcontext from dataclasses import dataclass from typing import Any, Coroutine @@ -20,7 +21,7 @@ RelationResolutionRuntime, resolve_project_relations, ) -from basic_memory.read_cache import ReadCache, invalidate_cache +from basic_memory.read_cache import ReadCacheInvalidator, invalidate_cache from basic_memory.runtime.vector_sync import EntityVectorSync # --- Background Task Machinery --- @@ -194,7 +195,7 @@ class LocalRelationResolutionScheduler: relation_runtime: RelationResolutionRuntime project_external_id: str - read_cache: ReadCache + read_cache: ReadCacheInvalidator | None test_mode: bool debounce_seconds: float = 0.5 @@ -226,7 +227,12 @@ async def _resolve_after_debounce(self, project_id: int) -> None: # Relation resolution commits entity changes after the index pass. # A second bump closes the window in which an intermediate entity # response could have populated the current generation. - async with invalidate_cache(self.read_cache, self.project_external_id): + invalidation_scope = ( + invalidate_cache(self.read_cache, self.project_external_id) + if self.read_cache is not None + else nullcontext() + ) + async with invalidation_scope: await resolve_project_relations(self.relation_runtime) finally: rerun = project_id in _dirty_relation_resolution diff --git a/src/basic_memory/index/note_content_materialization.py b/src/basic_memory/index/note_content_materialization.py index 0de4e9dde..7e0c4b658 100644 --- a/src/basic_memory/index/note_content_materialization.py +++ b/src/basic_memory/index/note_content_materialization.py @@ -4,7 +4,7 @@ import asyncio from collections.abc import Coroutine, Mapping -from contextlib import suppress +from contextlib import nullcontext, suppress from dataclasses import dataclass, replace from typing import Any @@ -51,7 +51,7 @@ NoteFileVacateRepository, RecoverableVacate, ) -from basic_memory.read_cache import ReadCache, invalidate_cache +from basic_memory.read_cache import ReadCacheInvalidator, invalidate_cache from basic_memory.schemas.response import ObservationResponse, RelationResponse from basic_memory.services.file_service import FileService @@ -584,7 +584,7 @@ class LocalNoteContentMaterializationProvider: session_maker: async_sessionmaker[AsyncSession] file_service: FileService project_external_id: str - read_cache: ReadCache + read_cache: ReadCacheInvalidator | None file_indexer: IndexFileExecutor | None = None test_mode: bool = False materialization_workers: int = 4 @@ -643,7 +643,12 @@ async def _materialize_write_now( # The accepted-write invalidation runs before deferred materialization. # Invalidate again after status publication and indexing so a read # filled during that window cannot survive the terminal state. - async with invalidate_cache(self.read_cache, self.project_external_id): + invalidation_scope = ( + invalidate_cache(self.read_cache, self.project_external_id) + if self.read_cache is not None + else nullcontext() + ) + async with invalidation_scope: storage = LocalNoteContentStorage(self.file_service) cleanup_enqueuer = InlineNoteFileDeleteEnqueuer( storage, diff --git a/src/basic_memory/index/watch_coordinator.py b/src/basic_memory/index/watch_coordinator.py index 4c527343b..8f3d0a07c 100644 --- a/src/basic_memory/index/watch_coordinator.py +++ b/src/basic_memory/index/watch_coordinator.py @@ -9,7 +9,7 @@ from loguru import logger from basic_memory.config import BasicMemoryConfig -from basic_memory.read_cache import NullReadCache, ReadCache +from basic_memory.read_cache import ReadCache class WatchStatus(Enum): @@ -31,7 +31,7 @@ class WatchCoordinator: should_watch: bool = True skip_reason: str | None = None quiet: bool = True - read_cache: ReadCache = field(default_factory=NullReadCache) + read_cache: ReadCache | None = None _status: WatchStatus = field(default=WatchStatus.NOT_STARTED, init=False) _watch_task: asyncio.Task[None] | None = field(default=None, init=False) diff --git a/src/basic_memory/read_cache/__init__.py b/src/basic_memory/read_cache/__init__.py index c0e005fcc..ff47c8521 100644 --- a/src/basic_memory/read_cache/__init__.py +++ b/src/basic_memory/read_cache/__init__.py @@ -17,12 +17,10 @@ invalidate_project_read_cache, ) from basic_memory.read_cache.keys import read_cache_request_digest -from basic_memory.read_cache.null import NullReadCache -from basic_memory.read_cache.read_through import ConfiguredReadCache +from basic_memory.read_cache.read_through import ModelReadCache, ReadCacheScope __all__ = [ - "ConfiguredReadCache", - "NullReadCache", + "ModelReadCache", "ReadCache", "ReadCacheDataError", "ReadCacheInvalidator", @@ -30,6 +28,7 @@ "ReadCacheKey", "ReadCacheLookup", "ReadCacheOperation", + "ReadCacheScope", "ReadCacheStoreStatus", "ReadCacheUnavailable", "finish_project_read_cache_invalidation", diff --git a/src/basic_memory/read_cache/contract.py b/src/basic_memory/read_cache/contract.py index 2e96c238b..1a33735a1 100644 --- a/src/basic_memory/read_cache/contract.py +++ b/src/basic_memory/read_cache/contract.py @@ -19,7 +19,6 @@ class ReadCacheStoreStatus(StrEnum): stored = "stored" superseded = "superseded" - disabled = "disabled" class ReadCacheInvalidationStatus(StrEnum): @@ -27,7 +26,6 @@ class ReadCacheInvalidationStatus(StrEnum): invalidated = "invalidated" unavailable = "unavailable" - disabled = "disabled" def canonical_read_cache_project_id(project_id: str) -> str: @@ -67,18 +65,14 @@ def __post_init__(self) -> None: @dataclass(frozen=True, slots=True) class ReadCacheLookup: - """Cache lookup result plus the generation observed by that read. + """Cache lookup result plus the generation observed by that read.""" - A missing generation means the cache implementation is disabled. Read-through - callers can then skip serialization and the store call entirely. - """ - - generation: str | None + generation: str payload: bytes | None = None def __post_init__(self) -> None: - if self.generation is None and self.payload is not None: - raise ValueError("read-cache payload requires a lookup generation") + if not self.generation: + raise ValueError("read-cache lookup generation must not be empty") @property def is_hit(self) -> bool: diff --git a/src/basic_memory/read_cache/null.py b/src/basic_memory/read_cache/null.py deleted file mode 100644 index e7dc957e3..000000000 --- a/src/basic_memory/read_cache/null.py +++ /dev/null @@ -1,28 +0,0 @@ -"""No-op cache used by default local-first installations.""" - -from basic_memory.read_cache.contract import ( - ReadCacheInvalidationStatus, - ReadCacheKey, - ReadCacheLookup, - ReadCacheStoreStatus, -) - - -class NullReadCache: - """A disabled cache implementation with no external dependencies.""" - - async def lookup(self, key: ReadCacheKey) -> ReadCacheLookup: - return ReadCacheLookup(generation=None) - - async def store( - self, - key: ReadCacheKey, - lookup: ReadCacheLookup, - payload: bytes, - *, - ttl_seconds: int, - ) -> ReadCacheStoreStatus: - return ReadCacheStoreStatus.disabled - - async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: - return ReadCacheInvalidationStatus.disabled diff --git a/src/basic_memory/read_cache/read_through.py b/src/basic_memory/read_cache/read_through.py index 973ea0174..719ae24d5 100644 --- a/src/basic_memory/read_cache/read_through.py +++ b/src/basic_memory/read_cache/read_through.py @@ -40,10 +40,11 @@ def require_value(self) -> ModelT: @dataclass(frozen=True, slots=True) -class ConfiguredReadCache: - """Request dependency that binds one cache backend to read policy.""" +class ModelReadCache[ModelT: BaseModel]: + """Typed facade that binds one cache backend to a response model and policy.""" backend: ReadCache + model_type: type[ModelT] ttl_seconds: int max_payload_bytes: int @@ -58,11 +59,10 @@ async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStat return await self.backend.invalidate_project(project_id) @asynccontextmanager - async def read[ModelT: BaseModel]( + async def read( self, *, key: ReadCacheKey, - model_type: type[ModelT], ) -> AsyncIterator[ReadCacheScope[ModelT]]: """Yield a cached model or store the authoritative value supplied by the route.""" with logfire.span( @@ -82,18 +82,6 @@ async def read[ModelT: BaseModel]( result.require_value() return - if lookup.generation is None: - # Trigger: the host selected the no-op cache implementation. - # Why: an optional cache must not serialize every response merely to - # discover that storage is disabled. - # Outcome: execute only the authoritative read path. - _record_event(key, "disabled") - span.set_attribute("cache.outcome", "disabled") - result = ReadCacheScope[ModelT]() - yield result - result.require_value() - return - if lookup.payload is not None: _record_event(key, "hit") span.set_attributes( @@ -103,7 +91,7 @@ async def read[ModelT: BaseModel]( } ) yield ReadCacheScope( - value=model_type.model_validate_json(lookup.payload), + value=self.model_type.model_validate_json(lookup.payload), cacheable=False, ) return diff --git a/src/basic_memory/read_cache/redis.py b/src/basic_memory/read_cache/redis.py index 9ba7e0046..b40dd6ca4 100644 --- a/src/basic_memory/read_cache/redis.py +++ b/src/basic_memory/read_cache/redis.py @@ -172,8 +172,6 @@ async def store( *, ttl_seconds: int, ) -> ReadCacheStoreStatus: - if lookup.generation is None: - raise ValueError("Redis cache store requires a lookup generation") if ttl_seconds <= 0: raise ValueError("read-cache ttl_seconds must be positive") diff --git a/src/basic_memory/services/directory_deletes.py b/src/basic_memory/services/directory_deletes.py index e1869b35e..6cc676df8 100644 --- a/src/basic_memory/services/directory_deletes.py +++ b/src/basic_memory/services/directory_deletes.py @@ -20,8 +20,7 @@ normalize_directory_delete_path, ) from basic_memory.read_cache import ( - NullReadCache, - ReadCache, + ReadCacheInvalidator, finish_project_read_cache_invalidation, ) @@ -62,7 +61,7 @@ async def delete_directory( *, project_external_id: str, directory: str, - read_cache: ReadCache | None = None, + read_cache: ReadCacheInvalidator | None = None, ) -> DirectoryDeleteAcceptedResult: """Delete directory entities immediately and queue file cleanup in the background. @@ -73,7 +72,6 @@ async def delete_directory( project_external_id=project_external_id, directory=directory, ) - active_read_cache = read_cache if read_cache is not None else NullReadCache() delete_may_have_committed = False try: # scoped_session enables `PRAGMA foreign_keys=ON` for SQLite; this bulk @@ -90,12 +88,12 @@ async def delete_directory( except DirectoryDeleteRejected as error: raise directory_delete_service_error_from_rejection(error.rejection) from error finally: - if delete_may_have_committed: + if delete_may_have_committed and read_cache is not None: # Acceptance commits entity and search deletion before storage cleanup. # Finish the generation bump even when cancellation interrupts the # transaction exit, then preserve that cancellation. await finish_project_read_cache_invalidation( - active_read_cache, + read_cache, project_external_id, ) @@ -119,11 +117,11 @@ async def delete_directory( return result finally: - if accepted.files: + if accepted.files and read_cache is not None: # Cleanup and relation refresh can publish additional state or fail # after partial progress. Close the fill window in either case. await finish_project_read_cache_invalidation( - active_read_cache, + read_cache, project_external_id, ) diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index 54e805e3e..d5f2c546e 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -1132,7 +1132,7 @@ async def move_directory( app_config: BasicMemoryConfig, *, project_external_id: str, - read_cache: ReadCache, + read_cache: ReadCache | None, ) -> DirectoryMoveResult: """Move all entities in a directory to a new location. @@ -1208,7 +1208,8 @@ async def move_directory( # Each move commits independently. Invalidate before the next file so a # long directory batch cannot serve early moves from the old generation. - await invalidate_project_read_cache(read_cache, project_external_id) + if read_cache is not None: + await invalidate_project_read_cache(read_cache, project_external_id) moved_files.append(new_path) successful_moves += 1 logger.debug(f"Moved entity: {old_path} -> {new_path}") diff --git a/src/basic_memory/services/initialization.py b/src/basic_memory/services/initialization.py index a4e3d18f6..77de17ecb 100644 --- a/src/basic_memory/services/initialization.py +++ b/src/basic_memory/services/initialization.py @@ -23,7 +23,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from basic_memory.index.local_project import LocalProjectIndexRuntimeProvider - from basic_memory.read_cache import ReadCache + from basic_memory.read_cache import ReadCache, ReadCacheInvalidator async def run_initial_project_index( @@ -52,7 +52,7 @@ async def recover_project_materializations( project: Project, session_maker: "async_sessionmaker[AsyncSession]", *, - read_cache: "ReadCache | None" = None, + read_cache: "ReadCacheInvalidator | None" = None, ) -> None: """Re-drive note materialization and move cleanup lost across a process exit. @@ -68,14 +68,12 @@ async def recover_project_materializations( recover_move_vacates, recover_stuck_materializations, ) - from basic_memory.read_cache import NullReadCache, invalidate_project_read_cache + from basic_memory.read_cache import invalidate_project_read_cache from basic_memory.services.file_service import FileService # FileService needs only base_path to write the accepted markdown bytes; # the markdown_processor/app_config are unused on the materialization path. file_service = FileService(Path(project.path)) - active_read_cache = read_cache if read_cache is not None else NullReadCache() - try: materialization_recovery = await recover_stuck_materializations( session_maker=session_maker, @@ -97,10 +95,11 @@ async def recover_project_materializations( # Redis can outlive the process that left this materialization unfinished. # Invalidate this committed phase before move-vacate recovery begins; a # later setup/query failure must not leave its published state cached. - await invalidate_project_read_cache( - active_read_cache, - str(project.external_id), - ) + if read_cache is not None: + await invalidate_project_read_cache( + read_cache, + str(project.external_id), + ) try: recovered_vacates = await recover_move_vacates( @@ -121,10 +120,11 @@ async def recover_project_materializations( recovered_move_vacates=recovered_vacates, ) - await invalidate_project_read_cache( - active_read_cache, - str(project.external_id), - ) + if read_cache is not None: + await invalidate_project_read_cache( + read_cache, + str(project.external_id), + ) async def initialize_database(app_config: BasicMemoryConfig) -> None: @@ -214,7 +214,6 @@ async def initialize_file_indexing( from basic_memory.index.local_project import LocalProjectIndexRuntimeFactory from basic_memory.index.local_runtime import LocalWatchEventIndexRuntimeFactory from basic_memory.index.watch_service import WatchService - from basic_memory.read_cache import NullReadCache # Get database session (migrations already run if needed) _, session_maker = await db.get_or_create_db( @@ -228,14 +227,12 @@ async def initialize_file_indexing( # running multiple `basic-memory mcp --project X` processes does not produce # duplicate watchers fighting over the same files. constrained_project = os.environ.get("BASIC_MEMORY_MCP_PROJECT") - active_read_cache = read_cache if read_cache is not None else NullReadCache() - event_index_runtime_factory = LocalWatchEventIndexRuntimeFactory( index_embeddings=app_config.semantic_search_enabled, - read_cache=active_read_cache, + read_cache=read_cache, ) project_index_runtime_factory = LocalProjectIndexRuntimeFactory( - read_cache=active_read_cache, + read_cache=read_cache, ) # Initialize watch service @@ -272,7 +269,7 @@ async def initialize_file_indexing( await recover_project_materializations( project, session_maker, - read_cache=active_read_cache, + read_cache=read_cache, ) # Trigger: the API/MCP lifespan is waiting for durable startup recovery. diff --git a/src/basic_memory/services/note_content_writes.py b/src/basic_memory/services/note_content_writes.py index 772d98e34..ba5822e89 100644 --- a/src/basic_memory/services/note_content_writes.py +++ b/src/basic_memory/services/note_content_writes.py @@ -31,8 +31,7 @@ RuntimeNoteContentResponsePayload, ) from basic_memory.read_cache import ( - NullReadCache, - ReadCache, + ReadCacheInvalidator, finish_project_read_cache_invalidation, ) from basic_memory.schemas.base import Entity as EntitySchema @@ -143,13 +142,13 @@ def __init__( mutation_dependencies: AcceptedNoteMutationDependencies, content_freshener: NoteContentMutationFreshener | None = None, actor_resolver: NoteContentMutationActorResolver | None = None, - read_cache: ReadCache | None = None, + read_cache: ReadCacheInvalidator | None = None, ) -> None: self.session_maker = session_maker self.mutation_dependencies = mutation_dependencies self.content_freshener = content_freshener self.actor_resolver = actor_resolver - self.read_cache = read_cache if read_cache is not None else NullReadCache() + self.read_cache = read_cache @asynccontextmanager async def _mutation_cache_scope( @@ -159,12 +158,17 @@ async def _mutation_cache_scope( invalidate_on_rejection: bool = False, ) -> AsyncIterator[None]: """Invalidate after a mutation can publish authoritative state.""" + read_cache = self.read_cache + if read_cache is None: + yield + return + try: yield except AcceptedNoteMutationRejected: if invalidate_on_rejection: await finish_project_read_cache_invalidation( - self.read_cache, + read_cache, project_external_id, ) raise @@ -173,13 +177,13 @@ async def _mutation_cache_scope( # Invalidation is safe after an earlier rollback and required after # any commit whose response was interrupted. await finish_project_read_cache_invalidation( - self.read_cache, + read_cache, project_external_id, ) raise else: await finish_project_read_cache_invalidation( - self.read_cache, + read_cache, project_external_id, ) @@ -224,10 +228,11 @@ async def freshen_existing_note_content( # Freshening can commit entity and note-content state before a later # indexing follow-up raises. Invalidate before propagating so those # partial publications cannot retain the previous cache generation. - await finish_project_read_cache_invalidation( - self.read_cache, - project_external_id, - ) + if self.read_cache is not None: + await finish_project_read_cache_invalidation( + self.read_cache, + project_external_id, + ) raise return True diff --git a/test-int/read_cache/test_read_cache_benchmark.py b/test-int/read_cache/test_read_cache_benchmark.py index 647901264..996ad9c35 100644 --- a/test-int/read_cache/test_read_cache_benchmark.py +++ b/test-int/read_cache/test_read_cache_benchmark.py @@ -15,7 +15,6 @@ from basic_memory.deps import get_read_cache from basic_memory.models import Project -from basic_memory.read_cache import NullReadCache from basic_memory.read_cache.redis import RedisReadCache @@ -77,7 +76,7 @@ def count_query(*_: object) -> None: engine, _ = engine_factory event.listen(engine.sync_engine, "before_cursor_execute", count_query) try: - app.dependency_overrides[get_read_cache] = NullReadCache + app.dependency_overrides[get_read_cache] = lambda: None await client.get(entity_url) query_count = 0 authoritative_latencies = await _entity_read_latencies( diff --git a/test-int/read_cache/test_redis_read_cache.py b/test-int/read_cache/test_redis_read_cache.py index 01347ac55..0c3e6758c 100644 --- a/test-int/read_cache/test_redis_read_cache.py +++ b/test-int/read_cache/test_redis_read_cache.py @@ -12,8 +12,7 @@ from redis.asyncio import Redis from basic_memory.read_cache import ( - ConfiguredReadCache, - NullReadCache, + ModelReadCache, ReadCacheDataError, ReadCacheInvalidationStatus, ReadCacheKey, @@ -54,6 +53,13 @@ class CachedEntity(BaseModel): title: str +class CachedResolution(BaseModel): + """A second boundary type used to prove facade-local serialization.""" + + external_id: str + resolution_method: str + + pytestmark = pytest.mark.redis PROJECT_ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" @@ -73,6 +79,20 @@ def _key( ) +def _model_cache( + backend: RedisReadCache, + *, + ttl_seconds: int = 60, + max_payload_bytes: int = 1_024, +) -> ModelReadCache[CachedEntity]: + return ModelReadCache( + backend=backend, + model_type=CachedEntity, + ttl_seconds=ttl_seconds, + max_payload_bytes=max_payload_bytes, + ) + + @pytest.mark.asyncio async def test_round_trip_and_ttl_expiry(redis_cache: RedisCacheHarness) -> None: key = _key() @@ -478,20 +498,13 @@ async def test_real_redis_capacity_failures_are_cache_unavailable( (await redis_cache.client.config_get("maxmemory-policy"))["maxmemory-policy"] ) await redis_cache.cache.lookup(key) - read_cache = ConfiguredReadCache( - backend=redis_cache.cache, - ttl_seconds=60, - max_payload_bytes=1_024, - ) + read_cache = _model_cache(redis_cache.cache) try: await redis_cache.client.config_set("maxmemory-policy", "noeviction") await redis_cache.client.config_set("maxmemory", "1") - async with read_cache.read( - key=key, - model_type=CachedEntity, - ) as cached: + async with read_cache.read(key=key) as cached: assert cached.value is None cached.value = CachedEntity( external_id="entity-1", @@ -510,65 +523,16 @@ async def test_real_redis_capacity_failures_are_cache_unavailable( assert invalidation_status is ReadCacheInvalidationStatus.unavailable -@pytest.mark.asyncio -async def test_null_cache_preserves_disabled_semantics() -> None: - class StoreMustNotRun(NullReadCache): - @override - async def store( - self, - key: ReadCacheKey, - lookup: ReadCacheLookup, - payload: bytes, - *, - ttl_seconds: int, - ) -> ReadCacheStoreStatus: - raise AssertionError("disabled read-through must skip serialization and storage") - - cache = StoreMustNotRun() - key = _key() - lookup = await cache.lookup(key) - - assert lookup == ReadCacheLookup(generation=None) - store_status = await NullReadCache().store(key, lookup, b"ignored", ttl_seconds=60) - assert store_status is ReadCacheStoreStatus.disabled - status = await cache.invalidate_project(key.project_id) - assert status is ReadCacheInvalidationStatus.disabled - read_cache = ConfiguredReadCache( - backend=cache, - ttl_seconds=60, - max_payload_bytes=1_024, - ) - - async with read_cache.read( - key=key, - model_type=CachedEntity, - ) as cached: - assert cached.value is None - cached.value = CachedEntity( - external_id="entity-1", - title="Authoritative", - ) - result = cached.value - assert result.title == "Authoritative" - - @pytest.mark.asyncio async def test_typed_read_through_uses_real_cached_representation( redis_cache: RedisCacheHarness, ) -> None: loads = 0 - read_cache = ConfiguredReadCache( - backend=redis_cache.cache, - ttl_seconds=60, - max_payload_bytes=1_024, - ) + read_cache = _model_cache(redis_cache.cache) results: list[CachedEntity] = [] for _ in range(2): - async with read_cache.read( - key=_key(), - model_type=CachedEntity, - ) as cached: + async with read_cache.read(key=_key()) as cached: if cached.value is None: loads += 1 cached.value = CachedEntity( @@ -584,21 +548,51 @@ async def test_typed_read_through_uses_real_cached_representation( @pytest.mark.asyncio -async def test_typed_read_through_does_not_cache_oversize_models( +async def test_typed_facades_share_backend_and_keep_model_types_local( redis_cache: RedisCacheHarness, ) -> None: - loads = 0 - read_cache = ConfiguredReadCache( + entity_cache = _model_cache(redis_cache.cache) + resolution_cache = ModelReadCache( backend=redis_cache.cache, + model_type=CachedResolution, ttl_seconds=60, - max_payload_bytes=1, + max_payload_bytes=1_024, + ) + entity_key = _key(request="typed-entity") + resolution_key = _key( + operation=ReadCacheOperation.resolve, + request="typed-resolution", ) + async with entity_cache.read(key=entity_key) as cached_entity: + cached_entity.value = CachedEntity(external_id="entity-1", title="First") + async with resolution_cache.read(key=resolution_key) as cached_resolution: + cached_resolution.value = CachedResolution( + external_id="entity-1", + resolution_method="permalink", + ) + + async with entity_cache.read(key=entity_key) as cached_entity: + assert cached_entity.value == CachedEntity(external_id="entity-1", title="First") + async with resolution_cache.read(key=resolution_key) as cached_resolution: + assert cached_resolution.value == CachedResolution( + external_id="entity-1", + resolution_method="permalink", + ) + + assert entity_cache.backend is redis_cache.cache + assert resolution_cache.backend is redis_cache.cache + + +@pytest.mark.asyncio +async def test_typed_read_through_does_not_cache_oversize_models( + redis_cache: RedisCacheHarness, +) -> None: + loads = 0 + read_cache = _model_cache(redis_cache.cache, max_payload_bytes=1) + for _ in range(2): - async with read_cache.read( - key=_key(), - model_type=CachedEntity, - ) as cached: + async with read_cache.read(key=_key()) as cached: assert cached.value is None loads += 1 cached.value = CachedEntity( @@ -614,16 +608,11 @@ async def test_typed_read_through_does_not_cache_ineligible_models( redis_cache: RedisCacheHarness, ) -> None: loads = 0 - read_cache = ConfiguredReadCache( - backend=redis_cache.cache, - ttl_seconds=60, - max_payload_bytes=1_024, - ) + read_cache = _model_cache(redis_cache.cache) for _ in range(2): async with read_cache.read( key=_key(operation=ReadCacheOperation.resolve), - model_type=CachedEntity, ) as cached: assert cached.value is None loads += 1 @@ -641,17 +630,10 @@ async def test_typed_read_through_does_not_store_after_body_error( redis_cache: RedisCacheHarness, ) -> None: key = _key(request="failed-authoritative-read") - read_cache = ConfiguredReadCache( - backend=redis_cache.cache, - ttl_seconds=60, - max_payload_bytes=1_024, - ) + read_cache = _model_cache(redis_cache.cache) with pytest.raises(RuntimeError, match="authoritative read failed"): - async with read_cache.read( - key=key, - model_type=CachedEntity, - ) as cached: + async with read_cache.read(key=key) as cached: cached.value = CachedEntity( external_id="entity-1", title="Must not be stored", @@ -668,17 +650,10 @@ async def test_typed_read_through_rejects_invalid_cached_models( key = _key() miss = await redis_cache.cache.lookup(key) await redis_cache.cache.store(key, miss, b'{"wrong":"shape"}', ttl_seconds=60) - read_cache = ConfiguredReadCache( - backend=redis_cache.cache, - ttl_seconds=60, - max_payload_bytes=1_024, - ) + read_cache = _model_cache(redis_cache.cache) with pytest.raises(ValidationError): - async with read_cache.read( - key=key, - model_type=CachedEntity, - ): + async with read_cache.read(key=key): raise AssertionError("invalid cache data must not enter the read scope") @@ -693,17 +668,10 @@ async def test_typed_read_through_bypasses_unavailable_real_redis() -> None: socket_timeout=0.05, ) cache = RedisReadCache(client=client, namespace="unavailable") - read_cache = ConfiguredReadCache( - backend=cache, - ttl_seconds=60, - max_payload_bytes=1_024, - ) + read_cache = _model_cache(cache) try: - async with read_cache.read( - key=_key(), - model_type=CachedEntity, - ) as cached: + async with read_cache.read(key=_key()) as cached: assert cached.value is None cached.value = CachedEntity( external_id="entity-1", @@ -747,17 +715,10 @@ async def test_typed_read_through_returns_data_when_real_redis_store_times_out( namespace="paused-store", prefix=prefix, ) - read_cache = ConfiguredReadCache( - backend=cache, - ttl_seconds=60, - max_payload_bytes=1_024, - ) + read_cache = _model_cache(cache) try: - async with read_cache.read( - key=_key(), - model_type=CachedEntity, - ) as cached: + async with read_cache.read(key=_key()) as cached: assert cached.value is None await redis_cache.client.execute_command("CLIENT", "PAUSE", 200, "WRITE") cached.value = CachedEntity( @@ -774,41 +735,30 @@ async def test_typed_read_through_returns_data_when_real_redis_store_times_out( await client.aclose() -@pytest.mark.asyncio -async def test_typed_read_through_validates_policy_before_loading() -> None: +def test_typed_read_through_validates_policy_before_loading( + redis_cache: RedisCacheHarness, +) -> None: with pytest.raises(ValueError, match="ttl_seconds"): - async with ConfiguredReadCache( - backend=NullReadCache(), + _model_cache( + redis_cache.cache, ttl_seconds=0, max_payload_bytes=1, - ).read( - key=_key(), - model_type=CachedEntity, - ): - raise AssertionError("invalid policy must fail before entering the read scope") + ) with pytest.raises(ValueError, match="max_payload_bytes"): - async with ConfiguredReadCache( - backend=NullReadCache(), + _model_cache( + redis_cache.cache, ttl_seconds=1, max_payload_bytes=0, - ).read( - key=_key(), - model_type=CachedEntity, - ): - raise AssertionError("invalid policy must fail before entering the read scope") + ) @pytest.mark.asyncio -async def test_typed_read_through_requires_an_authoritative_result() -> None: +async def test_typed_read_through_requires_an_authoritative_result( + redis_cache: RedisCacheHarness, +) -> None: + read_cache = _model_cache(redis_cache.cache) with pytest.raises(RuntimeError, match="exited without a result"): - async with ConfiguredReadCache( - backend=NullReadCache(), - ttl_seconds=1, - max_payload_bytes=1, - ).read( - key=_key(), - model_type=CachedEntity, - ): + async with read_cache.read(key=_key(request="missing-authoritative-result")): pass @@ -872,8 +822,8 @@ def test_key_validation_and_canonicalization() -> None: ).request_digest == uppercase_digest.lower() ) - with pytest.raises(ValueError, match="payload requires"): - ReadCacheLookup(generation=None, payload=b"orphaned") + with pytest.raises(ValueError, match="generation"): + ReadCacheLookup(generation="", payload=b"orphaned") with pytest.raises(ValueError, match="prefix"): redis_read_cache_generation_key( prefix="", @@ -902,13 +852,6 @@ async def test_invalid_store_inputs_fail_before_redis( ) -> None: key = _key() - with pytest.raises(ValueError, match="lookup generation"): - await redis_cache.cache.store( - key, - ReadCacheLookup(generation=None), - b"payload", - ttl_seconds=60, - ) with pytest.raises(ValueError, match="positive"): await redis_cache.cache.store( key, diff --git a/tests/cloud/test_note_content_materialization.py b/tests/cloud/test_note_content_materialization.py index 7587dc6d5..57df1248f 100644 --- a/tests/cloud/test_note_content_materialization.py +++ b/tests/cloud/test_note_content_materialization.py @@ -6,7 +6,7 @@ import os from datetime import UTC, datetime from hashlib import sha256 -from typing import Any, cast, override +from typing import Any, cast import pytest from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -28,8 +28,7 @@ ) from basic_memory.repository.note_file_vacate_repository import NoteFileVacateRepository from basic_memory.read_cache import ( - NullReadCache, - ReadCache, + ReadCacheInvalidator, ReadCacheInvalidationStatus, ) from basic_memory.runtime.cleanup import RuntimeNoteFileDeleteJobRequest @@ -67,11 +66,10 @@ async def index_file(self, file_path: str, *, source: str) -> FileIndexResult: ) -class RecordingReadCache(NullReadCache): +class RecordingReadCache: def __init__(self) -> None: self.invalidated_project_ids: list[str] = [] - @override async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: self.invalidated_project_ids.append(project_id) return ReadCacheInvalidationStatus.invalidated @@ -119,7 +117,7 @@ def local_materialization_provider( indexer: RecordingFileIndexer, *, test_mode: bool = True, - read_cache: ReadCache | None = None, + read_cache: ReadCacheInvalidator | None = None, ) -> LocalNoteContentMaterializationProvider: # test_mode=True keeps materialization inline so these tests can assert the # result synchronously; production defers it to a background task. @@ -127,7 +125,7 @@ def local_materialization_provider( session_maker=cast(async_sessionmaker[AsyncSession], object()), file_service=cast(FileService, object()), project_external_id=PROJECT_EXTERNAL_ID, - read_cache=read_cache if read_cache is not None else NullReadCache(), + read_cache=read_cache, file_indexer=indexer, test_mode=test_mode, ) diff --git a/tests/index/test_local_schedulers.py b/tests/index/test_local_schedulers.py index 53fa40ad6..3284f5d37 100644 --- a/tests/index/test_local_schedulers.py +++ b/tests/index/test_local_schedulers.py @@ -1,7 +1,7 @@ """Typed scheduler tests for derived async work.""" import asyncio -from typing import cast, override +from typing import cast import pytest @@ -13,10 +13,7 @@ LocalSearchReindexScheduler, drain_background_tasks, ) -from basic_memory.read_cache import ( - NullReadCache, - ReadCacheInvalidationStatus, -) +from basic_memory.read_cache import ReadCacheInvalidationStatus PROJECT_EXTERNAL_ID = "00000000-0000-0000-0000-000000000013" @@ -47,11 +44,10 @@ async def reindex_all(self) -> None: self.reindexed_project = True -class RecordingReadCache(NullReadCache): +class RecordingReadCache: def __init__(self) -> None: self.invalidated_project_ids: list[str] = [] - @override async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: self.invalidated_project_ids.append(project_id) return ReadCacheInvalidationStatus.invalidated @@ -288,7 +284,7 @@ async def test_relation_resolution_scheduler_coalesces_a_burst(): scheduler = LocalRelationResolutionScheduler( relation_runtime=runtime, project_external_id=PROJECT_EXTERNAL_ID, - read_cache=NullReadCache(), + read_cache=None, test_mode=False, debounce_seconds=0.02, ) @@ -399,7 +395,7 @@ async def resolve_relations(self, entity_id: int | None = None) -> set[int]: scheduler = LocalRelationResolutionScheduler( relation_runtime=runtime, project_external_id=PROJECT_EXTERNAL_ID, - read_cache=NullReadCache(), + read_cache=None, test_mode=False, debounce_seconds=0.0, ) diff --git a/tests/test_deps.py b/tests/test_deps.py index 2c1af30a6..cb106be13 100644 --- a/tests/test_deps.py +++ b/tests/test_deps.py @@ -7,7 +7,6 @@ from basic_memory.api.container import ApiContainer, resolve_container from basic_memory.deps import get_app_config, get_read_cache, validate_project_external_id from basic_memory.models.project import Project -from basic_memory.read_cache import NullReadCache from basic_memory.repository.project_repository import ProjectRepository from basic_memory.runtime.mode import resolve_runtime_mode @@ -46,30 +45,26 @@ def test_resolve_container_prefers_installed_container(app_config, monkeypatch): def test_get_read_cache_reads_lifespan_container(app_config): - """API requests get the cache the lifespan stored on app.state.""" + """API requests preserve the container's absent cache backend.""" app = FastAPI() - cache = NullReadCache() app.state.container = ApiContainer( config=app_config, mode=resolve_runtime_mode(is_test_env=True), - read_cache=cache, ) - assert get_read_cache(_request_for(app)) is cache + assert get_read_cache(_request_for(app)) is None def test_get_read_cache_falls_back_to_composition_root(app_config, monkeypatch): - """Off-lifespan requests resolve the cache from the API composition root.""" + """Off-lifespan requests preserve the composition root's absent cache backend.""" app = FastAPI() - cache = NullReadCache() installed = ApiContainer( config=app_config, mode=resolve_runtime_mode(is_test_env=True), - read_cache=cache, ) monkeypatch.setattr(container_module, "_container", installed) - assert get_read_cache(_request_for(app)) is cache + assert get_read_cache(_request_for(app)) is None @pytest.mark.asyncio From 53076d7731736590948f6561ec94705f9747f765 Mon Sep 17 00:00:00 2001 From: phernandez Date: Thu, 30 Jul 2026 10:14:03 -0500 Subject: [PATCH 18/28] test(index): cover optional project cache wiring Signed-off-by: phernandez --- .../test_local_project_vector_cleaner_wiring.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/tests/index/test_local_project_vector_cleaner_wiring.py b/tests/index/test_local_project_vector_cleaner_wiring.py index 0480fa891..ea4ca1428 100644 --- a/tests/index/test_local_project_vector_cleaner_wiring.py +++ b/tests/index/test_local_project_vector_cleaner_wiring.py @@ -5,7 +5,6 @@ from basic_memory.index.local_dependencies import LocalIndexProjectDependencies from basic_memory.index.local_project import LocalProjectIndexRuntimeFactory from basic_memory.indexing.project_index_maintenance import ( - InvalidatingProjectIndexBatchStore, RepositoryProjectIndexMaintenanceStore, StoreProjectIndexMaintenanceRunner, ) @@ -40,12 +39,6 @@ def test_full_project_runtime_forwards_external_vector_cleaner() -> None: ) assert isinstance(runtime.maintenance_runner, StoreProjectIndexMaintenanceRunner) - assert isinstance( - runtime.maintenance_runner.delete_store, - InvalidatingProjectIndexBatchStore, - ) - assert isinstance( - runtime.maintenance_runner.delete_store.delete_store, - RepositoryProjectIndexMaintenanceStore, - ) - assert runtime.maintenance_runner.delete_store.delete_store.external_vector_cleaner is cleaner + delete_store = runtime.maintenance_runner.delete_store + assert isinstance(delete_store, RepositoryProjectIndexMaintenanceStore) + assert delete_store.external_vector_cleaner is cleaner From 930b61f2de630a81f94a075d71c2cd1d0e6f9833 Mon Sep 17 00:00:00 2001 From: phernandez Date: Thu, 30 Jul 2026 10:31:35 -0500 Subject: [PATCH 19/28] fix(api): make move invalidation cancellation-safe Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 22 +-- src/basic_memory/read_cache/invalidation.py | 2 +- src/basic_memory/services/entity_service.py | 43 +++--- .../read_cache/test_runtime_invalidation.py | 133 +++++++++++++++++- 4 files changed, 173 insertions(+), 27 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index 74b211b33..031ba0bd2 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -87,10 +87,11 @@ tenant: the accepted-note transaction commits, again after terminal materialization/status publication and indexing, and again after relation resolution completes. This prevents a read filled between phases from surviving the later worker commit. -1. Make committed-mutation invalidation cancellation-safe. Accepted-note writes and directory - deletion can be cancelled after their database commit succeeds but before the transaction - context returns. Finish the namespace-bound generation bump before re-propagating cancellation - so committed state cannot remain hidden behind the previous generation. +1. Make committed-mutation invalidation cancellation-safe. Accepted-note writes, directory + deletion, and each file in a directory move can be cancelled after their database commit + succeeds but before the transaction context returns. Finish the namespace-bound generation + bump before re-propagating cancellation so committed state cannot remain hidden behind the + previous generation. 1. Invalidate after each committed project-index move, delete, and file-index batch, while retaining the final failure-safe completion boundary for later-phase errors. Local inline file batches invalidate when their runner returns; queued Cloud file batches invalidate in the child @@ -277,7 +278,9 @@ async with invalidate_cache(read_cache, project_id): ``` Callers enter this scope only when a backend is present. Conditional and multi-phase invalidation -remains explicit so the freshness boundary is visible. +remains explicit so the freshness boundary is visible. The scope finishes its generation bump +before re-propagating cancellation, which makes it safe around operations whose transaction can +commit during async context-manager exit. Primary integration points: @@ -383,9 +386,9 @@ The real-Redis suite must prove: - successful writes invalidate; a rejected write also invalidates when pre-write freshening may already have published external file state, while a rolled-back transaction without such a publication does not; -- cancellation after a real accepted-note or directory-delete transaction commits cannot - interrupt the real Redis generation bump, including repeated cancellation while invalidation is - in progress; +- cancellation after a real accepted-note, directory-delete, or per-file directory-move + transaction commits cannot interrupt the real Redis generation bump, including repeated + cancellation while invalidation is in progress; - real Redis no-eviction capacity failures bypass cache storage and cannot fail committed-write invalidation; - authoritative read exceptions propagate without populating the missed cache key; @@ -442,7 +445,8 @@ semantics themselves are asserted only against the real Redis integration fixtur - Invalidate every import attempt that may write files, including partial failures. - Finish directory-delete acceptance invalidation before re-propagating cancellation that lands after the delete transaction may have committed. -- Invalidate directory moves after every committed file and again after search/relation +- Finish each directory-move file invalidation before re-propagating cancellation that lands + after its move transaction may have committed, and invalidate again after search/relation follow-ups. - Invalidate project-root path changes in a failure-safe boundary because the filesystem source can change while every cache identity remains stable. diff --git a/src/basic_memory/read_cache/invalidation.py b/src/basic_memory/read_cache/invalidation.py index 776d88e65..0bd445fe2 100644 --- a/src/basic_memory/read_cache/invalidation.py +++ b/src/basic_memory/read_cache/invalidation.py @@ -91,4 +91,4 @@ async def invalidate_cache( try: yield finally: - await invalidate_project_read_cache(cache, project_id) + await finish_project_read_cache_invalidation(cache, project_id) diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index d5f2c546e..72f74ffd9 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -1,6 +1,7 @@ """Service for managing entities in the database.""" from collections.abc import Callable +from contextlib import nullcontext from dataclasses import dataclass from pathlib import Path from typing import Any, List, Optional, Sequence, Tuple, Union @@ -22,7 +23,7 @@ from basic_memory.models.knowledge import Entity from basic_memory.repository import ObservationRepository, RelationRepository from basic_memory.repository.entity_repository import EntityRepository -from basic_memory.read_cache import ReadCache, invalidate_project_read_cache +from basic_memory.read_cache import ReadCache, invalidate_cache from basic_memory.runtime.note_move import normalize_note_move_destination_path from basic_memory.schemas import Entity as EntitySchema from basic_memory.schemas.base import Permalink @@ -1192,24 +1193,34 @@ async def move_directory( # Entity is directly in the source directory (shouldn't happen with prefix match) new_path = f"{destination_directory}/{old_path}" - try: - # Move the individual entity - await self.move_entity( - identifier=entity.file_path, - destination_path=new_path, - project_config=project_config, - app_config=app_config, - ) - except Exception as e: # pragma: no cover + # Trigger: one file move can publish filesystem or database state before returning. + # Why: every move publishes independently and cached reads must not retain + # an earlier file's state while the remaining directory batch runs. + # Outcome: finish one generation bump before reporting the result or cancellation. + invalidation_scope = ( + invalidate_cache(read_cache, project_external_id) + if read_cache is not None + else nullcontext() + ) + move_error: Exception | None = None + async with invalidation_scope: + try: + # Move the individual entity + await self.move_entity( + identifier=entity.file_path, + destination_path=new_path, + project_config=project_config, + app_config=app_config, + ) + except Exception as error: # pragma: no cover + move_error = error + + if move_error is not None: # pragma: no cover failed_moves += 1 - errors.append(DirectoryMoveError(path=entity.file_path, error=str(e))) - logger.error(f"Failed to move entity {entity.file_path}: {e}") + errors.append(DirectoryMoveError(path=entity.file_path, error=str(move_error))) + logger.error(f"Failed to move entity {entity.file_path}: {move_error}") continue - # Each move commits independently. Invalidate before the next file so a - # long directory batch cannot serve early moves from the old generation. - if read_cache is not None: - await invalidate_project_read_cache(read_cache, project_external_id) moved_files.append(new_path) successful_moves += 1 logger.debug(f"Moved entity: {old_path} -> {new_path}") diff --git a/test-int/read_cache/test_runtime_invalidation.py b/test-int/read_cache/test_runtime_invalidation.py index c258b95b7..1387c4021 100644 --- a/test-int/read_cache/test_runtime_invalidation.py +++ b/test-int/read_cache/test_runtime_invalidation.py @@ -14,7 +14,9 @@ from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker import basic_memory.services.directory_deletes as directory_deletes +import basic_memory.services.entity_service as entity_service_module from basic_memory import db +from basic_memory.config import ProjectConfig, BasicMemoryConfig from basic_memory.index import note_content_materialization from basic_memory.index.local_moves import ( LocalMoveEntityRepository, @@ -51,6 +53,8 @@ StoreProjectIndexMaintenanceRunner, ) from basic_memory.indexing.relation_resolution import RelationResolutionRuntime +from basic_memory.markdown import EntityParser +from basic_memory.markdown.markdown_processor import MarkdownProcessor from basic_memory.models import Project from basic_memory.models.knowledge import Entity from basic_memory.read_cache import ( @@ -61,7 +65,11 @@ ) from basic_memory.read_cache.keys import redis_read_cache_generation_key from basic_memory.read_cache.redis import RedisReadCache -from basic_memory.repository import EntityRepository +from basic_memory.repository import ( + EntityRepository, + ObservationRepository, + RelationRepository, +) from basic_memory.repository.note_content_repository import ( AcceptedNoteContentWrite, NoteContentRepository, @@ -81,9 +89,13 @@ RuntimeStorageEventOperation, RuntimeStorageEventOperationKind, ) +from basic_memory.schemas import Entity as EntitySchema from basic_memory.services.directory_deletes import DirectoryDeleteService +from basic_memory.services.entity_service import EntityService from basic_memory.services.file_service import FileService from basic_memory.services.initialization import recover_project_materializations +from basic_memory.services.link_resolver import LinkResolver +from basic_memory.services.search_service import SearchService pytestmark = pytest.mark.redis @@ -868,3 +880,122 @@ async def pause_after_commit( async with original_scoped_session(session_maker) as session: deleted_entities = await entity_repository.get_by_title(session, title) assert deleted_entities == [] + + +@pytest.mark.asyncio +async def test_cancelled_committed_directory_move_finishes_real_redis_invalidation( + monkeypatch: pytest.MonkeyPatch, + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], + search_service: SearchService, + app_config: BasicMemoryConfig, + project_config: ProjectConfig, + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """Cancellation during a file-move commit cannot skip invalidation.""" + _, session_maker = engine_factory + entity_repository = EntityRepository(project_id=test_project.id) + entity_parser = EntityParser(project_config.home) + file_service = FileService(project_config.home, MarkdownProcessor(entity_parser)) + entity_service = EntityService( + entity_parser=entity_parser, + entity_repository=entity_repository, + observation_repository=ObservationRepository(project_id=test_project.id), + relation_repository=RelationRepository(project_id=test_project.id), + file_service=file_service, + link_resolver=LinkResolver( + entity_repository, + search_service, + session_maker=session_maker, + app_config=app_config, + ), + session_maker=session_maker, + search_service=search_service, + app_config=app_config, + ) + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="cancelled-committed-directory-move", + ) + entity = await entity_service.create_entity( + EntitySchema( + title="Cancelled Committed Directory Move", + directory="cancelled-move-source", + note_type="note", + content="Move must remain visible after cancellation.", + ) + ) + destination_path = "cancelled-move-destination/Cancelled Committed Directory Move.md" + blocking_cache = BlockingInvalidationRedisReadCache( + client=redis_cache.client, + namespace=redis_cache.namespace, + prefix=redis_cache.prefix, + ) + transaction_committed = asyncio.Event() + hold_after_commit = asyncio.Event() + original_scoped_session = entity_service_module.db.scoped_session + + @asynccontextmanager + async def pause_after_move_commit( + scoped_session_maker: async_sessionmaker[AsyncSession], + existing_session: AsyncSession | None = None, + ) -> AsyncIterator[AsyncSession]: + async with original_scoped_session(scoped_session_maker, existing_session) as session: + yield session + if transaction_committed.is_set(): + return + + async with original_scoped_session(scoped_session_maker) as session: + moved_entity = await entity_repository.get_by_id( + session, + entity.id, + load_relations=False, + ) + if moved_entity is not None and moved_entity.file_path == destination_path: + transaction_committed.set() + await hold_after_commit.wait() + + monkeypatch.setattr(entity_service_module.db, "scoped_session", pause_after_move_commit) + move_task = asyncio.create_task( + entity_service.move_directory( + source_directory="cancelled-move-source", + destination_directory="cancelled-move-destination", + project_config=project_config, + app_config=app_config.model_copy(update={"update_permalinks_on_move": False}), + project_external_id=project_external_id, + read_cache=blocking_cache, + ) + ) + + async with asyncio.timeout(5): + await transaction_committed.wait() + move_task.cancel() + async with asyncio.timeout(5): + await blocking_cache.invalidation_started.wait() + + # A second cancellation must not abandon the already-running generation bump. + move_task.cancel() + await asyncio.sleep(0) + assert not move_task.done() + + blocking_cache.release_invalidation.set() + with pytest.raises(asyncio.CancelledError): + await move_task + + generation_after = await _initialized_generation( + redis_cache, + project_external_id, + request="cancelled-committed-directory-move", + ) + assert generation_after != generation_before + + async with original_scoped_session(entity_service.session_maker) as session: + moved_entity = await entity_repository.get_by_id( + session, + entity.id, + load_relations=False, + ) + assert moved_entity is not None + assert moved_entity.file_path == destination_path From d46d4208051dc2f12a2fbacd9b37d2f35dcbc438 Mon Sep 17 00:00:00 2001 From: phernandez Date: Thu, 30 Jul 2026 10:47:56 -0500 Subject: [PATCH 20/28] fix(api): close read cache invalidation gaps Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 22 ++- src/basic_memory/deps/importers.py | 23 +++ src/basic_memory/importers/__init__.py | 3 +- src/basic_memory/importers/base.py | 29 +++- .../services/note_content_reads.py | 9 +- test-int/read_cache/test_api_read_cache.py | 148 +++++++++++++++++- 6 files changed, 216 insertions(+), 18 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index 031ba0bd2..cbc348a9e 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -101,14 +101,15 @@ tenant: failed index attempt can still publish cache-relevant state. 1. Pass the namespace-bound cache into hosted note-content read repair. A resource or entity read can bootstrap a missing accepted-content row; invalidate immediately after that commit and - before the repaired response is offered to read-through storage. + before the repaired response is offered to read-through storage. Finish that generation bump + before re-propagating cancellation that arrives after the repair transaction commits. 1. Pass the namespace-bound cache into pre-mutation content freshening. Freshening can index an externally edited file before the accepted mutation begins, so invalidate after every freshening attempt that may have published state, including when the later mutation is rejected or raises. 1. Inject the namespace-bound cache into import endpoints and workers. Invalidate after every - import attempt that may have written files, including importers that return a failed result or - raise after partial progress, so cached file-first resources cannot survive overwritten bytes. + attempted file write before the next item, including writes that raise after partial progress, + and retain the final failure-safe invalidation around the complete import attempt. 1. Invalidate directory deletion immediately after its acceptance transaction commits, then again after file cleanup and surviving-relation refresh. A slow or failed cleanup must not keep deleted entities reachable through the pre-acceptance generation. @@ -300,8 +301,9 @@ invalidates again so a value filled after an earlier generation bump cannot outl that phase publishes. Hosted read repair invalidates after bootstrapping accepted content and before a repaired entity or resource is stored. Directory moves invalidate after every committed file plus the final reindex; directory deletion invalidates after acceptance and after cleanup. -Imports invalidate after every attempt that may have written files. Project indexing invalidates -even after a partial failure. Direct single-file and watcher file indexing invalidate even when a +Imports invalidate after every attempted file write and again around the complete attempt so +partial failures cannot escape. Project indexing invalidates even after a partial failure. Direct +single-file and watcher file indexing invalidate even when a follow-up fails after the entity commit. Recovery phases invalidate independently before the serving barrier is released and include terminal conflict or failure publication. @@ -389,6 +391,8 @@ The real-Redis suite must prove: - cancellation after a real accepted-note, directory-delete, or per-file directory-move transaction commits cannot interrupt the real Redis generation bump, including repeated cancellation while invalidation is in progress; +- cancellation after a real hosted read-repair transaction commits cannot interrupt the real + Redis generation bump; - real Redis no-eviction capacity failures bypass cache storage and cannot fail committed-write invalidation; - authoritative read exceptions propagate without populating the missed cache key; @@ -401,7 +405,8 @@ The real-Redis suite must prove: - direct and watcher file-index failures invalidate any entity state committed before failed search or reconciliation follow-ups; - hosted read repair invalidates cached entity metadata before storing the repaired resource; -- import attempts invalidate cached file-first resources after success and partial failure; +- multi-item imports advance the real Redis generation after every attempted file write, while + retaining the final success and partial-failure bump; - directory moves invalidate after each committed file and again after final reindexing; - project-root path changes invalidate cached resources whose project/entity UUID keys remain stable; @@ -439,10 +444,11 @@ semantics themselves are asserted only against the real Redis integration fixtur - Invalidate direct and watcher file indexing from failure-safe boundaries because entity commits precede some search and reconciliation follow-ups. - Invalidate hosted note-content read repair after it commits and before returning its repaired - entity or resource to the read-through helper. + entity or resource to the read-through helper; finish that bump before cancellation propagates. - Invalidate pre-mutation content freshening even when a later accepted mutation is rejected or fails, because the freshening index may already have committed external file state. -- Invalidate every import attempt that may write files, including partial failures. +- Invalidate every imported file write before the next item, and retain whole-import invalidation + for partial failures. - Finish directory-delete acceptance invalidation before re-propagating cancellation that lands after the delete transaction may have committed. - Finish each directory-move file invalidation before re-propagating cancellation that lands diff --git a/src/basic_memory/deps/importers.py b/src/basic_memory/deps/importers.py index fc1c013bb..44c51d77e 100644 --- a/src/basic_memory/deps/importers.py +++ b/src/basic_memory/deps/importers.py @@ -12,6 +12,7 @@ from fastapi import Depends from basic_memory.deps.projects import ProjectConfigV2ExternalDep +from basic_memory.deps.read_cache import ReadCacheDep from basic_memory.deps.services import ( FileServiceV2ExternalDep, MarkdownProcessorV2ExternalDep, @@ -20,10 +21,24 @@ ChatGPTImporter, ClaudeConversationsImporter, ClaudeProjectsImporter, + ImportReadCache, MemoryJsonImporter, ) +async def get_import_read_cache( + read_cache: ReadCacheDep, + project_id: str, +) -> ImportReadCache | None: + """Bind the optional host cache to the requested project.""" + if read_cache is None: + return None + return ImportReadCache(backend=read_cache, project_id=project_id) + + +ImportReadCacheDep = Annotated[ImportReadCache | None, Depends(get_import_read_cache)] + + # --- ChatGPT Importer --- @@ -31,6 +46,7 @@ async def get_chatgpt_importer_v2_external( project_config: ProjectConfigV2ExternalDep, markdown_processor: MarkdownProcessorV2ExternalDep, file_service: FileServiceV2ExternalDep, + read_cache: ImportReadCacheDep, ) -> ChatGPTImporter: """Create ChatGPTImporter with v2 external_id dependencies.""" return ChatGPTImporter( @@ -38,6 +54,7 @@ async def get_chatgpt_importer_v2_external( markdown_processor, file_service, project_name=project_config.name, + read_cache=read_cache, ) @@ -51,6 +68,7 @@ async def get_claude_conversations_importer_v2_external( project_config: ProjectConfigV2ExternalDep, markdown_processor: MarkdownProcessorV2ExternalDep, file_service: FileServiceV2ExternalDep, + read_cache: ImportReadCacheDep, ) -> ClaudeConversationsImporter: """Create ClaudeConversationsImporter with v2 external_id dependencies.""" return ClaudeConversationsImporter( @@ -58,6 +76,7 @@ async def get_claude_conversations_importer_v2_external( markdown_processor, file_service, project_name=project_config.name, + read_cache=read_cache, ) @@ -73,6 +92,7 @@ async def get_claude_projects_importer_v2_external( project_config: ProjectConfigV2ExternalDep, markdown_processor: MarkdownProcessorV2ExternalDep, file_service: FileServiceV2ExternalDep, + read_cache: ImportReadCacheDep, ) -> ClaudeProjectsImporter: """Create ClaudeProjectsImporter with v2 external_id dependencies.""" return ClaudeProjectsImporter( @@ -80,6 +100,7 @@ async def get_claude_projects_importer_v2_external( markdown_processor, file_service, project_name=project_config.name, + read_cache=read_cache, ) @@ -95,6 +116,7 @@ async def get_memory_json_importer_v2_external( project_config: ProjectConfigV2ExternalDep, markdown_processor: MarkdownProcessorV2ExternalDep, file_service: FileServiceV2ExternalDep, + read_cache: ImportReadCacheDep, ) -> MemoryJsonImporter: """Create MemoryJsonImporter with v2 external_id dependencies.""" return MemoryJsonImporter( @@ -102,6 +124,7 @@ async def get_memory_json_importer_v2_external( markdown_processor, file_service, project_name=project_config.name, + read_cache=read_cache, ) diff --git a/src/basic_memory/importers/__init__.py b/src/basic_memory/importers/__init__.py index 3336cce02..ba7876afc 100644 --- a/src/basic_memory/importers/__init__.py +++ b/src/basic_memory/importers/__init__.py @@ -1,6 +1,6 @@ """Import services for Basic Memory.""" -from basic_memory.importers.base import Importer +from basic_memory.importers.base import Importer, ImportReadCache from basic_memory.importers.chatgpt_importer import ChatGPTImporter from basic_memory.importers.claude_conversations_importer import ( ClaudeConversationsImporter, @@ -22,6 +22,7 @@ __all__ = [ "Importer", + "ImportReadCache", "ChatGPTImporter", "ClaudeConversationsImporter", "ClaudeProjectsImporter", diff --git a/src/basic_memory/importers/base.py b/src/basic_memory/importers/base.py index d3ba81d25..280fb82f0 100644 --- a/src/basic_memory/importers/base.py +++ b/src/basic_memory/importers/base.py @@ -2,11 +2,14 @@ import logging from abc import abstractmethod +from contextlib import nullcontext +from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any, Optional, TypeVar from basic_memory.markdown.markdown_processor import MarkdownProcessor from basic_memory.markdown.schemas import EntityMarkdown +from basic_memory.read_cache import ReadCacheInvalidator, invalidate_cache from basic_memory.schemas.importer import ImportResult from basic_memory.utils import build_canonical_permalink, generate_permalink @@ -18,6 +21,14 @@ T = TypeVar("T", bound=ImportResult) +@dataclass(frozen=True, slots=True) +class ImportReadCache: + """Bind an importer's file writes to one project cache generation.""" + + backend: ReadCacheInvalidator + project_id: str + + class Importer[T: ImportResult]: """Base class for all import services. @@ -31,6 +42,8 @@ def __init__( markdown_processor: MarkdownProcessor, file_service: "FileService", project_name: Optional[str] = None, + *, + read_cache: ImportReadCache | None = None, ): """Initialize the import service. @@ -44,6 +57,7 @@ def __init__( self.file_service = file_service self.project_name = project_name self.project_permalink = generate_permalink(project_name) if project_name else None + self.read_cache = read_cache @abstractmethod async def import_data(self, source_data, destination_folder: str, **kwargs: Any) -> T: @@ -74,8 +88,19 @@ async def write_entity(self, entity: EntityMarkdown, file_path: str | Path) -> s Checksum of written file. """ content = self.markdown_processor.to_markdown_string(entity) - # FileService.write_file handles directory creation and returns checksum - return await self.file_service.write_file(file_path, content) + + # Trigger: one imported file can become visible before the remaining batch completes. + # Why: cached file-first resources must not retain the overwritten bytes until the + # import's final failure-safe invalidation. + # Outcome: advance the project generation after every attempted file write. + invalidation_scope = ( + invalidate_cache(self.read_cache.backend, self.read_cache.project_id) + if self.read_cache is not None + else nullcontext() + ) + async with invalidation_scope: + # FileService.write_file handles directory creation and returns checksum + return await self.file_service.write_file(file_path, content) def canonical_permalink(self, path: str) -> str: """Build a canonical permalink for imported content.""" diff --git a/src/basic_memory/services/note_content_reads.py b/src/basic_memory/services/note_content_reads.py index 0225d50d0..b644c144c 100644 --- a/src/basic_memory/services/note_content_reads.py +++ b/src/basic_memory/services/note_content_reads.py @@ -15,7 +15,10 @@ run_note_content_read_repair_with_default_reconciler, ) from basic_memory.models import Entity, NoteContent, Project -from basic_memory.read_cache import ReadCacheInvalidator, invalidate_project_read_cache +from basic_memory.read_cache import ( + ReadCacheInvalidator, + finish_project_read_cache_invalidation, +) from basic_memory.runtime.note_content import ( RuntimeNoteContentResource, RuntimeNoteContentResponsePayload, @@ -114,7 +117,7 @@ async def get_note_entity_payload_with_read_repair( # Read repair commits note_content before this method reloads the response. # Advance the generation first so the surrounding read-through cannot # publish the repaired payload under the pre-repair generation. - await invalidate_project_read_cache(read_cache, project_external_id) + await finish_project_read_cache_invalidation(read_cache, project_external_id) # The repair commits through a separate scoped session, so reopen the read to # avoid stale snapshots in caller-owned transactions. return await self.get_note_entity_payload( @@ -175,7 +178,7 @@ async def get_note_resource_with_read_repair( if read_cache is not None: # A resource read can repair the row used by cached entity responses. # Invalidate before returning the repaired resource to read-through. - await invalidate_project_read_cache(read_cache, project_external_id) + await finish_project_read_cache_invalidation(read_cache, project_external_id) # The repair commits through a separate scoped session, so reopen the read to # avoid stale snapshots in caller-owned transactions. return await self.get_note_resource( diff --git a/test-int/read_cache/test_api_read_cache.py b/test-int/read_cache/test_api_read_cache.py index 331de18ac..a873c9147 100644 --- a/test-int/read_cache/test_api_read_cache.py +++ b/test-int/read_cache/test_api_read_cache.py @@ -3,11 +3,12 @@ from __future__ import annotations import asyncio +import json from collections.abc import AsyncIterator from contextlib import asynccontextmanager from datetime import datetime, timezone from pathlib import Path -from typing import Protocol, override +from typing import Literal, Protocol, override from uuid import uuid4 import pytest @@ -45,6 +46,7 @@ ) from basic_memory.read_cache.redis import RedisReadCache from basic_memory.repository import EntityRepository +from basic_memory.repository.note_content_repository import NoteContentRepository from basic_memory.runtime.note_content import NOTE_CONTENT_BASE_CHECKSUM_HEADER from basic_memory.schemas.v2 import EntityResolveRequest from basic_memory.services.note_content_reads import NoteContentQueryService @@ -128,8 +130,8 @@ async def get_note_resource_with_read_repair( return None -class DirectoryMoveObservingRedisReadCache(RedisReadCache): - """Record how much of a directory move is durable at each real invalidation.""" +class DestinationObservingRedisReadCache(RedisReadCache): + """Record how many target files exist at each real invalidation.""" def __init__( self, @@ -142,10 +144,12 @@ def __init__( super().__init__(client=client, namespace=namespace, prefix=prefix) self.destination_paths = destination_paths self.destination_counts: list[int] = [] + self.project_ids: list[str] = [] @override async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: status = await super().invalidate_project(project_id) + self.project_ids.append(project_id) self.destination_counts.append( sum(destination.exists() for destination in self.destination_paths) ) @@ -649,6 +653,52 @@ async def test_partial_import_failure_invalidates_cached_resource_in_real_redis( assert refreshed_response.text == "# Imported before failure\n" +@pytest.mark.asyncio +async def test_import_invalidates_after_each_written_file_in_real_redis( + app: FastAPI, + client: AsyncClient, + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """Each imported file advances Redis before the next item is processed.""" + project_external_id = str(test_project.external_id) + destination_directory = "cache/import-per-file" + destination_paths = tuple( + Path(test_project.path) / destination_directory / "note" / f"{name}.md" + for name in ("First import", "Second import") + ) + observing_cache = DestinationObservingRedisReadCache( + client=redis_cache.client, + namespace=redis_cache.namespace, + prefix=redis_cache.prefix, + destination_paths=destination_paths, + ) + app.dependency_overrides[get_read_cache] = lambda: observing_cache + source_data = b"\n".join( + json.dumps( + { + "type": "entity", + "name": name, + "entityType": "note", + "observations": [], + } + ).encode() + for name in ("First import", "Second import") + ) + + response = await client.post( + f"/v2/projects/{project_external_id}/import/memory-json", + files={"file": ("memory.json", source_data, "application/json")}, + data={"directory": destination_directory}, + ) + + assert response.status_code == 200 + assert response.json()["entities"] == 2 + assert observing_cache.destination_counts[:2] == [1, 2] + assert observing_cache.project_ids[:2] == [project_external_id, project_external_id] + assert all(destination.exists() for destination in destination_paths) + + @pytest.mark.asyncio async def test_resource_read_repair_invalidates_cached_entity_in_real_redis( app: FastAPI, @@ -715,6 +765,96 @@ async def test_resource_read_repair_invalidates_cached_entity_in_real_redis( assert "Canonical file content." in refreshed_entity.json()["content"] +@pytest.mark.asyncio +@pytest.mark.parametrize("read_method", ["entity", "resource"]) +async def test_cancelled_committed_read_repair_finishes_real_redis_invalidation( + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], + test_project: Project, + redis_cache: RedisCacheHarness, + read_method: Literal["entity", "resource"], +) -> None: + """Cancellation after read-repair commit waits for invalidation.""" + project_external_id = str(test_project.external_id) + file_path = "cache/cancelled-read-repair.md" + markdown_content = "# Cancelled read repair\n\nCommitted canonical content.\n" + disk_path = Path(test_project.path) / file_path + disk_path.parent.mkdir(parents=True, exist_ok=True) + disk_path.write_text(markdown_content, encoding="utf-8") + + repository = EntityRepository(project_id=test_project.id) + _, session_maker = engine_factory + async with db.scoped_session(session_maker) as session: + entity = await repository.add( + session, + Entity( + title="Cancelled read repair", + note_type="note", + content_type="text/markdown", + file_path=file_path, + checksum="cancelled-read-repair-checksum", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + ) + + blocking_cache = BlockingInvalidationRedisReadCache( + client=redis_cache.client, + namespace=redis_cache.namespace, + prefix=redis_cache.prefix, + ) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="cancelled-committed-read-repair", + ) + + query_service = NoteContentQueryService( + session_maker=session_maker, + read_repair_file_reader=LocalReadRepairFileReader(), + ) + if read_method == "entity": + read_repair = query_service.get_note_entity_payload_with_read_repair( + project_external_id=project_external_id, + entity_external_id=entity.external_id, + read_cache=blocking_cache, + ) + else: + read_repair = query_service.get_note_resource_with_read_repair( + project_external_id=project_external_id, + entity_external_id=entity.external_id, + read_cache=blocking_cache, + ) + request_task = asyncio.create_task(read_repair) + + async with asyncio.timeout(5): + await blocking_cache.invalidation_started.wait() + request_task.cancel() + await asyncio.sleep(0) + request_task.cancel() + await asyncio.sleep(0) + assert not request_task.done() + + blocking_cache.release_invalidation.set() + with pytest.raises(asyncio.CancelledError): + await request_task + + generation_after = await _initialized_generation( + redis_cache, + project_external_id, + request="cancelled-committed-read-repair", + ) + assert generation_after != generation_before + + async with db.scoped_session(session_maker) as session: + repaired = await NoteContentRepository(project_id=test_project.id).get_by_entity_id( + session, + entity.id, + ) + assert repaired is not None + assert repaired.db_version == 1 + assert repaired.markdown_content == markdown_content + + @pytest.mark.asyncio async def test_directory_move_invalidates_after_each_committed_file_in_real_redis( app: FastAPI, @@ -742,7 +882,7 @@ async def test_directory_move_invalidates_after_each_committed_file_in_real_redi Path(test_project.path) / source_path.replace("move-source/", "move-destination/", 1) for source_path in created_paths ) - observing_cache = DirectoryMoveObservingRedisReadCache( + observing_cache = DestinationObservingRedisReadCache( client=redis_cache.client, namespace=redis_cache.namespace, prefix=redis_cache.prefix, From 596356038d0772ff970803bb40657f33878ffd1e Mon Sep 17 00:00:00 2001 From: phernandez Date: Thu, 30 Jul 2026 11:12:32 -0500 Subject: [PATCH 21/28] fix(api): close cancellation invalidation windows Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 39 ++++--- src/basic_memory/index/local_runtime.py | 8 +- .../services/note_content_reads.py | 35 +++--- test-int/read_cache/test_api_read_cache.py | 39 ++++++- .../read_cache/test_runtime_invalidation.py | 106 +++++++++++++++++- tests/cloud/test_cloud_services.py | 2 + 6 files changed, 189 insertions(+), 40 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index cbc348a9e..df0445d0a 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -98,11 +98,14 @@ tenant: worker after its durable commit, not merely when the coordinator enqueues the job. 1. Put direct single-file and watcher file-index invalidation in failure-safe boundaries. Entity transactions can commit before search refresh or note-content reconciliation raises, so a - failed index attempt can still publish cache-relevant state. + failed index attempt can still publish cache-relevant state. Once a watcher index or delete + completion callback runs after its durable event, finish the first generation bump before + propagating cancellation; relation/search cleanup may not begin after cancellation. 1. Pass the namespace-bound cache into hosted note-content read repair. A resource or entity read - can bootstrap a missing accepted-content row; invalidate immediately after that commit and - before the repaired response is offered to read-through storage. Finish that generation bump - before re-propagating cancellation that arrives after the repair transaction commits. + can bootstrap a missing accepted-content row. Put cancellation-safe invalidation around the + transaction-bearing repair call, not only after it returns, so cancellation during transaction + exit still advances the generation before the repaired response can reach read-through + storage. 1. Pass the namespace-bound cache into pre-mutation content freshening. Freshening can index an externally edited file before the accepted mutation begins, so invalidate after every freshening attempt that may have published state, including when the later mutation is @@ -299,13 +302,15 @@ mutations, imports, watcher-detected paired moves, startup recovery or reconcili storage events, and relation-resolution changes that affect cached responses. Each later phase invalidates again so a value filled after an earlier generation bump cannot outlive the state that phase publishes. Hosted read repair invalidates after bootstrapping accepted content and -before a repaired entity or resource is stored. Directory moves invalidate after every committed -file plus the final reindex; directory deletion invalidates after acceptance and after cleanup. -Imports invalidate after every attempted file write and again around the complete attempt so -partial failures cannot escape. Project indexing invalidates even after a partial failure. Direct -single-file and watcher file indexing invalidate even when a -follow-up fails after the entity commit. Recovery phases invalidate independently before the -serving barrier is released and include terminal conflict or failure publication. +before a repaired entity or resource is stored, including cancellation during the repair +transaction's exit. Directory moves invalidate after every committed file plus the final reindex; +directory deletion invalidates after acceptance and after cleanup. Imports invalidate after every +attempted file write and again around the complete attempt so partial failures cannot escape. +Project indexing invalidates even after a partial failure. Direct single-file and watcher file +indexing invalidate even when a follow-up fails after the entity commit. Watcher index and delete +completion callbacks finish their first post-event generation bump before cancellation can +escape. Recovery phases invalidate independently before the serving barrier is released and +include terminal conflict or failure publication. ## Dependency And Lifecycle @@ -391,8 +396,10 @@ The real-Redis suite must prove: - cancellation after a real accepted-note, directory-delete, or per-file directory-move transaction commits cannot interrupt the real Redis generation bump, including repeated cancellation while invalidation is in progress; -- cancellation after a real hosted read-repair transaction commits cannot interrupt the real - Redis generation bump; +- cancellation during a real hosted read-repair transaction's commit exit cannot interrupt the + real Redis generation bump; +- cancellation during watcher index or delete completion cannot interrupt the first real Redis + generation bump after the durable event; - real Redis no-eviction capacity failures bypass cache storage and cannot fail committed-write invalidation; - authoritative read exceptions propagate without populating the missed cache key; @@ -443,8 +450,10 @@ semantics themselves are asserted only against the real Redis integration fixtur deletion both after acceptance commit and after cleanup/relation refresh. - Invalidate direct and watcher file indexing from failure-safe boundaries because entity commits precede some search and reconciliation follow-ups. -- Invalidate hosted note-content read repair after it commits and before returning its repaired - entity or resource to the read-through helper; finish that bump before cancellation propagates. +- Finish watcher index and delete completion invalidation before cancellation propagates from the + post-event callback. +- Wrap hosted note-content read repair in cancellation-safe invalidation so cancellation during + transaction exit cannot skip the bump before a repaired entity or resource reaches read-through. - Invalidate pre-mutation content freshening even when a later accepted mutation is rejected or fails, because the freshening index may already have committed external file state. - Invalidate every imported file write before the next item, and retain whole-import invalidation diff --git a/src/basic_memory/index/local_runtime.py b/src/basic_memory/index/local_runtime.py index 7e1df799c..276576926 100644 --- a/src/basic_memory/index/local_runtime.py +++ b/src/basic_memory/index/local_runtime.py @@ -60,8 +60,8 @@ from basic_memory.models import Entity, Project from basic_memory.read_cache import ( ReadCache, + finish_project_read_cache_invalidation, invalidate_cache, - invalidate_project_read_cache, ) from basic_memory.repository import NoteContentRepository from basic_memory.runtime.projects import ProjectRuntimeReference @@ -167,7 +167,7 @@ async def index_file_completed( ) if result.status == IndexFileJobStatus.processed and self.read_cache is not None: - await invalidate_project_read_cache( + await finish_project_read_cache_invalidation( self.read_cache, self.project.project_external_id, ) @@ -236,7 +236,7 @@ async def delete_file_completed( if not result.entity_deleted: return if self.read_cache is not None: - await invalidate_project_read_cache( + await finish_project_read_cache_invalidation( self.read_cache, self.project.project_external_id, ) @@ -285,7 +285,7 @@ async def event_failed( # LocalMarkdownFileIndexer commits the entity before all search and # reconciliation follow-ups complete. A watcher failure can therefore # publish partial state even though the success callback never runs. - await invalidate_project_read_cache( + await finish_project_read_cache_invalidation( self.read_cache, self.project.project_external_id, ) diff --git a/src/basic_memory/services/note_content_reads.py b/src/basic_memory/services/note_content_reads.py index b644c144c..9ab50ff6e 100644 --- a/src/basic_memory/services/note_content_reads.py +++ b/src/basic_memory/services/note_content_reads.py @@ -2,6 +2,8 @@ from __future__ import annotations +from contextlib import nullcontext + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from basic_memory import db @@ -17,7 +19,7 @@ from basic_memory.models import Entity, NoteContent, Project from basic_memory.read_cache import ( ReadCacheInvalidator, - finish_project_read_cache_invalidation, + invalidate_cache, ) from basic_memory.runtime.note_content import ( RuntimeNoteContentResource, @@ -110,14 +112,10 @@ async def get_note_entity_payload_with_read_repair( project_external_id=project_external_id, entity_external_id=entity_external_id, source=source, + read_cache=read_cache, ) if not repaired: return None - if read_cache is not None: - # Read repair commits note_content before this method reloads the response. - # Advance the generation first so the surrounding read-through cannot - # publish the repaired payload under the pre-repair generation. - await finish_project_read_cache_invalidation(read_cache, project_external_id) # The repair commits through a separate scoped session, so reopen the read to # avoid stale snapshots in caller-owned transactions. return await self.get_note_entity_payload( @@ -172,13 +170,10 @@ async def get_note_resource_with_read_repair( project_external_id=project_external_id, entity_external_id=entity_external_id, source=source, + read_cache=read_cache, ) if not repaired: return None - if read_cache is not None: - # A resource read can repair the row used by cached entity responses. - # Invalidate before returning the repaired resource to read-through. - await finish_project_read_cache_invalidation(read_cache, project_external_id) # The repair commits through a separate scoped session, so reopen the read to # avoid stale snapshots in caller-owned transactions. return await self.get_note_resource( @@ -192,6 +187,7 @@ async def reconcile_note_content_from_file( project_external_id: str, entity_external_id: str, source: str, + read_cache: ReadCacheInvalidator | None = None, ) -> bool: """Repair a missing note_content row from the runtime's canonical file source.""" async with db.scoped_session(self.session_maker) as session: @@ -208,10 +204,19 @@ async def reconcile_note_content_from_file( if self.read_repair_file_reader is None: raise RuntimeError("note-content read repair requires a file reader") - repair_run = await run_note_content_read_repair_with_default_reconciler( - repair_preflight, - session_maker=self.session_maker, - file_reader=self.read_repair_file_reader, - source=source, + # The repair runner owns the transaction that may create note_content. + # Keep invalidation around that whole await so cancellation during commit + # exit still advances the cache generation before it propagates. + repair_scope = ( + invalidate_cache(read_cache, project_external_id) + if read_cache is not None + else nullcontext() ) + async with repair_scope: + repair_run = await run_note_content_read_repair_with_default_reconciler( + repair_preflight, + session_maker=self.session_maker, + file_reader=self.read_repair_file_reader, + source=source, + ) return repair_run.repaired diff --git a/test-int/read_cache/test_api_read_cache.py b/test-int/read_cache/test_api_read_cache.py index a873c9147..3964e0d51 100644 --- a/test-int/read_cache/test_api_read_cache.py +++ b/test-int/read_cache/test_api_read_cache.py @@ -17,6 +17,7 @@ from redis.asyncio import Redis from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker +import basic_memory.indexing.note_content_reconciler as note_content_reconciler import basic_memory.services.note_content_writes as note_content_writes from basic_memory import db from basic_memory.deps import ( @@ -772,8 +773,9 @@ async def test_cancelled_committed_read_repair_finishes_real_redis_invalidation( test_project: Project, redis_cache: RedisCacheHarness, read_method: Literal["entity", "resource"], + monkeypatch: pytest.MonkeyPatch, ) -> None: - """Cancellation after read-repair commit waits for invalidation.""" + """Cancellation during read-repair transaction exit waits for invalidation.""" project_external_id = str(test_project.external_id) file_path = "cache/cancelled-read-repair.md" markdown_content = "# Cancelled read repair\n\nCommitted canonical content.\n" @@ -807,6 +809,34 @@ async def test_cancelled_committed_read_repair_finishes_real_redis_invalidation( project_external_id, request="cancelled-committed-read-repair", ) + transaction_committed = asyncio.Event() + hold_after_commit = asyncio.Event() + original_scoped_session = note_content_reconciler.db.scoped_session + + @asynccontextmanager + async def pause_after_repair_commit( + scoped_session_maker: async_sessionmaker[AsyncSession], + session: AsyncSession | None = None, + ) -> AsyncIterator[AsyncSession]: + async with original_scoped_session(scoped_session_maker, session) as scoped_session: + yield scoped_session + if transaction_committed.is_set(): + return + + async with original_scoped_session(scoped_session_maker) as verification_session: + repaired = await NoteContentRepository(project_id=test_project.id).get_by_entity_id( + verification_session, + entity.id, + ) + if repaired is not None: + transaction_committed.set() + await hold_after_commit.wait() + + monkeypatch.setattr( + note_content_reconciler.db, + "scoped_session", + pause_after_repair_commit, + ) query_service = NoteContentQueryService( session_maker=session_maker, @@ -827,9 +857,12 @@ async def test_cancelled_committed_read_repair_finishes_real_redis_invalidation( request_task = asyncio.create_task(read_repair) async with asyncio.timeout(5): - await blocking_cache.invalidation_started.wait() + await transaction_committed.wait() request_task.cancel() - await asyncio.sleep(0) + async with asyncio.timeout(5): + await blocking_cache.invalidation_started.wait() + + # A second cancellation must not abandon the already-running generation bump. request_task.cancel() await asyncio.sleep(0) assert not request_task.done() diff --git a/test-int/read_cache/test_runtime_invalidation.py b/test-int/read_cache/test_runtime_invalidation.py index 1387c4021..dde2c3a85 100644 --- a/test-int/read_cache/test_runtime_invalidation.py +++ b/test-int/read_cache/test_runtime_invalidation.py @@ -7,7 +7,7 @@ from contextlib import asynccontextmanager from datetime import UTC, datetime from pathlib import Path -from typing import Protocol, cast, override +from typing import Literal, Protocol, cast, override import pytest from redis.asyncio import Redis @@ -29,6 +29,7 @@ run_local_project_index, ) from basic_memory.index.local_runtime import LocalInlineStorageEventResultRecorder +from basic_memory.indexing.external_file_delete_runner import ExternalFileDeleteResult from basic_memory.indexing.change_planning import ChangeReport from basic_memory.indexing.directory_delete_runner import ( DirectoryDeleteRuntime, @@ -40,7 +41,7 @@ IndexFileBatchIndexer, IndexFileBatchReader, ) -from basic_memory.indexing.models import IndexInputFile +from basic_memory.indexing.models import IndexFileJobResult, IndexFileJobStatus, IndexInputFile from basic_memory.indexing.project_index_maintenance import ( InvalidatingProjectIndexBatchStore, ProjectIndexDeleteBatch, @@ -74,7 +75,11 @@ AcceptedNoteContentWrite, NoteContentRepository, ) -from basic_memory.runtime.cleanup import RuntimeFileDeleteResult, RuntimeNoteFileDeleteJobRequest +from basic_memory.runtime.cleanup import ( + RuntimeExternalFileDeletePlan, + RuntimeFileDeleteResult, + RuntimeNoteFileDeleteJobRequest, +) from basic_memory.runtime.jobs import ( RuntimeIndexFileBatchJobRequest, RuntimeObservedIndexFile, @@ -476,6 +481,101 @@ async def test_watcher_index_failure_invalidates_real_redis( assert generation_after != generation_before +@pytest.mark.asyncio +@pytest.mark.parametrize("completion", ["index", "delete"]) +async def test_cancelled_watcher_completion_finishes_real_redis_invalidation( + test_project: Project, + redis_cache: RedisCacheHarness, + completion: Literal["index", "delete"], +) -> None: + """Cancellation after a durable watcher event waits for invalidation.""" + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request=f"cancelled-watcher-{completion}", + ) + blocking_cache = BlockingInvalidationRedisReadCache( + client=redis_cache.client, + namespace=redis_cache.namespace, + prefix=redis_cache.prefix, + ) + recorder = LocalInlineStorageEventResultRecorder( + project=ProjectRuntimeReference.from_project(test_project), + search_service=cast(LocalIndexSearchService, object()), + relation_cleanup_search_refresher=cast( + ProjectIndexMovedEntitySearchRefresher, + object(), + ), + relation_runtime=cast(RelationResolutionRuntime, object()), + index_embeddings=False, + read_cache=blocking_cache, + ) + + if completion == "index": + completion_call = recorder.index_file_completed( + _index_operation("notes/cancelled-index.md"), + IndexFileJobResult( + status=IndexFileJobStatus.processed, + reason="file indexed", + entity_id=42, + ), + ) + else: + deleted_entity = Entity( + id=42, + project_id=test_project.id, + external_id="cancelled-watcher-delete", + title="Cancelled watcher delete", + permalink="notes/cancelled-watcher-delete", + note_type="note", + content_type="text/markdown", + file_path="notes/cancelled-delete.md", + checksum="cancelled-watcher-delete-checksum", + created_at=datetime.now(UTC), + updated_at=datetime.now(UTC), + ) + completion_call = recorder.delete_file_completed( + RuntimeStorageEventOperation( + kind=RuntimeStorageEventOperationKind.delete_file, + storage_event=_move_event( + STORAGE_OBJECT_DELETED_EVENT, + deleted_entity.file_path, + ), + relative_path=deleted_entity.file_path, + ), + ExternalFileDeleteResult( + plan=RuntimeExternalFileDeletePlan.from_existing_entity( + deleted_entity, + file_path=deleted_entity.file_path, + object_exists=False, + ), + entity_deleted=True, + deleted_entity=deleted_entity, + ), + ) + + completion_task = asyncio.create_task(completion_call) + async with asyncio.timeout(5): + await blocking_cache.invalidation_started.wait() + completion_task.cancel() + await asyncio.sleep(0) + completion_task.cancel() + await asyncio.sleep(0) + assert not completion_task.done() + + blocking_cache.release_invalidation.set() + with pytest.raises(asyncio.CancelledError): + await completion_task + + generation_after = await _initialized_generation( + redis_cache, + project_external_id, + request=f"cancelled-watcher-{completion}", + ) + assert generation_after != generation_before + + @pytest.mark.asyncio async def test_startup_materialization_recovery_invalidates_real_redis( engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], diff --git a/tests/cloud/test_cloud_services.py b/tests/cloud/test_cloud_services.py index b37626cd9..4c08d8707 100644 --- a/tests/cloud/test_cloud_services.py +++ b/tests/cloud/test_cloud_services.py @@ -396,10 +396,12 @@ async def fake_reconcile_note_content_from_file( project_external_id: str, entity_external_id: str, source: str, + read_cache: object | None = None, ) -> bool: assert project_external_id == "project-123" assert entity_external_id == "note-456" assert source == "read_repair" + assert read_cache is None return True service = NoteContentQueryService( From 305c85879bae6bd87872aa38398ba162559a38e8 Mon Sep 17 00:00:00 2001 From: phernandez Date: Thu, 30 Jul 2026 11:29:37 -0500 Subject: [PATCH 22/28] fix(api): invalidate rebuild and watcher batches Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 42 ++++-- src/basic_memory/deps/services.py | 6 + src/basic_memory/index/local_runtime.py | 15 ++- src/basic_memory/index/local_schedulers.py | 16 ++- .../read_cache/test_runtime_invalidation.py | 123 ++++++++++++++++++ tests/index/test_local_schedulers.py | 4 + 6 files changed, 189 insertions(+), 17 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index df0445d0a..5784d366f 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -96,6 +96,9 @@ tenant: retaining the final failure-safe completion boundary for later-phase errors. Local inline file batches invalidate when their runner returns; queued Cloud file batches invalidate in the child worker after its durable commit, not merely when the coordinator enqueues the job. +1. Put full search reindexing in a failure-safe invalidation scope owned by the worker that runs + the rebuild. Reindexing drops and repopulates search rows incrementally, so fuzzy resolutions + filled during a partial or completed rebuild must not survive its final generation bump. 1. Put direct single-file and watcher file-index invalidation in failure-safe boundaries. Entity transactions can commit before search refresh or note-content reconciliation raises, so a failed index attempt can still publish cache-relevant state. Once a watcher index or delete @@ -116,8 +119,10 @@ tenant: 1. Invalidate directory deletion immediately after its acceptance transaction commits, then again after file cleanup and surviving-relation refresh. A slow or failed cleanup must not keep deleted entities reachable through the pre-acceptance generation. -1. Invalidate watcher-detected moves at their own completion boundary. Paired delete/create - events are consumed by move processing and therefore bypass the ordinary watcher callbacks. +1. Invalidate every committed watcher-detected move batch, then retain the final invalidation + after search refresh. Paired delete/create events are consumed by move processing and therefore + bypass the ordinary watcher callbacks; a large watcher run commits bounded batches before the + final refresh. 1. Invalidate directory moves after each individual file/database move commits, then again after the final search and relation follow-ups. Directory moves are incremental batches, so a long or partially failed request must not keep earlier files under the pre-move generation. @@ -299,18 +304,19 @@ Invalidation belongs at portable mutation and indexing completion boundaries, no FastAPI routes. It must cover accepted note writes, terminal deferred materialization and status publication, direct file indexing, filesystem watcher updates, project indexing, directory mutations, imports, watcher-detected paired moves, startup recovery or reconciliation, Cloud -storage events, and relation-resolution changes that affect cached responses. Each later phase -invalidates again so a value filled after an earlier generation bump cannot outlive the state -that phase publishes. Hosted read repair invalidates after bootstrapping accepted content and -before a repaired entity or resource is stored, including cancellation during the repair -transaction's exit. Directory moves invalidate after every committed file plus the final reindex; -directory deletion invalidates after acceptance and after cleanup. Imports invalidate after every -attempted file write and again around the complete attempt so partial failures cannot escape. -Project indexing invalidates even after a partial failure. Direct single-file and watcher file -indexing invalidate even when a follow-up fails after the entity commit. Watcher index and delete -completion callbacks finish their first post-event generation bump before cancellation can -escape. Recovery phases invalidate independently before the serving barrier is released and -include terminal conflict or failure publication. +storage events, full search reindexing, and relation-resolution changes that affect cached +responses. Each later phase invalidates again so a value filled after an earlier generation bump +cannot outlive the state that phase publishes. Hosted read repair invalidates after bootstrapping +accepted content and before a repaired entity or resource is stored, including cancellation +during the repair transaction's exit. Directory moves invalidate after every committed file plus +the final reindex; directory deletion invalidates after acceptance and after cleanup. Imports +invalidate after every attempted file write and again around the complete attempt so partial +failures cannot escape. Project indexing and full search reindexing invalidate even after a +partial failure. Direct single-file and watcher file indexing invalidate even when a follow-up +fails after the entity commit. Watcher move batches invalidate after every commit, and watcher +index/delete completion callbacks finish their first post-event generation bump before +cancellation can escape. Recovery phases invalidate independently before the serving barrier is +released and include terminal conflict or failure publication. ## Dependency And Lifecycle @@ -404,6 +410,10 @@ The real-Redis suite must prove: invalidation; - authoritative read exceptions propagate without populating the missed cache key; - watcher-detected paired moves invalidate even though their events bypass ordinary callbacks; +- consecutive watcher move batches each advance the real Redis generation before the next batch, + while retaining the final post-refresh bump; +- a partial or completed full search reindex invalidates fuzzy resolutions filled while the + search index was being rebuilt; - startup recovery that publishes written, conflict, or failed materialization state invalidates before serving resumes; - project-index failures invalidate any earlier committed batches; @@ -448,10 +458,14 @@ semantics themselves are asserted only against the real Redis integration fixtur resuming tenant traffic. - Invalidate project indexing from a failure-safe completion boundary, and invalidate directory deletion both after acceptance commit and after cleanup/relation refresh. +- Invalidate full search reindexing from the worker that executes the rebuild so partial or + completed search rows cannot leave cached fuzzy resolutions behind. - Invalidate direct and watcher file indexing from failure-safe boundaries because entity commits precede some search and reconciliation follow-ups. - Finish watcher index and delete completion invalidation before cancellation propagates from the post-event callback. +- Invalidate every committed watcher move batch before the next batch starts, then retain the + final post-refresh invalidation. - Wrap hosted note-content read repair in cancellation-safe invalidation so cancellation during transaction exit cannot skip the bump before a repaired entity or resource reaches read-through. - Invalidate pre-mutation content freshening even when a later accepted mutation is rejected or diff --git a/src/basic_memory/deps/services.py b/src/basic_memory/deps/services.py index 50395ec87..1cf33e030 100644 --- a/src/basic_memory/deps/services.py +++ b/src/basic_memory/deps/services.py @@ -417,11 +417,17 @@ async def get_project_index_scheduler( async def get_search_reindex_scheduler( + project_external_id: Annotated[ + str, FastAPIPath(alias="project_id", description="Project external UUID") + ], search_service: SearchServiceV2ExternalDep, app_config: AppConfigDep, + read_cache: ReadCacheDep, ) -> SearchReindexScheduler: return LocalSearchReindexScheduler( search_service=search_service, + project_external_id=project_external_id, + read_cache=read_cache, test_mode=app_config.is_test_env, ) diff --git a/src/basic_memory/index/local_runtime.py b/src/basic_memory/index/local_runtime.py index 276576926..28a587ccd 100644 --- a/src/basic_memory/index/local_runtime.py +++ b/src/basic_memory/index/local_runtime.py @@ -42,6 +42,7 @@ ) from basic_memory.indexing.models import IndexFileJobResult, IndexFileJobStatus from basic_memory.indexing.project_index_maintenance import ( + InvalidatingProjectIndexBatchStore, ProjectIndexMovedEntitySearchRefresher, RepositoryProjectIndexMaintenanceStore, RepositoryProjectIndexMovedEntitySearchRefresher, @@ -351,9 +352,19 @@ async def runtime_for_project(self, project: Project) -> StorageEventIndexRuntim file_service=dependencies.file_service, ), ) + maintenance_batch_store = ( + InvalidatingProjectIndexBatchStore( + move_store=maintenance_store, + delete_store=maintenance_store, + read_cache=self.read_cache, + project_external_id=project_ref.project_external_id, + ) + if self.read_cache is not None + else maintenance_store + ) maintenance_runner = StoreProjectIndexMaintenanceRunner( - move_store=maintenance_store, - delete_store=maintenance_store, + move_store=maintenance_batch_store, + delete_store=maintenance_batch_store, ) moved_entity_search_refresher = RepositoryProjectIndexMovedEntitySearchRefresher( session_maker=dependencies.session_maker, diff --git a/src/basic_memory/index/local_schedulers.py b/src/basic_memory/index/local_schedulers.py index 2ade6a102..f71dda676 100644 --- a/src/basic_memory/index/local_schedulers.py +++ b/src/basic_memory/index/local_schedulers.py @@ -156,15 +156,29 @@ async def _run_project_index(self, project_id: int, *, force_full: bool) -> None @dataclass(frozen=True, slots=True) class LocalSearchReindexScheduler: search_service: SearchReindexService + project_external_id: str + read_cache: ReadCacheInvalidator | None test_mode: bool def schedule_search_reindex(self, *, project_id: int) -> None: _ = project_id _schedule_background_coroutine( - self.search_service.reindex_all(), + self._run_search_reindex(), test_mode=self.test_mode, ) + async def _run_search_reindex(self) -> None: + # A rebuild publishes search rows incrementally after dropping the old + # index. Invalidate after success or partial failure so fuzzy resolutions + # cached during that window cannot survive the rebuild. + invalidation_scope = ( + invalidate_cache(self.read_cache, self.project_external_id) + if self.read_cache is not None + else nullcontext() + ) + async with invalidation_scope: + await self.search_service.reindex_all() + # Process-lifetime coalescing state: project ids with a relation-resolution # pass already pending or in flight. A burst of writes collapses to a single diff --git a/test-int/read_cache/test_runtime_invalidation.py b/test-int/read_cache/test_runtime_invalidation.py index dde2c3a85..7a847279a 100644 --- a/test-int/read_cache/test_runtime_invalidation.py +++ b/test-int/read_cache/test_runtime_invalidation.py @@ -23,6 +23,10 @@ LocalWatchMoveProcessor, ) from basic_memory.index.local_dependencies import LocalIndexSearchService +from basic_memory.index.local_schedulers import ( + LocalSearchReindexScheduler, + drain_background_tasks, +) from basic_memory.index.local_project import ( LocalProjectIndexBatchEnqueuer, LocalProjectIndexRuntime, @@ -159,6 +163,21 @@ async def detect_missing_entity_delete_events( return set() +class DetectedMoveBatchProcessor(DetectedMoveProcessor): + """Exercise multiple durable watcher move batches without detection I/O.""" + + @override + async def detect_moves( + self, + events: Sequence[StorageEventPayload], + ) -> tuple[dict[str, str], set[int]]: + del events + return { + "notes/old-1.md": "notes/new-1.md", + "notes/old-2.md": "notes/new-2.md", + }, {0, 1, 2, 3} + + class RecordingMoveMaintenance: def __init__(self) -> None: self.calls: list[tuple[dict[str, str], int]] = [] @@ -324,6 +343,26 @@ async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStat return await super().invalidate_project(project_id) +class PartiallyFailingSearchReindexService: + """Observe the generation inside a rebuild that fails after publishing work.""" + + def __init__( + self, + redis_cache: RedisCacheHarness, + project_external_id: str, + ) -> None: + self.redis_cache = redis_cache + self.project_external_id = project_external_id + self.generation_during_reindex: bytes | str | None = None + + async def reindex_all(self) -> None: + self.generation_during_reindex = await _current_generation( + self.redis_cache, + self.project_external_id, + ) + raise RuntimeError("partial search reindex failure") + + async def _initialized_generation( redis_cache: RedisCacheHarness, project_external_id: str, @@ -445,6 +484,59 @@ async def test_watcher_move_completion_invalidates_real_redis( assert generation_after != generation_before +@pytest.mark.asyncio +async def test_watcher_move_batches_invalidate_real_redis_before_next_batch( + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """Every committed watcher move batch advances Redis before the next batch.""" + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="watcher-move-batches", + ) + observed_store = GenerationObservingProjectIndexBatchStore( + redis_cache, + project_external_id, + ) + invalidating_store = InvalidatingProjectIndexBatchStore( + move_store=observed_store, + delete_store=observed_store, + read_cache=redis_cache.cache, + project_external_id=project_external_id, + ) + processor = DetectedMoveBatchProcessor( + session_maker=cast(async_sessionmaker[AsyncSession], object()), + file_service=cast(FileService, object()), + entity_repository=cast(LocalMoveEntityRepository, object()), + maintenance_runner=StoreProjectIndexMaintenanceRunner( + move_store=invalidating_store, + delete_store=invalidating_store, + ), + moved_entity_search_refresher=RecordingMovedEntitySearchRefresher(), + project_external_id=project_external_id, + read_cache=redis_cache.cache, + batch_size=1, + ) + + result = await processor.process_moves( + ( + _move_event(STORAGE_OBJECT_DELETED_EVENT, "notes/old-1.md"), + _move_event("OBJECT_CREATED_PUT", "notes/new-1.md"), + _move_event(STORAGE_OBJECT_DELETED_EVENT, "notes/old-2.md"), + _move_event("OBJECT_CREATED_PUT", "notes/new-2.md"), + ) + ) + + assert result.remaining_events == () + assert result.processed_moves == 2 + assert observed_store.move_generations[0] == generation_before + assert observed_store.move_generations[1] != observed_store.move_generations[0] + generation_after = await _current_generation(redis_cache, project_external_id) + assert generation_after != observed_store.move_generations[1] + + @pytest.mark.asyncio async def test_watcher_index_failure_invalidates_real_redis( test_project: Project, @@ -481,6 +573,37 @@ async def test_watcher_index_failure_invalidates_real_redis( assert generation_after != generation_before +@pytest.mark.asyncio +async def test_partial_search_reindex_invalidates_real_redis( + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """A partial search rebuild cannot retain resolutions filled during the run.""" + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="partial-search-reindex", + ) + search_service = PartiallyFailingSearchReindexService( + redis_cache, + project_external_id, + ) + scheduler = LocalSearchReindexScheduler( + search_service=search_service, + project_external_id=project_external_id, + read_cache=redis_cache.cache, + test_mode=False, + ) + + scheduler.schedule_search_reindex(project_id=test_project.id) + await drain_background_tasks() + + assert search_service.generation_during_reindex == generation_before + generation_after = await _current_generation(redis_cache, project_external_id) + assert generation_after != generation_before + + @pytest.mark.asyncio @pytest.mark.parametrize("completion", ["index", "delete"]) async def test_cancelled_watcher_completion_finishes_real_redis_invalidation( diff --git a/tests/index/test_local_schedulers.py b/tests/index/test_local_schedulers.py index 3284f5d37..c60872471 100644 --- a/tests/index/test_local_schedulers.py +++ b/tests/index/test_local_schedulers.py @@ -225,15 +225,19 @@ async def test_project_index_scheduler_is_noop_in_test_mode(): async def test_search_reindex_scheduler_maps_to_search_service(): """Search reindex scheduling should rebuild the search index.""" search_service = StubSearchService() + read_cache = RecordingReadCache() scheduler = LocalSearchReindexScheduler( search_service=search_service, + project_external_id=PROJECT_EXTERNAL_ID, + read_cache=read_cache, test_mode=False, ) scheduler.schedule_search_reindex(project_id=13) await asyncio.sleep(0.05) assert search_service.reindexed_project is True + assert read_cache.invalidated_project_ids == [PROJECT_EXTERNAL_ID] class StubRelationResolutionRuntime: From b35ba08ef86c22f1406008f5a35c93c9b82ee5fb Mon Sep 17 00:00:00 2001 From: phernandez Date: Thu, 30 Jul 2026 11:42:49 -0500 Subject: [PATCH 23/28] fix(api): shield startup recovery invalidation Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 12 +- src/basic_memory/services/initialization.py | 68 ++++---- .../read_cache/test_runtime_invalidation.py | 149 +++++++++++++++++- 3 files changed, 191 insertions(+), 38 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index 5784d366f..b5610717d 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -134,7 +134,9 @@ tenant: barrier or resuming tenant traffic. Treat materialization and move-vacate recovery as separate freshness phases: invalidate a completed first phase before starting the second so a later setup/query failure cannot skip the earlier generation bump. Terminal conflict and failure - publication count even when the recovery did not produce a written file. + publication count even when the recovery did not produce a written file. Put each + transaction-bearing phase inside a cancellation-safe invalidation scope so shutdown cannot + interrupt the generation bump after recovery commits. 1. Keep `bm:read:v1` separate from rate-limit and Cloud control-plane prefixes, metrics, timeouts, and failure policies. The clients may target one Redis deployment, but a read-cache timeout must bypass while a rate-limit decision keeps its Cloud-owned security behavior. @@ -316,7 +318,8 @@ partial failure. Direct single-file and watcher file indexing invalidate even wh fails after the entity commit. Watcher move batches invalidate after every commit, and watcher index/delete completion callbacks finish their first post-event generation bump before cancellation can escape. Recovery phases invalidate independently before the serving barrier is -released and include terminal conflict or failure publication. +released, include terminal conflict or failure publication, and finish their generation bump +before startup cancellation propagates. ## Dependency And Lifecycle @@ -416,6 +419,8 @@ The real-Redis suite must prove: search index was being rebuilt; - startup recovery that publishes written, conflict, or failed materialization state invalidates before serving resumes; +- cancellation after materialization or move-vacate recovery commits cannot interrupt the + phase-specific real Redis generation bump; - project-index failures invalidate any earlier committed batches; - consecutive project-index move, delete, and inline file batches each advance the real Redis generation before the next batch begins; @@ -479,7 +484,8 @@ semantics themselves are asserted only against the real Redis integration fixtur follow-ups. - Invalidate project-root path changes in a failure-safe boundary because the filesystem source can change while every cache identity remains stable. -- Invalidate each startup recovery phase before beginning the next phase. +- Put each startup recovery phase inside a cancellation-safe invalidation scope before beginning + the next phase. - Enable reads for a tenant only after every request, worker, partial-index, direct-index, read-repair, import, accepted-delete, move, and recovery boundary has namespace and invalidation parity. diff --git a/src/basic_memory/services/initialization.py b/src/basic_memory/services/initialization.py index 77de17ecb..7ff0a6e40 100644 --- a/src/basic_memory/services/initialization.py +++ b/src/basic_memory/services/initialization.py @@ -6,6 +6,7 @@ import asyncio import os +from contextlib import nullcontext from pathlib import Path from typing import TYPE_CHECKING @@ -68,21 +69,31 @@ async def recover_project_materializations( recover_move_vacates, recover_stuck_materializations, ) - from basic_memory.read_cache import invalidate_project_read_cache + from basic_memory.read_cache import invalidate_cache from basic_memory.services.file_service import FileService # FileService needs only base_path to write the accepted markdown bytes; # the markdown_processor/app_config are unused on the materialization path. file_service = FileService(Path(project.path)) - try: - materialization_recovery = await recover_stuck_materializations( - session_maker=session_maker, - file_service=file_service, - project_id=project.id, - ) - except Exception as e: # pragma: no cover - defensive startup guard - logger.error(f"Error recovering stuck materializations for project {project.name}: {e}") - return + project_external_id = str(project.external_id) + # Recovery reports whether it published state only after its transaction-bearing + # phase exits. Scope the whole phase so cancellation cannot land in that window; + # a harmless generation bump after a no-op recovery is the correctness tradeoff. + materialization_scope = ( + invalidate_cache(read_cache, project_external_id) + if read_cache is not None + else nullcontext() + ) + async with materialization_scope: + try: + materialization_recovery = await recover_stuck_materializations( + session_maker=session_maker, + file_service=file_service, + project_id=project.id, + ) + except Exception as e: # pragma: no cover - defensive startup guard + logger.error(f"Error recovering stuck materializations for project {project.name}: {e}") + return if materialization_recovery.attempted: logger.info( @@ -92,24 +103,21 @@ async def recover_project_materializations( recovered_materializations=materialization_recovery.written, ) - # Redis can outlive the process that left this materialization unfinished. - # Invalidate this committed phase before move-vacate recovery begins; a - # later setup/query failure must not leave its published state cached. - if read_cache is not None: - await invalidate_project_read_cache( - read_cache, - str(project.external_id), + vacate_scope = ( + invalidate_cache(read_cache, project_external_id) + if read_cache is not None + else nullcontext() + ) + async with vacate_scope: + try: + recovered_vacates = await recover_move_vacates( + session_maker=session_maker, + file_service=file_service, + project_id=project.id, ) - - try: - recovered_vacates = await recover_move_vacates( - session_maker=session_maker, - file_service=file_service, - project_id=project.id, - ) - except Exception as e: # pragma: no cover - defensive startup guard - logger.error(f"Error recovering move vacates for project {project.name}: {e}") - return + except Exception as e: # pragma: no cover - defensive startup guard + logger.error(f"Error recovering move vacates for project {project.name}: {e}") + return if not recovered_vacates: return @@ -120,12 +128,6 @@ async def recover_project_materializations( recovered_move_vacates=recovered_vacates, ) - if read_cache is not None: - await invalidate_project_read_cache( - read_cache, - str(project.external_id), - ) - async def initialize_database(app_config: BasicMemoryConfig) -> None: """Initialize database with migrations handled automatically by get_or_create_db. diff --git a/test-int/read_cache/test_runtime_invalidation.py b/test-int/read_cache/test_runtime_invalidation.py index 7a847279a..d623fd2f8 100644 --- a/test-int/read_cache/test_runtime_invalidation.py +++ b/test-int/read_cache/test_runtime_invalidation.py @@ -6,6 +6,7 @@ from collections.abc import AsyncIterator, Mapping, Sequence from contextlib import asynccontextmanager from datetime import UTC, datetime +from hashlib import sha256 from pathlib import Path from typing import Literal, Protocol, cast, override @@ -79,6 +80,7 @@ AcceptedNoteContentWrite, NoteContentRepository, ) +from basic_memory.repository.note_file_vacate_repository import NoteFileVacateRepository from basic_memory.runtime.cleanup import ( RuntimeExternalFileDeletePlan, RuntimeFileDeleteResult, @@ -331,15 +333,20 @@ def __init__( client: Redis, namespace: str, prefix: str, + block_on_call: int = 1, ) -> None: super().__init__(client=client, namespace=namespace, prefix=prefix) + self.block_on_call = block_on_call + self.invalidation_calls = 0 self.invalidation_started = asyncio.Event() self.release_invalidation = asyncio.Event() @override async def invalidate_project(self, project_id: str) -> ReadCacheInvalidationStatus: - self.invalidation_started.set() - await self.release_invalidation.wait() + self.invalidation_calls += 1 + if self.invalidation_calls == self.block_on_call: + self.invalidation_started.set() + await self.release_invalidation.wait() return await super().invalidate_project(project_id) @@ -736,6 +743,144 @@ async def test_startup_materialization_recovery_invalidates_real_redis( assert generation_after != generation_before +@pytest.mark.asyncio +async def test_cancelled_startup_materialization_recovery_finishes_real_redis_invalidation( + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """Cancellation after recovery commits waits for the Redis generation bump.""" + _, session_maker = engine_factory + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="cancelled-startup-materialization-recovery", + ) + entity = await _seed_recovery_note( + session_maker, + test_project, + title="Cancelled Startup Recovery", + file_path="notes/cancelled-startup-recovery.md", + markdown_content="# Recovered before cancellation\n", + ) + blocking_cache = BlockingInvalidationRedisReadCache( + client=redis_cache.client, + namespace=redis_cache.namespace, + prefix=redis_cache.prefix, + ) + + recovery_task = asyncio.create_task( + recover_project_materializations( + test_project, + session_maker, + read_cache=blocking_cache, + ) + ) + async with asyncio.timeout(5): + await blocking_cache.invalidation_started.wait() + + written = Path(test_project.path) / entity.file_path + assert written.read_text(encoding="utf-8") == "# Recovered before cancellation\n" + recovery_task.cancel() + await asyncio.sleep(0) + recovery_task.cancel() + await asyncio.sleep(0) + assert not recovery_task.done() + + blocking_cache.release_invalidation.set() + with pytest.raises(asyncio.CancelledError): + await recovery_task + + content_repository = NoteContentRepository(project_id=test_project.id) + async with db.scoped_session(session_maker) as session: + row = await content_repository.get_by_entity_id(session, entity.id) + assert row is not None + assert row.file_write_status == "synced" + generation_after = await _current_generation(redis_cache, project_external_id) + assert generation_after != generation_before + + +@pytest.mark.asyncio +async def test_cancelled_startup_move_vacate_recovery_finishes_real_redis_invalidation( + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """Cancellation after vacate cleanup commits waits for its own generation bump.""" + _, session_maker = engine_factory + markdown_content = "# Moved before cancellation\n" + entity = await _seed_recovery_note( + session_maker, + test_project, + title="Cancelled Startup Vacate Recovery", + file_path="notes/cancelled-startup-vacate-recovery.md", + markdown_content=markdown_content, + ) + await recover_project_materializations( + test_project, + session_maker, + read_cache=None, + ) + + source_relative = "old/cancelled-startup-vacate-recovery.md" + source = Path(test_project.path) / source_relative + source.parent.mkdir(parents=True, exist_ok=True) + source.write_text(markdown_content, encoding="utf-8") + source_checksum = sha256(markdown_content.encode()).hexdigest() + vacate_repository = NoteFileVacateRepository(project_id=test_project.id) + async with db.scoped_session(session_maker) as session: + await vacate_repository.record_vacate( + session, + entity_id=entity.id, + file_path=source_relative, + file_checksum=source_checksum, + ) + + project_external_id = str(test_project.external_id) + await _initialized_generation( + redis_cache, + project_external_id, + request="cancelled-startup-vacate-recovery", + ) + blocking_cache = BlockingInvalidationRedisReadCache( + client=redis_cache.client, + namespace=redis_cache.namespace, + prefix=redis_cache.prefix, + block_on_call=2, + ) + recovery_task = asyncio.create_task( + recover_project_materializations( + test_project, + session_maker, + read_cache=blocking_cache, + ) + ) + async with asyncio.timeout(5): + await blocking_cache.invalidation_started.wait() + + generation_before_vacate_invalidation = await _current_generation( + redis_cache, + project_external_id, + ) + assert not source.exists() + async with db.scoped_session(session_maker) as session: + assert await vacate_repository.load_vacate_markers(session, [source_relative]) == {} + + recovery_task.cancel() + await asyncio.sleep(0) + recovery_task.cancel() + await asyncio.sleep(0) + assert not recovery_task.done() + + blocking_cache.release_invalidation.set() + with pytest.raises(asyncio.CancelledError): + await recovery_task + + generation_after = await _current_generation(redis_cache, project_external_id) + assert generation_after != generation_before_vacate_invalidation + + @pytest.mark.asyncio async def test_startup_recovery_invalidates_before_later_vacate_failure_in_real_redis( monkeypatch: pytest.MonkeyPatch, From a5625747231802f1d0ffdd2e5b7712005639c7ab Mon Sep 17 00:00:00 2001 From: phernandez Date: Thu, 30 Jul 2026 11:54:05 -0500 Subject: [PATCH 24/28] fix(api): skip disabled cache identity Signed-off-by: phernandez --- src/basic_memory/services/initialization.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/basic_memory/services/initialization.py b/src/basic_memory/services/initialization.py index 7ff0a6e40..c25965874 100644 --- a/src/basic_memory/services/initialization.py +++ b/src/basic_memory/services/initialization.py @@ -75,12 +75,11 @@ async def recover_project_materializations( # FileService needs only base_path to write the accepted markdown bytes; # the markdown_processor/app_config are unused on the materialization path. file_service = FileService(Path(project.path)) - project_external_id = str(project.external_id) # Recovery reports whether it published state only after its transaction-bearing # phase exits. Scope the whole phase so cancellation cannot land in that window; # a harmless generation bump after a no-op recovery is the correctness tradeoff. materialization_scope = ( - invalidate_cache(read_cache, project_external_id) + invalidate_cache(read_cache, str(project.external_id)) if read_cache is not None else nullcontext() ) @@ -104,7 +103,7 @@ async def recover_project_materializations( ) vacate_scope = ( - invalidate_cache(read_cache, project_external_id) + invalidate_cache(read_cache, str(project.external_id)) if read_cache is not None else nullcontext() ) From efed2c8ab8575d46715ed6d5c070904ef53660e6 Mon Sep 17 00:00:00 2001 From: phernandez Date: Thu, 30 Jul 2026 12:11:13 -0500 Subject: [PATCH 25/28] fix(api): protect watcher delete invalidation Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 20 ++-- src/basic_memory/index/local_runtime.py | 25 +++-- .../indexing/external_file_delete_runner.py | 32 ++++++ .../read_cache/test_runtime_invalidation.py | 102 +++++++++++++++++- 4 files changed, 163 insertions(+), 16 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index b5610717d..d46a20d4e 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -101,9 +101,10 @@ tenant: filled during a partial or completed rebuild must not survive its final generation bump. 1. Put direct single-file and watcher file-index invalidation in failure-safe boundaries. Entity transactions can commit before search refresh or note-content reconciliation raises, so a - failed index attempt can still publish cache-relevant state. Once a watcher index or delete - completion callback runs after its durable event, finish the first generation bump before - propagating cancellation; relation/search cleanup may not begin after cancellation. + failed index attempt can still publish cache-relevant state. Wrap the transaction-bearing + watcher entity delete itself in cancellation-safe invalidation because cancellation can land + during transaction exit before its completion callback runs. Once a watcher index or delete + completion callback runs, retain its generation bump before relation and search cleanup. 1. Pass the namespace-bound cache into hosted note-content read repair. A resource or entity read can bootstrap a missing accepted-content row. Put cancellation-safe invalidation around the transaction-bearing repair call, not only after it returns, so cancellation during transaction @@ -316,10 +317,11 @@ invalidate after every attempted file write and again around the complete attemp failures cannot escape. Project indexing and full search reindexing invalidate even after a partial failure. Direct single-file and watcher file indexing invalidate even when a follow-up fails after the entity commit. Watcher move batches invalidate after every commit, and watcher -index/delete completion callbacks finish their first post-event generation bump before -cancellation can escape. Recovery phases invalidate independently before the serving barrier is -released, include terminal conflict or failure publication, and finish their generation bump -before startup cancellation propagates. +entity deletes wrap the transaction-bearing repository operation so cancellation cannot escape +between its commit and completion callback. Watcher index/delete completion callbacks retain a +later generation bump before relation and search cleanup. Recovery phases invalidate +independently before the serving barrier is released, include terminal conflict or failure +publication, and finish their generation bump before startup cancellation propagates. ## Dependency And Lifecycle @@ -409,6 +411,8 @@ The real-Redis suite must prove: real Redis generation bump; - cancellation during watcher index or delete completion cannot interrupt the first real Redis generation bump after the durable event; +- cancellation during a watcher entity-delete transaction's commit exit cannot escape before the + real Redis generation advances, even when the completion callback is never reached; - real Redis no-eviction capacity failures bypass cache storage and cannot fail committed-write invalidation; - authoritative read exceptions propagate without populating the missed cache key; @@ -467,6 +471,8 @@ semantics themselves are asserted only against the real Redis integration fixtur completed search rows cannot leave cached fuzzy resolutions behind. - Invalidate direct and watcher file indexing from failure-safe boundaries because entity commits precede some search and reconciliation follow-ups. +- Wrap the watcher entity-delete transaction itself in cancellation-safe invalidation because + cancellation can escape before its completion callback is reached. - Finish watcher index and delete completion invalidation before cancellation propagates from the post-event callback. - Invalidate every committed watcher move batch before the next batch starts, then retain the diff --git a/src/basic_memory/index/local_runtime.py b/src/basic_memory/index/local_runtime.py index 28a587ccd..5af07de5d 100644 --- a/src/basic_memory/index/local_runtime.py +++ b/src/basic_memory/index/local_runtime.py @@ -28,7 +28,12 @@ StorageEventOperationProcessorFactory, StorageEventProjectResolver, ) -from basic_memory.indexing.external_file_delete_runner import ExternalFileDeleteResult +from basic_memory.indexing.external_file_delete_runner import ( + ExternalFileDeleteEntities, + ExternalFileDeleteResult, + InvalidatingExternalFileDeleteEntities, + RepositoryExternalFileDeleteEntities, +) from basic_memory.indexing.file_index_checking import ( FileIndexChecker, RepositoryIndexedFileChecksumSource, @@ -55,9 +60,6 @@ plan_index_file_relation_resolution, resolve_project_relations, ) -from basic_memory.indexing.external_file_delete_runner import ( - RepositoryExternalFileDeleteEntities, -) from basic_memory.models import Entity, Project from basic_memory.read_cache import ( ReadCache, @@ -371,6 +373,16 @@ async def runtime_for_project(self, project: Project) -> StorageEventIndexRuntim entity_repository=dependencies.entity_repository, entity_indexer=dependencies.search_service, ) + delete_entities: ExternalFileDeleteEntities = RepositoryExternalFileDeleteEntities( + session_maker=dependencies.session_maker, + entity_repository=dependencies.entity_repository, + ) + if self.read_cache is not None: + delete_entities = InvalidatingExternalFileDeleteEntities( + entities=delete_entities, + read_cache=self.read_cache, + project_external_id=project_ref.project_external_id, + ) inline_runtime = InlineStorageEventIndexRuntime( project=project_ref, checker=checker, @@ -380,10 +392,7 @@ async def runtime_for_project(self, project: Project) -> StorageEventIndexRuntim entity_repository=dependencies.entity_repository, ), file_indexer=dependencies.file_indexer, - delete_entities=RepositoryExternalFileDeleteEntities( - session_maker=dependencies.session_maker, - entity_repository=dependencies.entity_repository, - ), + delete_entities=delete_entities, delete_objects=LocalExternalFileDeleteObjects(dependencies.file_service), result_recorder=LocalInlineStorageEventResultRecorder( project=project_ref, diff --git a/src/basic_memory/indexing/external_file_delete_runner.py b/src/basic_memory/indexing/external_file_delete_runner.py index e88d51f54..4e37a97bd 100644 --- a/src/basic_memory/indexing/external_file_delete_runner.py +++ b/src/basic_memory/indexing/external_file_delete_runner.py @@ -10,6 +10,7 @@ from basic_memory import db from basic_memory.models import Relation +from basic_memory.read_cache import ReadCacheInvalidator, invalidate_cache from basic_memory.runtime.cleanup import RuntimeExternalFileDeletePlan from basic_memory.runtime.note_content import ( RuntimeDeletedNoteEntityDeleteSource, @@ -126,6 +127,37 @@ async def delete_entity_if_file_path_matches( ) +@dataclass(frozen=True, slots=True) +class InvalidatingExternalFileDeleteEntities(ExternalFileDeleteEntities): + """Advance cached reads after a repository-backed delete attempt exits.""" + + entities: ExternalFileDeleteEntities + read_cache: ReadCacheInvalidator + project_external_id: str + + @override + async def find_entity_by_file_path( + self, + file_path: RuntimeFilePath, + ) -> RuntimeDeletedNoteEntityDeleteSource | None: + return await self.entities.find_entity_by_file_path(file_path) + + @override + async def delete_entity_if_file_path_matches( + self, + *, + entity_id: RuntimeEntityId, + file_path: RuntimeFilePath, + ) -> ExternalFileDeleteEntityDeleteResult: + # The repository transaction can commit while its context manager exits. + # Keeping that exit inside the scope makes cancellation wait for invalidation. + async with invalidate_cache(self.read_cache, self.project_external_id): + return await self.entities.delete_entity_if_file_path_matches( + entity_id=entity_id, + file_path=file_path, + ) + + class ExternalFileDeleteObjects(Protocol): """Storage capability required to detect stale delete notifications.""" diff --git a/test-int/read_cache/test_runtime_invalidation.py b/test-int/read_cache/test_runtime_invalidation.py index d623fd2f8..8db123194 100644 --- a/test-int/read_cache/test_runtime_invalidation.py +++ b/test-int/read_cache/test_runtime_invalidation.py @@ -14,6 +14,7 @@ from redis.asyncio import Redis from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker +import basic_memory.indexing.external_file_delete_runner as external_file_delete_runner import basic_memory.services.directory_deletes as directory_deletes import basic_memory.services.entity_service as entity_service_module from basic_memory import db @@ -34,7 +35,6 @@ run_local_project_index, ) from basic_memory.index.local_runtime import LocalInlineStorageEventResultRecorder -from basic_memory.indexing.external_file_delete_runner import ExternalFileDeleteResult from basic_memory.indexing.change_planning import ChangeReport from basic_memory.indexing.directory_delete_runner import ( DirectoryDeleteRuntime, @@ -46,6 +46,11 @@ IndexFileBatchIndexer, IndexFileBatchReader, ) +from basic_memory.indexing.external_file_delete_runner import ( + ExternalFileDeleteResult, + InvalidatingExternalFileDeleteEntities, + RepositoryExternalFileDeleteEntities, +) from basic_memory.indexing.models import IndexFileJobResult, IndexFileJobStatus, IndexInputFile from basic_memory.indexing.project_index_maintenance import ( InvalidatingProjectIndexBatchStore, @@ -706,6 +711,101 @@ async def test_cancelled_watcher_completion_finishes_real_redis_invalidation( assert generation_after != generation_before +@pytest.mark.asyncio +async def test_cancelled_watcher_delete_commit_finishes_real_redis_invalidation( + monkeypatch: pytest.MonkeyPatch, + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """Cancellation during watcher delete transaction exit cannot skip invalidation.""" + _, session_maker = engine_factory + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="cancelled-watcher-delete-commit", + ) + entity_repository = EntityRepository(project_id=test_project.id) + file_path = "notes/cancelled-watcher-delete-commit.md" + async with db.scoped_session(session_maker) as session: + entity = await entity_repository.add( + session, + Entity( + title="Cancelled Watcher Delete Commit", + note_type="note", + content_type="text/markdown", + file_path=file_path, + checksum="cancelled-watcher-delete-commit-checksum", + created_at=datetime.now(UTC), + updated_at=datetime.now(UTC), + ), + ) + + blocking_cache = BlockingInvalidationRedisReadCache( + client=redis_cache.client, + namespace=redis_cache.namespace, + prefix=redis_cache.prefix, + ) + delete_entities = InvalidatingExternalFileDeleteEntities( + entities=RepositoryExternalFileDeleteEntities( + session_maker=session_maker, + entity_repository=entity_repository, + ), + read_cache=blocking_cache, + project_external_id=project_external_id, + ) + found = await delete_entities.find_entity_by_file_path(file_path) + assert found is not None + assert found.id == entity.id + + transaction_committed = asyncio.Event() + hold_after_commit = asyncio.Event() + original_scoped_session = external_file_delete_runner.db.scoped_session + + @asynccontextmanager + async def pause_after_delete_commit( + scoped_session_maker: async_sessionmaker[AsyncSession], + ) -> AsyncIterator[AsyncSession]: + async with original_scoped_session(scoped_session_maker) as session: + yield session + transaction_committed.set() + await hold_after_commit.wait() + + monkeypatch.setattr( + external_file_delete_runner.db, + "scoped_session", + pause_after_delete_commit, + ) + delete_task = asyncio.create_task( + delete_entities.delete_entity_if_file_path_matches( + entity_id=entity.id, + file_path=file_path, + ) + ) + + async with asyncio.timeout(5): + await transaction_committed.wait() + async with original_scoped_session(session_maker) as session: + assert await entity_repository.get_by_id(session, entity.id) is None + + delete_task.cancel() + async with asyncio.timeout(5): + await blocking_cache.invalidation_started.wait() + + # Repeated cancellation must not abandon the in-flight Redis generation bump. + delete_task.cancel() + await asyncio.sleep(0) + assert not delete_task.done() + + blocking_cache.release_invalidation.set() + with pytest.raises(asyncio.CancelledError): + await delete_task + + generation_after = await _current_generation(redis_cache, project_external_id) + assert generation_after != generation_before + + @pytest.mark.asyncio async def test_startup_materialization_recovery_invalidates_real_redis( engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], From 7dd8175be5293e286ab5677f6bf7606defea969e Mon Sep 17 00:00:00 2001 From: phernandez Date: Thu, 30 Jul 2026 14:26:42 -0500 Subject: [PATCH 26/28] fix(sync): invalidate watcher index commits Signed-off-by: phernandez --- src/basic_memory/index/local_runtime.py | 10 +- .../indexing/index_file_runner.py | 24 +++- .../read_cache/test_runtime_invalidation.py | 109 ++++++++++++++++++ 3 files changed, 141 insertions(+), 2 deletions(-) diff --git a/src/basic_memory/index/local_runtime.py b/src/basic_memory/index/local_runtime.py index 5af07de5d..16d443eee 100644 --- a/src/basic_memory/index/local_runtime.py +++ b/src/basic_memory/index/local_runtime.py @@ -43,6 +43,8 @@ from basic_memory.repository.note_file_vacate_repository import NoteFileVacateRepository from basic_memory.indexing.index_file_runner import ( IndexFileObjectMetadata, + IndexFileExecutor, + InvalidatingIndexFileExecutor, RepositoryCurrentMaterializedNoteSource, ) from basic_memory.indexing.models import IndexFileJobResult, IndexFileJobStatus @@ -373,11 +375,17 @@ async def runtime_for_project(self, project: Project) -> StorageEventIndexRuntim entity_repository=dependencies.entity_repository, entity_indexer=dependencies.search_service, ) + file_indexer: IndexFileExecutor = dependencies.file_indexer delete_entities: ExternalFileDeleteEntities = RepositoryExternalFileDeleteEntities( session_maker=dependencies.session_maker, entity_repository=dependencies.entity_repository, ) if self.read_cache is not None: + file_indexer = InvalidatingIndexFileExecutor( + executor=file_indexer, + read_cache=self.read_cache, + project_external_id=project_ref.project_external_id, + ) delete_entities = InvalidatingExternalFileDeleteEntities( entities=delete_entities, read_cache=self.read_cache, @@ -391,7 +399,7 @@ async def runtime_for_project(self, project: Project) -> StorageEventIndexRuntim session_maker=dependencies.session_maker, entity_repository=dependencies.entity_repository, ), - file_indexer=dependencies.file_indexer, + file_indexer=file_indexer, delete_entities=delete_entities, delete_objects=LocalExternalFileDeleteObjects(dependencies.file_service), result_recorder=LocalInlineStorageEventResultRecorder( diff --git a/src/basic_memory/indexing/index_file_runner.py b/src/basic_memory/indexing/index_file_runner.py index aab43b1db..59f33f222 100644 --- a/src/basic_memory/indexing/index_file_runner.py +++ b/src/basic_memory/indexing/index_file_runner.py @@ -4,7 +4,7 @@ from collections.abc import Sequence from dataclasses import dataclass, field -from typing import Protocol +from typing import Protocol, override from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -25,6 +25,7 @@ plan_current_materialized_note_result, plan_indexed_file_live_update_metadata, ) +from basic_memory.read_cache import ReadCacheInvalidator, invalidate_cache from basic_memory.runtime.jobs import RuntimeStorageFileIndexMode from basic_memory.runtime.note_object_metadata import RuntimeNoteObjectMetadataMap from basic_memory.runtime.storage import RuntimeFileChecksum, RuntimeFilePath @@ -74,6 +75,27 @@ async def index_file( ) -> FileIndexResult: ... +@dataclass(frozen=True, slots=True) +class InvalidatingIndexFileExecutor(IndexFileExecutor): + """Advance cached reads after a transaction-bearing file index attempt exits.""" + + executor: IndexFileExecutor + read_cache: ReadCacheInvalidator + project_external_id: str + + @override + async def index_file( + self, + file_path: RuntimeFilePath, + *, + source: str, + ) -> FileIndexResult: + # File indexing can commit entity state while its session context exits. + # Keep that exit inside the scope so cancellation waits for invalidation. + async with invalidate_cache(self.read_cache, self.project_external_id): + return await self.executor.index_file(file_path, source=source) + + class CurrentMaterializedNoteEntityRepository(Protocol): """Repository capability needed to load the current materialized note entity.""" diff --git a/test-int/read_cache/test_runtime_invalidation.py b/test-int/read_cache/test_runtime_invalidation.py index 8db123194..14fefdd35 100644 --- a/test-int/read_cache/test_runtime_invalidation.py +++ b/test-int/read_cache/test_runtime_invalidation.py @@ -15,6 +15,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker import basic_memory.indexing.external_file_delete_runner as external_file_delete_runner +import basic_memory.indexing.index_file_runner as index_file_runner import basic_memory.services.directory_deletes as directory_deletes import basic_memory.services.entity_service as entity_service_module from basic_memory import db @@ -52,6 +53,8 @@ RepositoryExternalFileDeleteEntities, ) from basic_memory.indexing.models import IndexFileJobResult, IndexFileJobStatus, IndexInputFile +from basic_memory.indexing.models import FileIndexResult +from basic_memory.indexing.index_file_runner import InvalidatingIndexFileExecutor from basic_memory.indexing.project_index_maintenance import ( InvalidatingProjectIndexBatchStore, ProjectIndexDeleteBatch, @@ -375,6 +378,37 @@ async def reindex_all(self) -> None: raise RuntimeError("partial search reindex failure") +class CommittingIndexFileExecutor: + """Publish one real entity row before the caller cancels transaction exit.""" + + def __init__( + self, + session_maker: async_sessionmaker[AsyncSession], + entity_repository: EntityRepository, + ) -> None: + self.session_maker = session_maker + self.entity_repository = entity_repository + self.entity_id: int | None = None + + async def index_file(self, file_path: str, *, source: str) -> FileIndexResult: + del source + async with db.scoped_session(self.session_maker) as session: + entity = await self.entity_repository.add( + session, + Entity( + title="Cancelled Watcher Index Commit", + note_type="note", + content_type="text/markdown", + file_path=file_path, + checksum="cancelled-watcher-index-commit-checksum", + created_at=datetime.now(UTC), + updated_at=datetime.now(UTC), + ), + ) + self.entity_id = entity.id + raise AssertionError("test transaction-exit pause should be cancelled") + + async def _initialized_generation( redis_cache: RedisCacheHarness, project_external_id: str, @@ -806,6 +840,81 @@ async def pause_after_delete_commit( assert generation_after != generation_before +@pytest.mark.asyncio +async def test_cancelled_watcher_index_commit_finishes_real_redis_invalidation( + monkeypatch: pytest.MonkeyPatch, + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """Cancellation during watcher index transaction exit cannot skip invalidation.""" + _, session_maker = engine_factory + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation( + redis_cache, + project_external_id, + request="cancelled-watcher-index-commit", + ) + entity_repository = EntityRepository(project_id=test_project.id) + executor = CommittingIndexFileExecutor(session_maker, entity_repository) + blocking_cache = BlockingInvalidationRedisReadCache( + client=redis_cache.client, + namespace=redis_cache.namespace, + prefix=redis_cache.prefix, + ) + invalidating_executor = InvalidatingIndexFileExecutor( + executor=executor, + read_cache=blocking_cache, + project_external_id=project_external_id, + ) + + transaction_committed = asyncio.Event() + hold_after_commit = asyncio.Event() + original_scoped_session = index_file_runner.db.scoped_session + + @asynccontextmanager + async def pause_after_index_commit( + scoped_session_maker: async_sessionmaker[AsyncSession], + ) -> AsyncIterator[AsyncSession]: + async with original_scoped_session(scoped_session_maker) as session: + yield session + transaction_committed.set() + await hold_after_commit.wait() + + monkeypatch.setattr( + index_file_runner.db, + "scoped_session", + pause_after_index_commit, + ) + file_path = "notes/cancelled-watcher-index-commit.md" + index_task = asyncio.create_task( + invalidating_executor.index_file(file_path, source="s3_webhook") + ) + + async with asyncio.timeout(5): + await transaction_committed.wait() + assert executor.entity_id is not None + async with original_scoped_session(session_maker) as session: + indexed_entity = await entity_repository.get_by_id(session, executor.entity_id) + assert indexed_entity is not None + assert indexed_entity.file_path == file_path + + index_task.cancel() + async with asyncio.timeout(5): + await blocking_cache.invalidation_started.wait() + + index_task.cancel() + await asyncio.sleep(0) + assert not index_task.done() + + blocking_cache.release_invalidation.set() + with pytest.raises(asyncio.CancelledError): + await index_task + + generation_after = await _current_generation(redis_cache, project_external_id) + assert generation_after != generation_before + + @pytest.mark.asyncio async def test_startup_materialization_recovery_invalidates_real_redis( engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], From 7428125a10f65e987ef7878113339f36c91913ea Mon Sep 17 00:00:00 2001 From: phernandez Date: Thu, 30 Jul 2026 14:27:01 -0500 Subject: [PATCH 27/28] perf(api): cache directory read responses Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 101 ++++++++++--- .../api/v2/routers/directory_router.py | 136 +++++++++++++++--- src/basic_memory/deps/read_cache.py | 4 +- src/basic_memory/read_cache/contract.py | 3 + src/basic_memory/read_cache/policy.py | 1 + test-int/read_cache/test_api_read_cache.py | 122 ++++++++++++++++ 6 files changed, 326 insertions(+), 41 deletions(-) diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index d46a20d4e..ca04e3ed3 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -102,9 +102,9 @@ tenant: 1. Put direct single-file and watcher file-index invalidation in failure-safe boundaries. Entity transactions can commit before search refresh or note-content reconciliation raises, so a failed index attempt can still publish cache-relevant state. Wrap the transaction-bearing - watcher entity delete itself in cancellation-safe invalidation because cancellation can land - during transaction exit before its completion callback runs. Once a watcher index or delete - completion callback runs, retain its generation bump before relation and search cleanup. + watcher index and entity-delete operations themselves in cancellation-safe invalidation because + cancellation can land during transaction exit before their completion callbacks run. Once a + watcher callback runs, retain its generation bump before relation and search cleanup. 1. Pass the namespace-bound cache into hosted note-content read repair. A resource or entity read can bootstrap a missing accepted-content row. Put cancellation-safe invalidation around the transaction-bearing repair call, not only after it returns, so cancellation during transaction @@ -223,18 +223,20 @@ old token no longer matches. Phase one: -| Operation | Initial TTL | Constraints | -| ---------------------- | ----------: | --------------------------------------- | -| Entity by external ID | 60 seconds | Cache validated `EntityResponseV2` JSON | -| Identifier resolution | 60 seconds | Include body and workspace context | -| Markdown note resource | 60 seconds | Cache only below an explicit size limit | +| Operation | Initial TTL | Constraints | +| --------------------------- | ----------: | --------------------------------------------------- | +| Entity by external ID | 60 seconds | Cache validated `EntityResponseV2` JSON | +| Identifier resolution | 60 seconds | Include body and workspace context | +| Markdown note resource | 60 seconds | Cache only below an explicit size limit | +| Directory structure | 60 seconds | Folder-only tree; two MiB payload cap | +| Directory tree | 60 seconds | Full hierarchy; two MiB measured payload cap | +| Paginated directory listing | 60 seconds | Include path, depth, glob, page, and page-size keys | Phase two, after measuring phase one: | Operation | Initial TTL | Constraints | | --------------------------- | ------------: | ---------------------------------------------- | | Search | 30 seconds | Canonicalize the complete query and pagination | -| Directory reads | 30-60 seconds | Key every filtering and pagination input | | Context and recent activity | 15-30 seconds | Normalize or bound time-relative inputs | Do not initially cache failures, missing entities, graph/orphan responses, large or arbitrary @@ -299,6 +301,7 @@ Primary integration points: - `src/basic_memory/api/container.py` - `src/basic_memory/api/app.py` - `src/basic_memory/deps/read_cache.py` +- `src/basic_memory/api/v2/routers/directory_router.py` - `src/basic_memory/api/v2/routers/knowledge_router.py` - `src/basic_memory/api/v2/routers/resource_router.py` - later, `src/basic_memory/api/v2/routers/search_router.py` @@ -317,9 +320,9 @@ invalidate after every attempted file write and again around the complete attemp failures cannot escape. Project indexing and full search reindexing invalidate even after a partial failure. Direct single-file and watcher file indexing invalidate even when a follow-up fails after the entity commit. Watcher move batches invalidate after every commit, and watcher -entity deletes wrap the transaction-bearing repository operation so cancellation cannot escape -between its commit and completion callback. Watcher index/delete completion callbacks retain a -later generation bump before relation and search cleanup. Recovery phases invalidate +index and entity-delete operations wrap their transaction-bearing executors so cancellation cannot +escape between a commit and completion callback. Watcher index/delete completion callbacks retain +a later generation bump before relation and search cleanup. Recovery phases invalidate independently before the serving barrier is released, include terminal conflict or failure publication, and finish their generation bump before startup cancellation propagates. @@ -344,6 +347,58 @@ The FastAPI Redis SDK is not the foundational dependency for this work. The cach also participate in portable indexing and hosted storage-event invalidation, and Basic Memory's local ASGI transport does not run FastAPI lifespan. +## Cloud Production Calibration + +A bounded production snapshot from 2026-07-30 00:00-19:03 UTC changed directory reads from a +phase-two idea into a phase-one requirement: + +| Cloud web route | Calls | Average | p95 | +| --------------------------- | ----: | -------: | -------: | +| `GET /api/v2/projects/tree` | 1,073 | 1,794 ms | 4,656 ms | +| `GET /api/v2/notes` | 789 | 1,688 ms | 3,295 ms | +| `GET /api/v2/projects` | 145 | 1,536 ms | 3,269 ms | + +Cloud's existing user-scoped gateway response cache already proved that the directory work is +cacheable, but also exposed the misses that remain expensive: + +| Gateway route family | Attempts | Hits | Hit rate | Tenant dispatches | +| --------------------- | -------: | ----: | -------: | ----------------: | +| `directory_structure` | 14,163 | 8,845 | 62.5% | 5,318 | +| `directory_list` | 722 | 166 | 23.0% | 556 | +| `directory_tree` | 88 | 13 | 14.8% | 75 | +| `note_entity` | 2,779 | 877 | 31.6% | 1,896 | + +Across the preceding seven days, cached directory-tree response bodies had a p99 of 573 KiB and a +maximum of 1.29 MiB. Directory-node facades therefore use a two MiB payload cap rather than the +one MiB default; otherwise the largest and usually slowest tree in the measured workload would +always bypass storage. Paginated listings retain the one MiB default—their measured maximum was +183 KiB. + +For folder file navigation, a directory-list miss plus a tenant project-list dispatch averaged +2,407 ms; when both dependencies avoided tenant dispatch, the same composed endpoint averaged +636 ms. Project-tree requests with no directory miss and no project-list dispatch averaged +673 ms, while requests with six or more directory misses plus a project-list dispatch averaged +3,571 ms. These are associations within the snapshot, not a controlled benchmark, but they show +that cache locality materially changes end-to-end latency. + +The same snapshot contained 6,338 hosted MCP tool calls. `read_note` accounted for 2,636 calls at +2,782 ms average, `search` for 723 at 3,386 ms, and `list_directory` for 231 at 1,960 ms. Their +instrumented Basic Memory API dependencies included 3,947 identifier resolutions, 3,064 resource +reads, and 1,140 searches. This preserves resolution and resource as the primary MCP targets while +adding directory reads for the web explorer and `list_directory`. + +Project enumeration remains a Cloud-owned concern. `GET /api/v2/projects` combines access to +multiple workspaces, user visibility, project soft-delete state, and tenant database selection. +Even after loading a cached project-list body, the current Cloud service opens each tenant +database and queries active project IDs. Basic Memory's project-scoped semantic cache must not +absorb that authorization-aware composition. Cloud should optimize that active-project +reconciliation and its own project-list cache independently. + +Directory caching in Basic Memory is intended to replace overlapping route families after Cloud +reaches namespace, invalidation, and observability parity. During rollout, the inner cache can +also share tenant-project directory results across already-authorized users while Cloud's current +outer key remains user-specific. Do not keep both response-cache layers as the final design. + ## Failure Behavior - Operational Redis command failures, including connection, timeout, capacity, replica-read-only, @@ -413,6 +468,8 @@ The real-Redis suite must prove: generation bump after the durable event; - cancellation during a watcher entity-delete transaction's commit exit cannot escape before the real Redis generation advances, even when the completion callback is never reached; +- cancellation during a watcher file-index transaction's commit exit cannot escape before the + real Redis generation advances, even when the completion callback is never reached; - real Redis no-eviction capacity failures bypass cache storage and cannot fail committed-write invalidation; - authoritative read exceptions propagate without populating the missed cache key; @@ -450,11 +507,12 @@ semantics themselves are asserted only against the real Redis integration fixtur telemetry, and real Redis integration tests. - Do not cache production routes yet. -### 2. Hot entity reads +### 2. Hot semantic and directory reads -- Cache entity, resolution, and bounded markdown-resource reads behind default-off configuration. +- Cache entity, resolution, bounded markdown-resource, directory tree, directory structure, and + paginated directory-list reads behind default-off configuration. - Wire project invalidation through accepted writes and indexing paths. -- Add full-stack API and repeated `read_note` integration coverage. +- Add full-stack API, repeated `read_note`, and directory refresh integration coverage. ### 3. Cloud rollout @@ -471,8 +529,15 @@ semantics themselves are asserted only against the real Redis integration fixtur completed search rows cannot leave cached fuzzy resolutions behind. - Invalidate direct and watcher file indexing from failure-safe boundaries because entity commits precede some search and reconciliation follow-ups. -- Wrap the watcher entity-delete transaction itself in cancellation-safe invalidation because - cancellation can escape before its completion callback is reached. +- In Cloud's `build_cloud_index_file_runtime`, decorate the transaction-bearing `FileIndexer` + with `InvalidatingIndexFileExecutor` using the namespace-bound cache and canonical project UUID. + Both single-file jobs and every child of `index_file_batch` then invalidate at the same + committed-file boundary as the local filesystem watcher. +- Retain Cloud's existing post-entrypoint `GatewayCache` invalidation until directory/entity + route-family overlap is removed, but do not treat that later live-update side effect as the + Basic Memory cache correctness boundary. +- Wrap the watcher index and entity-delete transactions themselves in cancellation-safe + invalidation because cancellation can escape before their completion callbacks are reached. - Finish watcher index and delete completion invalidation before cancellation propagates from the post-event callback. - Invalidate every committed watcher move batch before the next batch starts, then retain the @@ -500,7 +565,7 @@ semantics themselves are asserted only against the real Redis integration fixtur ### 4. Expand from evidence -- Add search, directory, and graph-context reads when measured reuse supports them. +- Add search and graph-context reads when measured reuse supports them. - Refine project-wide invalidation only if unrelated writes materially reduce the entity hit rate. diff --git a/src/basic_memory/api/v2/routers/directory_router.py b/src/basic_memory/api/v2/routers/directory_router.py index 1f8637fed..c8c53518b 100644 --- a/src/basic_memory/api/v2/routers/directory_router.py +++ b/src/basic_memory/api/v2/routers/directory_router.py @@ -9,11 +9,24 @@ - Better performance through indexed queries """ -from typing import Optional +from contextlib import nullcontext +from typing import Annotated -from fastapi import APIRouter, Query, Path +from fastapi import APIRouter, Depends, Path, Query -from basic_memory.deps import DirectoryServiceV2ExternalDep +from basic_memory.deps import ( + create_model_read_cache, + DirectoryServiceV2ExternalDep, + ReadCacheDep, +) +from basic_memory.read_cache import ( + ModelReadCache, + ReadCacheKey, + ReadCacheOperation, + ReadCacheScope, + read_cache_request_digest, +) +from basic_memory.read_cache.policy import DIRECTORY_READ_CACHE_MAX_PAYLOAD_BYTES from basic_memory.schemas.directory import ( DEFAULT_DIRECTORY_PAGE_SIZE, MAX_DIRECTORY_PAGE_SIZE, @@ -24,11 +37,42 @@ router = APIRouter(prefix="/directory", tags=["directory-v2"]) +def get_directory_node_read_cache( + read_cache: ReadCacheDep, +) -> ModelReadCache[DirectoryNode] | None: + """Bind directory trees and structures to the optional cache backend.""" + return create_model_read_cache( + read_cache, + DirectoryNode, + max_payload_bytes=DIRECTORY_READ_CACHE_MAX_PAYLOAD_BYTES, + ) + + +DirectoryNodeReadCacheDep = Annotated[ + ModelReadCache[DirectoryNode] | None, + Depends(get_directory_node_read_cache), +] + + +def get_directory_list_read_cache( + read_cache: ReadCacheDep, +) -> ModelReadCache[DirectoryListResponse] | None: + """Bind paginated directory listings to the optional cache backend.""" + return create_model_read_cache(read_cache, DirectoryListResponse) + + +DirectoryListReadCacheDep = Annotated[ + ModelReadCache[DirectoryListResponse] | None, + Depends(get_directory_list_read_cache), +] + + @router.get("/tree", response_model=DirectoryNode, response_model_exclude_none=True) async def get_directory_tree( directory_service: DirectoryServiceV2ExternalDep, + read_cache: DirectoryNodeReadCacheDep, project_id: str = Path(..., description="Project external UUID"), -): +) -> DirectoryNode: """Get hierarchical directory structure from the knowledge base. Args: @@ -38,18 +82,31 @@ async def get_directory_tree( Returns: DirectoryNode representing the root of the hierarchical tree structure """ - # Get a hierarchical directory tree for the specific project - tree = await directory_service.get_directory_tree() + cache_key = ReadCacheKey( + project_id=project_id, + operation=ReadCacheOperation.directory_tree, + request_digest=read_cache_request_digest("tree"), + ) + cache_scope = ( + read_cache.read(key=cache_key) + if read_cache is not None + else nullcontext(ReadCacheScope[DirectoryNode]()) + ) + async with cache_scope as cached: + if cached.value is not None: + return cached.value - # Return the hierarchical tree - return tree + result = await directory_service.get_directory_tree() + cached.value = result + return result @router.get("/structure", response_model=DirectoryNode, response_model_exclude_none=True) async def get_directory_structure( directory_service: DirectoryServiceV2ExternalDep, + read_cache: DirectoryNodeReadCacheDep, project_id: str = Path(..., description="Project external UUID"), -): +) -> DirectoryNode: """Get folder structure for navigation (no files). Optimized endpoint for folder tree navigation. Returns only directory nodes @@ -62,8 +119,23 @@ async def get_directory_structure( Returns: DirectoryNode tree containing only folders (type="directory") """ - structure = await directory_service.get_directory_structure() - return structure + cache_key = ReadCacheKey( + project_id=project_id, + operation=ReadCacheOperation.directory_structure, + request_digest=read_cache_request_digest("structure"), + ) + cache_scope = ( + read_cache.read(key=cache_key) + if read_cache is not None + else nullcontext(ReadCacheScope[DirectoryNode]()) + ) + async with cache_scope as cached: + if cached.value is not None: + return cached.value + + result = await directory_service.get_directory_structure() + cached.value = result + return result @router.get( @@ -73,12 +145,11 @@ async def get_directory_structure( ) async def list_directory( directory_service: DirectoryServiceV2ExternalDep, + read_cache: DirectoryListReadCacheDep, project_id: str = Path(..., description="Project external UUID"), dir_name: str = Query("/", description="Directory path to list"), depth: int = Query(1, ge=1, le=10, description="Recursion depth (1-10)"), - file_name_glob: Optional[str] = Query( - None, description="Glob pattern for filtering file names" - ), + file_name_glob: str | None = Query(None, description="Glob pattern for filtering file names"), page: int = Query(1, ge=1, description="One-indexed result page"), page_size: int = Query( DEFAULT_DIRECTORY_PAGE_SIZE, @@ -86,7 +157,7 @@ async def list_directory( le=MAX_DIRECTORY_PAGE_SIZE, description="Number of nodes per page", ), -): +) -> DirectoryListResponse: """List directory contents with filtering and depth control. Args: @@ -101,11 +172,32 @@ async def list_directory( Returns: Bounded page of DirectoryNode objects matching the criteria """ - # Get directory listing with filtering - return await directory_service.list_directory( - dir_name=dir_name, - depth=depth, - file_name_glob=file_name_glob, - page=page, - page_size=page_size, + cache_key = ReadCacheKey( + project_id=project_id, + operation=ReadCacheOperation.directory_list, + request_digest=read_cache_request_digest( + dir_name, + str(depth), + file_name_glob or "", + str(page), + str(page_size), + ), + ) + cache_scope = ( + read_cache.read(key=cache_key) + if read_cache is not None + else nullcontext(ReadCacheScope[DirectoryListResponse]()) ) + async with cache_scope as cached: + if cached.value is not None: + return cached.value + + result = await directory_service.list_directory( + dir_name=dir_name, + depth=depth, + file_name_glob=file_name_glob, + page=page, + page_size=page_size, + ) + cached.value = result + return result diff --git a/src/basic_memory/deps/read_cache.py b/src/basic_memory/deps/read_cache.py index d786a6338..ec81b341e 100644 --- a/src/basic_memory/deps/read_cache.py +++ b/src/basic_memory/deps/read_cache.py @@ -33,6 +33,8 @@ def get_read_cache(request: Request) -> ReadCache | None: def create_model_read_cache[ModelT: BaseModel]( read_cache: ReadCache | None, model_type: type[ModelT], + *, + max_payload_bytes: int = READ_CACHE_MAX_PAYLOAD_BYTES, ) -> ModelReadCache[ModelT] | None: """Bind one response model to the host cache and Basic Memory's read policy.""" if read_cache is None: @@ -41,5 +43,5 @@ def create_model_read_cache[ModelT: BaseModel]( backend=read_cache, model_type=model_type, ttl_seconds=READ_CACHE_TTL_SECONDS, - max_payload_bytes=READ_CACHE_MAX_PAYLOAD_BYTES, + max_payload_bytes=max_payload_bytes, ) diff --git a/src/basic_memory/read_cache/contract.py b/src/basic_memory/read_cache/contract.py index 1a33735a1..1dc99dad4 100644 --- a/src/basic_memory/read_cache/contract.py +++ b/src/basic_memory/read_cache/contract.py @@ -9,6 +9,9 @@ class ReadCacheOperation(StrEnum): """Read operations supported by the initial cache rollout.""" + directory_list = "directory_list" + directory_structure = "directory_structure" + directory_tree = "directory_tree" entity = "entity" resolve = "resolve" resource = "resource" diff --git a/src/basic_memory/read_cache/policy.py b/src/basic_memory/read_cache/policy.py index 6ee02bffb..b269120cf 100644 --- a/src/basic_memory/read_cache/policy.py +++ b/src/basic_memory/read_cache/policy.py @@ -2,3 +2,4 @@ READ_CACHE_TTL_SECONDS = 60 READ_CACHE_MAX_PAYLOAD_BYTES = 1024 * 1024 +DIRECTORY_READ_CACHE_MAX_PAYLOAD_BYTES = 2 * 1024 * 1024 diff --git a/test-int/read_cache/test_api_read_cache.py b/test-int/read_cache/test_api_read_cache.py index 3964e0d51..1bd89d030 100644 --- a/test-int/read_cache/test_api_read_cache.py +++ b/test-int/read_cache/test_api_read_cache.py @@ -49,6 +49,7 @@ from basic_memory.repository import EntityRepository from basic_memory.repository.note_content_repository import NoteContentRepository from basic_memory.runtime.note_content import NOTE_CONTENT_BASE_CHECKSUM_HEADER +from basic_memory.schemas.directory import DirectoryListResponse, DirectoryNode from basic_memory.schemas.v2 import EntityResolveRequest from basic_memory.services.note_content_reads import NoteContentQueryService from basic_memory.workspace_context import ( @@ -192,6 +193,14 @@ def _cache_key( ) +def descendant_paths(node: DirectoryNode) -> set[str]: + """Collect one directory response's complete path set.""" + paths = {node.directory_path} + for child in node.children: + paths.update(descendant_paths(child)) + return paths + + async def _initialized_generation( redis_cache: RedisCacheHarness, project_id: str, @@ -435,6 +444,119 @@ async def test_entity_resolve_and_markdown_reads_cache_then_freshened_write_inva assert "Version two." in refreshed_resource.text +@pytest.mark.asyncio +async def test_directory_reads_cache_then_project_invalidation_refreshes( + app: FastAPI, + client: AsyncClient, + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """Tree, structure, and paginated list reads share real project invalidation.""" + project_external_id = str(test_project.external_id) + project_url = f"/v2/projects/{project_external_id}" + repository = EntityRepository(project_id=test_project.id) + _, session_maker = engine_factory + + async with db.scoped_session(session_maker) as session: + await repository.add( + session, + Entity( + title="Existing cached directory note", + note_type="note", + content_type="text/markdown", + file_path="cache-surface/existing.md", + checksum="existing-cached-directory-checksum", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + ) + + app.dependency_overrides[get_read_cache] = lambda: redis_cache.cache + tree_url = f"{project_url}/directory/tree" + structure_url = f"{project_url}/directory/structure" + list_url = f"{project_url}/directory/list" + list_params = { + "dir_name": "/cache-surface", + "depth": 2, + "page": 1, + "page_size": 200, + } + + first_tree_response = await client.get(tree_url) + first_structure_response = await client.get(structure_url) + first_list_response = await client.get(list_url, params=list_params) + assert first_tree_response.status_code == 200 + assert first_structure_response.status_code == 200 + assert first_list_response.status_code == 200 + + first_tree = DirectoryNode.model_validate(first_tree_response.json()) + first_structure = DirectoryNode.model_validate(first_structure_response.json()) + first_list = DirectoryListResponse.model_validate(first_list_response.json()) + assert "/cache-surface/after-cache" not in descendant_paths(first_tree) + assert "/cache-surface/after-cache" not in descendant_paths(first_structure) + assert "/cache-surface/after-cache" not in {node.directory_path for node in first_list.nodes} + + cache_keys = ( + _cache_key( + project_id=project_external_id, + operation=ReadCacheOperation.directory_tree, + request="tree", + ), + _cache_key( + project_id=project_external_id, + operation=ReadCacheOperation.directory_structure, + request="structure", + ), + _cache_key( + project_id=project_external_id, + operation=ReadCacheOperation.directory_list, + request="/cache-surface", + request_context=("2", "", "1", "200"), + ), + ) + for key in cache_keys: + redis_keys = redis_read_cache_keys( + prefix=redis_cache.prefix, + namespace=redis_cache.namespace, + key=key, + ) + assert await redis_cache.client.exists(redis_keys.data_key) == 1 + + async with db.scoped_session(session_maker) as session: + await repository.add( + session, + Entity( + title="Added after directory cache fill", + note_type="note", + content_type="text/markdown", + file_path="cache-surface/after-cache/new.md", + checksum="added-after-directory-cache-checksum", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + ) + + cached_tree_response = await client.get(tree_url) + cached_structure_response = await client.get(structure_url) + cached_list_response = await client.get(list_url, params=list_params) + assert cached_tree_response.json() == first_tree_response.json() + assert cached_structure_response.json() == first_structure_response.json() + assert cached_list_response.json() == first_list_response.json() + + await redis_cache.cache.invalidate_project(project_external_id) + + refreshed_tree = DirectoryNode.model_validate((await client.get(tree_url)).json()) + refreshed_structure = DirectoryNode.model_validate((await client.get(structure_url)).json()) + refreshed_list = DirectoryListResponse.model_validate( + (await client.get(list_url, params=list_params)).json() + ) + + assert "/cache-surface/after-cache" in descendant_paths(refreshed_tree) + assert "/cache-surface/after-cache" in descendant_paths(refreshed_structure) + assert "/cache-surface/after-cache" in {node.directory_path for node in refreshed_list.nodes} + + @pytest.mark.asyncio async def test_non_markdown_resource_is_never_cached( app: FastAPI, From 310c743fd495f6b8e671ae4084c4783b57dc7100 Mon Sep 17 00:00:00 2001 From: phernandez Date: Thu, 30 Jul 2026 14:47:12 -0500 Subject: [PATCH 28/28] fix(sync): invalidate terminal materialization state Signed-off-by: phernandez --- docs/REDIS_READ_CACHE_PLAN.md | 17 +- .../index/note_content_materialization.py | 42 ++-- .../test_materialization_invalidation.py | 202 ++++++++++++++++++ .../test_note_content_materialization.py | 10 +- 4 files changed, 247 insertions(+), 24 deletions(-) create mode 100644 test-int/read_cache/test_materialization_invalidation.py diff --git a/docs/REDIS_READ_CACHE_PLAN.md b/docs/REDIS_READ_CACHE_PLAN.md index ca04e3ed3..3207d4803 100644 --- a/docs/REDIS_READ_CACHE_PLAN.md +++ b/docs/REDIS_READ_CACHE_PLAN.md @@ -84,9 +84,10 @@ tenant: reconciliation. Request-path invalidation alone is insufficient because a worker can update a cached entity after the request returns. 1. Treat each asynchronous state transition as a separate freshness boundary. Invalidate after - the accepted-note transaction commits, again after terminal materialization/status publication - and indexing, and again after relation resolution completes. This prevents a read filled - between phases from surviving the later worker commit. + the accepted-note transaction commits, immediately after terminal materialization/status + publication before indexing begins, again after indexing, and again after relation resolution + completes. This prevents a read filled between phases from surviving the later worker commit + or remaining pending throughout a slow index. 1. Make committed-mutation invalidation cancellation-safe. Accepted-note writes, directory deletion, and each file in a directory move can be cancelled after their database commit succeeds but before the transaction context returns. Finish the namespace-bound generation @@ -325,6 +326,9 @@ escape between a commit and completion callback. Watcher index/delete completion a later generation bump before relation and search cleanup. Recovery phases invalidate independently before the serving barrier is released, include terminal conflict or failure publication, and finish their generation bump before startup cancellation propagates. +Deferred accepted-note materialization similarly advances the generation immediately after +terminal status publication, before potentially slow indexing, and retains the outer post-index +bump. ## Dependency And Lifecycle @@ -480,6 +484,8 @@ The real-Redis suite must prove: search index was being rebuilt; - startup recovery that publishes written, conflict, or failed materialization state invalidates before serving resumes; +- accepted-note materialization advances the generation after terminal status publication while + a following index is still blocked, then advances it again after indexing; - cancellation after materialization or move-vacate recovery commits cannot interrupt the phase-specific real Redis generation bump; - project-index failures invalidate any earlier committed batches; @@ -518,8 +524,9 @@ semantics themselves are asserted only against the real Redis integration fixtur - Inject a Basic Memory-specific Redis client and tenant namespace. - Derive that namespace from trusted request and worker context with one canonical function. -- Invalidate after accepted-note commit, terminal materialization/indexing, storage events, and - relation-resolution workers using the same tenant namespace as the request path. +- Invalidate after accepted-note commit, immediately after terminal materialization/status + publication before indexing, again after indexing, and after relation-resolution workers using + the same tenant namespace as the request path. - Invalidate watcher-detected paired moves at move completion, and invalidate any recovery or reconciliation attempt that can publish terminal state before releasing the serving barrier or resuming tenant traffic. diff --git a/src/basic_memory/index/note_content_materialization.py b/src/basic_memory/index/note_content_materialization.py index 7e0c4b658..8775781c2 100644 --- a/src/basic_memory/index/note_content_materialization.py +++ b/src/basic_memory/index/note_content_materialization.py @@ -641,33 +641,41 @@ async def _materialize_write_now( if accepted.materialization is None: # pragma: no cover - guarded by caller return accepted # The accepted-write invalidation runs before deferred materialization. - # Invalidate again after status publication and indexing so a read - # filled during that window cannot survive the terminal state. - invalidation_scope = ( + # The outer boundary retires reads filled during later indexing. + final_invalidation_scope = ( invalidate_cache(self.read_cache, self.project_external_id) if self.read_cache is not None else nullcontext() ) - async with invalidation_scope: + async with final_invalidation_scope: storage = LocalNoteContentStorage(self.file_service) cleanup_enqueuer = InlineNoteFileDeleteEnqueuer( storage, vacate_clearer=RepositoryMoveVacateClearer(session_maker=self.session_maker), ) - result = await run_note_materialization( - plan_note_materialization_job_request(accepted.materialization), - preflight=RepositoryNoteMaterializationPreflight( - session_maker=self.session_maker, - ), - writer=ContentStoreNoteMaterializationFileWriter(storage), - publisher=RepositoryNoteMaterializationPublisher( - session_maker=self.session_maker, - ), - status_publisher=RepositoryNoteMaterializationStatusPublisher( - session_maker=self.session_maker, - ), - cleanup_enqueuer=cleanup_enqueuer, + # Materialization commits terminal status before indexing begins. Retire + # pending/writing reads at that commit instead of holding them for the + # potentially slow index operation. + publication_invalidation_scope = ( + invalidate_cache(self.read_cache, self.project_external_id) + if self.read_cache is not None + else nullcontext() ) + async with publication_invalidation_scope: + result = await run_note_materialization( + plan_note_materialization_job_request(accepted.materialization), + preflight=RepositoryNoteMaterializationPreflight( + session_maker=self.session_maker, + ), + writer=ContentStoreNoteMaterializationFileWriter(storage), + publisher=RepositoryNoteMaterializationPublisher( + session_maker=self.session_maker, + ), + status_publisher=RepositoryNoteMaterializationStatusPublisher( + session_maker=self.session_maker, + ), + cleanup_enqueuer=cleanup_enqueuer, + ) if result.status is not RuntimeNoteMaterializationStatus.written: return replace( accepted, diff --git a/test-int/read_cache/test_materialization_invalidation.py b/test-int/read_cache/test_materialization_invalidation.py new file mode 100644 index 000000000..ec9eb6db6 --- /dev/null +++ b/test-int/read_cache/test_materialization_invalidation.py @@ -0,0 +1,202 @@ +"""Real Redis coverage for accepted-note materialization invalidation phases.""" + +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime +from hashlib import sha256 +from pathlib import Path +from typing import Protocol + +import pytest +from redis.asyncio import Redis +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker + +from basic_memory import db +from basic_memory.index.note_content_materialization import ( + LocalNoteContentMaterializationProvider, +) +from basic_memory.indexing.models import FileIndexOperation, FileIndexResult +from basic_memory.models import Entity, Project +from basic_memory.read_cache import ( + ReadCacheKey, + ReadCacheOperation, + read_cache_request_digest, +) +from basic_memory.read_cache.keys import redis_read_cache_generation_key +from basic_memory.read_cache.redis import RedisReadCache +from basic_memory.repository import EntityRepository, NoteContentRepository +from basic_memory.repository.note_content_repository import AcceptedNoteContentWrite +from basic_memory.runtime.note_content import ( + RuntimeAcceptedNoteChange, + RuntimeAcceptedNoteResponse, + RuntimePendingNoteMaterialization, + plan_accepted_note_response, +) +from basic_memory.services.file_service import FileService + +pytestmark = pytest.mark.redis + + +class RedisCacheHarness(Protocol): + cache: RedisReadCache + client: Redis + namespace: str + prefix: str + + +class BlockingIndexFileExecutor: + """Hold indexing after materialization has published its terminal state.""" + + def __init__(self, entity: Entity) -> None: + self.entity = entity + self.started = asyncio.Event() + self.release = asyncio.Event() + + async def index_file(self, file_path: str, *, source: str) -> FileIndexResult: + del source + self.started.set() + await self.release.wait() + checksum = self.entity.checksum + if checksum is None: + raise AssertionError("seeded materialization entity must carry a checksum") + return FileIndexResult( + file_path=file_path, + entity_id=self.entity.id, + external_id=str(self.entity.external_id), + title=self.entity.title, + permalink=self.entity.permalink, + checksum=checksum, + operation=FileIndexOperation.updated, + ) + + +async def _current_generation( + redis_cache: RedisCacheHarness, + project_external_id: str, +) -> bytes | str: + generation = await redis_cache.client.get( + redis_read_cache_generation_key( + prefix=redis_cache.prefix, + namespace=redis_cache.namespace, + project_id=project_external_id, + ) + ) + assert generation is not None + return generation + + +async def _initialized_generation( + redis_cache: RedisCacheHarness, + project_external_id: str, +) -> bytes | str: + await redis_cache.cache.lookup( + ReadCacheKey( + project_id=project_external_id, + operation=ReadCacheOperation.entity, + request_digest=read_cache_request_digest("materialization-publication"), + ) + ) + return await _current_generation(redis_cache, project_external_id) + + +async def _accepted_note( + session_maker: async_sessionmaker[AsyncSession], + project: Project, +) -> tuple[ + Entity, + RuntimeAcceptedNoteChange[RuntimeAcceptedNoteResponse], +]: + markdown_content = "# Terminal materialization\n" + checksum = sha256(markdown_content.encode()).hexdigest() + now = datetime.now(UTC) + entity_repository = EntityRepository(project_id=project.id) + content_repository = NoteContentRepository(project_id=project.id) + + async with db.scoped_session(session_maker) as session: + entity = await entity_repository.add( + session, + Entity( + title="Terminal Materialization", + note_type="note", + content_type="text/markdown", + file_path="notes/terminal-materialization.md", + checksum=checksum, + created_at=now, + updated_at=now, + ), + ) + await content_repository.accept_write( + session, + AcceptedNoteContentWrite( + entity_id=entity.id, + markdown_content=markdown_content, + db_version=1, + db_checksum=checksum, + last_source="api", + updated_at=now, + ), + ) + note_content = await content_repository.get_by_entity_id(session, entity.id) + assert note_content is not None + payload = plan_accepted_note_response( + entity=entity, + note_content=note_content, + fallback_source="api", + ) + + return entity, RuntimeAcceptedNoteChange( + status_code=202, + payload=payload, + materialization=RuntimePendingNoteMaterialization( + project_id=project.id, + entity_id=entity.id, + db_version=1, + db_checksum=checksum, + source="api", + ), + ) + + +@pytest.mark.asyncio +async def test_terminal_materialization_invalidates_before_index_completes( + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], + test_project: Project, + redis_cache: RedisCacheHarness, +) -> None: + """A synced publication advances Redis while the following index is still blocked.""" + _, session_maker = engine_factory + project_external_id = str(test_project.external_id) + generation_before = await _initialized_generation(redis_cache, project_external_id) + entity, accepted = await _accepted_note(session_maker, test_project) + indexer = BlockingIndexFileExecutor(entity) + provider = LocalNoteContentMaterializationProvider( + session_maker=session_maker, + file_service=FileService(Path(test_project.path)), + project_external_id=project_external_id, + read_cache=redis_cache.cache, + file_indexer=indexer, + test_mode=True, + ) + + materialization = asyncio.create_task(provider.materialize_write_change(accepted)) + async with asyncio.timeout(5): + await indexer.started.wait() + + try: + content_repository = NoteContentRepository(project_id=test_project.id) + async with db.scoped_session(session_maker) as session: + note_content = await content_repository.get_by_entity_id(session, entity.id) + generation_after_publication = await _current_generation( + redis_cache, + project_external_id, + ) + finally: + indexer.release.set() + await materialization + + assert note_content is not None + assert note_content.file_write_status == "synced" + assert generation_after_publication != generation_before + generation_after_index = await _current_generation(redis_cache, project_external_id) + assert generation_after_index != generation_after_publication diff --git a/tests/cloud/test_note_content_materialization.py b/tests/cloud/test_note_content_materialization.py index 57df1248f..124416ada 100644 --- a/tests/cloud/test_note_content_materialization.py +++ b/tests/cloud/test_note_content_materialization.py @@ -312,7 +312,10 @@ async def fake_run_note_materialization( # The write happens off the accept path via the bounded pool; drain to confirm. await pool.join() assert len(requests) == 1 - assert read_cache.invalidated_project_ids == [PROJECT_EXTERNAL_ID] + assert read_cache.invalidated_project_ids == [ + PROJECT_EXTERNAL_ID, + PROJECT_EXTERNAL_ID, + ] await pool.aclose() @@ -388,7 +391,10 @@ def schedule_relation_resolution(self, *, project_id: int) -> None: assert accepted.materialization is not None assert scheduled == [accepted.materialization.project_id] - assert read_cache.invalidated_project_ids == [PROJECT_EXTERNAL_ID] + assert read_cache.invalidated_project_ids == [ + PROJECT_EXTERNAL_ID, + PROJECT_EXTERNAL_ID, + ] @pytest.mark.asyncio