diff --git a/src/basic_memory/api/v2/routers/schema_router.py b/src/basic_memory/api/v2/routers/schema_router.py index 3b80cbbb9..bfb7ade03 100644 --- a/src/basic_memory/api/v2/routers/schema_router.py +++ b/src/basic_memory/api/v2/routers/schema_router.py @@ -22,6 +22,7 @@ SessionDep, ) from basic_memory.models.knowledge import Entity +from basic_memory.schemas.base import normalize_note_type from basic_memory.schemas.schema import ( ValidationReport, InferenceReport, @@ -196,10 +197,11 @@ async def search_fn(query: str) -> list[dict[str, Any]]: # --- Batch validation by note type --- if note_type: - entities = await _find_by_note_type(session, entity_repository, note_type) + canonical_note_type = normalize_note_type(note_type) + entities = await _find_by_note_type(session, entity_repository, canonical_note_type) results = await _validate_note_entities(session, entity_repository, file_service, entities) return ValidationReport( - note_type=note_type, + note_type=canonical_note_type, total_notes=len(results), total_entities=len(entities), valid_count=sum(1 for r in results if r.passed), @@ -217,10 +219,8 @@ async def search_fn(query: str) -> list[dict[str, Any]]: type_summaries: list[TypeValidationSummary] = [] total_entities = 0 - for target_type, stored_note_types in covered_types.items(): - entities = [] - for stored_type in stored_note_types: - entities.extend(await _find_by_note_type(session, entity_repository, stored_type)) + for target_type in covered_types: + entities = await _find_by_note_type(session, entity_repository, target_type) type_results = await _validate_note_entities( session, entity_repository, file_service, entities ) @@ -265,10 +265,11 @@ async def infer_schema_endpoint( Examines observation categories and relation types across all notes of the given type. Returns frequency analysis and suggested Picoschema. """ - entities = await _find_by_note_type(session, entity_repository, note_type) + canonical_note_type = normalize_note_type(note_type) + entities = await _find_by_note_type(session, entity_repository, canonical_note_type) notes_data = [_entity_to_note_data(entity) for entity in entities] - result = infer_schema(note_type, notes_data, optional_threshold=threshold) + result = infer_schema(canonical_note_type, notes_data, optional_threshold=threshold) return InferenceReport( note_type=result.note_type, @@ -316,14 +317,15 @@ async def search_fn(query: str) -> list[dict[str, Any]]: return [await _schema_frontmatter_from_file(file_service, e) for e in entities] # Resolve schema by note type - schema_frontmatter = {"type": note_type} + canonical_note_type = normalize_note_type(note_type) + schema_frontmatter = {"type": canonical_note_type} schema_def = await resolve_schema(schema_frontmatter, search_fn) if not schema_def: - return DriftReport(note_type=note_type, schema_found=False) + return DriftReport(note_type=canonical_note_type, schema_found=False) # Collect all notes of this type - entities = await _find_by_note_type(session, entity_repository, note_type) + entities = await _find_by_note_type(session, entity_repository, canonical_note_type) notes_data = [_entity_to_note_data(entity) for entity in entities] result = diff_schema(schema_def, notes_data) @@ -404,11 +406,10 @@ async def _schema_covered_note_types( """Map each schema-covered target type to the stored note_type values it covers. Coverage comes from both standalone schema notes and notes that carry inline - schemas or explicit schema references. Stored note types are snake_case while - schema authors may write "Person" or "person", so both sides are compared - through generate_permalink normalization — the same matching rule - _find_schema_entities uses for implicit type lookup. Standalone targets with no - matching notes map to an empty list so they still appear in the report. + schemas or explicit schema references. Schema authors and legacy database rows + may use different spellings, so both sides use the same note-type canonicalizer + as the write boundary. Standalone targets with no matching notes map to an empty + list so they still appear in the report. """ schema_query = entity_repository.select().where(Entity.note_type == "schema") schema_result = await entity_repository.execute_query(session, schema_query) @@ -418,7 +419,7 @@ async def _schema_covered_note_types( for schema_entity in schema_result.scalars().all(): target = (schema_entity.entity_metadata or {}).get("entity") if isinstance(target, str) and target: - targets.setdefault(generate_permalink(target), (target, set())) + targets.setdefault(normalize_note_type(target), (target, set())) # Column-only select: skip eager-load options, which apply only to full entities. # Reading metadata here also discovers inline schemas and explicit references; @@ -433,7 +434,7 @@ async def _schema_covered_note_types( if not stored_type: continue - normalized_type = generate_permalink(stored_type) + normalized_type = normalize_note_type(stored_type) if normalized_type in targets: targets[normalized_type][1].add(stored_type) @@ -456,8 +457,27 @@ async def _find_by_note_type( entity_repository: EntityRepositoryV2ExternalDep, note_type: str, ) -> list[Entity]: - """Find all entities of a given type using the repository's select pattern.""" - query = entity_repository.select().where(Entity.note_type == note_type) + """Find canonical and legacy spellings that represent one logical note type.""" + canonical_note_type = normalize_note_type(note_type) + + # Legacy databases may contain values written before note types were canonicalized. + # Resolve their exact stored spellings in Python, where the shared normalizer can + # handle camel-case as well as case and punctuation without backend-specific SQL. + stored_types_query = entity_repository.select(Entity.note_type).distinct() + stored_types_result = await entity_repository.execute_query( + session, + stored_types_query, + use_query_options=False, + ) + stored_types = { + stored_type + for stored_type in stored_types_result.scalars().all() + if stored_type and normalize_note_type(stored_type) == canonical_note_type + } + if not stored_types: + return [] + + query = entity_repository.select().where(Entity.note_type.in_(stored_types)) result = await entity_repository.execute_query(session, query) return list(result.scalars().all()) @@ -481,14 +501,14 @@ async def _find_schema_entities( result = await entity_repository.execute_query(session, query) entities = list(result.scalars().all()) - normalized_target = generate_permalink(target_note_type) + normalized_target_type = normalize_note_type(target_note_type) entity_matches = [ e for e in entities if e.entity_metadata and isinstance(e.entity_metadata.get("entity"), str) - and generate_permalink(e.entity_metadata["entity"]) == normalized_target + and normalize_note_type(e.entity_metadata["entity"]) == normalized_target_type ] if entity_matches: return entity_matches @@ -496,6 +516,7 @@ async def _find_schema_entities( if not allow_reference_match: return [] + normalized_target_reference = generate_permalink(target_note_type) reference_matches: list[Entity] = [] for entity in entities: candidate_refs: list[str] = [] @@ -505,7 +526,7 @@ async def _find_schema_entities( candidate_refs.append(entity.permalink) candidate_refs.append(FilePath(entity.permalink).name) - if any(generate_permalink(ref) == normalized_target for ref in candidate_refs): + if any(generate_permalink(ref) == normalized_target_reference for ref in candidate_refs): reference_matches.append(entity) return reference_matches diff --git a/src/basic_memory/indexing/accepted_note_search.py b/src/basic_memory/indexing/accepted_note_search.py index 856663e77..d1e51e29e 100644 --- a/src/basic_memory/indexing/accepted_note_search.py +++ b/src/basic_memory/indexing/accepted_note_search.py @@ -9,6 +9,7 @@ from basic_memory.file_utils import ParseError, remove_frontmatter from basic_memory.repository.accepted_note_search_row import AcceptedNoteSearchRow +from basic_memory.schemas.base import normalize_note_type MAX_ACCEPTED_SEARCH_CONTENT_STEMS_SIZE = 6000 @@ -125,7 +126,7 @@ def build_accepted_note_search_row( permalink=permalink, file_path=Path(file_path).as_posix(), item_type=item_type, - note_type=note_type, + note_type=normalize_note_type(note_type) if note_type is not None else None, entity_id=entity_id, created_at=created_at, updated_at=updated_at, diff --git a/src/basic_memory/mcp/tools/search.py b/src/basic_memory/mcp/tools/search.py index 7e79d108d..161c5597d 100644 --- a/src/basic_memory/mcp/tools/search.py +++ b/src/basic_memory/mcp/tools/search.py @@ -31,6 +31,7 @@ resolve_project_and_path, ) from basic_memory.mcp.server import mcp +from basic_memory.schemas.base import normalize_note_type from basic_memory.schemas.search import ( SearchItemType, SearchQuery, @@ -998,11 +999,12 @@ async def search_notes( categories = parse_str_list(categories) if categories is not None else [] # Avoid mutable-default-argument footguns. Treat None as "no filter". - # Lowercase note_types so "Chapter" matches the stored "chapter". - note_types = [t.lower() for t in note_types] if note_types else [] + # Note types use one snake_case identity at write and query boundaries. Lowercasing + # alone leaves multiword and camel-case inputs in a separate logical population. + note_types = [normalize_note_type(note_type) for note_type in note_types] entity_types = entity_types or [] # Categories are matched exactly against the indexed observation category, - # so preserve their original casing (unlike the lowercased note_types). + # so preserve their original casing (unlike the canonicalized note_types). categories = categories or [] # Trigger: tags arrived via a direct function call instead of the MCP layer. diff --git a/src/basic_memory/schemas/base.py b/src/basic_memory/schemas/base.py index 5337306d1..9aa1b22d0 100644 --- a/src/basic_memory/schemas/base.py +++ b/src/basic_memory/schemas/base.py @@ -72,6 +72,11 @@ def to_snake_case(name: str) -> str: return s2.lower() +def normalize_note_type(note_type: str) -> str: + """Return the canonical identity used for note types across all boundaries.""" + return to_snake_case(note_type) + + def parse_timeframe(timeframe: str) -> datetime: """Parse timeframe with special handling for 'today' and other natural language expressions. @@ -160,7 +165,7 @@ def validate_timeframe(timeframe: str) -> str: """Unique identifier in format '{path}/{normalized_name}'.""" -NoteType = Annotated[str, BeforeValidator(to_snake_case), MinLen(1), MaxLen(200)] +NoteType = Annotated[str, BeforeValidator(normalize_note_type), MinLen(1), MaxLen(200)] """Classification of note (e.g., 'note', 'person', 'spec', 'schema'). """ ALLOWED_CONTENT_TYPES = { diff --git a/src/basic_memory/schemas/search.py b/src/basic_memory/schemas/search.py index 675d5d266..d751239b4 100644 --- a/src/basic_memory/schemas/search.py +++ b/src/basic_memory/schemas/search.py @@ -11,7 +11,7 @@ from enum import Enum from pydantic import BaseModel, Field, field_validator -from basic_memory.schemas.base import Permalink +from basic_memory.schemas.base import Permalink, normalize_note_type class SearchItemType(str, Enum): @@ -80,6 +80,14 @@ def validate_date(cls, v: Optional[Union[datetime, str]]) -> Optional[str]: return v.isoformat() return v + @field_validator("note_types") + @classmethod + def normalize_note_types(cls, values: Optional[List[str]]) -> Optional[List[str]]: + """Apply the same canonical identity used when note types are written.""" + if values is None: + return None + return [normalize_note_type(value) for value in values] + def no_criteria(self) -> bool: text_is_empty = self.text is None or (isinstance(self.text, str) and not self.text.strip()) metadata_is_empty = not self.metadata_filters diff --git a/src/basic_memory/services/search_service.py b/src/basic_memory/services/search_service.py index f853cfb97..5f1df14ef 100644 --- a/src/basic_memory/services/search_service.py +++ b/src/basic_memory/services/search_service.py @@ -4,7 +4,7 @@ import ast import re from collections.abc import Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime from typing import Any, List, Optional, Set, Dict @@ -24,6 +24,7 @@ SearchRepository, ) from basic_memory.repository.search_query import relaxed_query_words +from basic_memory.schemas.base import normalize_note_type from basic_memory.schemas.search import SearchQuery, SearchItemType, SearchRetrievalMode from basic_memory.runtime.vector_sync import VectorSyncBatchResult from basic_memory.services import FileService @@ -143,7 +144,11 @@ def _prepare_query(self, query: SearchQuery) -> _PreparedSearchQuery | None: permalink=query.permalink, permalink_match=query.permalink_match, title=query.title, - note_types=query.note_types, + note_types=( + [normalize_note_type(note_type) for note_type in query.note_types] + if query.note_types + else None + ), search_item_types=query.entity_types, categories=query.categories, after_date=after_date, @@ -178,6 +183,35 @@ def _prepared_has_filters(prepared: _PreparedSearchQuery) -> bool: or prepared.after_date ) + async def _include_legacy_note_type_spellings( + self, + prepared: _PreparedSearchQuery, + *, + session: AsyncSession | None = None, + ) -> _PreparedSearchQuery: + """Expand canonical note-type filters to exact legacy entity spellings.""" + if not prepared.note_types: + return prepared + + canonical_note_types = set(prepared.note_types) + async with db.scoped_session(self.session_maker, session) as active_session: + stored_types_query = self.entity_repository.select(Entity.note_type).distinct() + stored_types_result = await self.entity_repository.execute_query( + active_session, + stored_types_query, + use_query_options=False, + ) + + # Search rows written before canonicalization preserve the owning entity's + # exact type spelling. Include those spellings alongside canonical values + # so an upgrade remains searchable without requiring an eager full reindex. + compatible_note_types = canonical_note_types | { + stored_type + for stored_type in stored_types_result.scalars().all() + if stored_type and normalize_note_type(stored_type) in canonical_note_types + } + return replace(prepared, note_types=sorted(compatible_note_types)) + async def _search_repository( self, prepared: _PreparedSearchQuery, @@ -245,6 +279,10 @@ async def search( prepared = self._prepare_query(query) if prepared is None: return [] + prepared = await self._include_legacy_note_type_spellings( + prepared, + session=session, + ) strict_search_text = prepared.search_text has_query = bool( @@ -284,6 +322,7 @@ async def count(self, query: SearchQuery) -> int: prepared = self._prepare_query(query) if prepared is None: return 0 + prepared = await self._include_legacy_note_type_spellings(prepared) strict_search_text = prepared.search_text has_query = bool( @@ -697,7 +736,7 @@ async def index_entity_file( permalink=entity.permalink, # Required for Postgres NOT NULL constraint file_path=entity.file_path, metadata={ - "note_type": entity.note_type, + "note_type": normalize_note_type(entity.note_type), }, created_at=entity.created_at, updated_at=entity.updated_at, @@ -779,7 +818,7 @@ async def index_entity_markdown( file_path=entity.file_path, entity_id=entity.id, metadata={ - "note_type": entity.note_type, + "note_type": normalize_note_type(entity.note_type), }, created_at=entity.created_at, updated_at=entity.updated_at, diff --git a/tests/api/v2/test_schema_router.py b/tests/api/v2/test_schema_router.py index 50931dd00..9c75d3a1a 100644 --- a/tests/api/v2/test_schema_router.py +++ b/tests/api/v2/test_schema_router.py @@ -3,8 +3,8 @@ Tests the integration layer where ORM entities are converted to NoteData and passed through the schema engine (infer, validate, diff). -Note: EntityType uses BeforeValidator(to_snake_case) so "Person" becomes "person" -in the database. All query params must use the stored (snake_case) form. +Note types use one snake_case identity at write and query boundaries. Legacy stored +spellings remain part of the same logical population. """ from pathlib import Path @@ -12,8 +12,9 @@ import pytest from httpx import AsyncClient +from sqlalchemy import update -from basic_memory.models import Project +from basic_memory.models import Entity, Project from basic_memory.schemas.base import Entity as EntitySchema from basic_memory.services.file_service import FileService @@ -366,6 +367,57 @@ async def test_validate_total_entities_without_schema( assert data["results"] == [] +@pytest.mark.asyncio +async def test_validate_note_type_includes_canonical_and_legacy_spellings( + client: AsyncClient, + test_project: Project, + v2_project_url: str, + entity_service, + session_maker, +): + """One explicit type validates canonical rows and legacy camel-case rows.""" + canonical_entity, _ = await entity_service.create_or_update_entity( + EntitySchema( + title="Canonical Task Item", + directory="tasks", + note_type="Task Item", + entity_metadata={"schema": {"status": "string"}}, + content="## Observations\n- [status] active\n", + ) + ) + legacy_entity, _ = await entity_service.create_or_update_entity( + EntitySchema( + title="Legacy Task Item", + directory="tasks", + note_type="task_item", + entity_metadata={"schema": {"status": "string"}}, + content="## Observations\n- [status] complete\n", + ) + ) + + # Simulate a row indexed by an older version before write-side canonicalization. + async with session_maker() as session: + await session.execute( + update(Entity).where(Entity.id == legacy_entity.id).values(note_type="TaskItem") + ) + await session.commit() + + response = await client.post( + f"{v2_project_url}/schema/validate", + params={"note_type": "task-item"}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["note_type"] == "task_item" + assert data["total_entities"] == 2 + assert data["total_notes"] == 2 + assert {result["note_identifier"] for result in data["results"]} == { + canonical_entity.title, + legacy_entity.title, + } + + # --- All-Types Validation Tests (#1013) --- diff --git a/tests/indexing/test_accepted_note_search.py b/tests/indexing/test_accepted_note_search.py index e677b8340..cfca1b183 100644 --- a/tests/indexing/test_accepted_note_search.py +++ b/tests/indexing/test_accepted_note_search.py @@ -102,3 +102,22 @@ def test_build_accepted_note_search_row_returns_immutable_hot_search_state() -> with pytest.raises(FrozenInstanceError): setattr(row, "title", "Changed") + + +def test_build_accepted_note_search_row_canonicalizes_legacy_note_type() -> None: + timestamp = datetime(2026, 6, 18, 12, 0, tzinfo=UTC) + + row = build_accepted_note_search_row( + entity_id=42, + title="Legacy task", + note_type="TaskItem", + entity_metadata=None, + permalink="tasks/legacy-task", + file_path="tasks/legacy-task.md", + search_content="Legacy body", + created_at=timestamp, + updated_at=timestamp, + project_id=7, + ) + + assert row.note_type == "task_item" diff --git a/tests/mcp/test_mcp_note_type_normalization.py b/tests/mcp/test_mcp_note_type_normalization.py new file mode 100644 index 000000000..d28c62630 --- /dev/null +++ b/tests/mcp/test_mcp_note_type_normalization.py @@ -0,0 +1,56 @@ +"""MCP boundary coverage for note-type canonicalization.""" + +from contextlib import asynccontextmanager +from typing import Any + +import pytest + +from basic_memory.schemas.search import SearchResponse + + +@pytest.mark.asyncio +async def test_search_notes_canonicalizes_multiword_note_types(monkeypatch): + """MCP filters use snake_case rather than lowercase-only normalization.""" + import importlib + + search_module = importlib.import_module("basic_memory.mcp.tools.search") + clients_module = importlib.import_module("basic_memory.mcp.clients") + + class StubProject: + name = "test-project" + external_id = "test-external-id" + + @asynccontextmanager + async def fake_get_project_client(*args, **kwargs): + yield object(), StubProject() + + async def fake_resolve_project_and_path( + client, identifier, project=None, context=None, headers=None + ): + return StubProject(), identifier, False + + captured_payload: dict[str, Any] = {} + + class MockSearchClient: + def __init__(self, *args, **kwargs): + pass + + async def search(self, payload, page, page_size): + captured_payload.update(payload) + return SearchResponse(results=[], current_page=page, page_size=page_size) + + monkeypatch.setattr(search_module, "get_project_client", fake_get_project_client) + monkeypatch.setattr( + search_module, + "resolve_project_and_path", + fake_resolve_project_and_path, + ) + monkeypatch.setattr(clients_module, "SearchClient", MockSearchClient) + + await search_module.search_notes( + project="test-project", + query="test", + note_types=["Task Item", "TaskItem", "task-item"], + ) + + assert captured_payload["note_types"] == ["task_item", "task_item", "task_item"] diff --git a/tests/schemas/test_search.py b/tests/schemas/test_search.py index ff9c4d648..0a5757407 100644 --- a/tests/schemas/test_search.py +++ b/tests/schemas/test_search.py @@ -42,6 +42,13 @@ def test_search_filters(): assert query.after_date == "2024-01-01T00:00:00" +def test_search_note_types_use_write_side_canonicalization(): + """Multiword and camel-case filters share the write-side note type identity.""" + query = SearchQuery(note_types=["Task Item", "TaskItem", "task-item"]) + + assert query.note_types == ["task_item", "task_item", "task_item"] + + def test_search_retrieval_mode_defaults_to_fts(): """Search retrieval mode defaults to FTS and accepts vector modes.""" query = SearchQuery(text="search implementation") diff --git a/tests/services/test_note_type_normalization.py b/tests/services/test_note_type_normalization.py new file mode 100644 index 000000000..c8ba3d725 --- /dev/null +++ b/tests/services/test_note_type_normalization.py @@ -0,0 +1,114 @@ +"""Search-service coverage for canonical note-type indexing and query preparation.""" + +from datetime import datetime, timezone +from types import SimpleNamespace +from typing import cast +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from basic_memory import db +from basic_memory.models import Entity, Project +from basic_memory.repository.entity_repository import EntityRepository +from basic_memory.repository.search_index_row import SearchIndexRow +from basic_memory.repository.search_repository import SearchRepository +from basic_memory.schemas.search import SearchQuery +from basic_memory.services.search_service import SearchService + + +def _search_service(repository: SearchRepository) -> SearchService: + return SearchService( + search_repository=repository, + entity_repository=MagicMock(), + file_service=MagicMock(), + session_maker=MagicMock(), + ) + + +def test_prepare_query_canonicalizes_directly_assigned_note_types(): + """Service callers cannot bypass canonicalization by mutating SearchQuery.""" + repository = cast(SearchRepository, MagicMock()) + service = _search_service(repository) + query = SearchQuery.model_construct(note_types=["TaskItem"]) + + prepared = service._prepare_query(query) + + assert prepared is not None + assert prepared.note_types == ["task_item"] + + +@pytest.mark.asyncio +async def test_reindex_canonicalizes_legacy_entity_note_type(): + """Reindexing gives legacy ORM rows the canonical search-filter identity.""" + repository_mock = MagicMock() + repository_mock.index_item = AsyncMock() + repository = cast(SearchRepository, repository_mock) + service = _search_service(repository) + entity = cast( + Entity, + SimpleNamespace( + id=1, + title="Legacy Task", + permalink="tasks/legacy-task", + file_path="tasks/legacy-task.pdf", + note_type="TaskItem", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + project_id=1, + ), + ) + + await service.index_entity_file(entity) + + repository_mock.index_item.assert_awaited_once() + index_call = repository_mock.index_item.await_args + assert index_call is not None + indexed_row = index_call.args[0] + assert indexed_row.metadata["note_type"] == "task_item" + + +@pytest.mark.asyncio +async def test_search_matches_legacy_note_type_projection_without_reindex( + search_service: SearchService, + entity_repository: EntityRepository, + session_maker, + test_project: Project, +) -> None: + """Canonical filters include exact legacy spellings still present on entities.""" + now = datetime.now(timezone.utc) + async with db.scoped_session(session_maker) as session: + entity = await entity_repository.add( + session, + Entity( + title="Legacy task", + note_type="TaskItem", + content_type="text/markdown", + file_path="tasks/legacy-task.md", + permalink="tasks/legacy-task", + created_at=now, + updated_at=now, + project_id=test_project.id, + ), + ) + + await search_service.repository.index_item( + SearchIndexRow( + id=entity.id, + entity_id=entity.id, + type="entity", + title=entity.title, + permalink=entity.permalink, + file_path=entity.file_path, + metadata={"note_type": "TaskItem"}, + created_at=now, + updated_at=now, + project_id=test_project.id, + ) + ) + + query = SearchQuery(note_types=["task-item"]) + + results = await search_service.search(query) + + assert [result.entity_id for result in results] == [entity.id] + assert await search_service.count(query) == 1