Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 44 additions & 23 deletions src/basic_memory/api/v2/routers/schema_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
SessionDep,
)
from basic_memory.models.knowledge import Entity
from basic_memory.schemas.base import normalize_note_type
from basic_memory.schemas.schema import (
ValidationReport,
InferenceReport,
Expand Down Expand Up @@ -196,10 +197,11 @@ async def search_fn(query: str) -> list[dict[str, Any]]:

# --- Batch validation by note type ---
if note_type:
entities = await _find_by_note_type(session, entity_repository, note_type)
canonical_note_type = normalize_note_type(note_type)
entities = await _find_by_note_type(session, entity_repository, canonical_note_type)
results = await _validate_note_entities(session, entity_repository, file_service, entities)
return ValidationReport(
note_type=note_type,
note_type=canonical_note_type,
total_notes=len(results),
total_entities=len(entities),
valid_count=sum(1 for r in results if r.passed),
Expand All @@ -217,10 +219,8 @@ async def search_fn(query: str) -> list[dict[str, Any]]:

type_summaries: list[TypeValidationSummary] = []
total_entities = 0
for target_type, stored_note_types in covered_types.items():
entities = []
for stored_type in stored_note_types:
entities.extend(await _find_by_note_type(session, entity_repository, stored_type))
for target_type in covered_types:
entities = await _find_by_note_type(session, entity_repository, target_type)
type_results = await _validate_note_entities(
session, entity_repository, file_service, entities
)
Expand Down Expand Up @@ -265,10 +265,11 @@ async def infer_schema_endpoint(
Examines observation categories and relation types across all notes
of the given type. Returns frequency analysis and suggested Picoschema.
"""
entities = await _find_by_note_type(session, entity_repository, note_type)
canonical_note_type = normalize_note_type(note_type)
entities = await _find_by_note_type(session, entity_repository, canonical_note_type)
notes_data = [_entity_to_note_data(entity) for entity in entities]

result = infer_schema(note_type, notes_data, optional_threshold=threshold)
result = infer_schema(canonical_note_type, notes_data, optional_threshold=threshold)

return InferenceReport(
note_type=result.note_type,
Expand Down Expand Up @@ -316,14 +317,15 @@ async def search_fn(query: str) -> list[dict[str, Any]]:
return [await _schema_frontmatter_from_file(file_service, e) for e in entities]

# Resolve schema by note type
schema_frontmatter = {"type": note_type}
canonical_note_type = normalize_note_type(note_type)
schema_frontmatter = {"type": canonical_note_type}
schema_def = await resolve_schema(schema_frontmatter, search_fn)

if not schema_def:
return DriftReport(note_type=note_type, schema_found=False)
return DriftReport(note_type=canonical_note_type, schema_found=False)

# Collect all notes of this type
entities = await _find_by_note_type(session, entity_repository, note_type)
entities = await _find_by_note_type(session, entity_repository, canonical_note_type)
notes_data = [_entity_to_note_data(entity) for entity in entities]

result = diff_schema(schema_def, notes_data)
Expand Down Expand Up @@ -404,11 +406,10 @@ async def _schema_covered_note_types(
"""Map each schema-covered target type to the stored note_type values it covers.

Coverage comes from both standalone schema notes and notes that carry inline
schemas or explicit schema references. Stored note types are snake_case while
schema authors may write "Person" or "person", so both sides are compared
through generate_permalink normalization — the same matching rule
_find_schema_entities uses for implicit type lookup. Standalone targets with no
matching notes map to an empty list so they still appear in the report.
schemas or explicit schema references. Schema authors and legacy database rows
may use different spellings, so both sides use the same note-type canonicalizer
as the write boundary. Standalone targets with no matching notes map to an empty
list so they still appear in the report.
"""
schema_query = entity_repository.select().where(Entity.note_type == "schema")
schema_result = await entity_repository.execute_query(session, schema_query)
Expand All @@ -418,7 +419,7 @@ async def _schema_covered_note_types(
for schema_entity in schema_result.scalars().all():
target = (schema_entity.entity_metadata or {}).get("entity")
if isinstance(target, str) and target:
targets.setdefault(generate_permalink(target), (target, set()))
targets.setdefault(normalize_note_type(target), (target, set()))

# Column-only select: skip eager-load options, which apply only to full entities.
# Reading metadata here also discovers inline schemas and explicit references;
Expand All @@ -433,7 +434,7 @@ async def _schema_covered_note_types(
if not stored_type:
continue

normalized_type = generate_permalink(stored_type)
normalized_type = normalize_note_type(stored_type)
if normalized_type in targets:
targets[normalized_type][1].add(stored_type)

Expand All @@ -456,8 +457,27 @@ async def _find_by_note_type(
entity_repository: EntityRepositoryV2ExternalDep,
note_type: str,
) -> list[Entity]:
"""Find all entities of a given type using the repository's select pattern."""
query = entity_repository.select().where(Entity.note_type == note_type)
"""Find canonical and legacy spellings that represent one logical note type."""
canonical_note_type = normalize_note_type(note_type)

# Legacy databases may contain values written before note types were canonicalized.
# Resolve their exact stored spellings in Python, where the shared normalizer can
# handle camel-case as well as case and punctuation without backend-specific SQL.
stored_types_query = entity_repository.select(Entity.note_type).distinct()
stored_types_result = await entity_repository.execute_query(
session,
stored_types_query,
use_query_options=False,
)
stored_types = {
stored_type
for stored_type in stored_types_result.scalars().all()
if stored_type and normalize_note_type(stored_type) == canonical_note_type
}
if not stored_types:
return []

query = entity_repository.select().where(Entity.note_type.in_(stored_types))
result = await entity_repository.execute_query(session, query)
return list(result.scalars().all())

Expand All @@ -481,21 +501,22 @@ async def _find_schema_entities(
result = await entity_repository.execute_query(session, query)
entities = list(result.scalars().all())

normalized_target = generate_permalink(target_note_type)
normalized_target_type = normalize_note_type(target_note_type)

entity_matches = [
e
for e in entities
if e.entity_metadata
and isinstance(e.entity_metadata.get("entity"), str)
and generate_permalink(e.entity_metadata["entity"]) == normalized_target
and normalize_note_type(e.entity_metadata["entity"]) == normalized_target_type
]
if entity_matches:
return entity_matches

if not allow_reference_match:
return []

normalized_target_reference = generate_permalink(target_note_type)
reference_matches: list[Entity] = []
for entity in entities:
candidate_refs: list[str] = []
Expand All @@ -505,7 +526,7 @@ async def _find_schema_entities(
candidate_refs.append(entity.permalink)
candidate_refs.append(FilePath(entity.permalink).name)

if any(generate_permalink(ref) == normalized_target for ref in candidate_refs):
if any(generate_permalink(ref) == normalized_target_reference for ref in candidate_refs):
reference_matches.append(entity)

return reference_matches
Expand Down
3 changes: 2 additions & 1 deletion src/basic_memory/indexing/accepted_note_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

from basic_memory.file_utils import ParseError, remove_frontmatter
from basic_memory.repository.accepted_note_search_row import AcceptedNoteSearchRow
from basic_memory.schemas.base import normalize_note_type

MAX_ACCEPTED_SEARCH_CONTENT_STEMS_SIZE = 6000

Expand Down Expand Up @@ -125,7 +126,7 @@ def build_accepted_note_search_row(
permalink=permalink,
file_path=Path(file_path).as_posix(),
item_type=item_type,
note_type=note_type,
note_type=normalize_note_type(note_type) if note_type is not None else None,
entity_id=entity_id,
created_at=created_at,
updated_at=updated_at,
Expand Down
8 changes: 5 additions & 3 deletions src/basic_memory/mcp/tools/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
resolve_project_and_path,
)
from basic_memory.mcp.server import mcp
from basic_memory.schemas.base import normalize_note_type
from basic_memory.schemas.search import (
SearchItemType,
SearchQuery,
Expand Down Expand Up @@ -998,11 +999,12 @@ async def search_notes(
categories = parse_str_list(categories) if categories is not None else []

# Avoid mutable-default-argument footguns. Treat None as "no filter".
# Lowercase note_types so "Chapter" matches the stored "chapter".
note_types = [t.lower() for t in note_types] if note_types else []
# Note types use one snake_case identity at write and query boundaries. Lowercasing
# alone leaves multiword and camel-case inputs in a separate logical population.
note_types = [normalize_note_type(note_type) for note_type in note_types]
entity_types = entity_types or []
# Categories are matched exactly against the indexed observation category,
# so preserve their original casing (unlike the lowercased note_types).
# so preserve their original casing (unlike the canonicalized note_types).
categories = categories or []

# Trigger: tags arrived via a direct function call instead of the MCP layer.
Expand Down
7 changes: 6 additions & 1 deletion src/basic_memory/schemas/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,11 @@ def to_snake_case(name: str) -> str:
return s2.lower()


def normalize_note_type(note_type: str) -> str:
"""Return the canonical identity used for note types across all boundaries."""
return to_snake_case(note_type)


def parse_timeframe(timeframe: str) -> datetime:
"""Parse timeframe with special handling for 'today' and other natural language expressions.

Expand Down Expand Up @@ -160,7 +165,7 @@ def validate_timeframe(timeframe: str) -> str:
"""Unique identifier in format '{path}/{normalized_name}'."""


NoteType = Annotated[str, BeforeValidator(to_snake_case), MinLen(1), MaxLen(200)]
NoteType = Annotated[str, BeforeValidator(normalize_note_type), MinLen(1), MaxLen(200)]
"""Classification of note (e.g., 'note', 'person', 'spec', 'schema'). """

ALLOWED_CONTENT_TYPES = {
Expand Down
10 changes: 9 additions & 1 deletion src/basic_memory/schemas/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from enum import Enum
from pydantic import BaseModel, Field, field_validator

from basic_memory.schemas.base import Permalink
from basic_memory.schemas.base import Permalink, normalize_note_type


class SearchItemType(str, Enum):
Expand Down Expand Up @@ -80,6 +80,14 @@ def validate_date(cls, v: Optional[Union[datetime, str]]) -> Optional[str]:
return v.isoformat()
return v

@field_validator("note_types")
@classmethod
def normalize_note_types(cls, values: Optional[List[str]]) -> Optional[List[str]]:
"""Apply the same canonical identity used when note types are written."""
if values is None:
return None
return [normalize_note_type(value) for value in values]

def no_criteria(self) -> bool:
text_is_empty = self.text is None or (isinstance(self.text, str) and not self.text.strip())
metadata_is_empty = not self.metadata_filters
Expand Down
47 changes: 43 additions & 4 deletions src/basic_memory/services/search_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import ast
import re
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from dataclasses import dataclass, replace
from datetime import datetime
from typing import Any, List, Optional, Set, Dict

Expand All @@ -24,6 +24,7 @@
SearchRepository,
)
from basic_memory.repository.search_query import relaxed_query_words
from basic_memory.schemas.base import normalize_note_type
from basic_memory.schemas.search import SearchQuery, SearchItemType, SearchRetrievalMode
from basic_memory.runtime.vector_sync import VectorSyncBatchResult
from basic_memory.services import FileService
Expand Down Expand Up @@ -143,7 +144,11 @@ def _prepare_query(self, query: SearchQuery) -> _PreparedSearchQuery | None:
permalink=query.permalink,
permalink_match=query.permalink_match,
title=query.title,
note_types=query.note_types,
note_types=(
[normalize_note_type(note_type) for note_type in query.note_types]
Comment thread
phernandez marked this conversation as resolved.
Comment thread
phernandez marked this conversation as resolved.
if query.note_types
else None
),
search_item_types=query.entity_types,
categories=query.categories,
after_date=after_date,
Expand Down Expand Up @@ -178,6 +183,35 @@ def _prepared_has_filters(prepared: _PreparedSearchQuery) -> bool:
or prepared.after_date
)

async def _include_legacy_note_type_spellings(
self,
prepared: _PreparedSearchQuery,
*,
session: AsyncSession | None = None,
) -> _PreparedSearchQuery:
"""Expand canonical note-type filters to exact legacy entity spellings."""
if not prepared.note_types:
return prepared

canonical_note_types = set(prepared.note_types)
async with db.scoped_session(self.session_maker, session) as active_session:
stored_types_query = self.entity_repository.select(Entity.note_type).distinct()
stored_types_result = await self.entity_repository.execute_query(
active_session,
stored_types_query,
use_query_options=False,
)

# Search rows written before canonicalization preserve the owning entity's
# exact type spelling. Include those spellings alongside canonical values
# so an upgrade remains searchable without requiring an eager full reindex.
compatible_note_types = canonical_note_types | {
stored_type
for stored_type in stored_types_result.scalars().all()
if stored_type and normalize_note_type(stored_type) in canonical_note_types
}
return replace(prepared, note_types=sorted(compatible_note_types))

async def _search_repository(
self,
prepared: _PreparedSearchQuery,
Expand Down Expand Up @@ -245,6 +279,10 @@ async def search(
prepared = self._prepare_query(query)
if prepared is None:
return []
prepared = await self._include_legacy_note_type_spellings(
prepared,
session=session,
)

strict_search_text = prepared.search_text
has_query = bool(
Expand Down Expand Up @@ -284,6 +322,7 @@ async def count(self, query: SearchQuery) -> int:
prepared = self._prepare_query(query)
if prepared is None:
return 0
prepared = await self._include_legacy_note_type_spellings(prepared)

strict_search_text = prepared.search_text
has_query = bool(
Expand Down Expand Up @@ -697,7 +736,7 @@ async def index_entity_file(
permalink=entity.permalink, # Required for Postgres NOT NULL constraint
file_path=entity.file_path,
metadata={
"note_type": entity.note_type,
"note_type": normalize_note_type(entity.note_type),
},
created_at=entity.created_at,
updated_at=entity.updated_at,
Expand Down Expand Up @@ -779,7 +818,7 @@ async def index_entity_markdown(
file_path=entity.file_path,
entity_id=entity.id,
metadata={
"note_type": entity.note_type,
"note_type": normalize_note_type(entity.note_type),
},
created_at=entity.created_at,
updated_at=entity.updated_at,
Expand Down
Loading
Loading