From 57c1a882d39c17cb075d24886e1b69a7a346a2a8 Mon Sep 17 00:00:00 2001 From: gingeard Date: Mon, 17 Aug 2026 16:27:59 +0100 Subject: [PATCH 01/19] fix(core): count non-Latin tokens when relaxing full-text queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `relaxed_query_words` decides whether a strict AND full-text query may be retried as an OR query. Its eligibility check counted tokens with `[A-Za-z0-9]+`, so any query written in a non-Latin alphabet produced zero tokens, tripped the "fewer than three tokens" guard, and never relaxed. Because only the hybrid path opts into relaxation, the effect was a silent degradation rather than an error: the FTS branch returned nothing for question-form queries, score fusion had a single non-zero side, and hybrid search became vector-only ranking. Nothing logs at default level, so the lexical half of hybrid search is simply absent for these languages. A dedicated CJK branch already worked around the same gate for Han, kana, and Hangul, which suggests the ASCII assumption was known but only patched for one script family. Switch the pattern to a Unicode-aware `[^\W_]+`. This keeps the alphanumeric intent (underscore stays excluded) and leaves every existing guard in place: short queries, quoted queries, explicit booleans, and pure-digit identifiers are rejected exactly as before, for Latin and non-Latin alike. Verified against a Russian corpus of 17 notes and 176 observations: before the change `fts_count=0` on every question-form query; after it the FTS branch contributes candidates and fusion has two sides again. Abugidas remain partially handled — Devanagari and Thai vowel signs are non-spacing marks outside `\w`, so words split into syllable fragments. Relaxation engages, but the OR terms are fragments; a test pins that behaviour so a future fix is deliberate. Tests: 12 added (6 alphabetic scripts relax, 4 guards still reject, 2 pin the abugida limitation). Existing search suites unchanged — 677 → 689 passing, same 32 pre-existing environment-dependent failures. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 19 +++++-- tests/repository/test_search_relaxation.py | 59 +++++++++++++++++++++ 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index 8544bdb47..2b56b911f 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -25,7 +25,16 @@ r"\uff65-\uff9f" # Halfwidth Katakana r"]" ) -RELAXATION_ASCII_TOKEN_PATTERN = re.compile(r"[A-Za-z0-9]+") +# Unicode-aware: `\w` minus underscore keeps the alphanumeric intent of the +# original ASCII pattern while also counting Cyrillic, Greek, Hebrew, Arabic, +# Armenian, and Georgian words. An ASCII-only gate made every such query look +# like it had zero tokens, so the three-token guard below rejected all of them +# and the hybrid FTS branch silently contributed nothing. +# +# Abugidas (Devanagari, Thai) are only partially handled: their vowel signs are +# non-spacing marks outside `\w`, so a word splits into syllable fragments. +# Relaxation still engages, but proper support needs grapheme segmentation. +RELAXATION_WORD_TOKEN_PATTERN = re.compile(r"[^\W_]+", re.UNICODE) RELAXATION_EDGE_PUNCTUATION = "?!.,;:，。！？；：、" @@ -59,9 +68,11 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: second-guessed); - fewer than three alphanumeric tokens (short queries like "New Feature" over-broaden under OR — and in hybrid the relaxed FTS-only rows normalize - to 1.0 and can outrank the vector result the user wanted); + to 1.0 and can outrank the vector result the user wanted). Tokens are + counted with a Unicode-aware pattern, so scripts other than Latin reach + the same guard instead of being read as zero tokens; - CJK terms separated by whitespace can relax with two or more terms because - the ASCII token gate would otherwise suppress the fallback entirely; + they are not whitespace-delimited the way the token guard assumes; - any pure-digit token ("root note 1", "SPEC 16") — identifier-like queries over-broaden and create false positives under OR. """ @@ -91,7 +102,7 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: # Outcome: preserve the short-query guard after pruning to avoid a broad retry. return relaxed_words if len(relaxed_words) >= 2 else None - tokens = RELAXATION_ASCII_TOKEN_PATTERN.findall(stripped.lower()) + tokens = RELAXATION_WORD_TOKEN_PATTERN.findall(stripped.lower()) if len(tokens) < 3 or any(token.isdigit() for token in tokens): return None pruned_words = [token for token in tokens if token not in RELAXATION_STOPWORDS] diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py index 9a7181d8b..01f0c2f9d 100644 --- a/tests/repository/test_search_relaxation.py +++ b/tests/repository/test_search_relaxation.py @@ -34,3 +34,62 @@ def test_relaxed_query_words_supports_whitespace_separated_cjk_scripts( def test_relaxed_query_words_preserves_short_query_guard_after_cjk_pruning(query: str) -> None: """Unsafe, duplicate, or stopword terms cannot pad a one-term CJK relaxation.""" assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("как отозвать выданный доступ", ["как", "отозвать", "выданный", "доступ"]), + ("як відкликати виданий доступ", ["як", "відкликати", "виданий", "доступ"]), + ("πώς να ανακαλέσετε πρόσβαση", ["πώς", "να", "ανακαλέσετε", "πρόσβαση"]), + ("כיצד לבטל גישה שניתנה", ["כיצד", "לבטל", "גישה", "שניתנה"]), + ("كيف تلغي الوصول الممنوح", ["كيف", "تلغي", "الوصول", "الممنوح"]), + ("ինչպես չեղարկել տրված մուտքը", ["ինչպես", "չեղարկել", "տրված", "մուտքը"]), + ], +) +def test_relaxed_query_words_supports_non_latin_alphabetic_scripts( + query: str, + expected: list[str], +) -> None: + """Non-Latin alphabetic queries reach the same guard as Latin ones. + + An ASCII-only token pattern found zero tokens in these queries, so the + three-token guard rejected every one of them and the hybrid FTS branch + contributed nothing — hybrid search silently became vector-only. + """ + assert relaxed_query_words(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "पहुंच कैसे रद्द करें", # Devanagari + "วิธี เพิกถอน การเข้าถึง", # Thai + ], +) +def test_relaxed_query_words_still_splits_scripts_with_combining_marks(query: str) -> None: + """Known limitation: abugidas split on combining marks, which `\\w` excludes. + + Vowel signs and viramas are non-spacing marks, so a `\\w`-based pattern cuts + each syllable cluster into fragments. Relaxation still engages — the token + count only grows — but the resulting OR terms are word fragments rather than + words. Proper support needs grapheme-cluster segmentation; this test pins the + current behaviour so a future change is a deliberate one. + """ + words = relaxed_query_words(query) + assert words is not None + assert len(words) > len(query.split()) + + +@pytest.mark.parametrize( + "query", + [ + "отозвать доступ", # fewer than three tokens + "спека 16 доступ", # pure-digit token + '"точная фраза"', # quoted: user intent is explicit + "доступ OR токен", # explicit boolean: user intent is explicit + ], +) +def test_relaxed_query_words_applies_existing_guards_to_non_latin(query: str) -> None: + """Non-Latin queries gain no exemption from the short-query and identifier guards.""" + assert relaxed_query_words(query) is None From 5576b5fb9c3b51c186c166f4d65f4acfa7dba0a0 Mon Sep 17 00:00:00 2001 From: gingeard Date: Mon, 17 Aug 2026 16:49:51 +0100 Subject: [PATCH 02/19] fix(core): keep combining marks with their base character in query tokens Combining marks are not alphanumeric, so counting them as token separators split abugida words (Devanagari, Thai) and NFD-decomposed text into syllable fragments. A single word then looked like several tokens, passed the three-token guard, and relaxed into a broad OR of one- and two-letter fragments whose top FTS row normalizes to 1.0 during hybrid fusion. Group marks with the base character they attach to. Single words in those scripts now stay one token and the short-query guard rejects them as intended, while multi-word queries relax into whole words. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 49 +++++++++++++++------ tests/repository/test_search_relaxation.py | 41 +++++++++++------ 2 files changed, 62 insertions(+), 28 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index 2b56b911f..6954d1dd3 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -1,6 +1,7 @@ """Shared full-text query preparation rules.""" import re +import unicodedata # Interrogative/function words contribute lexical noise when a strict # full-text query is relaxed: "when OR did OR a" matches loud wrong documents @@ -25,19 +26,39 @@ r"\uff65-\uff9f" # Halfwidth Katakana r"]" ) -# Unicode-aware: `\w` minus underscore keeps the alphanumeric intent of the -# original ASCII pattern while also counting Cyrillic, Greek, Hebrew, Arabic, -# Armenian, and Georgian words. An ASCII-only gate made every such query look -# like it had zero tokens, so the three-token guard below rejected all of them -# and the hybrid FTS branch silently contributed nothing. -# -# Abugidas (Devanagari, Thai) are only partially handled: their vowel signs are -# non-spacing marks outside `\w`, so a word splits into syllable fragments. -# Relaxation still engages, but proper support needs grapheme segmentation. -RELAXATION_WORD_TOKEN_PATTERN = re.compile(r"[^\W_]+", re.UNICODE) RELAXATION_EDGE_PUNCTUATION = "?!.,;:，。！？；：、" +def relaxation_word_tokens(text: str) -> list[str]: + """Split text into word tokens for the relaxation eligibility guards. + + A token is a run of alphanumeric characters together with any combining + marks attached to them. Counting this way matters twice over: + + - an ASCII-only rule saw zero tokens in Cyrillic, Greek, Hebrew, Arabic, + Armenian, and Georgian queries, so the three-token guard below rejected + every one of them and the hybrid FTS branch silently contributed nothing; + - combining marks are not alphanumeric, so counting them as separators cuts + abugidas (Devanagari, Thai) and decomposed text into syllable fragments. + One word then looks like several tokens, passes the three-token guard, and + relaxes into a broad OR of fragments — the opposite of what the guard is + for. Keeping marks with their base character preserves it. + """ + tokens: list[str] = [] + current: list[str] = [] + for char in text: + # A leading mark has no base character to attach to, so it cannot open + # a token; that keeps stray marks from forming fragment-only terms. + if char.isalnum() or (current and unicodedata.category(char).startswith("M")): + current.append(char) + elif current: + tokens.append("".join(current)) + current = [] + if current: + tokens.append("".join(current)) + return tokens + + def _dedupe_relaxation_words(words: list[str]) -> list[str]: """Preserve first-seen relaxed terms while removing duplicates case-insensitively.""" deduped_terms: list[str] = [] @@ -66,11 +87,11 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: - empty / quoted / explicit-boolean queries (user intent is not second-guessed); - - fewer than three alphanumeric tokens (short queries like "New Feature" + - fewer than three word tokens (short queries like "New Feature" over-broaden under OR — and in hybrid the relaxed FTS-only rows normalize to 1.0 and can outrank the vector result the user wanted). Tokens are - counted with a Unicode-aware pattern, so scripts other than Latin reach - the same guard instead of being read as zero tokens; + counted with relaxation_word_tokens, so scripts other than Latin reach the + same guard instead of being read as zero tokens; - CJK terms separated by whitespace can relax with two or more terms because they are not whitespace-delimited the way the token guard assumes; - any pure-digit token ("root note 1", "SPEC 16") — identifier-like queries @@ -102,7 +123,7 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: # Outcome: preserve the short-query guard after pruning to avoid a broad retry. return relaxed_words if len(relaxed_words) >= 2 else None - tokens = RELAXATION_WORD_TOKEN_PATTERN.findall(stripped.lower()) + tokens = relaxation_word_tokens(stripped.lower()) if len(tokens) < 3 or any(token.isdigit() for token in tokens): return None pruned_words = [token for token in tokens if token not in RELAXATION_STOPWORDS] diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py index 01f0c2f9d..9554939c5 100644 --- a/tests/repository/test_search_relaxation.py +++ b/tests/repository/test_search_relaxation.py @@ -61,24 +61,37 @@ def test_relaxed_query_words_supports_non_latin_alphabetic_scripts( @pytest.mark.parametrize( - "query", + ("query", "expected"), [ - "पहुंच कैसे रद्द करें", # Devanagari - "วิธี เพิกถอน การเข้าถึง", # Thai + ("पहुंच कैसे रद्द करें", ["पहुंच", "कैसे", "रद्द", "करें"]), # Devanagari + ("วิธี เพิกถอน การเข้าถึง", ["วิธี", "เพิกถอน", "การเข้าถึง"]), # Thai + ("como revogar acesso concedido", ["como", "revogar", "acesso", "concedido"]), ], ) -def test_relaxed_query_words_still_splits_scripts_with_combining_marks(query: str) -> None: - """Known limitation: abugidas split on combining marks, which `\\w` excludes. - - Vowel signs and viramas are non-spacing marks, so a `\\w`-based pattern cuts - each syllable cluster into fragments. Relaxation still engages — the token - count only grows — but the resulting OR terms are word fragments rather than - words. Proper support needs grapheme-cluster segmentation; this test pins the - current behaviour so a future change is a deliberate one. +def test_relaxed_query_words_keeps_combining_marks_with_their_base_character( + query: str, + expected: list[str], +) -> None: + """Vowel signs and diacritics stay inside the word they attach to. + + Combining marks are not alphanumeric, so treating them as separators splits + one abugida word into syllable fragments. The token count then inflates past + the three-token guard and relaxation ORs those fragments together. """ - words = relaxed_query_words(query) - assert words is not None - assert len(words) > len(query.split()) + assert relaxed_query_words(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "अंतर्राष्ट्रीयकरण", # one Devanagari word: 7 fragments if marks split it + "การเข้าถึง", # one Thai word + "pre\u0301sentation", # one word, NFD-decomposed acute accent + ], +) +def test_relaxed_query_words_guards_single_words_with_combining_marks(query: str) -> None: + """A single word stays one token, so the short-query guard still rejects it.""" + assert relaxed_query_words(query) is None @pytest.mark.parametrize( From 88b2453912f215879e08c60d0fc182d19ec8f387 Mon Sep 17 00:00:00 2001 From: gingeard Date: Tue, 18 Aug 2026 11:35:46 +0100 Subject: [PATCH 03/19] fix(core): keep join controls in tokens and reject Unicode numeric tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two eligibility-guard regressions from widening token recognition beyond ASCII. U+200C and U+200D are written inside an orthographic word, so terminating a token on them split Persian words and explicit Devanagari ZWJ conjuncts: the two-word query "می‌روم خانه" became three tokens and cleared the three-token guard. Join controls are now read as word-internal, and a trailing one is treated as a separator rather than kept in the term. isdigit() is false for Nl/No characters, so admitting every alphanumeric character let "SPEC Ⅻ design" and "spec ½ design" past the numeric-identifier guard that the old ASCII path rejected as too short. Both guards now classify numbers Unicode-wide with isnumeric(); the CJK branch is aligned for the same reason, since it carried the identical hole. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 65 ++++++++++++++------- tests/repository/test_search_relaxation.py | 37 ++++++++++++ 2 files changed, 80 insertions(+), 22 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index 6954d1dd3..9d826a152 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -27,35 +27,55 @@ r"]" ) RELAXATION_EDGE_PUNCTUATION = "?!.,;:，。！？；：、" +# Written inside a word (Persian "\u200c", Indic conjuncts) rather than between words. +RELAXATION_JOIN_CONTROLS = "\u200c\u200d" + + +def _is_token_continuation(char: str) -> bool: + """Whether a non-alphanumeric character belongs to the word being read. + + Combining marks and the zero-width join controls are written inside a word + but are not alphanumeric, so a naive scan would treat them as separators and + split one orthographic word into several tokens. + """ + return char in RELAXATION_JOIN_CONTROLS or unicodedata.category(char).startswith("M") def relaxation_word_tokens(text: str) -> list[str]: """Split text into word tokens for the relaxation eligibility guards. - A token is a run of alphanumeric characters together with any combining - marks attached to them. Counting this way matters twice over: - - - an ASCII-only rule saw zero tokens in Cyrillic, Greek, Hebrew, Arabic, - Armenian, and Georgian queries, so the three-token guard below rejected - every one of them and the hybrid FTS branch silently contributed nothing; - - combining marks are not alphanumeric, so counting them as separators cuts - abugidas (Devanagari, Thai) and decomposed text into syllable fragments. - One word then looks like several tokens, passes the three-token guard, and - relaxes into a broad OR of fragments — the opposite of what the guard is - for. Keeping marks with their base character preserves it. + A token is a run of alphanumeric characters together with the combining + marks and join controls written inside it. Counting this way matters because + an ASCII-only rule saw zero tokens in Cyrillic, Greek, Hebrew, Arabic, + Armenian, and Georgian queries, so the three-token guard below rejected every + one of them and the hybrid FTS branch silently contributed nothing. + + Counting characters that live inside a word as separators is just as wrong in + the other direction: it cuts abugidas (Devanagari, Thai), decomposed text, + and Persian or Indic words joined by U+200C/U+200D into fragments. One word + then looks like several tokens, clears the three-token guard, and relaxes + into a broad OR of fragments — the opposite of what the guard is for. """ tokens: list[str] = [] current: list[str] = [] + + def flush() -> None: + # Trailing join controls are word-internal by definition, so a token that + # ends in one is really a word followed by a separator. + token = "".join(current).rstrip(RELAXATION_JOIN_CONTROLS) + if token: + tokens.append(token) + current.clear() + for char in text: - # A leading mark has no base character to attach to, so it cannot open - # a token; that keeps stray marks from forming fragment-only terms. - if char.isalnum() or (current and unicodedata.category(char).startswith("M")): + # A leading mark or join control has no base character to attach to, so + # it cannot open a token; that keeps stray marks from forming + # fragment-only terms. + if char.isalnum() or (current and _is_token_continuation(char)): current.append(char) elif current: - tokens.append("".join(current)) - current = [] - if current: - tokens.append("".join(current)) + flush() + flush() return tokens @@ -94,8 +114,9 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: same guard instead of being read as zero tokens; - CJK terms separated by whitespace can relax with two or more terms because they are not whitespace-delimited the way the token guard assumes; - - any pure-digit token ("root note 1", "SPEC 16") — identifier-like queries - over-broaden and create false positives under OR. + - any numeric token ("root note 1", "SPEC 16", "SPEC \u216b") — identifier-like + queries over-broaden and create false positives under OR. Numeric-ness is + Unicode-wide, so Nl/No characters such as \u216b and \u00bd are caught too. """ if not search_text: return None @@ -109,7 +130,7 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: has_cjk_term = any(RELAXATION_CJK_PATTERN.search(word) for word in cjk_words) if has_cjk_term: - if len(cjk_words) < 2 or any(word.isdigit() for word in cjk_words): + if len(cjk_words) < 2 or any(word.isnumeric() for word in cjk_words): return None pruned_words = [ word @@ -124,7 +145,7 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: return relaxed_words if len(relaxed_words) >= 2 else None tokens = relaxation_word_tokens(stripped.lower()) - if len(tokens) < 3 or any(token.isdigit() for token in tokens): + if len(tokens) < 3 or any(token.isnumeric() for token in tokens): return None pruned_words = [token for token in tokens if token not in RELAXATION_STOPWORDS] return _dedupe_relaxation_words(pruned_words or tokens) or None diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py index 9554939c5..eeb299483 100644 --- a/tests/repository/test_search_relaxation.py +++ b/tests/repository/test_search_relaxation.py @@ -106,3 +106,40 @@ def test_relaxed_query_words_guards_single_words_with_combining_marks(query: str def test_relaxed_query_words_applies_existing_guards_to_non_latin(query: str) -> None: """Non-Latin queries gain no exemption from the short-query and identifier guards.""" assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("می‌روم خانه", None), # two Persian words, one joined by ZWNJ + ("نمی‌خواهم دسترسی را لغو", ["نمی‌خواهم", "دسترسی", "را", "لغو"]), + ("क‍ष विशेष पहुंच", ["क‍ष", "विशेष", "पहुंच"]), # explicit ZWJ conjunct + ], +) +def test_relaxed_query_words_keeps_join_controls_inside_words( + query: str, + expected: list[str] | None, +) -> None: + """U+200C/U+200D are written inside a word, so they must not split its token. + + Splitting on them inflates the token count: a two-word Persian query looks + like three tokens, clears the three-token guard, and relaxes into fragments. + """ + assert relaxed_query_words(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "SPEC Ⅻ design", # Nl: Roman numeral twelve + "spec ½ design", # No: vulgar fraction one half + "٣ ٤ ٥", # Arabic-Indic digits + ], +) +def test_relaxed_query_words_rejects_unicode_numeric_tokens(query: str) -> None: + """The identifier guard classifies numbers Unicode-wide, not just as ASCII digits. + + `isdigit()` is false for Nl/No characters, so admitting every alphanumeric + character would let identifier-like queries slip past the numeric guard. + """ + assert relaxed_query_words(query) is None From 401d5d45cc767e78b1c0967bd6b900dcafa030fd Mon Sep 17 00:00:00 2001 From: gingeard Date: Tue, 18 Aug 2026 12:04:27 +0100 Subject: [PATCH 04/19] fix(core): keep word-internal apostrophes inside query tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splitting on an apostrophe cut Ukrainian words apart: the two-word query "п’ять проектів" became three tokens, cleared the three-token guard, and relaxed into an OR containing one-letter fragments. Both U+2019 and the ASCII apostrophe are affected. An apostrophe now continues a token only between two letters. That keeps "SPEC 16's design" split on the digit, so the numeric-identifier guard still rejects it, and it leaves a leading or trailing apostrophe as a separator. One consequence for ASCII input: contractions become a single token, so "don't touch this" yields ["don't", "touch"] where it previously yielded ["don", "t", "touch"]. Merging can only lower the token count, so no query the guards used to reject can begin relaxing because of it, and the hyphen and slash cases named in #1022 are unchanged. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 36 ++++++++++++------ tests/repository/test_search_relaxation.py | 42 +++++++++++++++++++++ 2 files changed, 67 insertions(+), 11 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index 9d826a152..0f5e10148 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -29,23 +29,37 @@ RELAXATION_EDGE_PUNCTUATION = "?!.,;:，。！？；：、" # Written inside a word (Persian "\u200c", Indic conjuncts) rather than between words. RELAXATION_JOIN_CONTROLS = "\u200c\u200d" +# Word-internal only between letters: "\u043f\u2019\u044f\u0442\u044c", "don't" \u2014 but not "SPEC 16's", +# where the digit must stay its own token so the numeric guard still sees it. +RELAXATION_WORD_INTERNAL_PUNCTUATION = "'\u2019" -def _is_token_continuation(char: str) -> bool: +def _is_token_continuation(text: str, index: int, current: list[str]) -> bool: """Whether a non-alphanumeric character belongs to the word being read. - Combining marks and the zero-width join controls are written inside a word - but are not alphanumeric, so a naive scan would treat them as separators and - split one orthographic word into several tokens. + Combining marks, zero-width join controls, and apostrophes are written inside + a word but are not alphanumeric, so a naive scan treats them as separators + and splits one orthographic word into several tokens. + + An apostrophe counts only between two letters. That keeps "\u043f\u2019\u044f\u0442\u044c" whole while + leaving "SPEC 16's" split, so the digit stays a token of its own and the + numeric-identifier guard still rejects the query. """ - return char in RELAXATION_JOIN_CONTROLS or unicodedata.category(char).startswith("M") + char = text[index] + if char in RELAXATION_JOIN_CONTROLS or unicodedata.category(char).startswith("M"): + return True + if char in RELAXATION_WORD_INTERNAL_PUNCTUATION: + follows_letter = bool(current) and current[-1].isalpha() + precedes_letter = index + 1 < len(text) and text[index + 1].isalpha() + return follows_letter and precedes_letter + return False def relaxation_word_tokens(text: str) -> list[str]: """Split text into word tokens for the relaxation eligibility guards. A token is a run of alphanumeric characters together with the combining - marks and join controls written inside it. Counting this way matters because + marks, join controls, and apostrophes written inside it. Counting this way matters because an ASCII-only rule saw zero tokens in Cyrillic, Greek, Hebrew, Arabic, Armenian, and Georgian queries, so the three-token guard below rejected every one of them and the hybrid FTS branch silently contributed nothing. @@ -67,11 +81,11 @@ def flush() -> None: tokens.append(token) current.clear() - for char in text: - # A leading mark or join control has no base character to attach to, so - # it cannot open a token; that keeps stray marks from forming - # fragment-only terms. - if char.isalnum() or (current and _is_token_continuation(char)): + for index, char in enumerate(text): + # A leading mark, join control, or apostrophe has no base character to + # attach to, so it cannot open a token; that keeps stray punctuation from + # forming fragment-only terms. + if char.isalnum() or (current and _is_token_continuation(text, index, current)): current.append(char) elif current: flush() diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py index eeb299483..a5d9832a4 100644 --- a/tests/repository/test_search_relaxation.py +++ b/tests/repository/test_search_relaxation.py @@ -143,3 +143,45 @@ def test_relaxed_query_words_rejects_unicode_numeric_tokens(query: str) -> None: character would let identifier-like queries slip past the numeric guard. """ assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("п’ять проектів", None), # two Ukrainian words, U+2019 + ("об'єкт доступу", None), # two Ukrainian words, ASCII apostrophe + ( + "скасувати п’ять виданих об'єктів", + ["скасувати", "п’ять", "виданих", "об'єктів"], + ), + ], +) +def test_relaxed_query_words_keeps_apostrophes_inside_words( + query: str, + expected: list[str] | None, +) -> None: + """A word-internal apostrophe must not split one word into several tokens. + + Splitting on it turned a two-word Ukrainian query into three tokens, which + cleared the three-token guard and relaxed into one-letter fragments. + """ + assert relaxed_query_words(query) == expected + + +def test_relaxed_query_words_apostrophe_does_not_shield_numeric_tokens() -> None: + """An apostrophe joins letters only, so a digit stays a token of its own. + + Were `16's` read as one token it would not be numeric, and the query would + escape the identifier guard that rejects `SPEC 16 design`. + """ + assert relaxed_query_words("SPEC 16's design") is None + + +def test_relaxed_query_words_keeps_ascii_contractions_whole() -> None: + """ASCII contractions become one token instead of a word plus a stray letter. + + This is the one place where relaxed terms differ from the previous ASCII + behaviour. It only ever lowers the token count, so no query that the guards + used to reject can start relaxing because of it. + """ + assert relaxed_query_words("don't touch this") == ["don't", "touch"] From 1d5ed5d8977a326d59c53c6dce838cab3365eea1 Mon Sep 17 00:00:00 2001 From: gingeard Date: Tue, 18 Aug 2026 12:26:36 +0100 Subject: [PATCH 05/19] fix(core): quote relaxed terms that contain an apostrophe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keeping apostrophes inside tokens let them reach the relaxed renderers, which interpolate each word straight into backend query syntax. In FTS5 an ASCII apostrophe is syntax, not text: "don't touch this" rendered as `don't* OR touch*`, which fails to parse with `fts5: syntax error near "'"`. The caller treats a syntax error as an empty result, so the relaxed retry contributed nothing — the same silent-empty-FTS failure this fallback exists to prevent, reintroduced for English contractions and Ukrainian words. Both renderers now quote a word when it carries an apostrophe and leave every other word byte-identical. Postgres gets the matching treatment because it receives the same tokens; its escaping is the documented tsquery form, but I could not exercise it against a live server. Tests cover the rendered expression rather than only the token helper: the SQLite output is executed against a real FTS5 table, and one test pins that the unquoted form raises, so removing the quoting fails loudly. Signed-off-by: gingeard --- .../repository/postgres_search_repository.py | 14 ++- .../repository/sqlite_search_repository.py | 15 +++- .../test_search_relaxed_rendering.py | 85 +++++++++++++++++++ 3 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 tests/repository/test_search_relaxed_rendering.py diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index 6d05ca19f..8aa8e5bad 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -230,13 +230,25 @@ def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str: # For non-Boolean queries, prepare single term return self._prepare_single_term(term, is_prefix) + @staticmethod + def _relaxed_tsquery_term(word: str) -> str: + """Render one relaxed word as a tsquery-safe prefix expression. + + Mirrors the SQLite renderer: a word token can contain an apostrophe, and + tsquery reads that as lexeme-quoting syntax rather than text. Quoting the + lexeme and doubling any interior quote keeps it literal. + """ + if "'" in word: + return "'{}':*".format(word.replace("'", "''")) + return f"{word}:*" + @staticmethod def _relaxed_tsquery_text(search_text: Optional[str]) -> Optional[str]: """OR-relaxed tsquery expression for a failed strict query, or None.""" words = relaxed_query_words(search_text) if not words: return None - return " | ".join(f"{word}:*" for word in words) + return " | ".join(PostgresSearchRepository._relaxed_tsquery_term(word) for word in words) def _prepare_boolean_query(self, query: str) -> str: """Convert Boolean query to tsquery format. diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 91bcae435..9dced81f9 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -397,13 +397,26 @@ def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str: # For non-Boolean queries, use the single term preparation logic return self._prepare_single_term(term, is_prefix) + @staticmethod + def _relaxed_fts_term(word: str) -> str: + """Render one relaxed word as an FTS5-safe prefix expression. + + A word token can contain an apostrophe ("об'єкт", "don't"). Interpolated + bare it is FTS5 syntax, not text: the whole expression fails to parse, the + caller swallows the syntax error, and the relaxed retry returns nothing — + the exact silent-empty-FTS failure this fallback exists to prevent. + """ + if "'" in word or '"' in word: + return '"{}"*'.format(word.replace('"', '""')) + return f"{word}*" + @staticmethod def _relaxed_fts_text(search_text: Optional[str]) -> Optional[str]: """OR-relaxed FTS5 expression for a failed strict query, or None.""" words = relaxed_query_words(search_text) if not words: return None - return " OR ".join(f"{word}*" for word in words) + return " OR ".join(SQLiteSearchRepository._relaxed_fts_term(word) for word in words) @override async def semantic_effectively_enabled(self) -> bool: diff --git a/tests/repository/test_search_relaxed_rendering.py b/tests/repository/test_search_relaxed_rendering.py new file mode 100644 index 000000000..a1339320a --- /dev/null +++ b/tests/repository/test_search_relaxed_rendering.py @@ -0,0 +1,85 @@ +"""Relaxed-fallback rendering must survive the tokens the eligibility helper emits.""" + +import sqlite3 + +import pytest + +from basic_memory.repository.postgres_search_repository import PostgresSearchRepository +from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository + +CREATE_FTS = ( + "CREATE VIRTUAL TABLE t USING fts5(" + "body, tokenize='unicode61 tokenchars 0x2F', prefix='1,2,3,4')" +) +DOCUMENT = ( + "don't touch this п’ять проектів об'єкт доступу как отозвать выданный доступ पहुंच कैसे रद्द करें" +) + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("don't touch this", '"don\'t"* OR touch*'), + ("скасувати об'єкт виданий доступ", 'скасувати* OR "об\'єкт"* OR виданий* OR доступ*'), + ("п’ять виданих різних об’єктів", "п’ять* OR виданих* OR різних* OR об’єктів*"), + ("how to revoke granted access", "revoke* OR granted* OR access*"), + ], +) +def test_sqlite_relaxed_text_quotes_only_terms_that_need_it(query: str, expected: str) -> None: + """Apostrophe terms are quoted; every other term renders exactly as before.""" + assert SQLiteSearchRepository._relaxed_fts_text(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "don't touch this", + "скасувати об'єкт виданий доступ", + "п’ять виданих різних об’єктів", + "как отозвать выданный доступ", + "पहुंच कैसे रद्द करें", + ], +) +def test_sqlite_relaxed_text_is_accepted_by_fts5(query: str) -> None: + """The rendered expression must parse. + + An unquoted apostrophe raises `fts5: syntax error`, which the repository + catches and turns into an empty result — the relaxed retry then silently + contributes nothing, which is the failure this fallback exists to prevent. + """ + relaxed = SQLiteSearchRepository._relaxed_fts_text(query) + assert relaxed is not None + + connection = sqlite3.connect(":memory:") + try: + connection.execute(CREATE_FTS) + connection.execute("INSERT INTO t VALUES (?)", (DOCUMENT,)) + rows = connection.execute("SELECT rowid FROM t WHERE t MATCH ?", (relaxed,)).fetchall() + finally: + connection.close() + assert rows, f"relaxed expression matched nothing: {relaxed}" + + +def test_sqlite_relaxed_text_bare_apostrophe_would_be_rejected() -> None: + """Pin why the quoting exists, so removing it fails loudly rather than silently.""" + connection = sqlite3.connect(":memory:") + try: + connection.execute(CREATE_FTS) + connection.execute("INSERT INTO t VALUES (?)", (DOCUMENT,)) + with pytest.raises(sqlite3.OperationalError, match="fts5: syntax error"): + connection.execute("SELECT rowid FROM t WHERE t MATCH ?", ("don't* OR touch*",)) + finally: + connection.close() + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("don't touch this", "'don''t':* | touch:*"), + ("скасувати об'єкт виданий доступ", "скасувати:* | 'об''єкт':* | виданий:* | доступ:*"), + ("how to revoke granted access", "revoke:* | granted:* | access:*"), + ], +) +def test_postgres_relaxed_tsquery_quotes_apostrophe_lexemes(query: str, expected: str) -> None: + """Postgres carries the same token shapes, so it needs the same escaping.""" + assert PostgresSearchRepository._relaxed_tsquery_text(query) == expected From ab98d2fa4d91a28f9cf27582936835e289cd12db Mon Sep 17 00:00:00 2001 From: gingeard Date: Wed, 19 Aug 2026 10:43:25 +0100 Subject: [PATCH 06/19] fix(core): keep the CJK identifier guard on isdigit() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aligning the CJK branch to isnumeric() was wrong. 143 characters in U+3000–U+9FFF are isnumeric() without being isdigit(), and Han numerals are ordinary content words in CJK prose rather than identifiers. The swap therefore rejected queries like "数据 三 分析", "日本 十 経済 統計" and "データ 二 分析 結果", which main relaxes — switching relaxation off for the very queries #1022 turned it on for, in exactly the silent way this PR exists to remove. The identifier case the widening was for (SPEC 16, SPEC Ⅻ) is Latin, so the Unicode-wide classification stays where the tests are, on the general branch, and the CJK branch returns to isdigit(). Both directions are pinned by tests. Also names, in the tokenizer docstring, that scripts written without spaces between words — Thai, Lao, Khmer — arrive as a single token and so never reach the three-token guard. The docstring mentioned Thai in a way that could read as coverage; real support needs word segmentation. Reported by @tonydzi, verified here against main before changing anything. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 11 ++++++++- tests/repository/test_search_relaxation.py | 26 +++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index 0f5e10148..d64f2f056 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -69,6 +69,11 @@ def relaxation_word_tokens(text: str) -> list[str]: and Persian or Indic words joined by U+200C/U+200D into fragments. One word then looks like several tokens, clears the three-token guard, and relaxes into a broad OR of fragments — the opposite of what the guard is for. + + Scripts normally written without spaces between words — Thai, Lao, Khmer — + are counted, but a whole phrase arrives as a single token and so never + reaches the three-token guard. They are not in RELAXATION_CJK_PATTERN either, + so nothing relaxes for them. Fixing that needs real word segmentation. """ tokens: list[str] = [] current: list[str] = [] @@ -144,7 +149,11 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: has_cjk_term = any(RELAXATION_CJK_PATTERN.search(word) for word in cjk_words) if has_cjk_term: - if len(cjk_words) < 2 or any(word.isnumeric() for word in cjk_words): + # isdigit(), not isnumeric(): Han numerals are ordinary content words in + # CJK prose, not identifiers. Widening this guard the way the general + # branch needs would reject a query like "数据 三 分析" and silently switch + # off relaxation for the very queries #1022 turned it on for. + if len(cjk_words) < 2 or any(word.isdigit() for word in cjk_words): return None pruned_words = [ word diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py index a5d9832a4..e567c41fd 100644 --- a/tests/repository/test_search_relaxation.py +++ b/tests/repository/test_search_relaxation.py @@ -185,3 +185,29 @@ def test_relaxed_query_words_keeps_ascii_contractions_whole() -> None: used to reject can start relaxing because of it. """ assert relaxed_query_words("don't touch this") == ["don't", "touch"] + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("数据 三 分析", ["数据", "三", "分析"]), + ("日本 十 経済 統計", ["日本", "十", "経済", "統計"]), + ("データ 二 分析 結果", ["データ", "二", "分析", "結果"]), + ], +) +def test_relaxed_query_words_treats_han_numerals_as_content_words( + query: str, + expected: list[str], +) -> None: + """The CJK guard stays on `isdigit()`, so a numeral word does not veto relaxation. + + 143 characters in U+3000–U+9FFF are `isnumeric()` without being `isdigit()`. + Classifying them as identifiers would reject ordinary CJK prose and switch + relaxation back off for the queries it was turned on for. + """ + assert relaxed_query_words(query) == expected + + +def test_relaxed_query_words_still_rejects_ascii_digits_in_cjk_queries() -> None: + """The CJK identifier guard itself is unchanged: ASCII digits still veto.""" + assert relaxed_query_words("SPEC 16 设计") is None From 21dfdb880002ec792e03e7f66bbfbb887c0ca9a3 Mon Sep 17 00:00:00 2001 From: gingeard Date: Wed, 19 Aug 2026 11:17:57 +0100 Subject: [PATCH 07/19] fix(core): classify a token's number-ness without its combining marks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Marks stay inside a token so an orthographic word is counted once, but the guards classified the whole string. A decorated digit is therefore not isnumeric(): "SPEC 1️⃣ design" relaxed into a broad OR instead of being rejected as identifier-like. Classification now runs on the token core — the token minus combining marks and join controls — while the token itself is unchanged, so counting and the emitted terms stay exactly as they were. Both branches share the helper. Han numerals are unaffected: they carry no marks, so the core is the word itself and isdigit() stays false for it, as on main. The existing tests pin that "数据 三 分析" still relaxes. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 35 +++++++++++++++++++-- tests/repository/test_search_relaxation.py | 17 ++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index d64f2f056..6171e56f4 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -98,6 +98,37 @@ def flush() -> None: return tokens +def _token_core(token: str) -> str: + """The token without the characters that only ever attach to another one. + + Combining marks and join controls are kept inside a token so a word is + counted once, but they must not disguise what the token *is*: a keycap + digit is not `isnumeric()` as a whole string, which would walk + an identifier-like query straight past the numeric guard. + """ + return "".join( + char + for char in token + if char not in RELAXATION_JOIN_CONTROLS and not unicodedata.category(char).startswith("M") + ) + + +def _is_numeric_token(token: str) -> bool: + """Whether a token is a number once its attached characters are set aside.""" + core = _token_core(token) + return bool(core) and core.isnumeric() + + +def _is_digit_token(token: str) -> bool: + """Same, for the CJK branch, which classifies ASCII-style digits only. + + Han numerals stay content words here: they carry no combining marks, so the + core is the word itself and `isdigit()` is false for it, as on main. + """ + core = _token_core(token) + return bool(core) and core.isdigit() + + def _dedupe_relaxation_words(words: list[str]) -> list[str]: """Preserve first-seen relaxed terms while removing duplicates case-insensitively.""" deduped_terms: list[str] = [] @@ -153,7 +184,7 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: # CJK prose, not identifiers. Widening this guard the way the general # branch needs would reject a query like "数据 三 分析" and silently switch # off relaxation for the very queries #1022 turned it on for. - if len(cjk_words) < 2 or any(word.isdigit() for word in cjk_words): + if len(cjk_words) < 2 or any(_is_digit_token(word) for word in cjk_words): return None pruned_words = [ word @@ -168,7 +199,7 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: return relaxed_words if len(relaxed_words) >= 2 else None tokens = relaxation_word_tokens(stripped.lower()) - if len(tokens) < 3 or any(token.isnumeric() for token in tokens): + if len(tokens) < 3 or any(_is_numeric_token(token) for token in tokens): return None pruned_words = [token for token in tokens if token not in RELAXATION_STOPWORDS] return _dedupe_relaxation_words(pruned_words or tokens) or None diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py index e567c41fd..041c0bbc6 100644 --- a/tests/repository/test_search_relaxation.py +++ b/tests/repository/test_search_relaxation.py @@ -211,3 +211,20 @@ def test_relaxed_query_words_treats_han_numerals_as_content_words( def test_relaxed_query_words_still_rejects_ascii_digits_in_cjk_queries() -> None: """The CJK identifier guard itself is unchanged: ASCII digits still veto.""" assert relaxed_query_words("SPEC 16 设计") is None + + +@pytest.mark.parametrize( + "query", + [ + "SPEC 1️⃣ design", # keycap digit: digit plus VS16 plus enclosing keycap + "spec 1́ design", # digit carrying a combining acute + ], +) +def test_relaxed_query_words_sees_numbers_through_combining_marks(query: str) -> None: + """A mark attached to a digit must not disguise it from the numeric guard. + + Marks stay inside the token so the word is counted once, but classification + looks at the token without them — otherwise a decorated digit walks an + identifier-like query straight past the guard. + """ + assert relaxed_query_words(query) is None From 7f538d3d6b4e91de0af6dea760320801fead7bf7 Mon Sep 17 00:00:00 2001 From: gingeard Date: Wed, 19 Aug 2026 11:22:42 +0100 Subject: [PATCH 08/19] fix(core): keep invisible word-internal format characters inside tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only the two joiners were recognised, so a soft hyphen — which text pasted from formatted documents carries invisibly — still split a word: "пере­вод доступа" became three tokens and cleared the three-token guard. The set now covers the characters that sit inside a word: soft hyphen, both joiners, word joiner, and zero-width no-break space. U+200B is deliberately excluded and pinned by a test: zero-width space marks word boundaries in Thai and Khmer, so grouping it into the word would collapse a phrase into one token and switch relaxation off for the one form of those scripts that reaches the guard at all. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 20 +++++++----- tests/repository/test_search_relaxation.py | 34 +++++++++++++++++++++ 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index 6171e56f4..b156e44c8 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -28,7 +28,12 @@ ) RELAXATION_EDGE_PUNCTUATION = "?!.,;:，。！？；：、" # Written inside a word (Persian "\u200c", Indic conjuncts) rather than between words. -RELAXATION_JOIN_CONTROLS = "\u200c\u200d" +# Invisible format characters that sit *inside* a word: soft hyphen from copied +# formatted text, the Persian/Indic joiners, and the word joiners. U+200B is +# deliberately absent — zero-width space marks word *boundaries* in Thai and +# Khmer, so treating it as word-internal would glue a whole phrase into one +# token and switch relaxation off for the one form of those scripts that works. +RELAXATION_WORD_INTERNAL_FORMATS = "\u00ad\u200c\u200d\u2060\ufeff" # Word-internal only between letters: "\u043f\u2019\u044f\u0442\u044c", "don't" \u2014 but not "SPEC 16's", # where the digit must stay its own token so the numeric guard still sees it. RELAXATION_WORD_INTERNAL_PUNCTUATION = "'\u2019" @@ -37,7 +42,7 @@ def _is_token_continuation(text: str, index: int, current: list[str]) -> bool: """Whether a non-alphanumeric character belongs to the word being read. - Combining marks, zero-width join controls, and apostrophes are written inside + Combining marks, invisible word-internal format characters, and apostrophes are written inside a word but are not alphanumeric, so a naive scan treats them as separators and splits one orthographic word into several tokens. @@ -46,7 +51,7 @@ def _is_token_continuation(text: str, index: int, current: list[str]) -> bool: numeric-identifier guard still rejects the query. """ char = text[index] - if char in RELAXATION_JOIN_CONTROLS or unicodedata.category(char).startswith("M"): + if char in RELAXATION_WORD_INTERNAL_FORMATS or unicodedata.category(char).startswith("M"): return True if char in RELAXATION_WORD_INTERNAL_PUNCTUATION: follows_letter = bool(current) and current[-1].isalpha() @@ -79,9 +84,9 @@ def relaxation_word_tokens(text: str) -> list[str]: current: list[str] = [] def flush() -> None: - # Trailing join controls are word-internal by definition, so a token that - # ends in one is really a word followed by a separator. - token = "".join(current).rstrip(RELAXATION_JOIN_CONTROLS) + # A trailing format character is word-internal by definition, so a token + # that ends in one is really a word followed by a separator. + token = "".join(current).rstrip(RELAXATION_WORD_INTERNAL_FORMATS) if token: tokens.append(token) current.clear() @@ -109,7 +114,8 @@ def _token_core(token: str) -> str: return "".join( char for char in token - if char not in RELAXATION_JOIN_CONTROLS and not unicodedata.category(char).startswith("M") + if char not in RELAXATION_WORD_INTERNAL_FORMATS + and not unicodedata.category(char).startswith("M") ) diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py index 041c0bbc6..d6607fcd0 100644 --- a/tests/repository/test_search_relaxation.py +++ b/tests/repository/test_search_relaxation.py @@ -228,3 +228,37 @@ def test_relaxed_query_words_sees_numbers_through_combining_marks(query: str) -> identifier-like query straight past the guard. """ assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("пере­вод доступа", None), # soft hyphen from copied formatted text + ("сло⁠во доступа", None), # word joiner + ("слово доступа", None), # zero-width no-break space + ], +) +def test_relaxed_query_words_ignores_word_internal_format_characters( + query: str, + expected: list[str] | None, +) -> None: + """Invisible format characters inside a word must not split its token. + + Text pasted from formatted documents carries them, and splitting there + inflates the token count exactly as the join-control case did. + """ + assert relaxed_query_words(query) == expected + + +def test_relaxed_query_words_treats_zero_width_space_as_a_word_boundary() -> None: + """U+200B separates words in Thai and Khmer, so it must keep splitting. + + Grouping it into the word would collapse a whole phrase into one token and + switch relaxation off for the one form of those scripts that reaches the + guard at all. + """ + assert relaxed_query_words("ฉัน​จะลอง​ชำระเงิน") == [ + "ฉัน", + "จะลอง", + "ชำระเงิน", + ] From c86f05ba170b0d641168596c1dd756c675e0dd7c Mon Sep 17 00:00:00 2001 From: gingeard Date: Wed, 19 Aug 2026 11:30:27 +0100 Subject: [PATCH 09/19] fix(core): treat format characters as word-internal by rule, not by list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Naming individual characters was the wrong shape. The allowlist covered five; Unicode has 170 in category Cf, and each round of review surfaced another one that split a word — soft hyphen, then the Mongolian vowel separator, with bidi marks and the Arabic number signs waiting behind them. The rule is inverted: a format character is word-internal unless it is listed as a separator, and the only entry is U+200B, which marks word boundaries in Thai and Khmer. This is safe in the direction that matters — grouping a character into a word can only lower a token count, never inflate it past the guards, so the failure mode is a query that does not relax rather than one that relaxes into fragments. All 170 Cf characters now stay inside the word, U+200B still splits, and the existing cases are unchanged. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 45 ++++++++++++++------- tests/repository/test_search_relaxation.py | 2 + 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index b156e44c8..20c1cc86b 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -28,17 +28,35 @@ ) RELAXATION_EDGE_PUNCTUATION = "?!.,;:，。！？；：、" # Written inside a word (Persian "\u200c", Indic conjuncts) rather than between words. -# Invisible format characters that sit *inside* a word: soft hyphen from copied -# formatted text, the Persian/Indic joiners, and the word joiners. U+200B is -# deliberately absent — zero-width space marks word *boundaries* in Thai and -# Khmer, so treating it as word-internal would glue a whole phrase into one -# token and switch relaxation off for the one form of those scripts that works. -RELAXATION_WORD_INTERNAL_FORMATS = "\u00ad\u200c\u200d\u2060\ufeff" +# Format characters (Unicode Cf) are invisible and, with one exception, sit +# inside a word: soft hyphen from copied text, the Persian and Indic joiners, +# bidi marks, the Mongolian vowel separator, word joiners. Counting any of them +# as a separator splits one word into several tokens, which is the direction +# that defeats the guards below — grouping can only lower a token count, never +# inflate it past them. +# +# U+200B is the exception: zero-width space marks word *boundaries* in Thai and +# Khmer, so it must keep splitting, or a whole phrase collapses into one token +# and relaxation switches off for the one form of those scripts that reaches +# the guard at all. +RELAXATION_WORD_SEPARATOR_FORMATS = "\u200b" # Word-internal only between letters: "\u043f\u2019\u044f\u0442\u044c", "don't" \u2014 but not "SPEC 16's", # where the digit must stay its own token so the numeric guard still sees it. RELAXATION_WORD_INTERNAL_PUNCTUATION = "'\u2019" +def _is_word_internal_format(char: str) -> bool: + """Whether an invisible format character belongs to the word around it.""" + return unicodedata.category(char) == "Cf" and char not in RELAXATION_WORD_SEPARATOR_FORMATS + + +def _strip_trailing_formats(token: str) -> str: + """Drop format characters left at a token's end, where they separate rather than join.""" + while token and _is_word_internal_format(token[-1]): + token = token[:-1] + return token + + def _is_token_continuation(text: str, index: int, current: list[str]) -> bool: """Whether a non-alphanumeric character belongs to the word being read. @@ -51,7 +69,7 @@ def _is_token_continuation(text: str, index: int, current: list[str]) -> bool: numeric-identifier guard still rejects the query. """ char = text[index] - if char in RELAXATION_WORD_INTERNAL_FORMATS or unicodedata.category(char).startswith("M"): + if _is_word_internal_format(char) or unicodedata.category(char).startswith("M"): return True if char in RELAXATION_WORD_INTERNAL_PUNCTUATION: follows_letter = bool(current) and current[-1].isalpha() @@ -86,7 +104,7 @@ def relaxation_word_tokens(text: str) -> list[str]: def flush() -> None: # A trailing format character is word-internal by definition, so a token # that ends in one is really a word followed by a separator. - token = "".join(current).rstrip(RELAXATION_WORD_INTERNAL_FORMATS) + token = _strip_trailing_formats("".join(current)) if token: tokens.append(token) current.clear() @@ -106,16 +124,15 @@ def flush() -> None: def _token_core(token: str) -> str: """The token without the characters that only ever attach to another one. - Combining marks and join controls are kept inside a token so a word is - counted once, but they must not disguise what the token *is*: a keycap - digit is not `isnumeric()` as a whole string, which would walk - an identifier-like query straight past the numeric guard. + Combining marks and format characters are kept inside a token so a word is + counted once, but they must not disguise what the token *is*: a keycap digit + is not `isnumeric()` as a whole string, which would walk an identifier-like + query straight past the numeric guard. """ return "".join( char for char in token - if char not in RELAXATION_WORD_INTERNAL_FORMATS - and not unicodedata.category(char).startswith("M") + if not _is_word_internal_format(char) and not unicodedata.category(char).startswith("M") ) diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py index d6607fcd0..9cd9e0853 100644 --- a/tests/repository/test_search_relaxation.py +++ b/tests/repository/test_search_relaxation.py @@ -236,6 +236,8 @@ def test_relaxed_query_words_sees_numbers_through_combining_marks(query: str) -> ("пере­вод доступа", None), # soft hyphen from copied formatted text ("сло⁠во доступа", None), # word joiner ("слово доступа", None), # zero-width no-break space + ("сло᠎во доступа", None), # Mongolian vowel separator + ("сло‏во доступа", None), # right-to-left mark ], ) def test_relaxed_query_words_ignores_word_internal_format_characters( From 038d894377be994077754736289973d289f7e32d Mon Sep 17 00:00:00 2001 From: gingeard Date: Wed, 19 Aug 2026 11:35:09 +0100 Subject: [PATCH 10/19] fix(core): classify numeric tokens by Unicode category in both branches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The split between isdigit() on the CJK branch and isnumeric() on the general one was never the real distinction, and it left a hole: adding a CJK term let "SPEC Ⅻ 设计" and "SPEC ½ 设计" past a guard that rejects them in Latin. Unicode already draws the line. Han numerals are category Lo — letters, and ordinary content words in CJK prose — while ASCII and Arabic-Indic digits (Nd), Roman numerals (Nl) and vulgar fractions (No) are number characters. A token whose core is made only of category-N characters is the "SPEC 16" shape the guard exists to catch, in any script. One rule now serves both branches: 三 stays a word, every category-N token is rejected wherever it appears, and the isdigit()/isnumeric() asymmetry is gone. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 24 +++++++++------------ tests/repository/test_search_relaxation.py | 18 +++++++++++++--- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index 20c1cc86b..4dd6b6b2f 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -137,19 +137,19 @@ def _token_core(token: str) -> str: def _is_numeric_token(token: str) -> bool: - """Whether a token is a number once its attached characters are set aside.""" - core = _token_core(token) - return bool(core) and core.isnumeric() - + """Whether a token is a bare number, and so identifier-like rather than a word. -def _is_digit_token(token: str) -> bool: - """Same, for the CJK branch, which classifies ASCII-style digits only. + Classified by Unicode category, not by `isdigit()`/`isnumeric()`. Both of + those answer True for Han numerals, which are category Lo — letters, and + ordinary content words in CJK prose. Rejecting them would switch relaxation + off for the queries #1022 turned it on for. - Han numerals stay content words here: they carry no combining marks, so the - core is the word itself and `isdigit()` is false for it, as on main. + Everything in category N is a number character: ASCII and Arabic-Indic + digits (Nd), Roman numerals (Nl), vulgar fractions (No). A token made only + of those is the "SPEC 16" shape the guard exists to catch, in any script. """ core = _token_core(token) - return bool(core) and core.isdigit() + return bool(core) and all(unicodedata.category(char).startswith("N") for char in core) def _dedupe_relaxation_words(words: list[str]) -> list[str]: @@ -203,11 +203,7 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: has_cjk_term = any(RELAXATION_CJK_PATTERN.search(word) for word in cjk_words) if has_cjk_term: - # isdigit(), not isnumeric(): Han numerals are ordinary content words in - # CJK prose, not identifiers. Widening this guard the way the general - # branch needs would reject a query like "数据 三 分析" and silently switch - # off relaxation for the very queries #1022 turned it on for. - if len(cjk_words) < 2 or any(_is_digit_token(word) for word in cjk_words): + if len(cjk_words) < 2 or any(_is_numeric_token(word) for word in cjk_words): return None pruned_words = [ word diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py index 9cd9e0853..590931738 100644 --- a/tests/repository/test_search_relaxation.py +++ b/tests/repository/test_search_relaxation.py @@ -208,9 +208,21 @@ def test_relaxed_query_words_treats_han_numerals_as_content_words( assert relaxed_query_words(query) == expected -def test_relaxed_query_words_still_rejects_ascii_digits_in_cjk_queries() -> None: - """The CJK identifier guard itself is unchanged: ASCII digits still veto.""" - assert relaxed_query_words("SPEC 16 设计") is None +@pytest.mark.parametrize( + "query", + [ + "SPEC 16 设计", # ASCII digits + "SPEC Ⅻ 设计", # Roman numeral: a number character, not a Han word + "SPEC ½ 设计", # vulgar fraction + ], +) +def test_relaxed_query_words_rejects_number_characters_in_cjk_queries(query: str) -> None: + """Adding a CJK term must not smuggle an identifier past the numeric guard. + + Han numerals are category Lo — letters — so they stay content words, while + every category-N character is caught in both branches alike. + """ + assert relaxed_query_words(query) is None @pytest.mark.parametrize( From 03c5561a47b4880c03c249c4b16e4b87ec7002f5 Mon Sep 17 00:00:00 2001 From: gingeard Date: Wed, 19 Aug 2026 11:41:26 +0100 Subject: [PATCH 11/19] fix(core): emit relaxed terms that match the note as stored MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Format characters were kept in the emitted term as well as in the token, so a query pasted from a paginated document searched for something the note does not contain: "пере­vод*" does not match a note holding plain "перевод". Stripping them all would have broken the opposite case. Measured against a real FTS5 table: note "перевод" term with soft hyphen -> no match note "می‌روم" term without ZWNJ -> no match The joiners are orthography — written in the text, and therefore indexed with it — while a soft hyphen, a bidi hint or a stray BOM are rendering artifacts absent from the note. So the term keeps U+200C/U+200D and drops the rest, while the token keeps everything and the counting is unchanged. Tested through the renderer against a real FTS5 table, per the report. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 24 ++++++++++++++-- .../test_search_relaxed_rendering.py | 28 +++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index 4dd6b6b2f..f3a793ce3 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -40,6 +40,13 @@ # and relaxation switches off for the one form of those scripts that reaches # the guard at all. RELAXATION_WORD_SEPARATOR_FORMATS = "\u200b" +# Of the characters kept inside a token, only these two are orthography: the +# Persian and Indic joiners are written in the text and therefore sit in the +# index too, so a relaxed term must keep them or it stops matching. Every other +# format character is a rendering artifact — a soft hyphen from a paginated +# document, a bidi hint, a stray BOM — absent from the stored note, so carrying +# it into the term is what stops the term matching. +RELAXATION_ORTHOGRAPHIC_JOINERS = "\u200c\u200d" # Word-internal only between letters: "\u043f\u2019\u044f\u0442\u044c", "don't" \u2014 but not "SPEC 16's", # where the digit must stay its own token so the numeric guard still sees it. RELAXATION_WORD_INTERNAL_PUNCTUATION = "'\u2019" @@ -171,6 +178,19 @@ def _split_relaxation_words(search_text: str) -> list[str]: return [word for word in words if word] +def _emit_relaxation_terms(words: list[str]) -> list[str]: + """Clean the words for the backend, then drop duplicates the cleaning creates.""" + cleaned = [ + "".join( + char + for char in word + if char in RELAXATION_ORTHOGRAPHIC_JOINERS or not _is_word_internal_format(char) + ) + for word in words + ] + return _dedupe_relaxation_words([word for word in cleaned if word]) + + def relaxed_query_words(search_text: str | None) -> list[str] | None: """Return content-bearing words for OR-relaxing a strict full-text query. @@ -210,7 +230,7 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: for word in cjk_words if word.isalnum() and word.lower() not in RELAXATION_STOPWORDS ] - relaxed_words = _dedupe_relaxation_words(pruned_words) + relaxed_words = _emit_relaxation_terms(pruned_words) # Trigger: punctuation/stopword pruning or deduplication leaves only one term. # Why: the raw whitespace count can make an identifier-like mixed query # appear multi-term even though only one backend-safe CJK prefix remains. @@ -221,4 +241,4 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: if len(tokens) < 3 or any(_is_numeric_token(token) for token in tokens): return None pruned_words = [token for token in tokens if token not in RELAXATION_STOPWORDS] - return _dedupe_relaxation_words(pruned_words or tokens) or None + return _emit_relaxation_terms(pruned_words or tokens) or None diff --git a/tests/repository/test_search_relaxed_rendering.py b/tests/repository/test_search_relaxed_rendering.py index a1339320a..31f02aa96 100644 --- a/tests/repository/test_search_relaxed_rendering.py +++ b/tests/repository/test_search_relaxed_rendering.py @@ -83,3 +83,31 @@ def test_sqlite_relaxed_text_bare_apostrophe_would_be_rejected() -> None: def test_postgres_relaxed_tsquery_quotes_apostrophe_lexemes(query: str, expected: str) -> None: """Postgres carries the same token shapes, so it needs the same escaping.""" assert PostgresSearchRepository._relaxed_tsquery_text(query) == expected + + +@pytest.mark.parametrize( + ("document", "query"), + [ + ("перевод права доступа", "пере­vод права доступа"), # soft hyphen + ("слово права доступа", "сло⁠во права доступа"), # word joiner + ("نمی‌خواهم دسترسی را لغو", "نمی‌خواهم دسترسی را لغو"), # ZWNJ kept + ], +) +def test_relaxed_terms_match_the_stored_note(document: str, query: str) -> None: + """A term must match the note as stored, not as the query happened to be pasted. + + Rendering artifacts — a soft hyphen from a paginated document, a stray word + joiner — are absent from the note, so carrying them into the term stops it + matching. The Persian joiner is the opposite case: it is written in the text + and indexed with it, so removing it would break the match instead. + """ + connection = sqlite3.connect(":memory:") + try: + connection.execute(CREATE_FTS) + connection.execute("INSERT INTO t VALUES (?)", (document,)) + relaxed = SQLiteSearchRepository._relaxed_fts_text(query) + assert relaxed is not None + rows = connection.execute("SELECT rowid FROM t WHERE t MATCH ?", (relaxed,)).fetchall() + finally: + connection.close() + assert rows, f"relaxed expression did not match the stored note: {relaxed!r}" From eeba1deea911f5d2caa6abeb7d725ea53c75204c Mon Sep 17 00:00:00 2001 From: gingeard Date: Wed, 19 Aug 2026 12:52:29 +0100 Subject: [PATCH 12/19] test(core): sweep the Unicode classes the relaxation guards rely on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review found a soft hyphen, a Mongolian vowel separator and a keycap digit one at a time, each a member of a class the guards already reason about. This walks the classes instead of the examples: every format character (170), every combining mark (2501), and every number character (1911), plus the two deliberate departures — U+200B keeps splitting because it marks word boundaries in Thai and Khmer, and Han numerals stay content words. The rules match Unicode word segmentation, UAX #29 WB4 and WB6/WB7, so the sweeps double as a conformance statement without taking a dependency: a segmentation library would also have to be overridden for Thai, which it splits into syllables, and for the whitespace-delimited CJK terms #1022 relies on. Each sweep was checked against a mutant that breaks its rule. The zero-width-space test asserts the literal rather than reading the exception list, because a test parametrized over that list vanishes when the list is emptied — which is the change it exists to catch. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 10 +- .../test_search_relaxation_unicode_classes.py | 122 ++++++++++++++++++ 2 files changed, 127 insertions(+), 5 deletions(-) create mode 100644 tests/repository/test_search_relaxation_unicode_classes.py diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index f3a793ce3..c0cf20acb 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -27,7 +27,7 @@ r"]" ) RELAXATION_EDGE_PUNCTUATION = "?!.,;:，。！？；：、" -# Written inside a word (Persian "\u200c", Indic conjuncts) rather than between words. +# Written inside a word (Persian U+200C, Indic conjuncts) rather than between words. # Format characters (Unicode Cf) are invisible and, with one exception, sit # inside a word: soft hyphen from copied text, the Persian and Indic joiners, # bidi marks, the Mongolian vowel separator, word joiners. Counting any of them @@ -47,7 +47,7 @@ # document, a bidi hint, a stray BOM — absent from the stored note, so carrying # it into the term is what stops the term matching. RELAXATION_ORTHOGRAPHIC_JOINERS = "\u200c\u200d" -# Word-internal only between letters: "\u043f\u2019\u044f\u0442\u044c", "don't" \u2014 but not "SPEC 16's", +# Word-internal only between letters: "п’ять", "don't" — but not "SPEC 16's", # where the digit must stay its own token so the numeric guard still sees it. RELAXATION_WORD_INTERNAL_PUNCTUATION = "'\u2019" @@ -71,7 +71,7 @@ def _is_token_continuation(text: str, index: int, current: list[str]) -> bool: a word but are not alphanumeric, so a naive scan treats them as separators and splits one orthographic word into several tokens. - An apostrophe counts only between two letters. That keeps "\u043f\u2019\u044f\u0442\u044c" whole while + An apostrophe counts only between two letters. That keeps "п’ять" whole while leaving "SPEC 16's" split, so the digit stays a token of its own and the numeric-identifier guard still rejects the query. """ @@ -207,9 +207,9 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: same guard instead of being read as zero tokens; - CJK terms separated by whitespace can relax with two or more terms because they are not whitespace-delimited the way the token guard assumes; - - any numeric token ("root note 1", "SPEC 16", "SPEC \u216b") — identifier-like + - any numeric token ("root note 1", "SPEC 16", "SPEC Ⅻ") — identifier-like queries over-broaden and create false positives under OR. Numeric-ness is - Unicode-wide, so Nl/No characters such as \u216b and \u00bd are caught too. + Unicode-wide, so Nl/No characters such as Ⅻ and ½ are caught too. """ if not search_text: return None diff --git a/tests/repository/test_search_relaxation_unicode_classes.py b/tests/repository/test_search_relaxation_unicode_classes.py new file mode 100644 index 000000000..f6a03bcaa --- /dev/null +++ b/tests/repository/test_search_relaxation_unicode_classes.py @@ -0,0 +1,122 @@ +"""Exhaustive sweeps over the Unicode classes the relaxation guards depend on. + +The case-based tests next door pin individual queries. These pin the rules those +cases are instances of, by walking every character in the class rather than the +ones review happened to surface: a soft hyphen, a Mongolian vowel separator and +a keycap digit were each found one at a time, and each was one member of a class +already covered here. + +The rules match Unicode word segmentation (UAX #29) — WB4 ignores format and +combining characters inside a word, WB6/WB7 keep an apostrophe between letters — +with two deliberate departures, both pinned below: U+200B splits, because it +marks word boundaries in Thai and Khmer, and Han numerals stay content words +rather than identifiers. +""" + +import unicodedata + +import pytest + +from basic_memory.repository.search_query import ( + RELAXATION_WORD_SEPARATOR_FORMATS, + relaxation_word_tokens, + relaxed_query_words, +) + + +def _characters_in_categories(*categories: str) -> list[str]: + """Every assigned code point in the given general categories.""" + wanted = set(categories) + return [ + char + for code_point in range(0x110000) + if unicodedata.category(char := chr(code_point)) in wanted + ] + + +FORMAT_CHARACTERS = _characters_in_categories("Cf") +COMBINING_MARKS = _characters_in_categories("Mn", "Mc", "Me") +NUMBER_CHARACTERS = _characters_in_categories("Nd", "Nl", "No") + +# Numerals written as letters (category Lo). Unicode gives them a numeric value, +# so isdigit()/isnumeric() answer True, but they are ordinary words in CJK prose. +HAN_NUMERALS = ["三", "四", "五", "十", "百", "千", "万", "億", "零"] + + +def _describe(characters: list[str], limit: int = 8) -> str: + """Render failing characters as code points, since most are invisible.""" + shown = " ".join(f"U+{ord(char):04X}" for char in characters[:limit]) + return f"{len(characters)}: {shown}{' …' if len(characters) > limit else ''}" + + +def test_every_format_character_stays_inside_the_word() -> None: + """No format character may split a word, apart from the declared separators. + + Splitting inflates the token count, which is the direction that walks a + query past the three-token guard and relaxes it into fragments. + """ + splitting = [ + char + for char in FORMAT_CHARACTERS + if char not in RELAXATION_WORD_SEPARATOR_FORMATS + and len(relaxation_word_tokens(f"сло{char}во доступа")) != 2 + ] + assert not splitting, f"format characters that split a word — {_describe(splitting)}" + + +def test_zero_width_space_stays_a_word_separator() -> None: + """U+200B marks word boundaries in Thai and Khmer, so it must keep splitting. + + Grouping it into the word would collapse a whole phrase into one token and + switch relaxation off for the one form of those scripts that reaches the + guard at all. + + Written as a literal rather than read from the constant: a test parametrized + over the exception list disappears when the list is emptied, which is exactly + the change it exists to catch. + """ + assert len(relaxation_word_tokens("сло\u200bво доступа")) == 3 + + +def test_zero_width_space_is_the_only_declared_separator() -> None: + """The sweep above skips whatever this constant holds, so its contents are load-bearing. + + Adding a character here silently removes it from that sweep, so the addition + has to be a deliberate edit here rather than a side effect elsewhere. + """ + assert RELAXATION_WORD_SEPARATOR_FORMATS == "\u200b" + + +def test_no_combining_mark_splits_a_word() -> None: + """Marks attach to the character before them, so they cannot end a token. + + Counting them as separators cuts abugidas and decomposed text into syllable + fragments — one word then looks like several tokens. + """ + splitting = [ + char for char in COMBINING_MARKS if len(relaxation_word_tokens(f"сло{char}во доступа")) != 2 + ] + assert not splitting, f"combining marks that split a word — {_describe(splitting)}" + + +def test_every_number_character_is_caught_by_the_identifier_guard() -> None: + """A bare number term makes a query identifier-like, in any script. + + The guard exists for "SPEC 16"; Roman numerals, vulgar fractions and + non-ASCII digits are the same shape and must not slip through it. + """ + admitted = [ + char for char in NUMBER_CHARACTERS if relaxed_query_words(f"spec {char} design") is not None + ] + assert not admitted, f"number characters that cleared the guard — {_describe(admitted)}" + + +@pytest.mark.parametrize("numeral", HAN_NUMERALS) +def test_han_numerals_stay_content_words(numeral: str) -> None: + """Han numerals are letters (category Lo) and ordinary words in CJK prose. + + Classifying them as identifiers would reject "数据 三 分析" and switch + relaxation off for the queries it was turned on for. + """ + assert unicodedata.category(numeral) == "Lo" + assert relaxed_query_words(f"数据 {numeral} 分析") == ["数据", numeral, "分析"] From 4677f8787fa19cb7dce79aa88763f80405a26c60 Mon Sep 17 00:00:00 2001 From: gingeard Date: Wed, 19 Aug 2026 12:58:50 +0100 Subject: [PATCH 13/19] fix(core): join words on the UAX #29 MidLetter characters, not just apostrophes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The apostrophe rule was one instance of WB6/WB7. The rest of the class matters in a script this change exists to support: Hebrew writes geresh and gershayim inside words constantly, so "ג׳ון סמית כתב" tokenized to four terms including a one-letter fragment, cleared the three-token guard and relaxed. Catalan's middle dot behaved the same way. Colon and full stop are deliberately left out of the set, though the standard joins on them too. UAX #29 returns "1.2" as one token, and such a token is not category N, so it would walk an identifier-like query straight past the numeric guard — verified. Both also carry structure in permalinks, paths and versions. Checked against a UAX #29 implementation while writing this, which now agrees on Hebrew, Catalan, and both apostrophes; the dependency is not taken, since it still has to be overridden for Thai and for the CJK terms #1022 relies on. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 10 ++++-- tests/repository/test_search_relaxation.py | 38 +++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index c0cf20acb..bf0927429 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -47,9 +47,13 @@ # document, a bidi hint, a stray BOM — absent from the stored note, so carrying # it into the term is what stops the term matching. RELAXATION_ORTHOGRAPHIC_JOINERS = "\u200c\u200d" -# Word-internal only between letters: "п’ять", "don't" — but not "SPEC 16's", -# where the digit must stay its own token so the numeric guard still sees it. -RELAXATION_WORD_INTERNAL_PUNCTUATION = "'\u2019" +# Punctuation written inside a word, applied only between two letters: "п’ять", +# "don't", Hebrew ג׳ון and ר״ת, Catalan col·lecció. This is the UAX #29 MidLetter +# set minus colon and full stop, which the standard also joins on — "1.2" comes +# back as one token there, and a token like that is not category N, so it would +# walk an identifier-like query straight past the numeric guard. Both carry +# structure in permalinks, paths and versions here, so they keep splitting. +RELAXATION_WORD_INTERNAL_PUNCTUATION = "'\u2018\u2019\u00b7\u0387\u05f3\u05f4\u2027" def _is_word_internal_format(char: str) -> bool: diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py index 590931738..94be85a70 100644 --- a/tests/repository/test_search_relaxation.py +++ b/tests/repository/test_search_relaxation.py @@ -276,3 +276,41 @@ def test_relaxed_query_words_treats_zero_width_space_as_a_word_boundary() -> Non "จะลอง", "ชำระเงิน", ] + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("ג׳ון סמית כתב", ["ג׳ון", "סמית", "כתב"]), # geresh: foreign sounds + ("ר״ת של המשפט", ["ר״ת", "של", "המשפט"]), # gershayim: acronym + ("col·lecció de dades", ["col·lecció", "de", "dades"]), # Catalan middle dot + ], +) +def test_relaxed_query_words_keeps_letter_joining_punctuation( + query: str, + expected: list[str], +) -> None: + """The apostrophe rule covers every UAX #29 MidLetter character taken here. + + Hebrew writes geresh and gershayim inside words constantly, so splitting on + them turns a three-word query into fragments — the same failure the + apostrophe case had, in a script this change exists to support. + """ + assert relaxed_query_words(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "spec 1.2 design", # full stop: UAX #29 would join "1.2" into one token + "spec 1:2 design", # colon + ], +) +def test_relaxed_query_words_splits_on_structural_punctuation(query: str) -> None: + """Colon and full stop are deliberately outside the MidLetter set taken here. + + UAX #29 joins digits across a full stop, and "1.2" is not a category-N token, + so it would slip past the numeric guard. Both also carry structure in + permalinks, paths and version strings. + """ + assert relaxed_query_words(query) is None From 3fd21cb8528f3d940972504bee38c42b4232b28f Mon Sep 17 00:00:00 2001 From: gingeard Date: Wed, 19 Aug 2026 13:11:47 +0100 Subject: [PATCH 14/19] test(core): sweep the punctuation class the joining rule carves out of MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The joining set is a chosen subset of UAX #29 MidLetter, so the property worth pinning is that it stays a subset: of the 855 punctuation characters, only the eight declared ones join two letters, none of them joins digits, and colon and full stop keep splitting even though the standard joins on them — "1.2" as one token is not category N and would walk an identifier-like query past the numeric guard. The set's contents are asserted directly as well as swept. A sweep that iterates the constant stops covering whatever is removed from it, so removing a character would otherwise be silent; with the assertion, shrinking or extending the set has to be a deliberate edit. Verified against four mutants: adding colon, removing the Hebrew gershayim, joining on any punctuation, and dropping the between-letters requirement each turn a test red. Signed-off-by: gingeard --- .../test_search_relaxation_unicode_classes.py | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/tests/repository/test_search_relaxation_unicode_classes.py b/tests/repository/test_search_relaxation_unicode_classes.py index f6a03bcaa..28ec37f72 100644 --- a/tests/repository/test_search_relaxation_unicode_classes.py +++ b/tests/repository/test_search_relaxation_unicode_classes.py @@ -18,6 +18,7 @@ import pytest from basic_memory.repository.search_query import ( + RELAXATION_WORD_INTERNAL_PUNCTUATION, RELAXATION_WORD_SEPARATOR_FORMATS, relaxation_word_tokens, relaxed_query_words, @@ -37,6 +38,7 @@ def _characters_in_categories(*categories: str) -> list[str]: FORMAT_CHARACTERS = _characters_in_categories("Cf") COMBINING_MARKS = _characters_in_categories("Mn", "Mc", "Me") NUMBER_CHARACTERS = _characters_in_categories("Nd", "Nl", "No") +PUNCTUATION_CHARACTERS = _characters_in_categories("Pc", "Pd", "Ps", "Pe", "Pi", "Pf", "Po") # Numerals written as letters (category Lo). Unicode gives them a numeric value, # so isdigit()/isnumeric() answer True, but they are ordinary words in CJK prose. @@ -120,3 +122,66 @@ def test_han_numerals_stay_content_words(numeral: str) -> None: """ assert unicodedata.category(numeral) == "Lo" assert relaxed_query_words(f"数据 {numeral} 分析") == ["数据", numeral, "分析"] + + +def test_only_the_declared_punctuation_joins_a_word() -> None: + """Nothing else in the punctuation classes may join two letters. + + The joining set is a chosen subset of UAX #29 MidLetter, so it has to stay a + subset: any other punctuation that started joining would merge two words into + one term and quietly change what the backend searches for. + """ + joining = [ + char + for char in PUNCTUATION_CHARACTERS + if char not in RELAXATION_WORD_INTERNAL_PUNCTUATION + and len(relaxation_word_tokens(f"сло{char}во доступа")) != 3 + ] + assert not joining, f"punctuation that joined a word — {_describe(joining)}" + + +def test_declared_punctuation_is_exactly_the_chosen_midletter_subset() -> None: + """Pin the set itself: the sweep above skips whatever it holds. + + Removing a character silently drops it from every sweep here, and adding one + silently exempts it, so both have to be a deliberate edit rather than a side + effect. The named cases in test_search_relaxation.py pin what each is for. + """ + assert RELAXATION_WORD_INTERNAL_PUNCTUATION == "'\u2018\u2019\u00b7\u0387\u05f3\u05f4\u2027" + + +def test_declared_punctuation_joins_letters_only() -> None: + """Each joiner must join letters, and none may join digits. + + Joining digits is what makes the exclusions below necessary: a term like + "1.2" is not a category-N token, so it would walk an identifier-like query + past the numeric guard. + """ + not_joining = [ + char + for char in RELAXATION_WORD_INTERNAL_PUNCTUATION + if len(relaxation_word_tokens(f"сло{char}во доступа")) != 2 + ] + assert not not_joining, f"declared joiners that split letters — {_describe(not_joining)}" + + joining_digits = [ + char + for char in RELAXATION_WORD_INTERNAL_PUNCTUATION + if relaxed_query_words(f"spec 1{char}2 design") is not None + ] + assert not joining_digits, ( + f"declared joiners that shielded a digit — {_describe(joining_digits)}" + ) + + +@pytest.mark.parametrize("structural", [":", "."]) +def test_colon_and_full_stop_split_although_uax29_joins_them(structural: str) -> None: + """The two deliberate departures from MidLetter, written as literals. + + UAX #29 returns "1.2" as a single token. Such a token is not category N, so + it would slip past the numeric guard that rejects "SPEC 16". Both characters + also carry structure here — permalinks, paths, version strings — so both keep + splitting, and a query built on them stays ineligible for relaxation. + """ + assert len(relaxation_word_tokens(f"сло{structural}во доступа")) == 3 + assert relaxed_query_words(f"spec 1{structural}2 design") is None From 479728a0bafbb2446cfcb681d7de3a382ca7253b Mon Sep 17 00:00:00 2001 From: gingeard Date: Wed, 19 Aug 2026 13:27:21 +0100 Subject: [PATCH 15/19] fix(core): emit both stored forms of a word carrying a format character MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stripping format characters from the term fixed the note that does not have them and broke the note that does. A format character is invisible, so the same word gets stored either way, and the two index differently — "foo­bar" as two tokens, "foobar" as one. Measured on a note holding only the hyphenated form: foobar* -> no match foo­bar* -> match and the reverse on a note holding the plain form. Neither term covers both, so both are emitted; the OR that relaxation already builds does the rest. Orthographic joiners keep their single form: they are written in the text, so the stored word has them, and a stripped variant would only widen the OR with a term no note can hold. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 36 +++++++++++++------ .../test_search_relaxed_rendering.py | 29 +++++++++++++++ 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index bf0927429..07c0619f3 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -182,17 +182,33 @@ def _split_relaxation_words(search_text: str) -> list[str]: return [word for word in words if word] +def _relaxation_term_variants(word: str) -> list[str]: + """Every form of a word that could match how the note happens to be stored. + + A format character is invisible, so the same word may be stored with it or + without it, and the two index differently: a note holding "foo\u00adbar" is + indexed as "foo" and "bar", one holding "foobar" as a single token. Neither + term matches the other note, so both forms are emitted and the OR that + relaxation already builds covers whichever the note actually has. + + Orthographic joiners are not stripped: they are written in the text, so the + stored form has them and the cleaned variant would only add noise. + """ + cleaned = "".join( + char + for char in word + if char in RELAXATION_ORTHOGRAPHIC_JOINERS or not _is_word_internal_format(char) + ) + if not cleaned: + return [] + return [cleaned] if cleaned == word else [cleaned, word] + + def _emit_relaxation_terms(words: list[str]) -> list[str]: - """Clean the words for the backend, then drop duplicates the cleaning creates.""" - cleaned = [ - "".join( - char - for char in word - if char in RELAXATION_ORTHOGRAPHIC_JOINERS or not _is_word_internal_format(char) - ) - for word in words - ] - return _dedupe_relaxation_words([word for word in cleaned if word]) + """Expand the words into backend-ready terms, then drop duplicates.""" + return _dedupe_relaxation_words( + [variant for word in words for variant in _relaxation_term_variants(word)] + ) def relaxed_query_words(search_text: str | None) -> list[str] | None: diff --git a/tests/repository/test_search_relaxed_rendering.py b/tests/repository/test_search_relaxed_rendering.py index 31f02aa96..fc43d062c 100644 --- a/tests/repository/test_search_relaxed_rendering.py +++ b/tests/repository/test_search_relaxed_rendering.py @@ -111,3 +111,32 @@ def test_relaxed_terms_match_the_stored_note(document: str, query: str) -> None: finally: connection.close() assert rows, f"relaxed expression did not match the stored note: {relaxed!r}" + + +@pytest.mark.parametrize("document", ["foo­bar", "foobar"]) +def test_relaxed_terms_match_either_stored_form(document: str) -> None: + """A format character is invisible, so the note may hold it or not. + + The two forms index differently — "foo­bar" as two tokens, "foobar" as + one — and neither term matches the other note. Both forms are emitted, and + the OR relaxation already builds covers whichever the note actually has. + """ + connection = sqlite3.connect(":memory:") + try: + connection.execute(CREATE_FTS) + connection.execute("INSERT INTO t VALUES (?)", (document,)) + relaxed = SQLiteSearchRepository._relaxed_fts_text("foo­bar права доступа") + assert relaxed is not None + rows = connection.execute("SELECT rowid FROM t WHERE t MATCH ?", (relaxed,)).fetchall() + finally: + connection.close() + assert rows, f"relaxed expression missed the note {document!r}: {relaxed!r}" + + +def test_orthographic_joiners_are_not_duplicated_into_a_second_variant() -> None: + """Joiners are written in the text, so the stored form has them. + + A stripped variant would only widen the OR with a term no note can hold. + """ + relaxed = SQLiteSearchRepository._relaxed_fts_text("نمی‌خواهم دسترسی را لغو") + assert relaxed == "نمی‌خواهم* OR دسترسی* OR را* OR لغو*" From a6c5714bc5b4f69f27a82ed2222b13a75cfbe5d3 Mon Sep 17 00:00:00 2001 From: gingeard Date: Wed, 19 Aug 2026 13:33:46 +0100 Subject: [PATCH 16/19] fix(core): find the base letter through attached characters when joining MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The joining rule asked whether the previous character is a letter, but pointed Hebrew and decomposed Latin put a mark between the letter and the punctuation. "גּ׳ון סמית" therefore split into three tokens where the unpointed spelling stays two — a guarded two-word query turned into a broad three-term OR, and the spelling that triggers it is the more careful one. The rule now looks past marks and format characters on both sides to the base character, so it sees the letter it is asking about. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 32 +++++++++++++++++++-- tests/repository/test_search_relaxation.py | 20 +++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index 07c0619f3..9c461798c 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -68,6 +68,34 @@ def _strip_trailing_formats(token: str) -> str: return token +def _is_attached(char: str) -> bool: + """Whether a character hangs off the one before it rather than standing alone.""" + return _is_word_internal_format(char) or unicodedata.category(char).startswith("M") + + +def _base_before(current: list[str]) -> bool: + """Whether the token so far ends in a letter, looking past what hangs off it. + + Pointed Hebrew and decomposed Latin put a mark between the letter and the + punctuation, so reading only the last character sees the mark and splits a + word the joining rule is meant to keep whole. + """ + for char in reversed(current): + if _is_attached(char): + continue + return char.isalpha() + return False + + +def _base_after(text: str, index: int) -> bool: + """Whether a letter follows the punctuation, looking past what hangs off it.""" + for char in text[index + 1 :]: + if _is_attached(char): + continue + return char.isalpha() + return False + + def _is_token_continuation(text: str, index: int, current: list[str]) -> bool: """Whether a non-alphanumeric character belongs to the word being read. @@ -83,9 +111,7 @@ def _is_token_continuation(text: str, index: int, current: list[str]) -> bool: if _is_word_internal_format(char) or unicodedata.category(char).startswith("M"): return True if char in RELAXATION_WORD_INTERNAL_PUNCTUATION: - follows_letter = bool(current) and current[-1].isalpha() - precedes_letter = index + 1 < len(text) and text[index + 1].isalpha() - return follows_letter and precedes_letter + return _base_before(current) and _base_after(text, index) return False diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py index 94be85a70..0a21611a7 100644 --- a/tests/repository/test_search_relaxation.py +++ b/tests/repository/test_search_relaxation.py @@ -314,3 +314,23 @@ def test_relaxed_query_words_splits_on_structural_punctuation(query: str) -> Non permalinks, paths and version strings. """ assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("גּ׳ון סמית כתב", ["גּ׳ון", "סמית", "כתב"]), # dagesh between letter and geresh + ("pré d'accord test", ["pré", "d'accord", "test"]), # NFD before apostrophe + ], +) +def test_relaxed_query_words_finds_the_base_letter_through_marks( + query: str, + expected: list[str], +) -> None: + """The joining rule looks for a letter, not for the last character. + + Pointed Hebrew and decomposed Latin put a mark between the letter and the + punctuation, so reading only the character before it sees the mark and + splits a word the rule is meant to keep whole. + """ + assert relaxed_query_words(query) == expected From ca1be1529e45a5fbeaed60ec91967c25edd6a29b Mon Sep 17 00:00:00 2001 From: gingeard Date: Wed, 19 Aug 2026 13:44:06 +0100 Subject: [PATCH 17/19] fix(core): take the whole word-joining punctuation class, minus two families MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The joining set was picked by hand, so review kept finding the next member: first the Hebrew gershayim and Catalan middle dot, now the Armenian abbreviation mark and the fullwidth apostrophe — "об＇єкт доступу" split into three tokens and cleared the guard. It is now the whole of UAX #29 MidLetter, MidNumLet and Single_Quote, plus U+05F3 which the standard classes as a letter, minus two families excluded on purpose: the colons, because "tag:example" is documented query syntax, and the full stops, because permalinks and file names are built on them. Joining across either would merge a qualifier with its value or a name with its extension. The numeric guard needs no exclusions of its own — the rule only joins between two letters, so no character in the set can attach a digit to anything. A test transcribes the standard's class and partitions it: every member is either taken or named structural, and nothing is both. A character can now be excluded on purpose but not forgotten. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 19 +++--- .../test_search_relaxation_unicode_classes.py | 58 ++++++++++++++++++- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index 9c461798c..d34725621 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -47,13 +47,18 @@ # document, a bidi hint, a stray BOM — absent from the stored note, so carrying # it into the term is what stops the term matching. RELAXATION_ORTHOGRAPHIC_JOINERS = "\u200c\u200d" -# Punctuation written inside a word, applied only between two letters: "п’ять", -# "don't", Hebrew ג׳ון and ר״ת, Catalan col·lecció. This is the UAX #29 MidLetter -# set minus colon and full stop, which the standard also joins on — "1.2" comes -# back as one token there, and a token like that is not category N, so it would -# walk an identifier-like query straight past the numeric guard. Both carry -# structure in permalinks, paths and versions here, so they keep splitting. -RELAXATION_WORD_INTERNAL_PUNCTUATION = "'\u2018\u2019\u00b7\u0387\u05f3\u05f4\u2027" +# Punctuation written inside a word, joined only between two letters. This is +# the whole of UAX #29 MidLetter, MidNumLet and Single_Quote — plus U+05F3, which +# the standard classes as a letter — minus two families that carry structure in +# this project rather than inside words: +# +# colons U+003A U+FE13 U+FE55 U+FF1A "tag:example" is query syntax +# full stops U+002E U+2024 U+FE52 U+FF0E permalinks and file names +# +# Joining across those would merge a qualifier with its value, or a name with its +# extension, into one term. Everything else in the class is here, so a new +# member is a change to the standard rather than an oversight. +RELAXATION_WORD_INTERNAL_PUNCTUATION = "'\u2018\u2019\uff07\u00b7\u0387\u055f\u05f3\u05f4\u2027" def _is_word_internal_format(char: str) -> bool: diff --git a/tests/repository/test_search_relaxation_unicode_classes.py b/tests/repository/test_search_relaxation_unicode_classes.py index 28ec37f72..efbf7b2db 100644 --- a/tests/repository/test_search_relaxation_unicode_classes.py +++ b/tests/repository/test_search_relaxation_unicode_classes.py @@ -147,7 +147,10 @@ def test_declared_punctuation_is_exactly_the_chosen_midletter_subset() -> None: silently exempts it, so both have to be a deliberate edit rather than a side effect. The named cases in test_search_relaxation.py pin what each is for. """ - assert RELAXATION_WORD_INTERNAL_PUNCTUATION == "'\u2018\u2019\u00b7\u0387\u05f3\u05f4\u2027" + assert ( + RELAXATION_WORD_INTERNAL_PUNCTUATION + == "'\u2018\u2019\uff07\u00b7\u0387\u055f\u05f3\u05f4\u2027" + ) def test_declared_punctuation_joins_letters_only() -> None: @@ -185,3 +188,56 @@ def test_colon_and_full_stop_split_although_uax29_joins_them(structural: str) -> """ assert len(relaxation_word_tokens(f"сло{structural}во доступа")) == 3 assert relaxed_query_words(f"spec 1{structural}2 design") is None + + +# UAX #29 Word_Break values for the punctuation that joins words, transcribed +# from the standard. Python's unicodedata does not expose the property, so the +# class is pinned here as data: the partition below then has to account for every +# member, and a character cannot be forgotten, only deliberately excluded. +UAX29_WORD_JOINING_PUNCTUATION = { + "MidLetter": ":··՟״‧︓﹕：", + "MidNumLet": ".‘’․﹒＇．", + "Single_Quote": "'", +} +# Excluded on purpose: these carry structure in this project rather than sitting +# inside words — "tag:example" is documented query syntax, and permalinks and +# file names are built on the full stop. +STRUCTURAL_PUNCTUATION = ":︓﹕：.․﹒．" + + +def test_the_joining_set_accounts_for_every_word_joining_character() -> None: + """Every UAX #29 word-joining character is either taken or named structural. + + Review surfaced these one at a time — the Armenian abbreviation mark and the + fullwidth apostrophe were the last two. Partitioning the class means a + missing character fails here rather than in another round. + """ + standard = set("".join(UAX29_WORD_JOINING_PUNCTUATION.values())) + taken = set(RELAXATION_WORD_INTERNAL_PUNCTUATION) + excluded = set(STRUCTURAL_PUNCTUATION) + + unaccounted = standard - taken - excluded + assert not unaccounted, ( + f"word-joining characters neither taken nor excluded — {_describe(sorted(unaccounted))}" + ) + + contradictory = taken & excluded + assert not contradictory, ( + f"characters both taken and excluded — {_describe(sorted(contradictory))}" + ) + + +def test_taken_characters_outside_the_standard_are_justified() -> None: + """U+05F3 is the one addition: UAX #29 classes geresh as a letter, not punctuation. + + Python sees it as Po, so the joining rule has to name it explicitly to reach + the same result the standard does for Hebrew. + """ + standard = set("".join(UAX29_WORD_JOINING_PUNCTUATION.values())) + assert set(RELAXATION_WORD_INTERNAL_PUNCTUATION) - standard == {"׳"} + + +@pytest.mark.parametrize("structural", sorted(STRUCTURAL_PUNCTUATION)) +def test_structural_punctuation_keeps_splitting(structural: str) -> None: + """Joining across these would merge a qualifier with its value, or a name with its extension.""" + assert len(relaxation_word_tokens(f"сло{structural}во доступа")) == 3 From ec38525aa77b68fd7a297863e603271f6fa020e9 Mon Sep 17 00:00:00 2001 From: gingeard Date: Wed, 19 Aug 2026 14:00:13 +0100 Subject: [PATCH 18/19] perf(core): read ahead by index instead of slicing the rest of the query MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The joining rule looks at what follows the punctuation, and read it with a slice — which copies the remainder of the query at every joiner, so tokenizing is quadratic in the length of the input. Measured on an apostrophe-joined query: 100 KB 32 ms 200 KB 94 ms 400 KB 310 ms (×3.2 per doubling) after, reading by index: 100 KB 18 ms 200 KB 37 ms 400 KB 75 ms (×2.0 per doubling) Search text is not length-bounded on the way in, so a long query could hold a worker for the duration. The test asserts the ratio between one length and its double rather than a duration, so it does not depend on the speed of the machine, and it is sized at 128 KB: at 64 KB the sliced reader still comes in under the threshold and the regression slips through. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 5 +++- .../test_search_relaxation_unicode_classes.py | 28 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index d34725621..7af0e5b29 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -94,7 +94,10 @@ def _base_before(current: list[str]) -> bool: def _base_after(text: str, index: int) -> bool: """Whether a letter follows the punctuation, looking past what hangs off it.""" - for char in text[index + 1 :]: + # Indexed rather than sliced: a slice copies the rest of the query at every + # joiner, which makes tokenizing a long query quadratic in its length. + for position in range(index + 1, len(text)): + char = text[position] if _is_attached(char): continue return char.isalpha() diff --git a/tests/repository/test_search_relaxation_unicode_classes.py b/tests/repository/test_search_relaxation_unicode_classes.py index efbf7b2db..311a79a21 100644 --- a/tests/repository/test_search_relaxation_unicode_classes.py +++ b/tests/repository/test_search_relaxation_unicode_classes.py @@ -13,6 +13,7 @@ rather than identifiers. """ +import time import unicodedata import pytest @@ -241,3 +242,30 @@ def test_taken_characters_outside_the_standard_are_justified() -> None: def test_structural_punctuation_keeps_splitting(structural: str) -> None: """Joining across these would merge a qualifier with its value, or a name with its extension.""" assert len(relaxation_word_tokens(f"сло{structural}во доступа")) == 3 + + +def test_tokenizing_scales_linearly_with_query_length() -> None: + """Doubling the query must roughly double the work, not more. + + The joining rule has to look at what follows the punctuation. Reading that + with a slice copies the rest of the query at every joiner, which is + quadratic: an unbounded query full of apostrophes then ties up the worker + that tokenizes it. The ratio is asserted rather than a duration, so the test + does not depend on how fast the machine is. + """ + query = "a'b " * (128 * 1024 // 4) + + def elapsed(text: str) -> float: + start = time.perf_counter() + relaxation_word_tokens(text) + return time.perf_counter() - start + + # Fastest of three runs each: the ratio, not the duration, is what is being + # asserted, so a slow or busy machine does not turn this red. Measured here, + # the indexed reader scales at ×2.0 and the sliced one at ×3.2. + single = min(elapsed(query) for _ in range(3)) + double = min(elapsed(query * 2) for _ in range(3)) + + assert double < single * 2.5, ( + f"tokenizing scaled worse than linearly: {single * 1000:.1f} ms then {double * 1000:.1f} ms" + ) From 9eb72be4cc37b4070094e57788da6c5ef8c8705f Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 19 Aug 2026 21:53:49 -0500 Subject: [PATCH 19/19] perf(core): trim trailing formats by index instead of slicing Trimming one character at a time copies the shrinking token at every step, quadratic in a run of trailing format characters: 'a' followed by 400K soft hyphens took ~2.0s in relaxation_word_tokens and doubling the run scaled it ~3.2x. An index-based trim makes it linear (~0.1s), and a ratio-asserted regression covers the trailing-format input shape. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01G4rbaeHJN3L7CREp5v38J9 Signed-off-by: phernandez --- src/basic_memory/repository/search_query.py | 10 ++++--- .../test_search_relaxation_unicode_classes.py | 27 +++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index 7af0e5b29..d73d70c0c 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -68,9 +68,13 @@ def _is_word_internal_format(char: str) -> bool: def _strip_trailing_formats(token: str) -> str: """Drop format characters left at a token's end, where they separate rather than join.""" - while token and _is_word_internal_format(token[-1]): - token = token[:-1] - return token + # Indexed rather than sliced: trimming one character at a time copies the + # shrinking token at every step, which is quadratic in a run of trailing + # format characters. + end = len(token) + while end and _is_word_internal_format(token[end - 1]): + end -= 1 + return token[:end] def _is_attached(char: str) -> bool: diff --git a/tests/repository/test_search_relaxation_unicode_classes.py b/tests/repository/test_search_relaxation_unicode_classes.py index 311a79a21..5fb725b2a 100644 --- a/tests/repository/test_search_relaxation_unicode_classes.py +++ b/tests/repository/test_search_relaxation_unicode_classes.py @@ -269,3 +269,30 @@ def elapsed(text: str) -> float: assert double < single * 2.5, ( f"tokenizing scaled worse than linearly: {single * 1000:.1f} ms then {double * 1000:.1f} ms" ) + + +def test_trailing_format_trim_scales_linearly() -> None: + """A word ending in a long run of format characters must trim in one pass. + + Format characters are word-internal, so a token collects them all before the + trailing trim runs. Trimming one character at a time copies the shrinking + token at every step, which is quadratic: an unbounded query ending in enough + soft hyphens then ties up the worker that tokenizes it. As above, the ratio + is asserted rather than a duration. + """ + + def elapsed(count: int) -> float: + text = "a" + "\u00ad" * count + start = time.perf_counter() + relaxation_word_tokens(text) + return time.perf_counter() - start + + # Sized so the trim dominates the measurement: at shorter lengths the + # tokenizer's linear per-character work dilutes the quadratic term below + # the ratio threshold and a regression would pass unnoticed. + single = min(elapsed(256 * 1024) for _ in range(3)) + double = min(elapsed(512 * 1024) for _ in range(3)) + + assert double < single * 2.5, ( + f"trailing trim scaled worse than linearly: {single * 1000:.1f} ms then {double * 1000:.1f} ms" + )