From 141634b7aedab8e7408111bb19369fc550e07ba5 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 18 Sep 2026 10:41:37 +0200 Subject: [PATCH 01/16] rfc: land RFCs 0047 and 0048 from PR #606 with their receipts Takes the two RFC documents, the upstream-contract receipt, the Arrow pool probe, the lexical scoring oracle fixtures (moved under assets/) and the one green GQT case (ported to the corpus's `--- runner` header) from PR #606 at ce5a3012, on top of main's closed RFC number namespace. The agent-context copy of the RFC, the archived integration patches, the agent pilot record, the test-only staged compiler, the DataFusion composition probes, the selection bench and the oracle test stay on the `rfc-0047-search-plan-truth` branch as evidence; none of them tests a current OmniGraph boundary. --- ...5_statistics_ignore_eligibility_filter.gqt | 79 + docs/rfcs/0047-search-plan-truth.md | 511 +++ docs/rfcs/0048-search-contracts.md | 3121 +++++++++++++++++ docs/rfcs/README.md | 2 + docs/rfcs/assets/0048-arrow-pool-probe.py | 75 + docs/rfcs/assets/0048-arrow-pool-probe.rs | 55 + docs/rfcs/assets/0048-lexical-scoring-v1.json | 412 +++ docs/rfcs/assets/0048-lexical-scoring-v1.py | 81 + .../0048-upstream-contract-checkpoint.json | 408 +++ 9 files changed, 4744 insertions(+) create mode 100644 crates/omnigraph-gqt/cases/bm25_statistics_ignore_eligibility_filter.gqt create mode 100644 docs/rfcs/0047-search-plan-truth.md create mode 100644 docs/rfcs/0048-search-contracts.md create mode 100644 docs/rfcs/assets/0048-arrow-pool-probe.py create mode 100644 docs/rfcs/assets/0048-arrow-pool-probe.rs create mode 100644 docs/rfcs/assets/0048-lexical-scoring-v1.json create mode 100644 docs/rfcs/assets/0048-lexical-scoring-v1.py create mode 100644 docs/rfcs/assets/0048-upstream-contract-checkpoint.json diff --git a/crates/omnigraph-gqt/cases/bm25_statistics_ignore_eligibility_filter.gqt b/crates/omnigraph-gqt/cases/bm25_statistics_ignore_eligibility_filter.gqt new file mode 100644 index 000000000..7c65266bf --- /dev/null +++ b/crates/omnigraph-gqt/cases/bm25_statistics_ignore_eligibility_filter.gqt @@ -0,0 +1,79 @@ +# issue: none +# notes: `Doc` holds ten rows, four in group "e" (three "alpha", one "beta") and six "beta" rows in +# notes: group "o"; `Four` holds only the four group-"e" rows. The query "alpha beta" filtered to +# notes: group "e" ranks an alpha row first: the filter is a prefilter and BM25 statistics come from +# notes: the whole field (alpha is rare in ten). The same four rows as their own type rank the beta +# notes: row first (beta is rare in four). Same eligible rows, different winner; RFC 0048 chooses +# notes: the field corpus, so this pins the current answer as the contract. + +--- runner +timeout_ms: 10000 +environments: + - target: omnigraph-engine + storage: local-filesystem + +--- schema +node Doc { + slug: String @key + group: String + title: String @index +} + +node Four { + slug: String @key + title: String @index +} + +--- seed +{"type":"Doc","data":{"slug":"e1","group":"e","title":"alpha kiwi"}} +{"type":"Doc","data":{"slug":"e2","group":"e","title":"alpha mango"}} +{"type":"Doc","data":{"slug":"e3","group":"e","title":"alpha pear"}} +{"type":"Doc","data":{"slug":"e4","group":"e","title":"beta plum"}} +{"type":"Doc","data":{"slug":"o1","group":"o","title":"beta fig"}} +{"type":"Doc","data":{"slug":"o2","group":"o","title":"beta lime"}} +{"type":"Doc","data":{"slug":"o3","group":"o","title":"beta kale"}} +{"type":"Doc","data":{"slug":"o4","group":"o","title":"beta yam"}} +{"type":"Doc","data":{"slug":"o5","group":"o","title":"beta corn"}} +{"type":"Doc","data":{"slug":"o6","group":"o","title":"beta bean"}} +{"type":"Four","data":{"slug":"f1","title":"alpha kiwi"}} +{"type":"Four","data":{"slug":"f2","title":"alpha mango"}} +{"type":"Four","data":{"slug":"f3","title":"alpha pear"}} +{"type":"Four","data":{"slug":"f4","title":"beta plum"}} + +--- query +query filtered_in_ten($q: String) { + match { + $d: Doc { group: "e" } + } + return { $d.slug } + order { bm25($d.title, $q) } + limit 1 +} + +--- params +{"q": "alpha beta"} + +--- expect ordered +{"d.slug": "e1"} + +--- expect shape +d.slug: String + +--- query +query same_rows_as_four($q: String) { + match { + $f: Four + } + return { $f.slug } + order { bm25($f.title, $q) } + limit 1 +} + +--- params +{"q": "alpha beta"} + +--- expect ordered +{"f.slug": "f4"} + +--- expect shape +f.slug: String diff --git a/docs/rfcs/0047-search-plan-truth.md b/docs/rfcs/0047-search-plan-truth.md new file mode 100644 index 000000000..d5af74a09 --- /dev/null +++ b/docs/rfcs/0047-search-plan-truth.md @@ -0,0 +1,511 @@ +--- +rfc: "0047" +title: "Search plan truth: projectable ranking, deterministic order, and loud search failures" +track: public +status: draft +implementation: not-started +authors: + - Ragnor Comerford (@ragnorc) +created: 2026-09-01 +updated: 2026-09-09 +discussion: "https://github.com/ModernRelay/omnigraph/pull/606" +supersedes: [] +superseded_by: [] +blocked_on: + - "Complete boundary-tie handling at every native candidate cut, with bounded retention or explicit resource failure" +--- + +# RFC 0047: Search plan truth: projectable ranking, deterministic order, and loud search failures + +PR #606 reviews this correctness slice together with RFC 0048's final search +contract. Start with its [maintainer briefing](0048-search-contracts.md#maintainer-briefing) +for the combined design, migration tradeoffs and decisions still needed. +The [implementation handoff](0048-search-contracts.md#implementation-handoff-and-validation-checkpoint) +records the validated experiments and remaining gates; the interim mechanisms +in this RFC do not require a separate public release. + +## Summary + +Ranked reads become honest about what they executed, and search constructs +that silently do nothing become errors or warnings: + +1. A search filter or rank target on an unsupported traversal-introduced + binding is a stable `T26` compile diagnostic. The motivating failure + silently dropped the predicate or ranking and returned plausible rows. +2. The executed retrieval is stated once in the lowered plan + (`QueryIR::retrieval`) instead of being re-inferred from `order_by[0]` at + execution. +3. `bm25(...)`, `nearest(...)`, and `rrf(...)` become projectable in `return`, + observing the exact value the ordering used; a projected rank expression + that is not structurally identical to the executed retrieval is an error. +4. Every ranked result has a total, deterministic order — including `rrf()` + fusion (score, then trailing user keys applied inside score ties, then + stable ids) and aggregated orderings. +5. The canonical read envelope gains three additive arrays: `warnings` + (first use: full-text search on a column with no FTS index serves through + the case-sensitive flat fallback — now loud), `metrics` (descriptors for + projected rank columns), and `retrievals` (every executed source, with + known/unknown representation coverage for `@embed`-backed vector retrievals + over the prefiltered population, with exact counts when requested). + +Boundaries that do not change: no schema surface or storage-format change, no +change to BM25/vector scoring math, the deprecated `POST /read` envelope stays +byte-stable, and observable order changes only where scores tie (those orders +were previously run-dependent). + +`fuzzy()` remains available in this slice. It has working exact and typo +matches, but its analysis and matched set depend on index coverage. The +single breaking replacement of `fuzzy`, `search`, and `match_text` with +typed lexical matching and ranked retrieval belongs to +[RFC 0048](0048-search-contracts.md#user-and-operational-behavior), where the +schema-owned analyzer and exact lexical contract are defined. This RFC does +not introduce a `T25` retirement stage. + +This is the initial correctness slice. RFC 0048 owns the final staged query +language: explicit ranking targets, graph-defined source populations, named +arm metrics, and separate selection/output boundaries. `T26`, the single +`QueryIR::retrieval` field, and structural repetition of ordering expressions +are interim mechanisms, not permanent language requirements. The guarantees +can be implemented directly in the staged model if both RFCs land together; +there is no requirement to release an interim syntax first. RFC 0048 also +owns the deliberate changes to selection and tie semantics at that cutover. +For the combined release, the [user-facing migration +matrix](0048-search-contracts.md#user-facing-changes-and-migration) distinguishes +breaking query/schema changes from additive capabilities and unchanged graph +behavior. This RFC's narrower compatibility statements apply to its standalone +slice. + +## Motivation + +Two production bug classes and one structural gap motivate this. + +**Confident false negatives from silent text-search fallback.** Measured on a +real graph: `match_text($o.name, "Anthropic")` returns 2 rows while +`"anthropic"` returns 0, with no signal. The mechanism is source-validated: +OmniGraph's FTS indexes lowercase correctly, but a full-text function on a +column with **no** FTS index silently runs the substrate's flat scan with a +bare, case-sensitive tokenizer. This produced a real false report that an +entity was absent. + +**Silently dropped search on traversal targets.** A `search()`/`match_text()` +filter or `nearest()`/`bm25()` target whose variable is introduced by a +traversal is removed from the pipeline but never attached to any scan: the +query returns unranked or unfiltered rows with no error. The flat-traversal +form was fixed earlier; the traversal-introduced-binding form persists and is +indistinguishable from correct output. + +**Rank needs one plan identity.** The original baseline rejected ranking +expressions in projection and discarded the fused score after sorting. +Current code already carries some `_score`/`_distance` columns, as recorded +in the appendix; implementation must first reconcile the remaining gaps +against that baseline. Retrieval still needs one explicit lowered identity, +instead of rediscovery from `order_by[0]`, and complete tie handling must be +qualified before every candidate cut. The deny-list forbids discarding +retrieval rank before projection. Existing column materialization alone does +not establish source identity, arm membership, or the complete guarantees here. + +An issue-sized fix cannot close this: the failures span the compiler, the +executor, and the public read contract, and the cure requires new observable +surfaces (diagnostics, warnings, response metadata) that must be designed +once, coherently. + +## User and operational behavior + +**Compile diagnostics.** + +- `T26`: a search filter or rank expression targeting a non-scan-rooted + binding fails typecheck: "make the target the first-declared binding of + its match component, or target the scan-rooted variable." The engine also + refuses (rather than drops) the shape if reached with hand-built IR. + This diagnostic protects the interim executor. RFC 0048 replaces the + restriction for qualified graph-derived populations; unsupported shapes + must continue to fail explicitly. + +**Metric projection.** + +```gq +return { $d.slug, bm25($d.body, $q) as score } +order { bm25($d.body, $q) } +limit 20 +``` + +projects the score the ordering used — one computation, observed twice. The +projected rank expression must be structurally identical (source, target, +query argument) to the executed retrieval; a mismatch is a loud error, never +a NULL column. Projecting an `rrf()` yields the fused score. Named individual +arm metrics belong to RFC 0048's stage model and RFC 0040's namespace rules. +That model distinguishes an invalid metric reference from a valid arm metric +that is absent because the target never entered that arm; the latter is +ordinary missing membership, not a mismatched-projection error. + +**Determinism.** Ranked output order is total: score, then trailing `order` +keys (which now apply *inside* fused-score ties on the `rrf()` path — they +were previously ignored there), then stable entity/edge ids. Fusion winners +tied at the limit boundary are all retained as candidates so trailing keys +decide the cut. A search-ordered *aggregate* query applies its trailing keys +and carries a warning that the rank itself cannot order grouped rows +(previously the whole order clause was silently ignored). + +This guarantee requires complete boundary handling before every candidate +cut, including native FTS collectors and fusion arms. Sorting a bounded +over-fetched subset does not recover tied rows already discarded upstream. +The implementation must either apply the full required comparator at the +cut or preserve the complete boundary for later comparison. An unqualified +native path must use a complete fallback within the query budget or fail +explicitly. This requirement is not established by the retained prototype. +For approximate retrieval, deterministic ordering describes the candidates +actually selected. It does not promise that rerunning ANN on the same snapshot +discovers the same candidates; RFC 0048 keeps stable ranked pagination separate. + +**Response envelope (canonical `/query` and stored-query reads; additive).** + +```json +"warnings": [{ "code": "full_text_search_unindexed", "message": "…" }], +"metrics": [{ "column": "score", "kind": "score", "source": "bm25", + "variable": "d", "property": "body", "descending": true, + "recall": "exact" }], +"retrievals": [{ "variable": "d", "property": "embedding", "kind": "nearest", + "recall": "approximate", + "embedding_coverage": { "state": "known", "ready": 934, + "pending": 66, "missing": 0 } }] +``` + +- `warnings` never change rows, membership, or order. Human CLI formats print + them to stderr; full-JSON output carries them in-band. Initial codes: + `full_text_search_unindexed`, `search_order_ignored_by_aggregation`, + `embedding_coverage_pending`. +- `recall` reports the source *contract*: an index-accelerated `nearest` + reports `approximate` even when execution happened to be exact, so clients + do not acquire a guarantee that disappears when an index is built. +- `embedding_coverage` reports either exact counts over the **prefiltered** + population at the pinned snapshot, or an explicit unknown state such as + `{ "state": "unknown", "reason": "not_computed" }`. `pending` rows have + usable source text but no derived vector; `missing` rows have neither a + usable vector nor a source that can currently produce one. Known pending + data and unknown coverage must not be presented as complete representation. + Query completion remains a separate fact. By default, do not scan the full + population solely to compute these counts. A typed exact-coverage request + requires exact counts or failure within the query budget; it cannot degrade + to unknown. RFC 0048's [result metadata contract](0048-search-contracts.md#result-metadata-coherent-continuation-and-budgets) + owns the shared semantics and read-surface qualification. +- The deprecated `POST /read` envelope carries none of these fields, by + construction. + +**Operators** see `tracing` warnings for the unindexed-column condition and +remediate by declaring `@index` and running index reconciliation. No new +maintenance surface is added. + +## Design + +- **Retrieval in the plan.** `QueryIR` gains `retrieval: Option` + (`Nearest` / `Bm25` / `FuseRrf` with two leaf arms). Lowering fixes the + retrieval shape once — per-arm candidate counts and the bounded-bm25 scan + policy included — while parameter values and String-query embedding stay + execution-time, so one lowered plan serves every parameterization. The + executor's `order_by[0]` inference is deleted; the resolved mode feeds the + existing scan and fusion machinery unchanged (which is what let the #587 + prefilter gate compose with zero changes). +- **Scan-rooted targets.** The lowering component-root computation (first + declared binding of each traversal-connected component gets the scan) is + extracted and shared with typecheck's `T26` pass, so the rule and the plan + cannot drift. Negation scopes check their own roots. This implementation + guard does not establish a permanent first-declared-target rule for the + language; RFC 0048's stage target validation replaces it as shapes qualify. +- **Advisories.** Execution threads one explicit notice sink (deduplicating, + so the bounded-bm25 retry and fusion's forked arms cannot double-report); + results carry notices, metric descriptors, and retrieval descriptors as + first-class fields mapped additively onto the HTTP envelope. +- **Fusion.** The fused score is materialized as a real column on the fused + rows before projection; winner selection is deterministic (score, then + entity id) and retains the boundary tie plateau; ordering then applies + score, trailing keys, and id tie-breaks over fanout rows. A tie plateau can + span the entire candidate population; retention must stay within explicit + memory/work budgets, with spill or a typed failure if necessary. Its width + is not itself a fixed resource bound. +- **Coverage.** Default descriptors use exact counts already established by + required execution or qualified snapshot-bound metadata, otherwise explicit + unknown. Requested exact ready/pending/missing counts reuse the scan's own + structured predicate through a sealed, streaming count on the storage + boundary — no SQL strings or retained batches. Streaming bounds retained + batches, not rows examined: exact coverage may still scan the whole + prefiltered population and must be charged to the query's work budget. + Shared count results require the same population and representation identity. + +## Invariants + +- **Loud integrity failures (8):** strengthened for dropped search shapes: + `T26` refuses a predicate or ranking that cannot be attached to its target. + Warnings expose the absent-index fallback; they do not repair its matched + set. Exact lexical behavior, including typo tolerance, is RFC 0048's scope. +- **Query semantics are typed structures (9):** strengthened — retrieval + moves from execution-time re-inference into the typed lowered plan; rank + becomes an ordinary projected column, closing the recorded rank-carry gap. +- **Physical acceleration is derived (7):** the existing text-search + violation remains open in this slice: a warning does not make + index-dependent analysis correct. RFC 0048 closes it with schema-owned + analysis and exact evaluation across index states. This RFC's notices are + transitional visibility, not an exception to the invariant. Recall + reporting is contractual, not plan-derived. +- **Bounded, observable resource use (11):** requested coverage counts stream; + retries, complete tie handling, and exact coverage share explicit query budgets. + Streaming and finite tie plateaus alone do not prove bounded work or a + bounded memory footprint. Their qualification remains an implementation + requirement. +- Deny-list: no side channel for discarded rank remains; no new endpoint; no + string-built predicates (coverage uses structured expressions); no + logical precondition on index coverage is introduced. + +## Compatibility and reversibility + +- **Wire:** all three response fields are additive and serde-defaulted; + unknown-string tolerance is specified for `kind`/`source`/`recall`. The + legacy `/read` envelope is untouched. OpenAPI regenerates with the new + schemas. +- **Language:** `T26` rejects queries whose search predicate or ranking was + previously dropped. Existing lexical spellings remain available until + RFC 0048 replaces them together at its breaking release boundary. No + compatibility alias or deprecation window is required for that lexical + replacement while the language is pre-stable. +- **Order:** observable order changes only where scores tie; those orders + were run-dependent before, so nothing reproducible is broken. +- **Reverting** requires no storage or format work: the response fields are + additive, the IR field is internal, and the diagnostics can be relaxed — + at the cost of restoring the silent-failure classes this exists to remove. + +## Alternatives + +- **Retire `fuzzy()` before defining its replacement** — rejected: indexed + typo queries do match, while scan and indexed execution disagree. The + repair requires an explicit analyzer and matching contract. RFC 0048 + replaces the lexical surface in one breaking change; a standalone + retirement neither defines that contract nor supplies typo tolerance. +- **Fail closed on unindexed text search** — deferred, not chosen now: the + flat fallback serves correct exact-token matches; warning preserves + service while removing silence. A future schema-owned analyzed-search + contract may revisit the posture. +- **Silently NULL (or best-effort match) mismatched metric projections** — + rejected: a NULL column invites misreading; structural identity keeps the + projection an observation of the executed retrieval. +- **Trailing keys before fusion winner selection at entity level** — + rejected as ill-defined (trailing keys order fanout rows, not entities); + retaining the complete boundary tie plateau lets fanout rows determine + the cut, subject to the explicit resource policy above. +- **A separate search/rank response endpoint** — rejected: one GQ surface, + additive metadata on the existing envelope. +- **Doing nothing** — the two bug classes continue to produce confident + wrong answers, and rank remains unprojectable despite documentation + implying otherwise. + +## Evidence and tests + +A prototype of the preceding design exists (closed PR #595, branch +`search-contracts-p0-p1`, retained as evidence per the closure note). Its +historical record reports eleven staged commits, a green canonical workspace +graph (2,860 tests), both Clippy gates, regenerated OpenAPI, and a classified +vocabulary-guard inventory. These are historical results, not a fresh run on +the revised draft. Test +owners extended, not forked: compiler typecheck/lowering suites (including +the prototype's now-withdrawn T25 stage, T26, retrieval lowering, cap policy), +engine `search.rs` (projection, determinism, +fusion ties, coverage, warnings — including characterization goldens captured +*before* the executor refactor as the equivalence baseline), +`rrf_prefilter_gate.rs` (one fixture ported: the expand-dst shape now asserts +`T26`), `ordering.rs`/`aggregation.rs`, server `data_routes`/`openapi`, and +`forbidden_apis` (new storage count registered read-only). Three independent +review passes ran on the prototype; all six confirmed findings are fixed and +pinned by tests (see Decision log). + +The prototype is historical evidence, not qualification of this revised +proposal. The `fuzzy_does_not_match_under_default_tokenizer` characterization +in [the existing search owner](../../crates/omnigraph/tests/search.rs) tests +one capitalized query at one edit budget. The counterexamples in +[RFC 0048's evidence](0048-search-contracts.md#evidence-and-tests) disprove +the universal-failure premise. That RFC owns the matching and index-lifecycle +qualification required for the replacement. + +Revalidation against the pinned Lance source also leaves a separate +determinism gate: its plain FTS collector can drop equal-score boundary rows +before engine sorting. Extend `search.rs` with more tied rows than the native +candidate budget, reversed entity-ID/secondary-key order, and different +fragment/segment layouts. Verify complete winners and explicit resource +failure, not just sorted returned rows. Coverage cost needs an instrument +that measures rows examined as well as retained memory. + +The current source recheck also distinguishes BM25 from vector fusion arms: +`extract_sub_search_mode` leaves BM25 arms uncapped, while nearest arms derive +their candidate count from the query limit. `rrf_arms_scan_uncapped_in_one_pass` +guards the BM25 behavior. Current nearest/BM25 projection is checked by `T33`, +RRF projection is refused by `T37`, and the proposed `T26` target guard is not +present. `T25` now names duplicate output-column diagnostics; the withdrawn +prototype's retirement label must not be reused as a current-code fact. + +## Rollout + +For a standalone RFC 0047 implementation, use the ordered stages below after +acceptance. For the combined release, use +[RFC 0048's implementation phases](0048-search-contracts.md#implementation-phases): +plan identity and target validation belong to foundations/composition; +projectable metrics and complete ties to retrieval/composition qualification; +and warnings, retrieval descriptors, and coverage to the agent-facing read +path. These are the same correctness obligations, implemented once. + +The retained prototype supplies starting points; each stage must be checked +against the revised scope: + +1. Substrate fences for the Lance 11 update→optimize stale-vector window + (test-only; can land before acceptance as an ordinary change). +2. Warning carrier + `full_text_search_unindexed` (engine → API → CLI → + OpenAPI). +3. Characterization goldens, then the retrieval-IR refactor (behavior- + equivalent by construction; goldens prove it). +4. `T26` scan-rooted targets (compiler pass + engine backstops). +5. Projectable metrics and deterministic ties (single-search, fusion, + aggregated). +6. `metrics`/`retrievals` metadata with embedding coverage. + +For the standalone slice, `implementation` advances to `in-progress` at the +first landed stage and `complete` when stage 6 ships. For the combined release, +record completion against the corresponding RFC 0048 phase evidence instead +of requiring this interim sequence. Stages 2+ reference this RFC once accepted. +The lexical replacement and staged language are sequenced by RFC 0048, not +by a retirement commit in this rollout. Before porting prototype code, capture +the current compiler/engine baseline and identify which guarantees already +exist. If both designs are implemented together, build named stage IR and +metadata directly, preserving this RFC's correctness gates without an interim +language release or duplicate execution path. + +## Unresolved questions + +1. Is `warnings`' human-format contract (stderr for every non-full-JSON CLI + format) acceptable, or should the JSONL metadata record grow a warnings + field in the same change? + +## Decision log + +- 2026-08-20 — initial design draft produced from a source audit of + OmniGraph 0.10.0. +- 2026-08-31 — draft reconciled with a Lance 11 impact analysis, a + line-level source validation of every audit claim, and a fourteen-system + constraint-placement survey; this RFC is the repository-facing slice of + that larger draft (schema-owned analyzed search and format-boundary work + are explicitly out of scope here and will be proposed separately). +- 2026-09-01 — prototype PR #595 opened with the full implementation and + review record; six review findings (aggregated-fusion misalignment, + identifier normalization in coverage predicates, name-only projection + matching, post-cut trailing keys, unbounded coverage counts, JSONL + warning loss) confirmed, fixed, and pinned; one finding declined with + evidence. PR closed the same day under the governance process (size-L + requires an accepted RFC first); branch retained as evidence. +- 2026-09-01 — this RFC opened as the required public proposal. +- 2026-09-08 — withdrew standalone T25 retirement after matched-set probes + disproved universal fuzzy failure. RFC 0048 now owns a breaking lexical + replacement with explicit edit distance and exact scan/index parity; + warnings in this RFC do not claim to repair the existing analyzer cliff. +- 2026-09-08 — substrate revalidation made complete native tie handling an + explicit gate; removed claims that post-sorting, finite tie width, or + streaming counts alone prove the required result and resource bounds. +- 2026-09-08 — aligned with RFC 0048's staged retrieval design. The scan-root + rule, single retrieval field, and repeated-expression projection are + interim mechanisms; graph-derived targets and named arm metrics belong + to the final algebra. Resolved arm-metric ownership in RFC 0048, distinguished + absent membership from invalid projection, and required reconciliation + against current metric-carrying code before porting the historical prototype. + Both slices may land directly in one pre-stable language cutover. +- 2026-09-08 — rechecked current execution and diagnostics. Corrected the + blanket claim that all RRF arm windows follow the output limit: BM25 arms + are uncapped; vector arms inherit that limit. Distinguished historical + prototype results from current evidence and total ordering from ANN replay. +- 2026-09-09 — aligned representation coverage with RFC 0048: known counts or + explicit unknown by default, exact counts when explicitly requested. This + keeps query completion separate from representation knowledge and avoids + mandatory exhaustive counting solely for default discovery metadata. + +## Appendix: agent context (non-normative) + +Supporting context for implementers and coding agents. Nothing here is a +contract; the sections above are authoritative. + +**Prototype map.** Branch `search-contracts-p0-p1` @ `3e459aad` (closed +PR #595). Its commit order includes the withdrawn T25 stage and must not be +applied verbatim as the current rollout; the review-fix commit is +`3e459aad`, the #587 port is `858ce066`. This RFC is the first slice of a +larger search-contracts design program whose remaining scope includes +schema-owned analyzed search (`@analyzed`), vector-space identity, composable +ranking and graph stages, named metrics, and the SchemaIR boundary. That scope is +[RFC 0048](0048-search-contracts.md), blocked on this RFC (a Lance 11 +change-by-change impact analysis and a fourteen-system comparative survey +back both slices). + +**Substrate dependencies (pinned Lance 11.0.0, crates.io — validate against +the pin, not the GitHub tag; the two diverge).** + +- The motivating case-sensitivity mechanism: `full_text_search` on a column + with no FTS segments silently plans a flat scan with + `default_text_tokenizer()` (bare `SimpleTokenizer`, no lowercasing). The + warning detects per-column via `TableStore::has_fts_index_on`. +- The plain Match FTS path still compares score alone and leaf merges drop + equal-score boundary candidates by arrival order; the adapter's own + `.id`-column tie-breaks make the returned subset total, but cannot establish + that the subset contains the correct boundary winners. See the + [pinned collector](https://github.com/lance-format/lance/blob/ab6b5bbe46009ed78746b444df8db59a8bc5d842/rust/lance/src/io/exec/fts.rs#L310). + Do not assume Lance's compound-path + `(score, row_id)` ordering applies — row id is not the logical entity id. +- RFC 0043's fail-closed FTS certification is orthogonal: it gates + *uncertified indexes*; this RFC's warning covers *absent* indexes. Both + can fire on one graph. +- `search_score_orderings` (from the #544 work) already synthesizes + `{var}._distance`/`{var}._score` orderings; projection resolves the same + columns. `_distance`/`_score` are schema-reserved property names. + +**Assumptions that were validated (and where they broke in review).** + +- Scan-rootedness must be computed by the *same* code as lowering's + deferred-binding walk (`scan_root_variables`); the naive + "declared = scanned" rule is wrong for explicitly declared second bindings + of a component — pinned by a dedicated T26 test. +- Fused-score ties are bit-identical (same rank arithmetic per entity), so + exact `==` comparison for boundary-tie retention is sound. +- Current RRF vector-arm candidate counts follow the query limit. BM25 arms + scan uncapped; `rrf_arms_scan_uncapped_in_one_pass` guards that distinct + behavior. The older prototype's limit-sensitive tie fixture is historical + evidence, not proof that current lexical arms have a bounded window. +- Embedding-coverage predicates must reuse the scan's own `filter_expr` + (coverage describes the *prefiltered* population) and must use + DataFusion `ident()`, not `col()` — `col()` lowercases unquoted + identifiers and breaks camelCase properties (#283 precedent). +- Coverage counting must stream (fold `num_rows`, retain nothing): a + `try_collect` of id batches is O(population) memory on a bounded query. +- `render_jsonl` emits a slim metadata record without `warnings`; only full + JSON carries advisories in-band. +- Column-name matching is insufficient for metric projection: every nearest + on one binding shares `{var}._distance`, so structural fingerprints + (kind + target + query argument) are required. + +**Cross-PR composition contracts.** + +- #587 (rrf prefilter gate, merged): composes untouched because the gate + operates on the resolved `SearchMode` and this RFC only changes where the + mode comes from. Its `ranked_var_is_expand_dst` fixture is the T26 shape; + the port asserts the diagnostic instead of the gate fallback. The gate's + own expand-dst check stays as the engine backstop. +- RFC 0040 (system columns): the `__` reserved namespace is where + engine-owned metric columns migrate. RFC 0048 owns named stage/arm metrics + and must coordinate their public spelling with that namespace. +- The remainder of the search-contracts program + ([RFC 0048](0048-search-contracts.md)) depends on decisions not made + here: analyzer/scorer schema surface, SchemaIR version-facet coordination + with RFCs 0040/0044, analyzed-search index posture, target identity, stage + composition, and coherent follow-up reads. The dependency is on plan-truth + guarantees, not this slice's interim `Option` shape. + +**Build/CI traps observed while producing the evidence.** + +- The repo pins Rust 1.97.1; an exported `RUSTUP_TOOLCHAIN` env var + silently overrides the pin and changes which Clippy lints fire. +- New OpenAPI prose/property names using guarded vocabulary (rows/columns) + need classified rows in the vocabulary-guard inventory (strictly sorted + by occurrence id, exact content hashes from the failing test's output). +- New `TableStore` methods must be classified in the `forbidden_apis` + registry (`count_rows_matching` → read-only). +- Workspace test runs piped through `tail` mask non-final suite failures + and the exit code; capture full logs. diff --git a/docs/rfcs/0048-search-contracts.md b/docs/rfcs/0048-search-contracts.md new file mode 100644 index 000000000..55df526be --- /dev/null +++ b/docs/rfcs/0048-search-contracts.md @@ -0,0 +1,3121 @@ +--- +rfc: "0048" +title: "Search contracts and retrieval algebra" +track: public +status: draft +implementation: not-started +authors: + - Ragnor Comerford (@ragnorc) +created: 2026-09-03 +updated: 2026-09-13 +discussion: "https://github.com/ModernRelay/omnigraph/pull/606" +supersedes: [] +superseded_by: [] +blocked_on: + - "Phase 0 language/type/plan proofs for staged scope, C1–C4, identity, grouping and future unions; reconcile RFC 0047 without freezing its interim IR restrictions" + - "Phase 0 resolved-schema syntax/serialization prototype, default/reapplication fixtures, qualified encoder identity and format coordination with RFCs 0040/0043/0044" + - "Phase 0 vector/fusion numeric policies and independent oracles; bounded NFC, live-statistics and exact-tie route dispositions against pinned upstream code" + - "Phase 0 shared resource/read interfaces and discriminating admission, fingerprint, follow-up and refusal prototypes" + - "Phase 0 minimal compiler/engine/GQT vertical slice with later-stage discovery and shared-resource refusal" + - "Phase 0 fixed mixed-workload corpus, judgments, budgets and acceptance protocol; production qualification belongs to Phases 1–6" +--- + +# RFC 0048: Search contracts and retrieval algebra + +## Maintainer briefing + +This is the concise decision document. The companion +[agent context](assets/0048-agent-context.md) preserves the full long-form +rationale, experiments and handoff detail. Use this RFC for current decisions +and the context document for the evidence behind them. + +OmniGraph needs search that composes with exact lookup, graph traversal, +aggregation and projection in one typed query language. The immediate defects +are correctness failures: indexed and unindexed text can match differently, +fuzzy predicates and ranking can disagree, and retrieval inside `order` +hides candidate cuts. + +This RFC proposes explicit stages and schema-owned representations: + +- Predicates define eligibility; retrievers select and rank targets; scorers + add features without changing membership. +- Candidate windows, physical search effort and final output limits have + separate meanings. Moving a filter or aggregate across a cut changes the + question unless an equivalence proof says otherwise. +- Named metrics retain their source and target identity through graph fan-out. + Duplicate paths cannot create extra fusion votes. +- One accepted snapshot and execution budget cover the whole query, including + fallback, projection and source reads. + +Applications keep their own node types. A document or passage is an ordinary +node; attribution uses entity identity, selected properties and snapshot +context. No built-in `Document`, `EvidenceReference`, search-profile registry +or separate search endpoint is required. Typed stored queries provide concise +agent-facing recipes. + +The initial release covers analyzed Boolean matching, exact/fuzzy lexical +retrieval, exact `knn`, approximate `ann`, named fusion, graph-defined +populations, per-group selection, terminal aggregates and coherent source +reads. The [capability matrix](#query-capability-matrix) distinguishes this +scope from deferred operators. Cross-type, all-node and all-edge search are +deferred; their typed union/projection and source contracts remain required +language-evolution work, not an implied capability of same-type fusion. + +The pre-stable cutover deliberately breaks search queries and representation +declarations, with one coordinated format rebuild. The +[migration table](#user-facing-changes-and-migration) owns those consequences. + +**Phase 0 is incomplete.** There are checked logical plans, native probes and +a narrow archived compiler → engine → GQT integration. None establishes the +complete production feature; four recorded GQT regressions remain open. +[Evidence](#implementation-handoff-and-validation-checkpoint) distinguishes +those boundaries. The [phase handoffs](#implementation-phases) name the +remaining decisions, owners and required proof. Implementers must investigate +open claims rather than treating proposed syntax or passing probes as truth. +Frontmatter lists remaining design-acceptance gates; the +[release gates](#phase-0-acceptance-versus-release-qualification) retain full +production obligations. Moving a check to its implementation phase does not +waive it or make the current code correct. + +## Summary + +| Authority | Owns | +|---|---| +| Accepted schema | Analyzers, scoring defaults, vector geometry/space, source mapping and resolved encoding recipes | +| Typed query plan | Populations, targets, sources, windows, fusion, selection, metric origin and stage placement | +| Physical execution | Qualified Lance scans/indexes, DataFusion operators, existing graph traversal and shared accounting | + +Lance owns versioned datasets and indexes; OmniGraph owns graph semantics, +snapshot coordination and admission. DataFusion supplies relational operators +where qualified. Missing acceleration may change cost, never logical meaning. + +### Agent workload and design objective + +Optimize reliable task completion within latency, execution and working-context +budgets. An agent's context can contain exact computed facts, comparisons, +graph neighborhoods and supporting passages. An aggregate over retrieved +candidates describes that selected population; relevance is neither confidence +nor evidence that no other facts exist. + +| Workload | Required composition | +|---|---| +| Lookup and verification | Keys/logical identity → predicates → selective properties → coherent follow-up | +| Discovery | Graph scope → lexical/fuzzy/vector candidates → fusion → selection | +| Analytical investigation | Exact graph population → aggregate → select entities → retrieve evidence | +| Evidence assembly | Computed facts + optional relationships + locally ordered, bounded source lists | +| Adaptive investigation | Inspect, narrow/broaden, traverse and revisit identities at the declared snapshot | + +These requirements guide grammar evolution; they do not make deferred +operators available. An exact five-row answer can examine millions of records. +Streaming/spilling or explicit refusal must preserve that exact population. +Stored-query descriptions must explain recipe semantics so agents can choose +and adapt queries without reconstructing engine rules in client code. +[Mixed workload qualification](#mixed-workload-qualification) measures task +correctness, query repairs, round trips, context size and execution cost. + +## Motivation + +```text +Best 10 organizations within Project A + != best 10 organizations overall, then keep members of Project A + +Best 20 passages, then group by source + != select passages while enforcing a per-source quota +``` + +A later filter or reranker cannot recover a discarded candidate. Explicit +stage boundaries expose this information loss and allow exhaustive analytics +and bounded discovery to compose honestly. + +Schema-owned analysis and an exact baseline also fix the current index-state +divergence: indexed `beto` can match `beta` while identical appended values +are missed; `running` can disappear when indexing stores `run`. Boolean +matching and fuzzy retrieval therefore consume one typed lexical description. + +[RFC 0047](0047-search-plan-truth.md) supplies plan truth, projectable metrics +and complete boundaries. Its first-scan restriction and single retrieval field +are transitional implementation choices, not permanent language rules. + +## User and operational behavior + +### User-facing changes and migration + +This section describes the proposed combined RFC 0047/0048 release. It is a +draft migration contract, not a description of features already shipped. +Search queries and representation declarations change together in one +pre-stable release. Ordinary graph queries keep their language, but their +graphs still cross the storage-format upgrade described below. + +#### Breaking changes + +| Existing usage or expectation | Proposed change | What users must do | +|---|---|---| +| `fuzzy`, `search`, or `match_text` | These spellings are removed without compatibility aliases. Boolean matching and ranked lexical retrieval consume the same typed `terms` query. | Choose `match_terms` for filtering or `lexical` for retrieval; set term combination and edit tolerance deliberately. | +| `nearest`, retrieval expressions inside `order`, or positional RRF | Retrieval moves into explicit `rank` stages. Vector retrieval distinguishes exact `knn` from approximate `ann`; fusion names its inputs. | Rewrite inline and stored queries, choose exact versus approximate retrieval, and project named metrics instead of repeating retrieval expressions. | +| Vector candidate depth inherited from final `limit`, while BM25 fusion arms scan uncapped | Each source and fusion stage has its own candidate window. Final `limit` counts output rows; graph fan-out can produce several rows per selected target. Stage comparators determine selection before final ordering. | Choose source/fusion windows explicitly and review tie keys and expected row counts. A migration cannot infer the intended recall/cost tradeoff from the old limit. | +| `@index` or `@key` implicitly makes a String searchable by analyzed text | Analyzed matching requires `@analyzed`, which enables BM25 ranking by default. Exact key/index annotations keep their separate meaning. | Declare `@analyzed` on searchable text, choose another analyzer when needed, or explicitly opt out of ranking with `scorer="none"`. An exact-only slug does not need an analyzer. | +| Implicit vector geometry or an unresolved `@embed` model | A field's encoding recipe and geometry must resolve at schema acceptance. The model may inherit a schema-owned default recipe; distance may inherit that recipe's declared default. Raw vectors require explicit distance. | Declare the source and dimensions, then resolve a compatible recipe and geometry. A new default cannot identify how old vectors were produced; unresolved legacy vectors still need operator resolution or regeneration. | +| Existing search rows, scores, or ordering survive a spelling-only rewrite | `terms` defaults to all terms and zero edits. Analysis, complete fuzzy matching, and explicit selection boundaries can change results. Lexical ranking deduplicates query terms, groups fuzzy alternatives, and uses float64 scores with snapshot-visible field statistics independent of eligibility. | Review analyzer choices, fuzzy relevance, score thresholds, and tie fixtures. Repeating a query term no longer increases its weight. The rewrite does not promise equivalent results to legacy search. | +| Queries relying on silently ignored search constructs or permissive parameter handling | Invalid shapes, incompatible representations, token-empty queries, and exhausted budgets produce typed failures. A successful partial candidate set cannot stand in for an exact result. | Handle the declared errors and size queries explicitly; do not interpret a failure as an empty successful search. | +| Existing graph files open directly after the upgrade | The accepted-schema change requires an export/init/load rebuild. Compatible values and logical graph content are carried over; commit history, branches, and physical indexes are not preserved by that rebuild. | Plan the data upgrade even if no query uses search. Retain the predecessor graph if its history is needed, rebuild indexes explicitly, and obtain fresh snapshot references from the rebuilt graph. | + +#### Additive capabilities + +These capabilities extend the query language without requiring ordinary graph +queries to adopt ranking. Existing search queries still need the rewrites above. + +- Explicit exact vector retrieval and fuzzy lexical ranking, with the same + lexical matching definition available as a Boolean predicate. +- Short schema declarations with fully resolved semantics: bare `@analyzed` + enables ranking, and embedding fields can inherit one schema-owned default + recipe without repeating its model on each field. +- Named lexical/vector sources and weighted fusion, with each source's rank, + score or distance available for projection. Missing arm membership remains + distinguishable from a computed score. +- Ranking within a graph-defined population, further traversal of selected + targets, and `take` selection with per-group quotas. Local and final ordering + can explicitly place missing values with `nulls first` or `nulls last`; + omitting the modifier retains current ascending/descending defaults. +- Inspectable query definitions and plans, retrieval/coverage metadata, and + completion of snapshot-coherent selective reads through existing read and + stored-query facilities. Stable ranked pagination remains deferred. + +#### What remains unchanged + +- Ordinary graph matching, traversal, exact property predicates, aggregates, + projection, and non-retrieval ordering/limits retain their existing roles. + String equality, `starts_with`, and String `contains` remain exact and + case-sensitive; they require no analyzed declaration. +- Applications keep their own node, edge, and property model. Source texts, + passages, and their relationships remain application data; there is no + required `Document` type or `EvidenceReference` wrapper. +- Queries still use the existing query endpoint and typed parameter/stored-query + mechanisms. Stored-query bodies that use removed search constructs must be + updated, even though the invocation mechanism is retained. +- Graph commits, branches, coherent reads, and existing authorization retain + their contracts. Index maintenance remains explicit. These guarantees do not + imply preservation of old branches or snapshots across the format rebuild. + +#### HTTP, CLI, and SDK compatibility still to confirm + +Keeping the query endpoint does not by itself establish wire compatibility. +The combined release extends result metadata and coherent follow-up behavior; +its exact response fields, metric serialization, diagnostics, and CLI output +must be reviewed against API types, OpenAPI, and client parsers. In particular, +clients that reject unknown fields or parse human output may need updates. +RFC 0047's narrower additive-envelope and legacy `/read` commitments must be +reconciled at this boundary. Until that review, this RFC makes no blanket +claim that existing HTTP/SDK clients work unchanged. + +#### Migration sequence + +1. Review the schema and resolve analyzer, scoring, vector geometry, and + encoding choices before rebuilding data. +2. Rewrite application and stored queries against that schema, choosing + matching modes, exact/approximate retrieval, and candidate windows. +3. Use the existing export/init/load upgrade path, regenerate incompatible + representations, and reconcile indexes explicitly. +4. Validate rewritten queries, expected results, client response/error handling, + and snapshot follow-up before switching applications to the rebuilt graph. + +The release must include the schema/query migration diagnostics, updated +examples, user guides, and release notes. Detailed semantics and remaining +acceptance gates follow; the migration tools must not guess unresolved choices. + +### Schema and analyzers + +The annotations below are proposed syntax. The representation identity +requirements in Design also apply; a provider/model label alone is not a +complete embedding-space declaration. This example assumes the schema +declares a qualified default embedding recipe compatible with 1536 dimensions: + +```pg +node Organization { + slug: String @key // exact only + name: String @analyzed @index // matching and ranking + notes: String? @analyzed // matching and ranking + embedding: Vector(1536, distance="cosine")? + @embed("name") @index // inherits the recipe +} +``` + +#### Directive defaults and omission rules + +Defaults shorten the authored schema. They are resolved before acceptance +and persisted as field semantics, never looked up afresh during a read or +content write. Unknown arguments and unresolved required choices are errors. + +| Surface | Omitted value | Explicit choice or failure | +|---|---|---| +| `@analyzed` | No annotation means no analyzed capability; `@index` and `@key` do not supply it. | Exact predicates remain available without it. | +| `@analyzed.analyzer` | `standard_v1` | A named immutable profile such as `standard_folded_v1` or `english_v1`. | +| `@analyzed.scorer` | `bm25_v1` capability and default policy for exact or tolerant `terms` | `scorer="none"` permits analyzed filtering only; a lexical ranking source on that field is a typed error. | +| `@embed` source | No default | The source property is mandatory and resolves to its stable identity. | +| `Vector` dimensions | No default | A positive dimension is mandatory and must be compatible with the resolved encoding recipe when present. | +| `@embed.model` | The schema's declared default embedding recipe | Without that default, require an explicit model that resolves to a complete qualified recipe. A model label alone is insufficient. | +| `Vector.distance` on an `@embed` field | The selected recipe's declared default distance | An explicit field distance takes precedence. If neither provides one, schema acceptance fails. Persist and validate the resulting geometry. | +| `Vector.distance` without `@embed` | No default | Require explicit `l2`, `cosine`, or `dot`; the graph's embedding default does not assign geometry to raw vectors. | +| Embedding normalization and query/document roles | The selected recipe's resolved choices | There is no global normalization fallback. An incomplete recipe fails acceptance. | + +Thus bare `@analyzed` expands to +`@analyzed(analyzer="standard_v1", scorer="bm25_v1")`. +The initial scoring capability supplies one default policy, `bm25_v1`. +`lexical` may omit `scoring`; spelling `scoring: bm25_v1` explicitly is +equivalent. Exact and fuzzy search use that same policy, with edit tolerance +specified once in `terms`. There is no separate `fuzzy_bm25_v1` selector. +This default does not enable edit tolerance, execute ranking, create an +index, or remove the need for the policy's qualification. An explicit +`@analyzed(scorer="none")` retains the former matching-only use case. + +The graph schema may declare one default embedding recipe in its accepted +metadata. The recipe identifies the model revision, compatible query/record +encoding behavior, dimension constraints, and any default distance. Its exact +top-level `.pg` declaration is a parser/typechecker acceptance gate; the +inheritance rules above do not depend on the final spelling. It is not a +deployment setting or a separately managed retrieval-profile registry. +An explicit field model selects its own complete qualified recipe; it cannot +borrow unrelated normalization or role settings from the schema default. + +`schema plan` must show each field's resolved analyzer, scoring capability, +model/recipe identity, normalization, and geometry, including inherited values. +Exports must carry the schema default and enough resolved field information +to reproduce those bindings without the original deployment's defaults. +Changes to a schema default must +expose any proposed field rebinding as a semantic migration, subject to the +existing refusal/rebuild rules. A changed runtime provider configuration or a +no-op reapplication cannot silently rebind an accepted field. + +**Default lifetime decision:** a default supplies a recipe when a field first +acquires an embedding binding. Changing or removing the schema default alone +does not rebind existing fields; it affects newly introduced bindings. Rebinding +an existing field requires an explicit field recipe change, visible in +`schema plan`. An unrelated property edit and a no-op apply retain the accepted +recipe. Drop/re-add creates a new field lifetime and resolves the current +default. Canonical export spells out every accepted field's resolved choice, +including previously inherited choices, so reinitialization does not require +the old deployment's defaults. This rule requires accepted-state-aware schema +resolution; parsing the desired source alone cannot implement it. + +#### Analyzer profiles + +Initial immutable analyzer profiles resolve these settings explicitly; +unspecified Lance defaults are never part of the schema contract: + +| Setting | `standard_v1` (default) | `standard_folded_v1` | `english_v1` | +|---|---|---|---| +| Document / base tokenizer | Text / `simple` | Text / `simple` | Text / `simple` | +| Language setting | English (inactive) | English (inactive) | English | +| Lowercase | Yes | Yes | Yes | +| Stemming / stop words | Neither | Neither | English stemmer, then built-in English stop words | +| ASCII folding | No | Yes | Yes, after stemming and stop words | +| Token-length filter | Disabled (`max_token_length: None`) | Disabled | Disabled | +| Unicode normalization | NFC before tokenization; no NFKC | Same | Same | + +All three profiles first apply NFC to lexical analysis input, then run the +pinned `simple` tokenizer and Lance's filter order: lowercase, optional +stemmer, optional stop words, optional ASCII folding. Scalar String fields use +row document granularity; positions do not affect membership. These profiles +use no custom stop words, external dictionaries, n-grams, or code-tokenizer flags. +Posting positions and block layout remain derived index settings. + +NFC makes canonically equivalent spellings reach tokenization as the same +string while retaining distinctions that compatibility normalization can +erase. See the [Unicode normalization specification](https://www.unicode.org/reports/tr15/). +This preprocessing is part of the proposed analyzer, not a claim about the +unmodified Lance tokenizer. Query, scan, and index construction must use the +same qualified pipeline, with normalization work charged to the query/build +budget. A native path without that proof stays disabled. Stored String values +and exact predicates are unchanged; embedding input preprocessing remains +owned by its encoding recipe and does not inherit lexical normalization. + +Disabling the token-length filter is deliberate. Pinned Lance defaults to +`Some(40)`, whose filter retains only tokens shorter than 40 **UTF-8 bytes**, +before lowercasing or folding. Resource limits must reject excessive work, +not silently erase a long name or query term. Thus `english_v1` retains +the pinned linguistic filter choices but intentionally changes normalization +and long-token matching at this breaking boundary. + +`simple` splits at non-alphanumeric Unicode scalar values. Lowercasing is +not full case folding, and ASCII folding is distinct from Unicode +normalization. NFC preprocessing makes composed `résumé` and its canonically +equivalent decomposed spelling analyze identically. Accent removal is still +opt-in: with `english_v1`, `résumé` and `resume` become `resume` and +`resum`, respectively, because stemming precedes folding. These limitations +are explicit profile behavior, not promises of language-independent typo +equivalence. Once a profile is accepted, changing segmentation, normalization, +or filter order requires a new profile. Fingerprints include the normalizer +implementation and Unicode data identity as well as the Rust Unicode behavior +used by `simple` and lowercase. These are revised, unshipped `v1` definitions; +the earlier no-NFC probe does not qualify them. Adding a profile or scorer +version requires an RFC; accepted identities are never mutated. +Query-time analyzer or vector-distance overrides do not exist. A lexical +source resolves a versioned scoring policy compatible with the field's +declared capability, using its accepted default when omitted. That resolution +does not change analysis. The first release defines only `bm25_v1`; additional +policies need explicit semantics and qualification rather than runtime aliases. + +For spelling tolerance on names and titles, the non-stemming profiles keep +edit distance close to the spelling the user supplied. Under `english_v1`, +distance is measured after stemming; a one-character typo in the original +word need not remain one edit after analysis. The field's declared profile +decides this for every query and execution path. + +### One lexical query, two consumers + +A typed lexical query describes matching independently of its consumer. +The initial variant is `Terms { text, mode, max_edits }`; the grammar sketch +spells it `terms($q, mode: all, max_edits: 1)`. `mode` defaults to `all` and +`max_edits` to zero in both consumers. Candidate-oriented stored queries +usually choose `mode: any` explicitly. Phrase, prefix, and Boolean query +composition are future typed variants, not embedded vendor query strings. + +`match_terms(field, terms(...))` is a Boolean predicate on a scalar String +property with `@analyzed`. A `lexical` source consumes the same description +to select and rank matches with a resolved scoring policy. The Boolean +predicate introduces no score, ranking, candidate window, or implicit +retrieval. Its contract is: + +| Aspect | Rule | +|---|---| +| Query text | A non-null String literal or query parameter, constant for one execution; row-dependent query text is rejected. | +| Analysis | Apply the same resolved field analyzer to document and query text, at every edit budget including zero. | +| Distance | Minimum insertions, deletions, and substitutions over analyzed Unicode scalar values; each costs one. An adjacent transposition costs two. There is no implicit prefix restriction or length-based automatic tolerance. | +| `max_edits` | Optional integer literal or query parameter in `0..=2`, default `0`. Check literals at compile time and bound parameters before execution; reject negative, oversized, and non-integer values without narrowing casts. | +| `mode` | `all` by default: every analyzed query term has a document term within the budget. `any`: at least one does. | +| Term identity | Repeated query terms do not require repeated occurrences; a document term can satisfy multiple query terms. Reordering analyzed terms cannot change membership. Matching is neither phrase matching nor distance over the whole field value. | +| Empty text | A query yielding no searchable terms is a typed error. A null or token-empty document does not match. | +| Completeness | Every successful result satisfies the predicate exactly; an edit-tolerant predicate does not advertise approximate recall. | + +For a fixed schema, query, and document population, these are normative laws: + +```text +matches(edits=0) ⊆ matches(edits=1) ⊆ matches(edits=2) +matches(indexed) = matches(unindexed) = matches(partially indexed) +``` + +Adding an analyzed query term cannot widen `mode: all`. Appending unrelated +documents cannot remove existing predicate matches. A resource failure is a +typed query failure, not an empty or truncated successful matched set. + +### Clause composition and named stages + +The current grammar has one `match`, followed by `return`, optional `order`, +and optional `limit`. The proposed extension admits explicit rank boundaries +between graph blocks. Each `match` retains graph-pattern semantics. Stage +aliases are scoped plan names, not ordinary variables that hold relations. +The following proposed syntax is exercised by the test-only compiler +prototype described below. The production parser still rejects rank stages: + +```gq +query find_organizations($q: String) + @description("Find organizations by name or meaning") { + match { + $o: Organization + } + rank $o { + lexical($o.name, terms($q, mode: any, max_edits: 1), + candidates: 100) as words + ann($o.embedding, $q, oversample: 4, candidates: 100) as meaning + rrf(arm(words), arm(meaning, weight: 1.5), + k: 60, candidates: 20) as combined + yield combined + } + return { + $o.slug, + $o.name, + metric(combined, score) as score, + metric(words, rank) as lexical_rank + } + order { score desc, $o.@id asc } + limit 8 +} +``` + +This sketch selects distinct `$o` identities. The source declarations read +the rank block's incoming eligible population; a fusion declaration reads its +named preceding inputs **within the same rank block**. The final declaration +does not implicitly become the block output. Exactly one terminal +`yield ` selects a source declared in this block, even for a one-source +block. Adding or reordering independent declarations does not change that +selection. Missing, repeated, unknown and earlier-block outputs are errors. +Source aliases are unique across the query; duplicate +aliases, duplicate arm references, and forward references are errors. +Referencing an earlier block's arm in a new fusion is rejected: that arm's +population precedes any intervening graph filter, and reusing its candidates +could restore targets the filter removed. Cross-block candidate reuse needs +an explicit bounded intersection/remapping contract; a common target type +alone does not establish population compatibility. + +`arm(name)` resolves a current-block source, while `metric(name, field)` can +read an earlier source's metric with its original target and population. +Neither resolves a result alias. A bare name in final `order` resolves a +projected result alias; `$name` remains a graph binding or query parameter. +A result alias and a source alias may share a spelling because these syntactic +positions distinguish them. Aliasing a metric must preserve its domain and +source-instance identity, not just its underlying numeric type. +`$o.@id` follows RFC 0040's system-field namespace; query-specific scores are +not node properties or new system fields. These constructors do not imply +arbitrary function calls, closures, or relation-valued transport parameters. + +Here, `candidates: 100` bounds each source, `candidates: 20` bounds fusion, +and `limit 8` bounds final output rows. `oversample` controls physical ANN +effort. Changing projection or final limit does not resize the source windows. +Final `order` orders surviving rows; it cannot change an earlier cutoff. +Each ranked stage has its own total comparator, ending in stable target +identity. Any custom selection tie keys must belong to that stage, not to a +later row-order clause. + +Further graph composition has this shape (schematic, not current syntax): + +```text +match { bind projects and their organizations; filter project } +rank organization { select organization candidates } +match { follow selected organizations to related incidents } +rank incident { select incident candidates } +return { organization identity, incident identity, selected properties } +order { final output keys } +limit final row count +``` + +All rank-block inputs and targets must already be bound. The target must be a +named node or edge binding; anonymous `$_` cannot name a reusable rank target. +Searching a traversal-introduced target is supported through its graph-defined +eligible population; making it the first textual declaration is not required. A filter +in the second graph block filters the selected organizations or expanded rows; +it does not retroactively change the first search population. Policy applies +before selection and at every source read and expansion. + +Current Cedar enforcement admits graph/branch reads and stored-query +invocations; it is not a row- or field-filtering policy engine. In this RFC, +an authorized population means the graph query's eligible targets after those +existing gates. Row-level security is not introduced here. If such predicates +are added later, they must constrain selection, statistics, and metadata +before retrieval, rather than removing denied hits after top-k. + +The initial lexical/vector sources read a property of the declared target. +A source cannot silently rank another binding or interpret one query vector +as a batch of per-row queries. Correlated per-target retrieval requires a +separate bounded operator contract; it is not implicit in repeated `match`. + +Analyzed filtering alone remains an ordinary match predicate: + +```gq +query organization_names($q: String) { + match { + $o: Organization + match_terms($o.name, terms($q, mode: all, max_edits: 1)) + } + return { $o.slug, $o.name } + order { $o.slug asc } +} +``` + +No adjacent source inherits that edit budget. `lexical` with exact terms +continues to select only its own exact matches; fuzzy retrieval must consume +an explicitly tolerant query. Both resolve the same default scoring policy. + +### Language evolution and compatibility + +Extend one typed stage sequence followed by `return`, optional final `order` +and `limit`. Graph patterns remain declarative within `match`; source text +does not prescribe a physical scan/join schedule. The production single-match +grammar and test-only staged grammar are not the complete stable language. + +Use one expression model for properties, parameters, literals, aliases and +metrics wherever their types/scopes permit them. Keep graph patterns and stages +as dedicated nodes; a call-shaped retriever is not a scalar. Unknown +constructors/options fail. Fix precedence, associativity, null rules, name +resolution and positional/named arguments before stabilization. Scalar +negation must not inherit correlated graph `not` semantics; score arithmetic +retains explicit domain policies. The following Phase 0 expression decisions +apply when the corresponding operators are implemented: + +| Boundary | Decision | +|---|---| +| Precedence | Parentheses/calls, then `*` and `/`, then `+` and `-`, then one comparison, then scalar `not(...)`, then `and`, then `or`. Arithmetic at one level associates left; chained comparisons fail. Graph `not { ... }` retains its separate scope. | +| Numeric types | Arithmetic requires the same scalar numeric type; no implicit widening, integer/float mixing or conversion of metrics to numbers. Integer division remains unavailable until an explicit quotient/rounding contract; future casts must be named. | +| Null values | Arithmetic/comparison with null yields null. Boolean composition uses three-valued logic: false dominates `and`, true dominates `or`, otherwise an unknown operand yields null; `not(null)` is null. `is_null` returns non-null Bool; filters and `count_if` accept only true. | +| Numeric errors | Integer overflow/underflow is a typed failure, never wrapping, saturation or null. New floating arithmetic rejects non-finite operands/results and division by either signed zero. Null operands yield null without applying that arithmetic operation. These rules do not retroactively redefine stored scalar values or existing aggregate signatures. | +| Evaluation | Operands have no promised left-to-right or short-circuit evaluation. Boolean guards cannot make an invalid arithmetic expression safe; future conditional/try expressions need explicit evaluation rules. Constant folding must use the same types, null and error rules as execution. | +| Aliases | Sibling `let`/return expressions read the incoming scope; they cannot read an alias declared beside them. A later stage can read exported values. Final ordering can read output aliases. | + +The staged compiler checks precedence, same-type arithmetic and nullable +results. The native `staged_composition_numeric_and_null_contracts` probe +checks all nine Boolean pairs and exposes two adapter obligations: +DataFusion 54's default integer `BinaryExpr` wraps; its explicit +`with_fail_on_overflow(true)` path refuses the tested I64 overflows. That +switch still permits floating infinity/NaN, so finite arithmetic needs a +checked expression adapter. This is a physical-expression proof, not proof +that logical optimization, aggregate accumulators or GQ lowering preserve +these rules. Those remain qualification gates; a session default is +insufficient. See the pinned [binary evaluator](https://docs.rs/crate/datafusion-physical-expr/54.0.0/source/src/expressions/binary.rs) +and [Arrow arithmetic](https://docs.rs/crate/arrow-arith/58.3.0/source/src/numeric.rs). + +Every operator declares its scope and population effect: + +| Operation | Binding/identity rule | +|---|---| +| Predicate / ordinary projection | Preserve rows, duplicates, bindings and metric origins | +| Traversal | Extend bindings with existing endpoint/edge-instance semantics; bounded reachability does not become path enumeration | +| `rank` | Select distinct targets and retain their incoming binding rows | +| `take` | Select target/group pairs; it does not create reusable group features | +| Group/reduction | Export keys and reduced values, discard unreduced member bindings/metrics | +| Future optional/branch/collection | Explicit imports/exports, nullability, multiplicity and local scope | + +Terminal aggregate-return shorthand retains its existing implicit grouping: +adding a non-aggregate projection can change the groups. Future intermediate +grouping uses the same equality/null/duplicate rules with explicit exports; +ordinary intermediate projection must not acquire that implicit reduction. + +Each rank block ends with `yield `, resolving a declaration in that +block. Sources share the incoming population and explicit fusion dependencies. +Adding an unused declaration cannot select a different output or comparator; +every declaration is still validated and requested metrics consume resources. +Earlier metrics retain their original scope; reusing earlier candidates needs +explicit remapping/intersection. + +Keywords are contextual and token-bounded (`yield yield` can select an alias +named `yield`). Preserve property, system and source namespaces: +`$binding.property`, `$binding.@id` and explicit metric references. +New nested scopes must not capture unrelated bindings implicitly. Future +user-defined namespaces require collision rules. + +Admit count literals/parameters consistently while retaining their units: +positive source-target windows, nonnegative group-pair quotas and final +binding-row limits. Extending final-limit parameters is additive, not row-dependent +query batching. Physical effort, memory, I/O and output bytes stay separate. + +After the coordinated cutover, additive syntax must preserve valid queries' +parsing/name resolution, typing, populations, metric meaning and result schema +under the same accepted identities. Exact results stay exact; ANN preserves +its approximation/effort contract rather than a fixed candidate set. +All-node expansion after a schema change follows its explicit scope contract. + +Changing an analyzer, scorer, missing-value rule, default window, encoding +recipe or ANN mapping is a semantic change even if queries still parse. +Version policies/accepted bindings and preserve resolved choices. Unavoidable +future breaks require an explicit language/format boundary and migration. + +Before freezing syntax, qualify [C1–C4](#required-composition-examples) and a +compatibility corpus covering contextual identifiers, mixed/null expressions, +alias collisions, unused sources, projection/window independence, both sides +of cuts and discarded group/union metrics. Deferred operators need coherent +type/plan proofs now and executable GQT/resource tests when implemented. +Stored queries describe supported combinations; they cannot replace missing +language operators with unbounded client-side computation. + +### Query capability matrix + +This inventory covers the current public `.gq` surface, the initial release +described by this RFC, and the general graph/query/retrieval extensions the +grammar must leave room for. It separates language support from implementation +evidence; parser acceptance and native substrate support alone are insufficient. +It is a review checklist, not a promise to implement every conceivable query +operator or every future row in the initial release. + +**Current** describes the production path in this PR's source baseline, not +the test-only staged parser or the archived integration prototype. `Limited` +means a narrower capability or a known correctness gap, described in the cell. +The four [GQT search regressions](#ci-checkpoint-and-regression-disposition) +remain open. **RFC release** uses these dispositions: + +- `Keep`: preserve the existing operation and its meaning. +- `Deliver`: required for the coordinated release, subject to its stated gates. +- `Replace`: an existing surface migrates to the new contract at the cutover. +- `Foundation`: fix the shared grammar/type/scope rules now; the broader + user-facing operator remains deferred. +- `Defer`: outside the initial implementation; preserve the stated extension. +- `Separate`: an adjacent capability governed by its own contract, not a new + search feature. + +The last column states what can be added later and what the syntax must +preserve. Examples such as `yield`, `group`, `collect` and nested object +construction discussed during design are illustrative; this matrix does not +accept their spelling, introduce a general scalar-variable declaration, or +change the tested RFC query examples. + +#### Declarations, expressions and scope + +| Capability | Current | RFC release | Future extension / syntax constraint | +|---|---|---|---| +| Named queries; multiple query declarations per file | Yes | Keep | Query-body extensions retain the declaration and invocation model. | +| Typed required and nullable parameters | Yes; scalar, list and vector forms; nullable omission accepted | Keep | Relation/object parameters would need separate type and transport contracts; a list is not implicit query batching. | +| Stored-query tool metadata | Yes; `@description` and `@instruction` | Keep | Metadata describes the same query; it does not carry hidden execution semantics. | +| Literals, parameters, property values, `date`, `datetime`, `now` | Yes, with current context restrictions | Keep | New expression variants preserve existing types, time semantics and literal parsing. | +| One shared expression model across stages | Limited; fixed expression variants and context-specific handling | Foundation | Deliver shared parsing/typing for admitted expressions, including metrics; this does not deliver every scalar function below. | +| Scalar comparison, exact String prefix/substring and list membership | Yes | Keep | Exact String behavior remains case-sensitive; `contains` on a list remains membership. | +| General scalar Boolean composition and negation | Limited; match conditions conjoin and graph `not` tests pattern absence | Foundation | Add typed Boolean expressions with fixed precedence and null rules; graph absence keeps its correlation semantics. | +| Explicit null tests/replacement, conditionals, arithmetic, casts and general scalar functions | No general surface | Foundation | Extend shared expressions through typed signatures; preserve overflow, null and score-domain rules. | +| Intermediate named computations/projection | No | Foundation | Fix explicit value bindings and retained/exported scope through composition example C1; operator implementation remains deferred. | +| Result aliases used in final ordering | Yes | Keep | Result aliases remain distinct from graph parameters/bindings and source aliases. | +| Reusing an earlier result alias in another projection | No; production rejects `T36` | Foundation | Decide alias scope explicitly before relaxing this rule; shared expression parsing alone is not alias reuse. | +| Nested scopes, reusable query branches and bounded subqueries | Limited; correlated graph negation only | Defer | Declare imports, exports, shadowing, snapshot and shared budgets; distinguish reuse from re-execution. | +| User-defined functions or query-valued parameters | No | Separate | If justified later, define typed namespaces, versioning and execution limits; arbitrary evaluation is not an initial requirement. | + +#### Graph matching and identity + +| Capability | Current | RFC release | Future extension / syntax constraint | +|---|---|---|---| +| Concrete typed node scans and inline property constraints | Yes | Keep | New type selectors must not reinterpret an existing concrete type. | +| Directed traversal and binding joins within `match` | Yes | Keep | Preserve endpoint-pair versus edge-instance multiplicity and graph eligibility. | +| Undirected traversal | Yes, between the same endpoint type | Keep | Preserve orientation and duplicate rules when extending edge-pattern syntax. | +| Single-hop edge bindings and edge-property access | Yes | Keep | Node and edge kinds remain distinct even when names or ids coincide. | +| Bounded multi-hop reachability | Yes; shortest-distance semantics, no bound path/edge sequence | Keep | A future path value or path enumeration needs an explicit operator and bounds; it cannot redefine existing hop syntax. | +| Anonymous bindings and repeated-binding constraints | Yes | Keep | Anonymous bindings cannot name reusable rank/take targets; existing bindings cannot change type through rebinding. | +| Correlated pattern absence | Yes; `not { ... }` | Keep | Inner bindings do not escape; future scalar negation does not acquire this scope behavior. | +| Positive existence without outer-row fan-out | No dedicated positive-existence construct | Defer | Add a correlated pattern predicate; ordinary traversal continues to expose its matches. | +| Optional graph enrichment | No | Foundation | Example C4 must preserve unmatched inputs and nullable exports; distinguish inner predicates from later filters. Multiple matches may still fan out; implementation remains deferred. | +| Graph matching before, between and after retrieval cuts | Limited; one `match` and terminal search ordering | Deliver | Repeated stages retain their input population; later filters cannot move before an earlier selection cut. | +| Logical system identity projection and filtering | Limited; whole-node objects expose `id`, but `.id` is an ordinary declared property lookup | Deliver with RFC 0040 | Qualify `$binding.@id` and type/incarnation-aware follow-up; never expose native row IDs as graph identity. | +| Explicit unions of typed graph bindings | No; interfaces do not provide polymorphic query scans | Foundation | Execution deferred. Preserve typed branches, explicit exports, bag/set semantics and original entity identity. | +| All-node discovery, representation expansion and type narrowing | No | Foundation | Execution deferred. Resolve type scope separately from searchable representations at one snapshot; common fields and type-specific projections must typecheck. | +| Global all-edge or mixed node/edge discovery | No; a concrete edge can be reached through traversal | Defer | Make entity kind and selected scope explicit; an all-node selector must not silently start including edges. | +| Cross-graph federation | No combined `.gq` population; graph selection is external | Separate | Requires explicit authority, identity, snapshot and budget rules; global search in this RFC means one graph. | + +#### Projection and result shape + +| Capability | Current | RFC release | Future extension / syntax constraint | +|---|---|---|---| +| Flat projection of properties, parameters and literals | Yes | Keep | Ordinary projection preserves binding rows, duplicates and candidate windows. | +| Explicit column aliases and unique output names | Yes; duplicate names rejected; some unaliased inferred/executed names still differ | Keep; qualify schema agreement | Freeze naming and result-schema agreement; extra syntax must not capture existing aliases. | +| Whole-node object projection | Yes; id and properties except Blob/Vector | Keep | This remains schema-shaped shorthand; explicit fields provide stable compact results across schema additions. | +| Bare edge object projection | No; project edge properties explicitly | Defer | Define an edge object/reference type and endpoints deliberately; do not assume node-object behavior. | +| Existing list and vector property projection | Yes; each remains one column value | Keep | Projection does not unnest a list or execute vector retrieval. | +| Blob values in ordinary read projection | No; dedicated Blob API | Separate | Preserve the Blob access/resource contract; object shorthand must not fetch Blob contents. | +| Source score, distance, rank and fused-score projection | Limited; only a repeated leading `nearest`/`bm25` expression can expose its metric | Replace | Named `metric(source, field)` preserves domain, origin and missing membership through aliases and later stages. | +| Selected nested object construction | No; whole-node objects are the only node-object shorthand | Foundation | Example C4 fixes typed row reshaping separately from traversal/grouping/collection; constructor implementation remains deferred. | +| New list/object construction from expressions | Limited; list literals and existing list values | Defer | Define element/field types and nullability; constructing a value is separate from collecting rows. | +| List unnesting, mapping or comprehensions | No general surface | Defer | Unnest is an explicit population-changing operation; local value mapping needs its own typed scope and bounds. | +| Nested related-entity/evidence collections | No | Foundation | Example C4 fixes correlation, exports, duplicate and empty-result rules, local order, item limit and shared byte/work budgets; implementation remains deferred. | +| Snippets, highlights and source ranges | No dedicated query construct | Defer | Preserve source property/version, offset unit and bounds; generated text must remain distinguishable from stored source. | +| Token-budget result packing | No | Defer | Declare tokenizer, selection policy, attribution and completeness; row limits must not become token limits. | + +#### Aggregation, selection and ordering + +| Capability | Current | RFC release | Future extension / syntax constraint | +|---|---|---|---| +| Terminal `count`, `sum`, `avg`, `min`, `max` | Yes | Keep | Preserve admitted input types, null behavior and duplicates; `count($node)` counts binding rows. | +| Implicit grouping by non-aggregate return values | Yes, with current type restrictions | Keep | Adding a group projection can change row count; do not give ordinary intermediate projection this implicit effect. | +| Grouped metric projection and explicit metric reductions | No staged metrics; legacy search/aggregate combinations restricted | Deliver | Projected metrics can be group keys or explicit reductions; discarded source metrics cannot order a new group. | +| Reusable intermediate grouping/reduction | No | Foundation | Examples C1–C2 export group keys/entities and reduced values; drop unreduced member bindings and active ranks. Group-stage implementation remains deferred. | +| General distinct rows/entities and distinct aggregates | No dedicated syntax | Defer | Specify the identity/value tuple and duplicate equivalence; retrieval deduplication is not general `DISTINCT`. | +| Distinct-target ranking with retained binding rows | Limited; legacy search exists without the new stage contract | Deliver | Each target gets one source rank; selecting it retains its associated binding rows. | +| Quotas per explicit group, including ordinary non-search selection | No | Deliver through `take` | Select target/group pairs; use explicit reductions when order varies per pair; require local order without an active rank. | +| Parent selection using reduced child metrics | Limited; terminal aggregates cannot feed a quota | Deliver through `take` reductions | Selection does not create reusable group features or implicit parent-source ranks. | +| Top-N retrieval within every full group; row-dependent query inputs | No | Foundation | Examples C1/C4 require explicit correlation/partition scope, empty-group behavior and one total budget; implementation remains deferred. Global top-K plus `take` is different. | +| Final ordering by properties/aliases and identity tie-breaks | Yes, with documented legacy search boundary-tie gaps | Keep; repair retrieval ties | Final ordering cannot change a previous source cutoff; each selection owns its complete comparator. | +| Explicit `nulls first` / `nulls last` | No syntax; fixed ascending/descending defaults | Deliver | Apply consistently in local and final ordering; omission preserves existing defaults. | +| Final row limit, including zero | Yes; integer literal | Keep | Counts output rows; it does not resize retrieval or bound all intermediate work. | +| Parameterized final row limit | No | Foundation | An additive count-parameter extension remains possible; this matrix does not add it to the initial release. | +| Parameterized source windows and group quotas | No corresponding stage syntax | Deliver | Admit integers before execution; positive source windows and nonnegative quotas have different units. | +| General analytic windows, running aggregates and partition ranks | No public surface | Defer | Typed stage/expression rules must define partitions, frames and order; internal window use does not expose a language feature. | +| Stable ranked cursors/pagination | No replayable ranked-execution contract | Defer | Preserve actual candidate order or qualify reproducibility, retention and policy; snapshot identity alone is insufficient. | + +#### Retrieval and ranking + +| Capability | Current | RFC release | Future extension / syntax constraint | +|---|---|---|---| +| Analyzed lexical membership | Limited; `search`/`match_text` have known execution-path inconsistencies | Replace | One typed `terms` description consumed by `match_terms`; filtering introduces no rank/window. | +| Edit-tolerant lexical membership | Limited; `fuzzy` exists with analyzer/coverage defects | Replace | Shared analyzed edit semantics, explicit `mode` and `max_edits`, complete membership across index states. | +| Exact-term lexical ranking | Limited; legacy `bm25` exists | Replace | Explicit bounded `lexical` source with qualified `bm25_v1` statistics, numeric rules and ties. | +| Fuzzy lexical ranking | No unified ranked tolerant-query contract | Deliver | `lexical` consumes tolerant `terms`; no separate fuzzy scorer or inherited edit budget. | +| Exact vector top-K as an explicit user choice | No exactness selector; some physical paths scan exactly | Deliver through `knn` | Exact top-K over eligible valid vectors; an index is an acceleration choice. | +| Approximate vector top-K | Yes through legacy `nearest`, with implicit effort/window behavior | Replace with `ann` | Explicit approximation and bounded effort contract; exact rescoring does not imply complete recall. | +| String or raw-vector retrieval inputs | Yes | Keep with resolved encoding | Validate dimensions and compatible encoders; inputs are constant per execution, not implicit per-row batches. | +| Named sources, explicit candidate windows and explicit block output | No | Deliver | Sources share the incoming target population; select output independently of declaration order before freezing syntax. | +| Weighted multi-source rank fusion | Limited; legacy RRF has two inline arms and asymmetric windows | Replace | Named 2–16-arm RRF, explicit weights/windows, checked arithmetic and one vote per target per arm. | +| Search a traversal-introduced node or a bound edge | Limited; node search shapes have gaps and edge search is rejected | Deliver | Every source targets a property of the declared eligible binding; do not require a textual scan root. | +| Search, traverse and search again | No explicit staged surface | Deliver | Each stage retains population and metric origin; earlier candidates cannot be reopened implicitly. | +| Multiple fields on the same target as separate sources | Limited by legacy inline source forms | Deliver | Named sources may select different compatible field capabilities; fusion remains explicit. | +| One logical multi-field/cross-type lexical corpus | No unified contract | Defer | Specify field reduction and shared live statistics before candidate cuts; per-table BM25 values are not globally comparable by default. Multiple named fields of one target type can still use explicit fusion. | +| Phrase, token-prefix, proximity and Boolean lexical queries | No portable typed contract | Defer | Add `LexicalQuery` variants shared by matching/ranking consumers; avoid a vendor query-string sublanguage. | +| Geometric range retrieval and exact distance predicates | No dedicated public contract | Defer | Distinguish a complete eligible-population range query from a filter on ANN candidates. | +| Candidate rescoring, learned reranking and general feature combination | No general stage | Foundation | Example C3 fixes membership-preserving scoring, domains, statistics/normalization populations and resource behavior. Scoring/model/formula operators remain deferred. | +| General cross-identity fusion, such as passages with parent sources | No | Defer | Require graph mapping, reduction and fresh target ranks; matching id strings do not establish compatibility. | +| Semantic diversification / complementary evidence selection | No | Defer | Add a set-selection objective and qualification; per-group quotas alone do not supply it. | + +#### Representation and execution contracts + +These rows are part of a query's meaning or execution interface, even when +their authored syntax belongs to `.pg` or request options rather than `.gq`. + +| Capability | Current | RFC release | Future extension / syntax constraint | +|---|---|---|---| +| Analyzed String capability with immutable analyzer/scorer defaults | Limited; legacy FTS/index configuration and compatibility proofs | Deliver | Resolve `@analyzed` independently of `@index`; accepted semantics cannot follow mutable runtime defaults. | +| Dense vector fields and embedding-source declarations | Yes; existing `Vector`/`@embed` | Replace unresolved semantics | Resolve dimension, geometry, source mapping and compatible immutable encoding recipes; qualify providers. | +| Schema-owned default embedding recipe and reproducible exports | No complete resolved-default contract | Deliver | Final declaration spelling remains open; omission, explicit override and migration preserve accepted field bindings. | +| Sparse vectors, multivectors/late interaction and named analyzed views | No retrieval surface | Defer | Extend typed representation/capability variants; do not define every retriever input permanently as one dense vector. | +| Additional modalities or external retrieval sources | No general query source | Separate | Any later source needs explicit typing, authority, coherence and budget guarantees; no universal content object is introduced. | +| Coherent branch/snapshot reads | Yes through existing execution interfaces | Keep | Use the graph's accepted snapshot and existing authority; stage syntax must not reopen a fresh view. | +| Snapshot-coherent identity follow-up through all read transports | Limited; snapshot fields exist, system-id lookup and output paths have gaps | Deliver | Complete RFC 0040 lookup and JSON/JSONL continuation; unavailable/expired identity is explicit. | +| Inline/stored query invocation and typed result descriptors | Yes; result-schema agreement has known gaps | Keep and qualify staged reads | Derive reads, result shape and fingerprints from every stage; aliases and hidden demand must agree with execution. | +| Inspectable staged plans, metric origins and selection descriptors | No complete staged contract | Deliver | Expose resolved sources, inputs and semantics through the existing query interface; keep transport naming separate from graph properties. | +| Representation coverage, including explicitly requested exact counts | No complete source-level contract | Deliver | Known/unknown readiness differs from completion and index coverage; exact counts share the query budget. | +| Whole-query resource admission, cancellation and output accounting | Limited; existing local bounds do not establish the RFC guarantee | Deliver | All stages, encoders, fallbacks, nested work and projection share one execution context; no silent truncation. | +| Graph/branch and stored-query authorization | Yes | Keep | All source reads and graph expansions obey existing gates; a snapshot is not a retained-access capability. | +| Row/field-level security predicates | No policy engine for this scope | Separate | If added, constrain eligibility, statistics and metadata before selection; post-top-K filtering is insufficient. | + +#### Adjacent `.gq` operations + +| Capability | Current | RFC release | Future extension / syntax constraint | +|---|---|---|---| +| Named insert/update/delete query bodies | Yes, with current target and constructive/destructive restrictions | Keep | Read stages remain separate from mutation bodies; no implicit search-to-write pipeline. | +| Several mutation statements in one graph publication | Yes within admitted mutation forms | Keep | A future write extension must preserve the single publication and recovery contracts. | +| Standalone branch create/delete/merge/list statements | Yes; one per file, not beside query declarations | Keep | These control operations do not become data-query stages or typed unions. | +| Conditional writes after a coherent read | Yes through the execution interface | Keep | Preserve the returned graph commit precondition; do not infer a new transaction from projection syntax. | + +#### Syntax decisions needed now + +Every `Deliver`/`Replace` row needs accepted syntax and typing/lowering. +`Foundation` requires C1–C4 and global-extension design proofs now while its +broader implementation stays deferred. +All extensions follow the [language rules](#language-evolution-and-compatibility). + +Current-state evidence is the production +[grammar](../../crates/omnigraph-compiler/src/query/query.pest), +[typechecker](../../crates/omnigraph-compiler/src/query/typecheck.rs) and +[projection executor](../../crates/omnigraph/src/exec/projection.rs). +The query guide's earlier-alias-reuse claim is stale: `T36` and the +[score-projection case](../../crates/omnigraph-gqt/cases/issue_640_search_score_projection.gqt) +refuse it. `executed_column_name` also records unaliased inferred/executed +name drift. The [staged parser](../../crates/omnigraph-compiler/src/query/staged_probe.pest) +qualifies only its documented prototype subset. + +### Vector behavior and agent recipes + +A String argument to `knn` or `ann` uses the field's resolved compatible query +encoder. A raw vector is an explicit assertion that the caller supplied the +correct space; dimension checks cannot verify its provenance. Geometry comes +from the accepted field's resolved `Vector` declaration, never a query +override. An omitted authoring-time distance follows the +[directive defaults](#directive-defaults-and-omission-rules); execution always +sees an explicit `l2`, `cosine`, or `dot` value. + +The proposed scalar distance values follow the pinned Lance kernels, all +ordered ascending: + +| Geometry | Value for vectors x and q | +|---|---| +| `l2` | Sum of squared component differences; no square root | +| `cosine` | `1 - dot(x, q) / (norm(x) * norm(q))` | +| `dot` | `1 - dot(x, q)`; may be negative and is not a mathematical metric | + +Normalization is an encoding-recipe choice. Current generated embeddings are +L2-normalized by the shared resolver; carrying that behavior unchanged into +every new space could erase magnitudes needed by dot-product ranking. Raw +vectors and generated vectors need consistent, explicit recipe semantics. +Dimension, null-component, non-finite, overflow, and zero-norm behavior must +be specified at ingestion and query admission. A null property is missing +representation; an invalid present vector must not silently become a pending +embedding or a score tie. Precision and kernel/rounding compatibility remain +qualification gates, particularly for near ties and threshold boundaries. + +`knn` selects exact top-k over the eligible targets with valid vectors under +every index state. `ann` advertises approximation even if a particular plan +runs exactly. Uncovered population segments require a qualified exact path; +missing coverage alone is not an error. Incompatible or corrupt artifacts +must not be used, and integrity/resource failures remain typed failures. +Every returned ANN distance is recomputed from the stored vector under the +accepted numeric contract; exact rescoring does not repair missing candidates. + +`oversample` raises configured candidate/work budgets within an immutable, +bounded family-specific mapping. Increasing it cannot lower those budgets; +it does not promise nested candidates or monotone recall for each query. +`ef`, `nprobes`, and quantizer choices remain physical execution details. +Representation readiness is reported separately from index coverage and ANN +approximation. + +These are required adapter behaviors, not scanner defaults: the pinned +indexed path performs raw-vector rescoring when `refine_factor` is set, and +its `fast_search` option can skip uncovered rows. The qualified adapter must +request the necessary refinement and coverage paths explicitly. + +A stored query supplies deliberate source choices, windows, and selection +rules, with a small typed parameter list for the agent. Its resolved definition +and semantic fingerprints must be inspectable; a mutable stored-query name is +not an immutable recipe version. `explain` must show stage inputs, targets, +matching/scoring identities, windows, defaults, and physical fallback choices. +Definitions are resolved once per execution. The simple and advanced query +surfaces compile to the same operators. + +Compiled-plan identity is distinct from resolved execution identity. The +latter must distinguish parameter values and resolved query vectors/encoding +recipes; two invocations of the same source text need not produce the same +candidates. Embedding reuse must key on the complete query-encoding identity +and input, not only text and dimensions. + +An agent can first request identities, titles, and metrics, then read selected +properties or traverse related nodes using the same graph snapshot. Exact +fact verification uses ordinary predicates, negation, and aggregates over +the intended population. A completed search with no hits cannot prove that +population contains no relevant fact. + +### Errors and operational changes + +Typed errors include missing analyzed capability, disabled or incompatible +scoring, token-empty queries, invalid edit/window/weight/effort parameters, +incompatible encoding spaces, invalid stage references, ambiguous target +mappings or inherited metrics, and exhausted resources. Unsupported plan shapes are refused before +results are presented as complete. Ordinary absence of a hit in one fusion +arm is represented as missing membership, not a query failure. + +All search-language replacements ship together. Diagnostics point callers +from `fuzzy`, `search`, and `match_text` to typed lexical matching or retrieval; +from retrieval in `order` and `nearest` to explicit rank stages and `knn`/`ann`; +and from positional RRF to named inputs with declared candidate windows. +Rewritten application/stored queries and examples ship with the implementation. +A rewrite must ask the caller to choose exact versus approximate retrieval +and semantic windows when the old query never specified them. + +The accepted-schema boundary uses the existing export/init/load rebuild. +An offline rewrite makes implicit analyzer and L2 choices explicit and +preserves exact key/index annotations. Legacy embeddings without recoverable +revision/recipe identity require operator resolution; a newly declared default +cannot establish historical coordinate spaces. For new or regenerated values, +`@embed("source")` may omit its model when the schema default resolves it. +The generated schema and every expanded default are reviewed before init. +Rebuilt vectors are needed when the chosen encoding recipe differs from that +which produced existing values. The NFC/profile change also requires index +rebuild or proven parity; an old certificate does not establish new analysis. + +Index reconciliation remains explicit and schema-profile-targeted, with +property selectors and RFC 0043's certification/publication discipline. +Generalizing rebuild support must retain its recovery ownership. Reads and +ordinary content writes never build indexes inline. + +## Design + +### Logical operators and composition laws + +Every stage consumes and produces a typed relation. Its schema identifies +available graph bindings and computed values, with explicit multiplicity, +scope and snapshot context. Ordinary graph and analytical relations need no +relevance score or retrieval target. A ranked relation additionally carries +declared target identity, order, named metrics with origin and source +membership under its selection contract. Relations are internal plan values, +not new stored objects or public graph entities. +The logical plan needs multiple stages rather than one global +`Option`; concrete Rust shapes remain an implementation decision. + +| Operator | Population, scope and multiplicity | Order and metric contract | +|---|---|---| +| Graph match / predicate | Establish or extend bindings; predicates retain only eligible rows | Preserve inherited metric origins; a filter cannot create source membership | +| Retriever | Select distinct eligible target identities and retain their associated incoming bindings | Establish source ranks/comparator; do not score duplicate paths as separate targets | +| Scorer | Add a named feature to existing candidates while preserving membership and binding rows | Keep active order and original source membership; output has its own typed domain and origin | +| Fusion | Combine named ranked inputs on a declared common identity | Compute new score/rank; retain original arm ranks and absence | +| Reranker | Reorder its declared input; any membership cutoff is explicit | Establish a new comparator; preserve earlier metric origins | +| Graph expansion | Extend bindings under graph traversal semantics; may produce several rows per input | Carry metrics on their original binding; never create additional arm votes | +| Select per group | Select target/group pairs and retain their associated bindings | Use the declared local comparator; preserve source ranks and incoming active order | +| Aggregate | Produce group keys/entities and explicit reductions; discard unreduced member bindings | Establish new group scope; no inherited member rank or implicit representative score | +| Intermediate computation / projection | Add or export explicitly named values; declare retained/dropped bindings | Preserve row multiplicity and demanded metric origin; no implicit grouping | +| Optional expansion / correlated collection | Declare imports/exports, nullable bindings or typed lists, duplicates and empty-result behavior | Local selection/order belongs to the declared input; nested work shares the query budget | +| Final projection / property read | Materialize requested values from surviving rows | Preserve selection; aggregate-return shorthand follows the Aggregate contract | + +All operators use the same accepted snapshot and cumulative resource context. +Each lowering must declare input column demand, read descriptors, output +types/nullability, identity, ordering guarantees and failure behavior. This +table includes extension contracts; release availability remains in the matrix. + +The normative laws are: + +1. Predicates determine eligibility; retrievers select and rank eligible + targets. Ordinary projection changes neither selection nor stage order. + Aggregate projection is an aggregation operator and is outside this law. +2. Stage order is semantic. Filters, expansion, grouping, and selection cannot + cross a candidate boundary without an equivalence proof. Policy cannot be + postponed to a post-top-k cleanup filter. +3. Source windows, fusion/rerank windows, graph traversal bounds, physical + effort, and final row/byte limits are distinct. A downstream limit cannot + silently resize an upstream source. +4. Target identity, deduplication, metric origin, and multiplicity are + explicit. Traversal fan-out cannot create extra votes in an earlier arm. +5. Exact membership, exact lexical ranking, and `knn` preserve their logical + results across physical index states. ANN's declared approximation does + not permit dropping an uncovered population segment silently. +6. Every exact ranking cut honors the declared comparator over its whole + logical input. ANN has a total order over its selected candidates, without + claiming exhaustive candidate discovery or reproducible reruns. +7. Partitioning cannot redefine a source population or fusion scope. Global + ranking means the declared logical population, independent of physical + sharding; a shard-local candidate cutoff needs a proven exact merge or an + explicitly approximate source contract. +8. Resolved semantics are versioned: analyzer implementation/Unicode behavior, + scoring math, encoding compatibility, defaults, and effort mappings. + Index presence never chooses those meanings. +9. One accepted snapshot covers all arms, graph stages, statistics, and reads + in an execution. Retry starts a fresh attempt, never a mixture of views. +10. Completion, representation coverage, approximation, and task usefulness + are different facts. None is an implicit probability of answer correctness. +11. Admission, execution, fallback, and output use one bounded resource + protocol. Silent partial results and uncharged fallback work are forbidden. + +### Analytical populations and selection boundaries + +Four populations have separate identities in a composed query: + +| Population | Meaning | +|---|---| +| Eligible population | Distinct targets admitted by the graph/property constraints at a source, with their incoming binding rows | +| Scoring corpus | The data used to establish declared statistics, such as the snapshot-visible field corpus for `bm25_v1` | +| Retrieved candidates | Targets surviving a particular source/fusion/selection boundary | +| Aggregate input | The binding rows or explicitly deduplicated entities consumed by that reduction | + +Existing aggregates consume their complete declared input under their numeric +and null rules; a future approximate aggregate needs its own explicit contract. +Counting an ANN-selected set does not establish a complete count of relevant entities in the eligible +graph. Candidate selection is not statistical sampling, and the engine must +not infer an extrapolated total or error bar from it. A complete semantic +range/count would first need a defined predicate and complete evaluation over +the stated population; a relevance label alone supplies neither. + +The plan must retain the aggregate's input stage and duplicate semantics, +including any upstream approximate selection. A typed descriptor can expose +that lineage without adding a score to ordinary analytical rows or inventing +a graph-wide completeness flag. Exact result computation, retrieval recall, +representation coverage and answer correctness remain separate properties. +For BM25, ordinary eligibility and aggregate grouping do not redefine the +accepted scoring corpus; the lexical section owns that statistics contract. + +Qualification must include these counterexamples: + +- With three eligible incidents, one binding per incident, and a cut selecting + two incidents, a count before selection is three and a count afterward is two. +- After graph fan-out, three binding rows can represent two distinct incidents; + `count($incident)` still counts rows. Distinct counting needs explicit syntax + or a declared deduplication stage, not retrieval's implicit target identity. +- A global candidate set can omit an entire group; a later per-group quota + cannot refill that group from the full population. +- An exact aggregate over a large input may return one row. Final row or byte + limits cannot authorize an earlier truncation of the aggregate input. +- Changing eligibility or grouping leaves the accepted BM25 corpus unchanged; + the existing statistics oracle must continue to distinguish these scopes. + +Selection, grouping, projection and graph expansion may be reordered only +with an equivalence proof that preserves these populations, duplicate/null +semantics and metric origins. Cost optimization cannot change which question +the aggregate answers. + +### Required composition examples + +C1–C4 are grammar-evolution acceptance cases. The test-only compiler parses and +checks these examples; the production parser rejects them. Their logical plans +and native probes are design evidence. Deferred operators require integrated +GQT/resource qualification when implemented. + +**C1 — Aggregate, select entities, then retrieve evidence.** A has prior/current +incident counts 2/8; B has 6/7. Select A's increase of six, irrespective of B's +report relevance. Missing periods count zero; incident-rooted input does not +invent services with no incidents. Including those needs a service population +and enrichment. + +```gq +query composition_c1($q: String) { + match { + $i: Incident + $s hasIncident $i + $i.period = "prior" or $i.period = "current" + } + group { + per { $s } + reduce { + count_if($i.period = "prior") as prior_count, + count_if($i.period = "current") as current_count + } + } + let { current_count - prior_count as increase } + select { order { increase desc, $s.@id asc } limit 1 } + match { $s hasReport $p } + rank $p { + lexical($p.text, terms($q), candidates: 2) as reports + yield reports + } + return { + $s as service, prior_count as prior_count, current_count as current_count, + increase as increase, $p.@id as report_id, $p.text as report_text + } +} +``` + +`group` exports entity keys under their binding names and named reductions, +discarding incident bindings. `let` adds row values whose sibling expressions +read the incoming scope; numeric results do not become identities. `select` +cuts rows, while `take` selects target/group pairs. Retain computed facts +through the service cut and later report reads. Multiple selected services +require C4's explicit correlation, not repeated global top-K plus `take`. + +**C2 — Retrieve, traverse, then aggregate the selected population.** Selected +p1/p2 have three bindings to project P (two from p1); eligible p3 falls outside +the cut. P must report three binding rows and two distinct passages. + +```gq +query composition_c2($q: String) { + match { $p: Passage } + rank $p { + lexical($p.text, terms($q), candidates: 2) as candidates + yield candidates + } + match { $p $membership:inProject $project } + group { + per { $project } + reduce { + count($p) as binding_rows, + count_distinct($p.@id) as passages, + min(metric(candidates, rank)) as best_rank + } + } + return { + $project as project, binding_rows as binding_rows, + passages as passages, best_rank as best_rank + } + order { $project.@id asc } +} +``` + +Group exports project identity/counts and explicitly reduced metrics; member +bindings and active rank disappear. In terminal aggregate shorthand, an +unreduced projected metric instead adds a grouping key and changes the question. +Keep the candidate barrier before traversal/aggregation. General distinct/ +intermediate grouping is deferred; the initial path must qualify terminal +binding counts and metric reductions. + +**C3 — Score existing candidates without changing membership.** Dense retrieval +selects d1/d2, both lexical matches, while the lexical arm's window contains only +d1. A fresh feature scores both without manufacturing lexical-arm membership. + +```gq +query composition_c3($q: String, $vector: Vector(3)) { + match { $p: Passage } + rank $p { + lexical($p.text, terms($q), candidates: 1) as words + knn($p.embedding, $vector, candidates: 2) as dense + yield dense + } + score $p { + lexical($p.text, terms($q), scoring: bm25_v1) as words_feature + } + return { + $p.@id as passage_id, metric(words, rank) as lexical_rank, + metric(words, score) as lexical_score, feature(words_feature) as lexical_feature + } +} +``` + +`score` preserves targets, bindings and comparator; it has no candidate window, +output selector, rank or fusion vote. `feature` names a scorer; `metric` +names retrieval membership. Reordering/cutting requires an explicit operation. +A hidden lexical top-K followed by a join can drop d2 and is invalid. +Scoring uses accepted corpus statistics and charges their work even for a +small target set. + +**Decision:** scoring evaluates the term contributions independently of +`all`/`any` membership. A partial `all` match can have a positive feature; +a present value with no matching terms, including a token-empty value, has +zero; a null field has a null feature. The +[lexical contract](#lexical-scoring-and-shared-matching-semantics) owns the +complete rules and numerical oracle. The native composition fixture still +uses precomputed features: it proves preservation, not their production. + +**C4 — Computed facts, optional graph facts and bounded evidence.** A has one +owner and three reports; B has neither. Both survive with their original +counts; A gets the first two reports under the local comparator and B gets a +null owner and empty typed list. + +```gq +query composition_c4($q: String) { + match { + $i: Incident + $s hasIncident $i + $i.period = "prior" or $i.period = "current" + } + group { + per { $s } + reduce { + count_if($i.period = "prior") as prior_count, + count_if($i.period = "current") as current_count + } + } + let { current_count - prior_count as increase } + select { order { increase desc, $s.@id asc } limit 2 } + optional ($s) as owner { + match { $s ownedBy $person } + return { $person as person } + } + collect ($s) as reports { + match { $s hasReport $p } + rank $p { + lexical($p.text, terms($q), candidates: 2) as relevant + yield relevant + } + return { $p.@id as id, $p.text as text } + order { metric(relevant, rank) asc, $p.@id asc } + limit 2 + } + return { + $s as service, prior_count as prior_count, current_count as current_count, + increase as increase, owner as owner, reports as reports + } +} +``` + +Imports explicitly distinguish `$entity` from computed row values; query +parameters stay available. Child bindings/sources do not escape. `optional` +returns a nullable object: zero rows produce null, one produces an object, +and more than one requires explicit selection/collection or a cardinality +error. `OwnedBy` is zero-or-one in this example. + +`collect` consumes ordered binding rows and requires a local comparator and +output limit. Unique reports need an explicit identity reduction; source +windows do not bound later fan-out or cumulative parent work. Both children +preserve the parent row, accepted snapshot and one whole-query budget. + +Correlation is per parent **row**, not merely entity ID: repeated service +bindings can carry different facts. Compiled source ID alone does not identify +all runtime per-parent rankings. Reuse requires equivalent imports, parameters, +representation and snapshot, preserving every parent row and charging actual +shared work. Edge imports/group exports, total row ties, nested-field access, +cardinality/resource enforcement and safe decorrelation remain unqualified. + +**Evidence and open boundaries.** The +[compiler](../../crates/omnigraph-compiler/src/query/staged_probe.rs) reads the +four examples directly. Its [derived logical views](../../crates/omnigraph-compiler/src/query/staged_probe/plan.rs) +retain relations, barriers, sources, group populations, projection types and +imports. They are neither executable IR nor a public serialization; their +snapshot/budget/statistics statements are requirements, not measured behavior. +Symbolic bindings and omitted options do not freeze defaults/fingerprints. + +| Example / checked logical plan | Prototype evidence | Still unproved / required falsifier | +|---|---|---| +| [C1](../../crates/omnigraph-compiler/src/query/staged_probe/composition_c1.json) | Group retains `$s`, exports counts and drops `$i`; selection precedes report retrieval. Native filtered counts produce A=2/8, B=6/7, C=1/0, selecting A. A report-driven prefilter selects B instead; `count(Boolean)` counts false; a duplicate path changes A's count. | Actual GQ lowering and optimized population barriers, entity rehydration, enforcement of the decided arithmetic/null/overflow rules and resource ownership. The native probe models the report filter; it does not retrieve reports. | +| [C2](../../crates/omnigraph-compiler/src/query/staged_probe/composition_c2.json) | Group retains project identity and reduced metric origin while dropping member bindings/order. Native selection of p1/p2 yields three binding rows and two distinct passages; removing the cut admits p3 and changes both counts. | Graph target-ID mapping, GQ aggregate lowering, general equality/null semantics, distinct-state memory and the complete optimized graph/retrieval plan. | +| [C3](../../crates/omnigraph-compiler/src/query/staged_probe/composition_c3.json) | The scorer creates a separate feature with no candidate window or rank; dense output remains the comparator. A native fixture preserves p2 with absent lexical-arm rank and a positive feature; filtering on lexical membership incorrectly drops it. Zero/null feature inputs also survive. The Decimal oracle separately checks partial/zero/null feature values. | Production scorer and enforcement of the decided nonmatch/empty/missing-field policy, fixed live statistics and numeric parity. Precomputed native features prove neither BM25 values nor scorer cost. | +| [C4](../../crates/omnigraph-compiler/src/query/staged_probe/composition_c4.json) | Checked imports/exports and separate child source IDs. Native ordered object lists preserve two rows for A and an empty B; presence markers distinguish absent objects from present null payloads. Duplicate paths consume collection rows, and fan-out exceeds the source window. An aggregate detects multiple owners. | Correlated GQ lowering, generation/preservation of parent-row identity, actual typed cardinality refusal, full entity-object projection, total row ties and shared budget/cancellation under many parents. Detection of ambiguous owners is not the query refusal path. | + +The [native probe](../../crates/omnigraph/tests/rrf_prefilter_gate/composition.rs) +uses DataFusion 54.0.0 / Arrow 58.3.0, memory inputs, one/four partitions, +forward/reversed one-row batches and two zero hash-join thresholds. It does +not qualify Lance providers, arbitrary optimizer configurations, snapshot +pinning, cancellation or allocations. See the +[checkpoint](#composition-plan-and-primitive-checkpoint) for reproduction; +GQT takes ownership when these operators become executable. + +C1/C2 use typed +[aggregation](https://docs.rs/crate/datafusion-expr/54.0.0/source/src/logical_plan/builder.rs), +[filtered expressions](https://docs.rs/crate/datafusion-expr/54.0.0/source/src/expr_fn.rs) +and [distinct count](https://docs.rs/crate/datafusion-functions-aggregate/54.0.0/source/src/count.rs). +Count matching rows: `count(predicate)` also counts false. Entity rehydration +and preservation of cuts remain OmniGraph lowering responsibilities. + +C4 must bridge [ArrayAgg's empty-input null](https://docs.rs/crate/datafusion-functions-aggregate/54.0.0/source/src/array_agg.rs) +to a typed empty list. Aggregate actual child rows by parent-row identity with +explicit order, then left-join to parents and construct empty lists for absent +groups. A left join before aggregation can create phantom `[null]` or `[{}]`. +Optional objects need a presence marker: null properties do not mean no row. +The native object-list probe exercises this distinction; the current JSON +writer omits null fields. These tests do not decide a new wire contract or +prove bounded nested execution. + +### Target identity, fan-out, grouping, and metrics + +In `rank $p`, the initial target is a distinct bound node or edge identity, +including its accepted type/incarnation context. A target is eligible when +at least one incoming binding row satisfies the preceding graph pattern. +Repeated incoming paths do not repeat its source score or rank. Selection +retains the associated binding rows for each winning target, subject to the +query's explicit resource bounds; it does not arbitrarily pick one path. +Consequently, selecting 20 targets can produce more than 20 output rows. +Final `limit` counts those output rows. Exact bag/path behavior must follow +the existing graph language and be pinned by the grammar/plan fixtures. + +Sources in the initial fusion target the same binding identity. Fusing +passages with organizations requires an explicit graph mapping, a duplicate +reduction rule, and an order/score rule before fusion. It cannot be an implicit +ID cast. General cross-identity remapping is an extension; unsupported mappings +fail. Applications can model a passage as an ordinary node with an edge to its +source and retrieve either unit through separate stages. + +Metrics belong to `(stage, source, target identity, query, population, +representation, scoring version)` within one execution. Human aliases refer +to those instances. A vector-only hit has no lexical-arm rank or score from +that execution; projections expose absence. A later lexical rescore produces +a new named feature and does not retroactively enter the hit into the arm. +RRF consumes one ordinal rank per target per arm: + +```text +rrf(target) = sum(weight[arm] / (k + rank[arm, target])) + over arms containing target; rank starts at 1 +``` + +`rrf_v1` admits 2–16 arms, positive finite weights, positive `k` (default 60), +and positive candidate windows at most 10,000. Literal bounds are checked at +compile time and parameter bounds before execution; aggregate work across +arms/stages is additionally budgeted. Missing membership contributes no term. +Each input stage occurs once in a fusion; repeated weighting uses its explicit +weight rather than duplicate arm references. +Finite positive weights alone do not ensure a finite fused score: 16 maximum +finite floating-point weights with `k=1` overflow their sum at rank one. +The numeric contract must define accumulation order and checked arithmetic, +with typed failure for overflow/non-finite results; silent weight rescaling +would change the public score. Window/effort products also require checked +arithmetic before allocation. These are acceptance gates for `rrf_v1`. +Fusion assigns new ranks after sorting by fused score and stable identity. +Ranks are established before traversal fan-out, never by first-seen row order. + +After expansion, inherited metrics remain attached to the binding that was +ranked. A second ranking has separate metrics. Aggregation follows existing +return semantics: non-aggregate projected values become grouping keys. A +metric explicitly projected beside `count` is therefore a grouping key, not +an arbitrarily chosen representative score. To obtain one score per otherwise +defined group, request a supported explicit reduction, such as +`min(metric(hits, rank))`, or omit the metric. The group has a new result +identity; it does not inherit source membership or a constituent's active +rank. Ordering by a discarded source metric is rejected. Ordering by a +projected grouping key or explicit reduction remains valid. + +Without a final `order`, ranked output follows the latest ranking stage's +order. Row-preserving filters and projection retain it; expansion orders +surviving rows by their originating target rank and stable binding identities. +A later rank stage establishes a new active order while earlier metrics keep +their origin. Aggregation does not implicitly inherit a constituent target's +rank. These ordering rules require golden plans and fan-out fixtures. + +Selection per group uses a `take` stage, with a named node/edge target, a +nonempty `per` key tuple, an optional local `order`, and a required `limit`: + +```gq +query incidents_per_organization($q: String) { + match { $o: Organization $o hasIncident $i } + rank $i { + lexical($i.title, terms($q), candidates: 100) as incidents + yield incidents + } + take $i { per { $o.slug } limit 2 } + return { $o.slug, $i.slug, metric(incidents, rank) as rank } + order { $o.slug asc, rank asc, $i.@id } + limit 20 +} +``` + +Here the source selects up to 100 distinct incidents globally from the graph +population, the quota retains at most two of those incidents per organization, +and the final limit returns at most 20 binding rows. Neither later limit refills +the source. `per { $o.slug, $o.category }` illustrates a composite group key. + +An omitted local `order` uses the latest ranking stage's rank. An explicit +local order, such as `order { metric(incidents, rank) asc nulls last }`, chooses +the pair winners without establishing a new global output order. With no +ranking stage in scope, a local comparator is required; this also permits +ordinary graph selection such as latest events per entity. Such input remains +subject to whole-query budgets rather than requiring a synthetic search stage. +Quota `limit` takes a non-null integer literal or parameter, must be nonnegative, +and permits zero, consistent with ordinary limit semantics. It does not impose +a source candidate-window cap; resource admission and checked arithmetic still +apply. Without a final order or an earlier ranking, output remains unordered. + +Ordering expressions in either location may append `nulls first` or +`nulls last`. Omission preserves current GQ defaults: ascending puts nulls first, +descending puts them last. A missing arm metric remains null; choosing its +placement never gives that target source membership or a computed score. + +Selection per group consumes the admitted incoming relation, partitions by an +explicit bound key tuple, and takes at most N distinct targets per group under +a declared total order. Its selection unit is `(target identity, group key)`: + +- Evaluate group keys on incoming binding rows. A target can participate in + several groups through those bindings; there is no implicit array expansion. + Within one group, duplicate paths for the same target consume one slot. +- Select winning target/group pairs, then retain every incoming binding row + belonging to a winning pair. Winning one group does not restore that target's + losing bindings in another group. A target winning several groups retains + its bindings in each; selected pairs, distinct targets, and output rows are + separate counts. +- Use the language's grouping equivalence for each admitted key type. Null + components group together, as in existing aggregate grouping; they do not + mean “no group.” Reattachment must use that same equivalence, including + null-safe equality. Supported key types and tuple ordering need compiler + and execution fixtures; stringified keys are not the public contract. +- A selection comparator must have one value per target/group pair. If an + expression varies across the pair's binding rows, require an explicit + supported reduction or reject it; never choose the first path's value. + The inherited default comparator follows this same rule. Identity, complete + non-null key/unique tuples, and enforced relationship cardinalities can prove + constancy; partial composite keys and nullable unique values cannot. Cardinality + is per native edge source, so a maximum of one does not imply a unique reverse + endpoint. Directed one-hop edge identity fixes both endpoints; undirected + same-type endpoints can still exchange roles. Unproved cases require an + explicit reduction. Reductions operate over the pair's incoming binding rows + using ordinary aggregate null and duplicate semantics. Append stable target + identity to break ties within each group. +- Quota selection preserves metric values and their origins. Removing a + higher-ranked pair does not renumber surviving source ranks or create a + new source-membership vote. By default it filters the incoming active order; + an explicit final order can replace that order. Final `limit` still counts + binding rows, not pairs, and applies after the quota. + +For example, if `a` belongs to `g1` and `g2`, `c` beats `a` in `g1`, and `a` +beats `b` in `g2`, quota one selects `(c, g1)` and `(a, g2)`. Restoring every +binding for target `a` would also restore `(a, g1)` and violate `g1`'s quota. +Global target selection can use a target-only semi-join; per-group selection +requires the winning pair keys. + +This operator is distinct from `count`/`sum` grouping inferred by current +aggregate returns. A global candidate cut before it can leave groups empty; +there is no implied refill. Retrieving top-N within every group of the full +population is a different, separately bounded shape. Selecting a global set +of targets subject to quotas on *all* their overlapping memberships is also +a different optimization problem; independent per-group winners do not claim +that guarantee. The prototype checks this spelling and a subset of the type +rules; full compiler/lowering enforcement remains an acceptance gate. + +Per-group quotas and identity deduplication address concentration, but do not +establish complementary reasoning coverage. Semantic diversification and +selection relative to already known information are extension requirements +and evaluation concerns. Agent-side exclusions and follow-up queries remain +ordinary graph operations. + +### Lexical scoring and shared matching semantics + +Lower both lexical consumers through `LexicalQueryIR`, bound to the +rename-stable property identity and accepted analyzer fingerprint. Resolve +parameters and analyze once per compatible field/query instance per execution, +including on empty populations. Reuse resolved matching state across scan and +indexed paths. The consumers differ in role: matching returns a Boolean; +retrieval selects matches and computes ranking under its declared policy. +Removed spellings have no separate IR variants or compatibility evaluators. + +`bm25_v1` is one versioned term scorer for both exact and tolerant queries. +Zero edits reduces to BM25 with unit weight per distinct analyzed query term; +nonzero edits adds alternatives and an edit penalty within the same formula. +Candidate membership is always the shared `Terms` relation, evaluated before +the ranking cutoff. A score is not a second membership predicate. This removes +the earlier requirement to coordinate `terms.max_edits` with a separate exact +or fuzzy scorer selector. Query-term repetition and order carry no extra +weight; intentional source weighting belongs to explicit fusion or a separately +specified future scoring operator. + +The deferred membership-preserving `score` operator uses this same kernel +over its incoming targets, including targets that fail `Terms` membership: + +| Field/query outcome | Retrieval membership before its cut | Lexical feature | +|---|---|---| +| At least one matching term group, but not every group of an `all` query | False | Positive sum of the matching contributions | +| Nonempty field with no matching group | False | `0.0` | +| Present field with zero analyzed tokens | False | `0.0` | +| Null field | False | Null: no field value to score | +| Full query match | True | Same finite positive value as retrieval scoring | + +`mode` controls the Boolean consumer; it never masks term contributions. +Changing `all` to `any` alone cannot change a feature value. Changing the terms, +edit budget, analyzer or statistics can. A score of zero is observed absence +of term evidence; null remains missing representation. Neither adds a source +rank or fusion vote. Feature nullability follows field nullability, while a +retrieval metric can also be absent because its target missed that source's +window. A future explicit conditional can gate features by membership; no +second implicitly gated BM25 formula is introduced. + +An empty analyzed **query** is an error before any target evaluation, even for +an empty population or all-null fields. An empty corpus has no positive +contributions: return zero for present token-empty fields and null for null +fields without dividing by corpus length. Invalid representations/statistics, +non-finite arithmetic and resource failures remain errors. Scoring never +turns them into zero, null or dropped targets. + +The statistics population is part of source identity, independently of its +eligible population and candidate window. The initial contract uses the +snapshot-visible field corpus: distinct, policy-visible targets owning that +accepted type/property identity with nonempty analyzed values, before ordinary +graph or property filters and the source's lexical query. Null and token-empty +values contribute neither a document nor tokens. Graph fan-out contributes one +target, not repeated field values. A physical dataset, index segment, or shard +cannot choose a different logical corpus. Polymorphic fields must resolve +their corpus identities explicitly before that source shape is supported. + +At one snapshot, with the same query, analyzer, and scoring policy, narrowing +eligibility therefore leaves a surviving target's lexical score unchanged. +This makes scoring compose with graph filters and keeps it independent of +candidate windows. Selection still depends on eligibility: filtering before +top-k and filtering after top-k are different operations. Scores may change +across snapshots or policy-visible corpora; this is not a globally calibrated +score or a reason to compare unrelated source instances. + +The existing `fts_prefilter_does_not_change_covered_fragment_scores` guard +confirms that a native prefilter does not rescope covered-index statistics. +The new `fts_statistics_scope_can_reverse_ranking` probe goes further: with +the same four eligible targets, an alpha/beta query ranks an alpha target +first using the full ten-target corpus, but the beta target first using an +index containing only those four eligible targets. Eligibility and statistics +must remain separate typed plan facts. Implicitly recomputing statistics for +each match block would change score meaning with graph scope. An explicit +alternative corpus could be a future contract; it is not an initial query +option. + +This choice does not qualify native BM25 or make field statistics free. +Deleted/updated rows, unindexed tails, null/token-empty handling, arithmetic, +and complete boundary ties still require a checked-in score oracle and +scan/index parity. Reusable statistics must be derived from the accepted +snapshot and representation; a stale aggregate is not authority. Exact +statistics work consumes the query budget and may require a corpus scan. +The prefilter probe proves neither live-row statistics parity nor a cost +advantage for every query. Policy constrains the corpus before statistics; +ordinary query filters do not replace the access-control boundary. + +For each distinct analyzed query term `q`, treat all stored terms within the +declared edit budget as alternatives for that term. Let `N` be the number of +documents in the field corpus, `df_q` the number containing at least one such +alternative, `len_d` the total analyzed token count of document `d`, `avg_len` +the corpus mean length, and `tf(t,d)` the occurrence count of stored term `t`. +All analyzed occurrences contribute to length, including repetitions and +terms unrelated to the query. A field value is the document unit; graph paths +and index segments never multiply its frequency. + +The mathematical definition is: + +```text +idf(q) = log1p((N - df_q + 0.5) / (df_q + 0.5)) +norm(d) = 1.2 * (0.25 + 0.75 * len_d / avg_len) +weight(t,d) = 2.2 * tf(t,d) / (tf(t,d) + norm(d)) +part(q,d) = idf(q) * max [ 2^(-edit(q,t)) * weight(t,d) ] + over stored terms t within the edit budget +score(d) = sum part(q,d) over distinct query terms q +``` + +An empty maximum contributes zero. `df_q` counts a document once even when +several alternatives occur, before the full query's `all`/`any` membership +test and ordinary eligibility filters. Alternative-specific rarity cannot +give a misspelling extra weight: all alternatives share the term group's IDF. +Using the maximum or sum of individual term document frequencies would not +count the group's matched population: maximum misses disjoint occurrences, +while sum double-counts documents containing several alternatives. Union DF +also stays unchanged when spellings redistribute without changing which +documents match the group. This semantic benefit has a cost: native per-term +frequency metadata alone generally cannot produce it. +Within a document, maximum contribution prevents adding different spellings +from summing several pieces of evidence for one query term. Repeated occurrences +of a single stored term still affect its BM25 term frequency. A stored term +may contribute to several distinct query terms, matching the existing `Terms` +membership rule; the score is not a count of independent facts. + +Edit weights are `1`, `1/2`, and `1/4` for zero, one, and two edits. These are +versioned design choices, not values inferred from a backend or established +as globally optimal. Exact preference is a contribution boost at equal term +frequency and length, not a strict ordering tier above every fuzzy document. +Length, term frequency, and other query terms can still change the winner. +Changing `max_edits` may change group IDF and ranking, while membership retains +the declared monotonicity. At zero edits each group contains only the exact +term, so the same kernel supplies the exact-scorer reduction. + +The numeric profile uses float64 and `libm 0.2.16`'s scalar `log1p` +implementation, with its dependency identity recorded in the scorer +fingerprint. It does not delegate to the platform's logarithm. Count documents, +tokens, and frequencies with checked `u64` arithmetic; compute `N - df_q` +before conversion, reject inconsistent statistics, +and handle an empty corpus before division. Convert counts to float64 for +scoring, evaluate the parenthesized expressions above without algebraic +reassociation or fused operations, and sum contributions in ascending UTF-8 +query-term order. Zero-contribution terms add zero; matching rows must produce +a finite positive score or a typed numeric failure. Final ties use the source's +stable target comparator. One canonical numeric implementation must serve all +qualified paths; testing that pinned kernel across supported targets is +required before acceptance. The Decimal oracle verifies mathematical values +within its stated tolerance, not cross-platform bit identity. A future kernel +change needs a new scorer identity unless score/order equivalence is proved. + +Native Lance BM25 does not supply this contract. The +`native_fuzzy_bm25_rewards_a_rare_expansion` probe gives `beto` about 38 times +the score of exact `beta` in a 100-document fixture and doubles scores for a +repeated query term. The public float32 scorer also rounds a common term's +positive IDF to zero at `N = 2^24`; the +`native_bm25_idf_can_round_a_common_term_to_zero` guard reproduces this without +allocating that corpus. Native scores or top-k cuts require new qualification; +rescoring an incomplete native candidate set cannot establish exact winners. + +The [numerical fixtures](../../crates/omnigraph/tests/fixtures/lexical_scoring_v1.json) +and `lexical_scoring_v1_reference_oracle` in the existing search test owner +cover the chosen formula, zero-edit behavior, repeated/reordered terms, +alternative aggregation, overlapping groups, `all`/`any`, null/token-empty +values, and two-edit membership. They are design oracles over already-analyzed +terms, not a production evaluator or relevance benchmark. The initial complete +path must implement fuzzy ranking against these oracles; leaving it to a later +release is not completion. Scan-based correctness may ship before native +acceleration, with bounded failure when the work cannot complete. + +The same oracle now checks separate feature expectations, including partial +`all` matches, all-null/token-empty corpora, unchanged features under `all`/`any` +and eligibility changes, and identical retrieval/feature values for matches. +These settle C3's numeric meaning; they do not deliver the deferred scorer +operator or qualify analysis, live-row accounting and execution bounds. + +An exact physical baseline can compute field statistics in one snapshot-pinned +pass, then score eligible values in a second pass. For each query term, count +at most one family occurrence per document in the first pass; the second uses +that document's term frequencies and the same edit relation for its maximum. +Keep query-term statistics and bounded per-document state, not a retained +corpus or persistent expansion dictionary. N/mean-length metadata may be reused +when snapshot and analyzer identity qualify it; fuzzy family DF generally +depends on the query and edit budget. Both passes, normalization, matching, +and top-k selection share the execution budget. + +This is an implementation route to qualify, not a claim that two full scans +meet production latency goals. Indexed acceleration must supply complete +membership, live-row counts, term frequencies and group reductions, or prove +conservative bounds for every pruned candidate. Lance's public scorer trait +returns float32 additive term weights; merely substituting a scorer cannot +provide the group's maximum reduction or this float64 contract. Reuse Lance +postings through qualified upstream/adapter work and DataFusion group operators +where appropriate; do not introduce another stored index. A numeric or quota +post-pass over native top-k cannot restore discarded winners. + +### Explicit ranking features and distance predicates + +Metrics retain typed domains such as `Score`, `Score`, and +`Distance`, together with source-instance identity. Plain +cross-domain arithmetic does not become valid merely because storage uses +floating-point columns. RRF consumes ranks rather than raw score magnitudes. + +Explicit feature combination remains a supported extension of the algebra. +A scoring/reranking stage must declare inputs, normalization population and +method, missing/non-finite handling, model or formula version, output domain, +and resource/failure behavior. Per-window normalization names that window; +changing it can legitimately change scores. It must not silently normalize +against the entire corpus. Learned model execution and general feature +combination syntax are deferred until these contracts are qualified. + +A threshold on a declared geometric distance is legitimate and need not be +called confidence. An exact range query evaluates the distance predicate over +the specified eligible population under defined inclusive/exclusive bounds. +Applying that predicate to ANN candidates is only a filter on those candidates; +it does not promise complete range results. Preserve a typed distance-scorer +extension point and resolve its grammar before exposing it. Initial release +scope does not include a dedicated range-search operator. A raw relevance +score threshold likewise requires an explicit source/domain contract and must +never be presented as a probability that an answer is correct. + +### Graph-wide discovery across entity types + +**Phase 0 disposition:** defer cross-type union execution, all-node search, +all-edge search and mixed-kind search from the initial release. Concrete +node targets, and concrete edge targets reached through graph traversal, +retain their specified scope. A loop of independent client queries is not +the promised one-snapshot global query. This decision keeps the coordinated +release focused on the existing search correctness failures while the common +language acquires the union, narrowing and heterogeneous result machinery. + +Before stabilizing the initial grammar/IR, preserve these extension boundaries: +target identity carries entity kind and accepted type/incarnation as well as +entity ID; a logical source/window is independent of physical table count; +type scope and representation selection are separate typed plan facts. +Common-property projection requires compatible property types in every union +arm, with nullability widened when necessary. A property absent from an arm +requires explicit type narrowing or branch projection, never implicit null +or stringification. Union construction preserves binding rows; deduplication +and target selection remain explicit. An all-node expansion is the union of +the snapshot's concrete node scans, with each node identity once before any +graph fan-out. No wildcard or union punctuation is accepted by this decision; +its parser/type/plan compatibility proof remains a Phase 0 gate. + +Here, global search means discovery across eligible entity types in one +logical graph at one accepted snapshot. Its scope includes the target types, +searchable properties, representation policies and authorization rules. A +convenience stored query can own that scope; schema-driven expansion must +resolve it against the accepted catalog and include the resolved scope in its +query identity. Adding a searchable type can then change that explicitly +defined population. Ordinary unindexed or non-searchable fields do not acquire +an implicit text conversion or embedding recipe. + +An all-node selector must expand to every accepted node type without requiring +applications to enumerate the alternatives. Keep this type scope separate from +the representation selector: the latter resolves analyzed properties, field +reductions and compatible embedding spaces. A node type with no selected usable +representation remains in the declared scope but cannot contribute a ranked +hit. Coverage must distinguish this absence from an evaluated non-match; +unknown counts remain explicit unless exact coverage was requested. An index +is not what makes a property searchable, and missing indexes must not remove a +type from the logical scope. All-edge scope is a separate explicit choice. + +Resolve both selectors against the query's accepted snapshot and include their +expanded identities in execution identity. A newly added node type is therefore +included in a subsequent all-node query, while a read pinned to an older snapshot +retains that snapshot's type and representation scope. This automatic scope +expansion does not authorize implicit stringification, mixing vector spaces, or +resetting resource allowances per type. The wildcard spelling and typed +representation-selector grammar remain part of the general query-language +extension; a wildcard alone does not make `$hit.title` valid on every type. + +The current compiler binds a variable to one concrete node type. Schema +interfaces provide declarations and inheritance, but the query binder does +not yet execute polymorphic scans or a heterogeneous union. The initial +same-binding fusion contract alone therefore does not provide a single +cross-type global search query. This is an expressiveness gap to resolve +before advertising that capability. + +The preferred extension is a general typed union of graph bindings. Each +branch preserves the entity kind, accepted type/incarnation identity and +entity id; a hit remains its original node or edge. Common projections and +type-specific property access need ordinary query-language typing and type +narrowing. No synthetic stored `Document` entity or separate search registry +is needed. Source metrics and matched-property attribution remain attached to +their producing source. Two types with equal id strings are still different +targets. Passage-to-owner mapping remains the separate explicit graph mapping +and reduction problem described above. + +Ranking this union requires a declared cross-type policy: + +- A lexical source over a common logical text representation needs compatible + analysis, a defined treatment of multiple fields per target, and statistics + over that logical corpus. Per-table BM25 scores cannot be merged as if each + table had used those shared statistics. Multiple property hits must not + accidentally duplicate a target or let traversal fan-out add relevance. +- Compatible vector representations can share a distance comparator when + space/revision, query encoding, dimensions, metric and numeric rules agree. + Different spaces remain separate named sources, with their query encoding + calls included in the shared execution budget. +- Fusion can combine sources that now name this same typed target universe. + RRF over disjoint per-type lists is also a possible explicit policy, but it + gives each equally weighted type's first result the same contribution. It + is a source-balancing choice, not proof of comparable relevance. A common + scorer or qualified reranker is needed when the product requires a single + relevance ordering that those source ranks do not supply. + +Logical source count and physical table count are separate. Compatible tables +can be partitions of one logical source with one global candidate window. +Creating an independent fusion vote for every table would let physical schema +layout influence relevance. All partition scans and merges still consume the +same query budget; a small logical source count does not bound physical work. + +Physically, Lance remains responsible for each version-pinned dataset and +qualified index/scan path. DataFusion can union narrow candidate relations, +group by the full typed identity, rank/fuse and apply the global cut; hydrate +payloads from the corresponding pinned datasets after selection. For an exact +source split into disjoint physical partitions, local top-k followed by a +global top-k is valid only when every partition uses the same score and total +comparator as the global source. A partition cannot substitute local corpus +statistics or a different tie rule. Complete per-target field reduction and +deduplication before such a cut. If a target's score still depends on +contributions from several partitions, pruning those contributions requires a +valid global-score bound; ordinary local top-k is insufficient. ANN remains +explicitly approximate. + +Rescoring a shortlist selected with table-local BM25 statistics cannot repair +this error: the global winner may already have been discarded. Lance 11's +public `InvertedIndex::bm25_stats_for_terms` and `bm25_search` with a supplied +`MemBM25Scorer` provide a lower-level shared-statistics path to investigate. +They do not establish the complete contract: the scorer is native float32, +the RFC requires different fuzzy-group scoring, and immutable index statistics +can still count deleted rows. Reconcile live corpus statistics, uncovered +rows and invalidated coverage before using them for a global candidate cut. +Any cache remains derived from the accepted snapshot and representation +identities. Native index statistics alone are not a new source of truth. + +Before enabling that extension, qualification must cover two unrelated node +types, an explicitly selected edge type, equal id strings across types, multiple searchable fields, +incompatible vector spaces, empty/unavailable sources, snapshot changes, +policy and shared-resource refusal. Compare the physical fan-out plan with an +independent evaluator over the full logical union. Include disjoint-source +RRF behavior in the retrieval-task evaluation. The grammar, typed result and +global-corpus rules require an explicit extension; this section records its +design direction and does not claim implemented support. + +### Representation identity and source attribution + +Accepted SchemaIR owns resolved analyzer/default-scorer fingerprints, +`VectorSpec { dimensions, distance, embedding_space }`, rename-stable source +property references, and encoding compatibility. Physical indexes remain +derived artifacts checked against that authority through RFC 0043's proofs. +The version/stamp assignment must coordinate with RFCs 0040 and 0044. + +Schema-owned authoring defaults are resolved into those field bindings before +publication. Equivalent shorthand and fully explicit declarations have the +same field semantics and representation fingerprint; whether a value was +inherited can be shown as provenance without changing its meaning. Persisted +execution bindings never depend on a mutable default name. Schema export and +reapplication must preserve them, including on another deployment. + +Embedding space identity includes immutable model revision and the compatible +record/query encoding recipes: preprocessing, prompts or role selection, +pooling/normalization, and representation shape where applicable. +Query and record encoders may differ intentionally while producing compatible +representations. Equal dimensions, matching labels, or identical provider +names are insufficient. Mutable aliases must resolve to a pinned identity or +fail when compatibility cannot be established. Credentials and endpoint +configuration remain runtime concerns, not accepted-schema secrets. + +Keep the following identities separate. All are versioned typed descriptors; +their hashes are derived views, not an alternative source of authority. + +| Identity | Included meaning | Excluded distinctions | +|---|---|---| +| Encoding recipe / embedding space | Qualified immutable model revision, compatible query/record transforms and roles, pooling, normalization, output shape/component type and numeric policy | Owning node type, field name, source-property identity, index layout, endpoint and credentials | +| Field representation binding | Graph schema identity domain, owner/type/incarnation/property identities, source mapping, resolved analyzer/scorer or encoding recipe, and field geometry | Display names, inherited-versus-explicit authoring provenance, physical indexes and current source values | +| Query encoding invocation | Query-side recipe identity and exact typed input/options, with actual resulting vector values in resolved execution identity | Stored-query name alone, model label alone, dimensions alone | + +Two fields can have different bindings and the same space. Reading `title` +instead of `body` changes the binding; identical encoder transforms can still +produce comparable vectors. Changing a prompt or normalization changes the +space. Distance comparability additionally requires the same field geometry +and numeric comparator; equal space identity alone does not authorize mixing +`dot` and `cosine`. Renames preserve field identity; drop/re-add and a new graph +root do not. Export/rebuild can preserve encoding semantics and vector values +while minting new graph-bound identities. It does not preserve old snapshots. + +Use domain-separated SHA-256 over a versioned, canonical typed encoding, +consistent with the repository's existing hash family. Freeze field ordering, +enum tags, integer widths, optional values and float bit encoding in fixtures +before serialization is accepted; never hash `Debug` text or incidental map +iteration. The existing `schema_ir_hash` covers the whole accepted IR and is +not a substitute for these narrower equivalence relations. Source mapping is +not evidence that a supplied vector was generated from the current row value; +representation-generation validity still needs its own admission contract. + +Do not reserve a numeric SchemaIR stamp in this draft. Extend the existing +`required_ir_version`/validation owner for the combined declared features and +prove mutual format refusal. The current owner accepts base version 2 and +edge-key version 4; version 3 is already burned. A parallel version selector +or reusing an apparently available number would bypass that authority. + +The current provider adapters check model labels and already distinguish +Gemini query/document roles, but they do not prove an immutable hosted-model +revision. An operator-declared recipe fingerprint is not proof of a remote +provider's model contents. Qualification must say which providers can satisfy +the proposed identity contract and how unverifiable aliases are refused; +neither an explicit provider/model label nor an inherited schema recipe is +itself that proof. Default inheritance does not weaken this qualification. + +The existing CLI embedding/export owner now exercises a controlled provider +counterexample. Identical requests with the same model label and graph snapshot +return different query vectors and select different nodes. Schema/data export +and reapplication preserve the label and stored vectors, but do not restore +the original text-query answer. Explicit-vector controls remain consistent; +a mismatched model label is rejected before another provider call. This +qualifies the distinction between graph-snapshot coherence and resolved encoder +identity. It does not claim that a particular public provider changes a model +under a fixed label. + +Initial representations are scalar analyzed String fields and single dense +vectors. Multiple fields can provide different views. Future sparse vectors, +multivectors/late interaction, and named analyzed views need typed representation +and capability variants, without redefining every source as a single dense +vector. Their formats and algorithms are not accepted or implemented by this +RFC. No universal multi-modal storage object is introduced. + +Generated context used during indexing belongs to the encoding recipe. It +must remain distinguishable from original source values when returned. An +application can store source text, generated context, passage nodes, and edges +using its schema. Ordinary graph identity plus property selection and graph +snapshot context provide attribution. A later snippet/range facility must +identify property, source version, offset unit, and bounds; it is separately +scoped and cannot invent a passage reference from a native Lance row ID. + +### Result metadata, coherent continuation, and budgets + +Extend RFC 0047's read envelope through the existing `/query` and stored-query +surfaces. The logical result contract includes these separate facts; concrete +wire types and namespace choices must be checked before implementation: + +| Fact | Required meaning | +|---|---| +| Completion | Every declared stage completed, or a typed failure; a delivered prefix is not complete success | +| Representation coverage | Explicitly known counts or an unknown state for usable representation, pending derivation, and missing source/value among distinct eligible targets at that source's snapshot | +| Selection | Stage/input identities, semantic windows, actual returned counts, comparator and exact/approximate source contract | +| Metric origin | Named source instance, target, domain, scoring/encoding fingerprints, and missing-arm membership | +| Attribution | Graph snapshot context, graph binding identities, and selected properties; application-defined source relationships remain ordinary data | +| Follow-up | A supported way to read/expand those bindings at that snapshot, or an explicit expired/unavailable outcome | + +**Interface decision:** add one typed `execution` option to canonical inline +and stored-query requests, with the same options on the embedded entry point: +`coverage: report | exact` (default `report`), `require_replay: Bool` (default +false), and optional `limits`. This is request admission/metadata policy; +retrieval sources, windows and effort remain in the typed query. Unknown keys, +invalid units and unsupported requirements fail before query work. An omitted +limit inherits a finite operator-configured default; an explicit request may +tighten that cap; a value above the cap is an admission error. Zero means no +allowance for a counted resource, never unlimited. A deadline must be positive. + +Retain `ReadOutput.graph_commit_id` as the replay identifier and `target` as +the requested target. Add an `execution` result descriptor with its format +version, completion, effective limits, usage, fingerprints and replay state. +Replay is `available` when `graph_commit_id` is present, or `unavailable` with +a reason such as `no_graph_commit`; do not manufacture a second snapshot ID. +`require_replay: true` rejects unavailable identity after snapshot resolution +and before source execution. Available means addressable under current +retention, not reserved against subsequent cleanup. + +JSONL's metadata record and Arrow response metadata must carry the same +descriptor and graph commit as JSON. Human tables/CSV remain presentations, +not a lossless continuation protocol. Versioned wire structs, OpenAPI and +renderer/client fixtures must prove these additions; naming them here does +not make them accepted request fields today. +The current owners are [API types](../../crates/omnigraph-api-types/src/lib.rs), +[engine query admission](../../crates/omnigraph/src/exec/query.rs) and +[CLI renderers](../../crates/omnigraph-cli/src/read_format.rs). The engine +currently captures its view before compiling and has no such shared execution +argument; implementing this decision requires changing that entry sequence. + +Coverage knowledge is separate from query completion. By default, report exact +counts only when required retrieval work or qualified snapshot-bound metadata +establishes them; otherwise report `unknown` with a reason such as +`not_computed`. Do not add an exhaustive population scan solely to fill the +default descriptor. A sampled candidate set does not establish representation +coverage, and unknown is neither zero pending nor complete coverage. An empty +successful retrieval with unknown coverage cannot establish that the graph +contains no relevant unrepresented data. + +Callers can explicitly require exact coverage through a typed read-execution +option shared by embedded, CLI, inline-query, and stored-query surfaces. It +changes the requested metadata work, not eligibility, score meaning, or +candidate windows. Its wire spelling remains part of the read-envelope gate. +In exact mode, every representation-consuming source must produce exact +ready/pending/missing counts over its distinct eligible population or the query +fails; resource exhaustion cannot silently downgrade the request to unknown. Counts share the +retrieval budget. Reuse a count only when snapshot, policy, population, and +representation identities agree across sources. Fusion reports its input +sources rather than inventing one combined representation count. A source +without derivation has zero pending; null or token-empty values without usable +representation are missing. An invalid present representation remains a typed +error, not a missing or pending value. + +Lance's unfiltered row count has a metadata path; its filtered count builds a +scanner. Neither a total row count nor an ANN result supplies exact counts for +arbitrary graph-defined eligibility and representation status. Metadata-only +or index-assisted counting needs its own equivalence proof. This is a +structural cost distinction, not a measured latency claim. Counts and +statistics obey policy; metadata must not disclose hidden rows. Representation +absence is separate from a lagging index and from approximate candidate +selection. Retrieval quality and answer confidence are not inferred from +these descriptors. RFC 0047 uses the same known/unknown coverage contract. + +The inline and stored-query request types already accept `snapshot`, mutually +exclusive with `branch`, and canonical reads already return a same-read +`graph_commit_id` when one exists. Reuse those fields for follow-up reads; +do not add a parallel snapshot-token mechanism. `target.snapshot` echoes the +request and is not automatically a resolved token for a branch read. A +synthetic identity for a graph with no commit is not a replayable commit id. +The initial result contract must make unavailable replay identity explicit. +Verify the actual output renderers as well as their shared response type: +JSON and JSONL outputs used for continuation must preserve that same-read +identity. The current JSONL renderer omits `graph_commit_id` from its metadata +line even when the read envelope contains it. Qualify both known and unavailable +identity through the CLI; a passing JSON response does not cover JSONL. + +Identity-based follow-up also depends on RFC 0040's typed meta-field access +and logical result identity. Today a projected node object includes its id, +but `$p.id` resolves only declared properties and does not address the system +identity. Returning an id and a snapshot therefore does not by itself prove +that `.gq` can follow that id. Qualify `$p.@id` projection and filtering through +the compiler, embedded, inline, stored-query and CLI paths once the shared +namespace is available. A declared-key lookup can validate snapshot coherence +now, but does not establish arbitrary entity-id lookup. Cross-type follow-up +must also retain the accepted type/incarnation identity described above. + +Follow-up reads must preserve the relevant graph snapshot across participating +tables using the existing snapshot/retention machinery. They recheck access +under the applicable policy; a snapshot token is not a capability to retain +revoked access. Expired snapshots fail explicitly. The transport binding and +retention behavior, including deleted branches or cleaned-up versions, are +acceptance gates for completing the coherent-read path; +no new durable search-result store is implied. + +**Retention decision:** a read creates no persistent pin, native tag or lease. +Within an execution, every participant uses the captured accepted view; a +concurrent destructive maintenance operation may cause an explicit unavailable +history failure, never substitution with current data. Between calls, replay +depends on the existing graph retention policy. Return an expiry timestamp +only if an actual retention guarantee supplies one. Recheck authorization +before resolving or disclosing historical state. Unknown identifiers, reclaimed +history and authorization refusal remain distinct typed outcomes internally; +public mapping must preserve the existing policy's disclosure boundary. + +**Fingerprint decision:** distinguish the selected stored/inline definition, +the resolved semantic plan and one execution. The definition digest covers the +submitted source, selected query name and tool metadata; whitespace or +instructions may change it. The semantic digest covers the versioned typed +plan, resolved identities/defaults, result shape, sources and metric origins. +Execution identity additionally binds parameters, snapshot, authorization +context, actual query vectors and effective execution options. Neither a +definition digest nor an execution identifier promises repeatable ANN +candidates or deterministic hosted encoding. Keep sensitive parameter/vector +material and authorization details out of public descriptors; an opaque +execution ID can refer to internal diagnostics. Public fingerprints never +authorize cache reuse across actors. The canonical encodings and compatibility +fixtures remain an acceptance gate, shared with representation fingerprints. + +Ranked pagination is deferred. Snapshot, query digest, and semantic +fingerprints alone do not freeze an ANN candidate execution. A stable cursor +needs bounded preservation of the actual candidate order or a qualified +reproducible execution, plus retention, policy, expiry, and failure contracts. +A deterministic sort of one returned subset does not establish that guarantee. + +A small final `limit` does not bound intermediate work, including graph stages +after the final ranking cut. Fallback spends the remaining query budget. +Compact discovery and selective property reads support context control; +token-budget packing and source snippets remain deferred, with their +encoding/source attribution requirements retained. + +**Resource interface decision:** create one request-owned execution context +before snapshot capture/compilation and retain it through serialization. It +owns the accepted view once resolved, cancellation/deadline, resource ledger +and diagnostics. Every stage, nested invocation, scanner, encoder and fallback +borrows that context; cloning a handle does not clone its allowance. Extend +the sealed storage/embedding adapters and DataFusion runtime integration, +rather than adding a parallel executor or storage manager. + +| Limit field / unit | Admission boundary | +|---|---| +| `deadline_ms` / elapsed milliseconds from engine admission | Includes queueing after admission, compilation, reads, retries, encoding and output preparation; transport/body admission has its own earlier bounds. No fresh deadline per stage. | +| `input_bytes` / decoded query and parameter bytes | Bound transport input before parsing; bound internal expansion and retained values through memory/work reservations. | +| `memory_bytes` / simultaneously reserved query-owned allocation bytes | Reserve before allocation/decode, including operator state, native buffers and output. Shared buffers count once while retained; process caches also need a separate finite service cap. This is not a whole-process RSS promise. | +| `storage_requests`, `storage_read_bytes` / admitted storage operations and read payload bytes | Reserve before dispatch; range sizes and bounded metadata bodies determine reservations. Record hidden retry behavior separately and qualify it before promising attempt/transfer caps. | +| `binding_rows` / cumulative rows produced by population-changing operators | Charge scans, traversal fan-out, joins and grouping output before retaining/emitting each batch. Counts measure operator work, not distinct entities or final result rows. | +| `analysis_bytes`, `edit_cells`, `distance_components` / processed text bytes, edit-DP cells, vector components | Bound normalization/tokenization, complete fuzzy evaluation and vector distance work in their adapters. Accelerated algorithms need an explicit conservative charging rule; these are separate units, not one weighted cost score. | +| `embedding_calls` / provider request attempts | Reserve before each call, including retries; bound request/response bytes and concurrency as well. Cache hits do not consume a provider attempt. | +| `spill_bytes` / simultaneously reserved scratch bytes | Reserve before writes; release only when files are retired. Spill reads/writes and processing still consume deadline and relevant I/O/work budgets. | +| `output_bytes` / uncompressed serialized response bytes in the selected format | Enforce with a bounded encoder before publishing a complete success, including descriptors. A small row count never permits an unbounded field. | + +The ledger separates cumulative charges from live reservations. A failed +reservation performs no corresponding allocation, dispatch or batch emission; +checked counter overflow fails. Releasing memory does not refund consumed +work. Multi-resource reservation must either acquire all required allowances +or unwind them before effects. Cancellation stops new work and drains owned +tasks/permits under a bounded cleanup policy; an already dispatched remote +operation cannot be described as undone. + +These units settle the interface, not numeric defaults or enforcement proof. +Default values and adapter chunk/reservation bounds must be fixed in the +implementation and qualification configuration. A native route that cannot +reserve its material allocations/work is unavailable under this contract; +use a qualified bounded route or fail admission. Ordinary Lance metrics, +sampled RSS and charging a completed batch are insufficient proof. In the +pinned `lance-io` GET wrapper, byte counters are recorded after the body is +drained or dropped; they are observations rather than pre-dispatch controls. +See the [pinned wrapper](https://docs.rs/crate/lance-io/11.0.0/source/src/object_store/metrics.rs) +and the [upstream metric definitions](https://lance.org/guide/observability/). + +Extend the existing `OmniError::ResourceLimitExceeded` owner and add typed +deadline/cancellation outcomes through that same error surface; these are +not all existing variants. Include stage, resource, effective limit and +attempted reservation where safe to disclose. +Failures carry no complete-success descriptor. The initial protocol buffers +or boundedly spools the result before success; transport interruption still +means the client did not receive a complete result. A future streaming-success +protocol needs an explicit terminal completion record, not silent partial rows. + +### Exact lexical execution and qualified acceleration + +**Analyzer parity without index coupling.** Instantiate the analyzer from +accepted SchemaIR, even when no FTS artifact exists. Eager empty indexes are +not the analyzer carrier: overwrite, index removal, or an incomplete rebuild +must not erase logical semantics. Use the pinned substrate tokenizer +implementation through one analyzer binding that includes the declared NFC +preprocessing step. Do not reimplement its filters or pass already analyzed +tokens into another path that analyzes them again. Normalization must precede +tokenization consistently in the query, scan, and index-builder paths; +normalizing only a query does not qualify an index built from raw spellings. +Creating or changing an analyzer profile continues to require the existing +schema publication and compatibility protocol; ordinary content writes do +not build indexes inline. + +**Exact scan baseline.** Evaluate the typed predicate over streamed document +tokens using that analyzer and the declared edit relation. Build bounded +query matching state once; honor cancellation and the query's execution +budgets while processing batches. This path is the correctness oracle and +remains available with full, partial, or absent index coverage. It composes +with typed graph/property filters at their declared stage. The first-declared +scan restriction is transitional; supported graph-derived populations must +retain their predicates. Unsupported placement fails validation rather than +losing the predicate. + +This is a new typed Boolean evaluator, not a wrapper around Lance's flat +BM25 scanner. `InvertedIndexParams::build()` already exposes the analyzer +without a dataset or index; its Text tokenizer shares query/document +tokenization. That construction alone does not supply the newly specified NFC +pipeline; its integration is an explicit qualification gate. Lance's flat +BM25 helper accepts a tokenizer but collects per-row scoring counts, does not +implement edit matching, and its fuzzy post-filter path rejects execution. +Reuse the public analyzer in a typed +engine/DataFusion filter over the sealed scan stream. Native index +acceleration additionally needs an analyzer-consistent query path and a +complete-expansion outcome; those scanner capabilities are not supplied by +the pin. Boolean evaluation introduces no dictionary or posting storage. + +The implementation must bound query bytes, distinct terms, individual +materialized values, matching state, and execution work. Enforce admission +before allocations that can exceed the budget, including Unicode normalization +and token construction. NFC can expand UTF-8 bytes: the pinned normalizer turns +the two-byte U+0344 into four bytes. It can also consume a long combining-mark +sequence before yielding its first output scalar. Bound source consumption, +internal normalization buffers/work, and output capacity; an input-byte limit +or cancellation checks only between emitted characters or tokens are +insufficient. A bounded implementation must enforce these limits at the +normalizer's input and internal work boundaries. +Use a bounded exact edit matcher, with cancellation checkpoints inside large +comparisons and across successive small comparisons. A work quantum that +resets for every token pair does not bound uninterrupted aggregate work. +Executor cooperation, cancellation propagation from the request boundary, +and cleanup of native tasks are separate qualification obligations. Marking a +function async or adding a yield does not establish end-to-end cancellation. +The pinned `fst` Levenshtein automaton agrees with the declared scalar-value distance, +but construction can consume substantial memory and hit its state limit. +Its default per-automaton cap is not a whole-query resource protocol. Numeric +limits, accounting units, and fallback charging must be specified and +qualified before shipping this evaluator. + +**Qualified index acceleration.** Lance continues to own dictionaries, +postings, and physical index state. A native path is eligible only when its +artifact passes RFC 0043's proof checks against the accepted profile and its +complete normalization/tokenization/filter pipeline is qualified for the +requested mode and edit budget. +Covered rows use complete term expansion and posting evaluation; uncovered +or rewritten rows use the same exact predicate on their accepted values. +Combine them at one snapshot with the existing visibility and row-identity +rules. Final limits and retrieval arm windows cannot change which terms or +rows satisfy the predicate; ordinary early termination is allowed when the +query plan proves the requested result complete. Post-verifying a truncated +candidate set cannot repair omitted matches. + +An indexed expansion must distinguish **complete** from **overflow**. The +current substrate shares a default budget of 50 expansions across query +terms within each segment, selecting lexically across that segment's +partitions. It returns tokens without a completeness flag. Earlier terms +can exhaust the budget before later terms are considered; changing the +vocabulary or segment layout can change the selected terms. This is not an +exact membership contract, and increasing the cap is not a completeness proof. +When complete acceleration cannot be established, use the exact scan within +the remaining query budget. If that budget is exhausted, fail the whole +query with a typed resource outcome; do not return partial success. Bound +token/automaton construction, memory, and execution work without silently +discarding analyzed terms or increasing budgets on fallback. This introduces +no query-level `max_expansions` knob or separate approximate predicate. + +The pinned Lance fuzzy scanner does not yet satisfy these requirements. +Until an upstream implementation and adapter pass the qualification matrix, +use the exact baseline for the affected shapes even when an index exists. +Keep native matching changes upstream where possible; do not build a second +index subsystem or a persistent shadow vocabulary in OmniGraph. BM25's +separate analyzer and corpus-statistics parity gate remains required; a +membership matcher alone does not qualify ranked scores. + +### Lance, DataFusion, and graph execution + +The audited pin is Lance 11.0.0, DataFusion 54.0.0 and Arrow 58.3.0. Current +graph execution uses Arrow batches and engine-owned fusion/aggregation; this +proposal does not assume a complete DataFusion logical plan already exists. + +| Logical need | Physical mechanism | Required integration or proof | +|---|---|---| +| Property eligibility | Structured DataFusion expressions and qualified Lance filter pushdown | Preserve binding identity, policy, and stage boundaries | +| Graph-defined candidate population | Existing traversal, typed semi-join or Lance external row mask | Map graph identities into the pinned target dataset's native row-ID domain | +| Vector candidates | Qualified Lance nearest scanner and exact scan paths | Geometry/space, tail coverage, raw-vector scoring, ties, and effort bounds | +| Analyzed matching | Qualified NFC preprocessing, public Lance tokenizer, and typed Boolean evaluation | One accepted pipeline through query/scan/index construction, complete edit matching, admission and cancellation | +| Lexical ranking | Structured Lance FTS where qualified; exact scoring fallback | Declared corpus statistics, fuzzy formula, numeric parity, complete boundaries | +| Fusion | Explicit arm ranks, union, aggregate and sort | Common identity, missing-arm semantics, one snapshot, shared budgets | +| Target selection and binding preservation | Distinct target stream for ranking; semi-join selected identities back to the incoming bindings | Deduplicate before candidate cuts; preserve every surviving graph binding and its metric origin | +| Selection per group | DataFusion `dense_rank` over bindings, or distinct pairs plus `row_number` and a null-safe semi-join | Pair-constant comparator, explicit target tie key, pair quotas, binding multiplicity, key equality, metric and incoming-order preservation | +| Graph expansion | Existing CSR/CSC and indexed edge paths | Retain traversal/path semantics, bound fan-out, carry metric origin | +| Intermediate analytical results (future stage) | Qualified aggregation, expressions and entity-key joins | Export typed group values and accepted entity identities; preserve aggregate input lineage and selection barriers | +| Correlated evidence and optional facts (future operators) | Qualified outer joins, grouped selection and typed collection | Define local populations, empty/null behavior, order, duplicate semantics and cumulative nested work | +| Learned reranking | Future bounded scoring/model operator | Model identity, batched input, cancellation, resource and failure contracts | + +Use the sealed `TableStore` boundary for native search nodes. It must preserve +snapshot selection, analyzer certificates, policy and visibility. Public +`Scanner::create_plan()` takes no caller session; `create_plan_with_session()` +is crate-private and used internally by `LanceTableProvider::scan`. Executing +a configured vector/FTS plan under a shared `TaskContext` does not prove shared +planning configuration. Qualify each source or obtain the missing upstream hook. + +`Scanner::with_row_addr_prefilter` consumes the selected dataset's native +`_rowid` domain, including stable row IDs. Resolve graph IDs against the pinned +dataset and charge that mapping; the current `ScanTuning` wrapper does not +expose this route. Provider registration alone does not implement retrieval. + +For group quotas, retain two equivalent physical choices: + +- Reduce to distinct target/group pairs, rank with `row_number`, and null-safe + semi-join winning pairs back to bindings. +- Use `dense_rank` directly on bindings when every comparator value is constant + per pair, with target identity as the final tie key. Duplicate paths share a + rank; restore the incoming logical order after the physical window. + +A varying path comparator requires the declared pair reduction first. +`row_number` directly on bindings counts paths, while omitting the identity +tie key from `dense_rank` can exceed the quota. Reattaching selected pairs +directly to incoming bindings can avoid repeating an already implied cutoff. +These are qualified rewrite candidates, not a universal planner choice. + +The existing +[selection owner](../../crates/omnigraph/tests/rrf_prefilter_gate.rs) and +[independent scenario oracle](../../crates/omnigraph/benches/scenarios/search_selection.rs) +cover duplicate paths, multiple memberships, null/composite keys, filter/cut +placement, memory and persisted Lance sources, input reversal and partition +changes. Retain two native compatibility fences: + +- With four partitions, the tested default memory-source plan gives + `CollectLeft` a multi-partition build input and fails sanity checking. + Default real-Lance variants pass. Do not infer a global planner override. +- Nullable integer joins can lose winning bindings under hash-join dynamic + filters (20 expected become 18). Staged sessions containing these joins must + disable `enable_join_dynamic_filter_pushdown` until the upstream path passes + the oracle. `NullEqualsNull` alone is insufficient. The dense route avoids + that pair join but does not qualify other nullable joins. + +Carry narrow rows through selection and hydrate payloads afterward where +qualified. The instrument uses native `_rowid` with public Lance `TakeExec`, +checks payloads and final binding order, and retains both dense and distinct-pair +routes because their costs differ with fan-out and payload width. Downstream +reliance on Take's ordering metadata remains unqualified. + +The [historical physical experiments](assets/0048-agent-context.md#lance-datafusion-and-graph-execution) +record configurations, timing/I/O/spill tables and controls. They support these +implementation candidates, not production latency or a universal winner. +Reproduce through the existing instrument: + +```sh +cargo bench -p omnigraph-engine --bench scenarios -- \ + --scenario search-selection --rows 10000 --fanout 8 --groups 64 \ + --selectivity 0.1 --k 100 --quota 2 --text-bytes 2048 \ + --selection-plan hash --group-select dense --join-filters false --reattach-cut false \ + --late-payload true \ + --partitions 4 --query-memory-mb 128 --scratch-mb 1024 +``` + +Vary plan, group-selection algorithm, repeated-cut reattachment, eligibility, +fan-out, windows, quotas, payload width, partitions and memory/scratch. Oracle +mismatches retain records and fail; resource refusals have no successful oracle +verdict. Local Lance reads can bypass object-store wrappers, parent RSS includes +fixture/oracle setup, and OS cache state is uncontrolled. Keep these measurement +boundaries with any comparison. + +The [contract-to-code matrix](#contract-to-code-qualification) owns native +resource counterexamples and their required dispositions. The adapter must +distinguish processed input/work from unique live allocations, track shared +buffer lifetimes, and govern native decode, queued/dispatched I/O, analyzer +state, output and encoders as well as participating DataFusion operators. +Summing batch allocation sizes double-counts shared buffers; post-decode +charging cannot bound allocation peaks. + +Preserve typed resource refusals through engine/transport wrappers. A fair-share +consumer can refuse below the total pool limit; report measured usage only +where available, never fabricate it from an error string. Cleanup includes +physical plans and shared hash-build state, not just streams. Dispose of a +failed query's runtime when scratch accounting is stale; fallback retains the +same spent allowance. Disable spilling for a no-write policy, and qualify +positive-limit write overshoot separately. + +A dropped request or stream does not establish stopped work. Qualify actual +reader/client, decode tasks, plans, output queues and encoder calls through +bounded shutdown. The standard/lite scheduler difference is a compatibility +fence, not justification to switch every backend. Materializing everything in +a `MemTable` or replacing graph traversal with eager cross products does not +satisfy these resource obligations. + +## Invariants + +The design strengthens schema authority, coherent snapshots, typed query +semantics, derived acceleration, and bounded observable execution. Graph +publication remains one manifest publication; the read algebra adds no writer, +transaction manager, queue, shadow vocabulary, or durable result authority. +Schema and artifact changes retain their existing publication/recovery owners. + +Policy precedes candidate selection and covers follow-up reads. Stable graph +identity survives supported renames but not drop/re-add. No native row address, +index state, or mutable query/profile name becomes logical authority. Physical +fallback preserves the declared exact contract or fails explicitly; it cannot +hide a partial candidate population. No invariant exception is requested. + +## Compatibility and reversibility + +The [user-facing migration matrix](#user-facing-changes-and-migration) owns +the breaking/additive classification, client compatibility limits, and upgrade +sequence. Representation semantics require coordinated accepted SchemaIR and +internal manifest versions. Incompatible binaries refuse rather than +reinterpret data; implementation must regenerate OpenAPI and update current +developer guides alongside the user-facing material listed above. + +Grammar remains changeable before stable release. Accepted representation +identities, scoring meanings, and format compatibility still require evidence +because deployed data and queries depend on them. Reverting a format change +requires another explicit rebuild. + +## Alternatives + +- **Keep adding retrieval functions inside `order`.** This leaves candidate + selection hidden in scalar-looking expressions and makes multi-stage scope, + metrics, and windows special cases. Explicit rank boundaries own selection. +- **General relation-valued variables and a separate search-profile registry.** + The existing clause language and stored queries provide the needed entry + points with less new language and registry machinery. +- **Built-in document/evidence entities.** Applications already define nodes, + properties, and relationships. Preserve graph identity and source version + without imposing a second data model. +- **Fuzzy filtering as the complete typo-search feature.** An exact lexical + source can discard admitted typo matches. Shared lexical descriptions and + explicit fuzzy scoring close the ranked path. +- **Native fuzzy defaults or truncated expansion as exact semantics.** They + change results with analyzer/index/segment state. Complete evaluation or + typed failure is required; a larger cap is not a completeness proof. +- **Empty indexes as analyzer authority.** Artifact removal cannot erase + logical matching semantics. Accepted SchemaIR owns the analyzer. +- **Implicitly rescope lexical statistics with every graph filter.** This can + reverse the ordering of the same eligible targets and couples score meaning + to match-block placement. A snapshot-visible field corpus gives filtering + and scoring separate meanings; qualified statistics still have a cost. +- **Implicit score blending or a blanket ban on all feature combination.** + Named domains prevent accidental mixing while explicit normalized/model + stages can define valid combinations. Geometric range predicates remain + distinct from calibrated relevance judgments. +- **Treat grouping as semantic diversity.** A quota constrains concentration; + it cannot establish that the selected facts cover a reasoning task. +- **Require exact population coverage on every discovery query.** Truthful + reporting can state that coverage is unknown. Exact counting remains an + explicit, budgeted request; a candidate sample must not impersonate a count. +- **Infer stable pagination from a snapshot.** A snapshot fixes source data, + not an approximate candidate execution. Preserve that execution or defer + the stable cursor promise. +- **Let physical indexes choose meaning or expose per-query `ef`/`nprobes`.** + Logical contracts are stable; the bounded `oversample` mapping and qualified + capabilities control physical effort. + +## Evidence and tests + +Evidence is tied to recorded source revisions and configurations. A source +audit establishes API behavior; a probe establishes its tested fixtures; +production qualification requires the actual compiler/engine/transport path. +Neither test counts nor API availability establish retrieval quality or +whole-query resource bounds. + +The [upstream receipt](assets/0048-upstream-contract-checkpoint.json) records +checksum-matched crate sources and probe results. The +[agent context](assets/0048-agent-context.md#evidence-and-tests) +retains the detailed validation history. The current obligations are summarized +below; historical pass counts must not be reported as fresh evidence. + +### Contract-to-code qualification + +This matrix is the implementation decision record for Lance 11.0.0, +DataFusion 54.0.0 and Arrow 58.3.0. **Reuse** means use the public primitive +within its established semantics; **adapter** means OmniGraph must supply the +missing contract; **bounded fallback** means complete evaluation under the +same query budget or typed refusal. **Upstream** identifies a limitation that +needs a qualified workaround or upstream fix before enabling that route. +None of these labels means the full search feature has shipped. + +The [source and probe receipt](assets/0048-upstream-contract-checkpoint.json) +records the inspected source files, archive checksums, lockfile and executable +probe results. Every recorded source file was compared byte-for-byte with its +crate archive, whose checksum matched `Cargo.lock`. Full upstream guides were +read alongside the code, including [DataFusion integration](https://lance.org/integrations/datafusion/), +[object stores](https://lance.org/guide/object_store/) and +[observability](https://lance.org/guide/observability/). Live documentation +describes available concepts; the exact crate source and qualified tests +determine what this pin can promise. + +| Search promise | Pinned interface and limit | Decision | Minimal falsifier and disposition | +|---|---|---|---| +| Accepted analyzer, including fuzzy input | `InvertedIndexParams::build`; native fuzzy and flat paths choose different analysis/expansion behavior; NFC needs preprocessing. | Reuse tokenizer; adapter plus bounded fallback for complete membership. | Same composed/decomposed or capitalized typo before/after indexing must have identical membership. Native Unicode/analyzer guards expose the limits; the four current GQT regressions remain open. | +| One exact/fuzzy lexical score | `InvertedIndex::bm25_stats_for_terms`, `MemBM25Scorer`; native float32 BM25 does not implement the proposed grouped fuzzy formula or numeric policy. | Bounded exact scorer first; native acceleration only after parity qualification. | Rare expansion, repeated terms, common-term IDF and deletion change native scores. Existing native and independent Decimal fixtures exercise these counterexamples; full indexed parity remains open. | +| Cross-table lexical ranking | Public shared scorer can be supplied before native per-table cuts; immutable index statistics can include deleted rows. | Adapter owns live corpus aggregation, scoring policy and complete cut boundaries. | Two local top-1 cuts lose the global winner; the shared-statistics probe recovers its score band. Canonical ties and live multi-type statistics remain open. | +| Exact `knn` and declared `ann` effort | `Scanner::nearest`, `use_index(false)`, metric/refinement controls and `scan_stats_callback`; refinement scores only retrieved candidates. | Reuse qualified scanner paths; adapter owns geometry, coverage, ties and effort. | ANN omits a true neighbor or returns an unreached `+inf` row. Existing vector guards fence the latter; family-specific recall, arithmetic and complete-tie qualification remain required. | +| Graph-scoped candidate population | `Scanner::with_row_addr_prefilter` consumes the selected dataset's native `_rowid` domain. | Adapter resolves accepted graph identity to pinned native identity. | Delete/compact a target, then search an older snapshot with a graph mask. Native identity guards exist; the full graph-mask retriever remains open. | +| Independent candidate windows and final output size | Scanner limits and FTS collectors can cut before OmniGraph sees candidates. Native row-ID or score-only ties are not the declared entity comparator. | Adapter owns every semantic cut; bounded fallback for an unqualified boundary. | More tied targets than the native window, with reversed graph/native identity order. Sorting a truncated subset cannot pass this gate; production qualification remains open. | + +| Composition promise | Pinned interface and limit | Decision | Minimal falsifier and disposition | +|---|---|---|---| +| Configured retrieval shares planning and execution context | Public `Scanner::create_plan()` takes no session; `create_plan_with_session()` is crate-private, used internally by public `LanceTableProvider::scan`. | Reuse provider for qualified scans; adapter or upstream hook for required configured-source controls. | Execute vector/FTS plus downstream operators under one tiny allowance and nondefault planning options. Ordinary provider pool/control probes pass; configured retrieval-source propagation remains open. | +| Filters stay on their side of a stage | `TableProvider::scan` applies filters before limit before projection. `UserDefinedLogicalNodeCore` provides predicate and limit pushdown controls. | Reuse ordinary relational nodes; conservative adapter barriers for graph/search nodes. | Filter before/after a target cut or binding limit must produce different fixture results. Extended staged probe passes all twelve native/memory configurations; new GQ nodes still need the same oracle. | +| Hidden ranking and identity columns survive optimization | `necessary_children_exprs` defines extension input demand; its default retains all input columns. Provider filters may reference unprojected columns. | Adapter declares every stage dependency; prune only after its last use. | Project only `binding_id` after target ranking, offset/limit and a nullable-group filter. Extended probe returns exactly `path-7`; moving a later filter before the limit changes empty output into one row. | +| Group quotas preserve graph bindings | DataFusion `dense_rank`, distinct, `row_number` and typed null-safe joins; nullable join dynamic filters have a measured defect. | Reuse windows; qualify each join route and retain the dynamic-filter fence. | Two paths to one target consume one slot; null-key pairs survive. Existing native selection oracles pass the fenced routes and expose 20 expected bindings becoming 18 on the affected route. | +| Global order and late payload reads | `ExecutionPlan` ordering/distribution properties and Lance `TakeExec`; partition-local order is insufficient for a global cut. | Adapter restores declared order and validates properties after hydration. | Shuffle partitions and tie keys, then hydrate and apply another ordered stage. Existing terminal-take result checks pass; downstream reliance on take's ordering metadata remains open. | +| Fusion and all-node discovery retain type/metric identity | DataFusion union, aggregate and sort can compose compatible Arrow relations; no native operator resolves OmniGraph's accepted all-type scope or score comparability. | Adapter expands the schema scope, carries type/entity/source identity and performs declared fusion. | Same entity ID text in two types, incompatible vector spaces, duplicated paths, missing arms and overflowing weights. Scalar controls cover some arithmetic/multiplicity cases; cross-type grammar and complete integration remain open. | +| Coherent source reads and truthful metadata | Pinned Lance datasets/tags and native takes supply per-table reads; statistics and task metrics have narrower scopes than graph completion. | Reuse snapshot carriers; adapter owns schema/graph identity, authorization, retention and response classification. | Change head or drop/re-add a type between search/read; ask for exact counts under insufficient budget. Existing snapshot mechanisms are qualified; proposed reference/error/count envelopes remain integration work. | + +| Resource promise | Pinned interface and limit | Decision | Minimal falsifier and disposition | +|---|---|---|---| +| One memory allowance across arms and fallback | `RuntimeEnv`/`TaskContext` share participating `MemoryPool` reservations; Lance decoded batches can remain outside them. | Reuse pool; adapter owns admission for native input, retained state and output. | One-byte pool retains a 304-byte native scan result while aggregation refuses. Existing guard passes; preallocation and whole-query accounting remain open. | +| Shared Arrow buffers count once without stealing another owner's allowance | Optional `Array::claim` and `ArrowMemoryPool::reserve` track buffers infallibly and replace prior reservations. | Tracking primitive only; adapter or upstream fallible allocation/ownership support required. | New isolated probe overfills 1 byte with 20 bytes; second query claims the buffer and first pool falls to zero while its array remains live. Last-alias cleanup and fallible refusal controls pass. | +| I/O buffering and cancellation stay bounded | `ScanScheduler`, `FileScheduler`, `SchedulerConfig`; priority progress can exceed the byte setting; standard scheduler dispatches detached tasks. | Adapter or upstream support for admission, task lifetime and actual transport shutdown. | New native guard admits 16 bytes against 1; queued read never starts, standard in-flight read survives all public owners, lite read future is dropped. Cloud/decode/whole-query shutdown remains open. | +| Spill and post-error cleanup preserve allowances | DataFusion `DiskManager`; positive limits are checked after writes and a refused write can leave stale accounting. Plans may retain hash-build reservations after stream completion. | Disable spilling for no-write policy; adapter plus upstream qualification for positive limits and cleanup. | Existing seven-byte refusal and collected-plan probes expose the boundaries. Dispose of query-owned runtime/plan; qualify bounded write overshoot before advertising a hard scratch quota. | +| External encoding and CPU work respect the same attempt | DataFusion owned-task/cooperation utilities help; neither Lance nor Arrow freezes an external provider or budgets arbitrary analysis/model work. | Adapter owns resolved encoder identity, cancellation and cumulative work. | Same provider label changes vectors; long normalization/token work delays yielding. Existing provider and CPU controls expose both; end-to-end request/encoder cancellation remains open. | + +The pinned [table-provider contract](https://docs.rs/crate/datafusion-catalog/54.0.0/source/src/table.rs) +allows an advisory scan limit to return extra rows and disallows pushing it +through inexact filters. Lance's ordinary provider reports exact filters +because its scan evaluates them; that is not authority to move a filter across +an OmniGraph candidate cut. For extension stages, retain +`supports_limit_pushdown=false` and conservative +`prevent_predicate_push_down_columns` until a rewrite is proven. Implement +`necessary_children_exprs` only with target, metric, comparator, group and +payload dependencies accounted for. The [extension API](https://docs.rs/crate/datafusion-expr/54.0.0/source/src/logical_plan/extension.rs) +already offers these hooks; a separate query optimizer is not required. +Validate parameters before optimization can erase an empty branch. + +The extended stage probe also calls `LanceTableProvider::scan` directly in +its eight native configurations with projection `binding_id`, filter +`score < 10` and limit one. It returns one eligible row with no score column. +This checks the actual provider limit separately from an optimizer's sort +top-K rewrite. The composed checks additionally exercise twelve combinations +of memory/native source, input reversal and one/four target partitions. + +DataFusion's [execution-plan contract](https://docs.rs/crate/datafusion-physical-plan/54.0.0/source/src/execution_plan.rs) +requires cooperative streams and owned background tasks that stop when +dropped. Lance's pinned [standard scheduler](https://github.com/lance-format/lance/blob/ab6b5bbe46009ed78746b444df8db59a8bc5d842/rust/lance-io/src/scheduler.rs) +spawns dispatched reads without retaining their join handles; its drop path +closes the queue. The [lightweight scheduler](https://github.com/lance-format/lance/blob/ab6b5bbe46009ed78746b444df8db59a8bc5d842/rust/lance-io/src/scheduler/lite.rs) +keeps reader futures in task state and drops them on abandonment. The new +guard establishes that difference at the native scheduling boundary. It does +not establish that dropping any particular OS or cloud request cancels its +underlying operation, or that per-request timeouts impose a query deadline. + +Reproduce these native checks from the repository root: + +```sh +cargo test -p omnigraph-engine --test lance_surface_guards --locked +cargo test -p omnigraph-engine --test rrf_prefilter_gate --locked +python3 docs/rfcs/assets/0048-arrow-pool-probe.py +``` + +The Arrow runner creates an isolated temporary crate, checks all resolved +registry versions/checksums against the workspace lockfile, and enables only +there the optional Arrow pool features. Its [Rust probe](assets/0048-arrow-pool-probe.rs) +and result receipt are reviewable; production dependency features are unchanged. +These deterministic mechanism checks do not add latency, peak-memory or +retrieval-quality claims. They supplement the frozen integrated prototype and +agent evaluation rather than changing their inputs or results. + +The recorded focused run reports 43 Lance surface guards, 13 RRF/prefilter tests and +11 benchmark contracts passing, plus the isolated Arrow assertions. The S3 +same-version guard returns early without configured storage credentials, so +its passing libtest entry is not remote-storage evidence. Both workspace +Clippy feature graphs pass. The four previously recorded GQT failures remain +open; these native checks neither repair them nor replace release validation. + +### Assumption audit + +The pinned tokenizer/edit-distance probe passed 73,008 comparisons against an +independent Unicode-scalar evaluator at budgets zero through two. This covers +the primitive, not the revised NFC pipeline, indexed completeness or budgets. + +The [Decimal oracle](../../crates/omnigraph/tests/fixtures/lexical_scoring_v1.py) +generates thirteen 80-digit reference cases. The float64 evaluator uses pinned +`libm 0.2.16`, tolerance `2e-14 * max(1, expected)`, exact fixture order, +repeated-term invariance, eligibility-independent scores/features, all/any +feature invariance and edit-budget inclusion. Inputs already represent +analyzed tokens. Native/indexed numeric parity, complete ties and relevance +defaults remain unqualified. + +The [native matrix](#contract-to-code-qualification) and +[composition evidence](#required-composition-examples) own the remaining +assumptions and falsifiers; avoid duplicating their status here. + +#### CI checkpoint and regression disposition + +The 2026-09-09 GitHub Actions runs for PR head +`bf5a77e55bc1dcd8415f28d5900d0a9cd218d78e` confirm the same four failures in +both [GQ Logic Tests](https://github.com/ModernRelay/omnigraph/actions/runs/34371510788/job/102533700021) +and [Test Workspace](https://github.com/ModernRelay/omnigraph/actions/runs/34371511424/job/102534255348): +67 of 71 cases pass, and all 127 runner self-tests pass. The workspace job +tests GitHub's merge candidate `f4ce7e895eb5c29b34c70cb1dbfbf95fba2f8a27`, +which combines that PR head with base `ed3ea5006f55ef703a3331f29ac45a2262f13300`. +This identifies the tested source; it does not claim validation against a +later `main` revision. + +| Existing GQT owner | First observed failure | Required implementation and migration proof | +|---|---|---| +| [`fuzzy_query_bypasses_index_analyzer`](../../crates/omnigraph-gqt/cases/fuzzy_query_bypasses_index_analyzer.gqt) | Step 2: capitalized `Introductio` returns no rows; `intro` is expected. | Phase 2 must apply the accepted field analyzer at every edit budget. Retain the lowercase and zero-edit controls and reach the later assertions. | +| [`index_state_changes_text_matches`](../../crates/omnigraph-gqt/cases/index_state_changes_text_matches.gqt) | Step 5, including the two mutation steps: `running` finds only the appended row and loses the indexed row. | Phase 2 must use one matching definition for indexed and uncovered rows. Complete the later `beto` assertion as well; passing the first repaired step is insufficient. | +| [`search_on_traversal_target_is_dropped`](../../crates/omnigraph-gqt/cases/search_on_traversal_target_is_dropped.gqt) | Step 2: traversal returns B, C and D where only B and D match. | Phase 3 must retain the target predicate and rank the traversal target. The later ranking assertion must return D, rather than fail for a missing score column. | +| [`unindexed_search_is_case_sensitive`](../../crates/omnigraph-gqt/cases/unindexed_search_is_case_sensitive.gqt) | Step 3: unindexed `deep` finds only the lowercase row; both rows are expected. | Phase 2 must preserve matching with and without an index. Migrate both field declarations to the intended accepted analyzer, and run both query-case controls. | + +These cases were added in `b1df2041` and remain active regressions. During the +coordinated query migration, rewrite their legacy `search`, `fuzzy` and `bm25` +forms through the new typed terms and ranking constructs while preserving +their expected membership and ordering. Declare analyzer intent in the schema; +index presence cannot provide it. Removing the old syntax, blessing the +incorrect rows, or running only the isolated staged case does not close these +regressions. A passing implementation needs every step and the full corpus to +run through the production parser and engine. + +The same workspace run passes 361 compiler tests, 42 Lance surface guards, +56 search tests, 13 RRF/prefilter tests, 88 CLI data tests and 23 CLI parity +tests before failing at GQT. These counts describe the PR's checked-in code; +the archived integration experiment has different test counts and is outside +this CI build. Environment-specific skips inside native guards retain their +existing limitations. The workspace process stops on the GQT failure, so +later workspace test targets are not established by this run. Format and +Clippy checks pass independently. These historical CI failures require the +implementation above; later native probes do not close them. + +### Implementation handoff and validation checkpoint + +Use the existing owners and archived experiments below. The checked-in staged +compiler is test-only; the production parser does not accept these examples. +Its shared expression root, precedence, contextual names, argument/refusal +rules, scope transitions and metric identities are partial design evidence. +Numeric/null evaluation, all result types, resolved identities and integrated +lowering still need qualification. + +#### Explicit-output integration checkpoint + +The [updated patch](assets/0048-phase0-integration.patch) applies directly at +`b87068cb452ac4c2af0c4056ed3a5cde0b678604`; its +[receipt](assets/0048-phase0-integration-checkpoint.json) records exact inputs, +commands, results and limitations. Do not apply the older patch first. + +This isolated experiment carries explicit `yield` through the real compiler, +IR, engine and GQT for multiple scalar-String lexical sources. It tests +independent windows, declaration reorder, unused sources, output choice, +inherited order and nullable missing metrics. Read descriptors, column demand +and GQT detection see later stages. Invalid parameters fail before scanning; +a suffix with 100,200 bindings refuses despite final `limit 1`. + +The pinned Rust 1.97.1 checks pass 368 compiler, 127 GQT unit, one staged GQT, +ten column-demand and four staged resource/scoring tests, plus the extended +admission owner; one diagnostic instrument remains ignored. The full 57-test +search owner predates the final admission assertions, whose owner was rerun. +Historical CLI/server counts do not qualify this patch revision. + +Limits remain explicit: alias-wrapper source IDs, experimental admission caps, +post-decode accounting and incomplete ownership of analyzed state, I/O and +output. Boolean matching, vector/fusion execution, full expressions, C1–C4, +global search, schema/default persistence and read/error contracts are not +implemented by this patch. + +#### Composition plan and primitive checkpoint + +Starting from `1fb0423d`, the checked-in C1–C4 extension passes 367 compiler +tests and adds four golden assertions to existing tests. One native composition +test passes four memory/partition/order configurations under Rust 1.97.1, +DataFusion 54.0.0 and Arrow 58.3.0. Focused Clippy, formatting and docs checks +pass. [C1–C4](#required-composition-examples) links the plans, probe and per-case +limits; those are the evidence authority. + +The logical goldens and hand-built native plans are separate. Materialized +ranks/features and fixture parent IDs do not prove retrieval, scoring, +correlation or generated identity. Ambiguity detection does not implement +typed cardinality refusal. This checkpoint neither completes Phase 0 nor +repairs the four GQT regressions. + +```bash +cargo +1.97.1 test --locked -p omnigraph-compiler +cargo +1.97.1 test --locked -p omnigraph-engine --test rrf_prefilter_gate staged_composition +``` + +#### Historical integration and diagnostic pilot + +The [historical patch](assets/0048-staged-integration.patch) applies at +`b1df2041c93e03aa13cee8308a0a574689baf210`. Its +[validation inventory](assets/0048-validation-checkpoint.json) records the +frozen sources and tested compiler, engine, GQT, stored-query and CLI boundaries. +It uses one lexical source per rank block and is outside this PR's production +build. Inspect it in an isolated checkout; prefer the later explicit-output +patch for ongoing integration work. + +Its evidence includes forty generated graph comparisons against an independent +evaluator, twelve Decimal/lifecycle fixtures, snapshot/current-read controls, +authorization refusals and resource counterexamples. These are finite +experimental proofs, not complete language, transport or resource qualification. +The [original checkpoint](assets/0048-agent-context.md#historical-integration-and-diagnostic-pilot) +retains exact commands and the distinction between fresh and earlier runs. + +#### Lessons the implementation must retain + +Walk every stage for column demand, read descriptors, GQT detection and +resource accounting. Validate inputs before optimization removes an empty +branch. Keep graph rewrites inside selection barriers and preserve expected +results in the four existing regressions. + +Shared Arrow buffers exposed repeated allocation charging; repeated vocabulary +exposed redundant per-document edit-distance work. Exact token lookup and +query-local distance reuse improved the prototype, but cache/state construction, +native allocation and cancellation still need admission. Experimental caps +are not product defaults. + +Refresh SchemaIR version ownership before assigning a format; the audited +baseline recognizes versions 2 and 4 and burns 3. Export/reapplication must +preserve resolved recipes. Keyed follow-up does not qualify general system-ID +lookup; archived JSONL metadata changes need their actual transport owner. + +#### Completed agent pilot and interpretation + +The frozen pilot uses 109 repository Markdown documents at +`bf1e5ca15868c9ce2444062e9d9d02b539d786e3`, split into 1,439 passages, with one +development question and eleven held-out questions across four recipes. +The [final record](assets/0048-agent-pilot-results.json) preserves all 44 trials, +model/configuration identities, answers, source reviews, refusals and failures. + +| Frozen recipe | Trials | Final answers, including abstentions | Answers with citations, all read | Strict supported tasks | +|---|---:|---:|---:|---:| +| Exact lexical, all terms | 11 | 11 | 4 | 3 | +| One-edit lexical, all terms | 11 | 10 | 5 | 3 | +| Dense | 11 | 11 | 11 | 8 | +| Exact lexical + dense, application RRF | 11 | 9 | 9 | 8 | + +Review was by the assistant, without independent human adjudication. Strict +success required complete supported claims, no draft mistaken for current +behavior, and full reads of citations; failures stayed in the denominator. +Independent arithmetic agreed on all 23 completed dense top-20 lists and ten +hybrid top-10 lists, but does not qualify near ties or ANN. + +The lexical recipes used `mode: all`, whose conjunction was not explained in +the shared tool description. Hybrid used application RRF over two engine +queries, not staged fusion or one shared native budget. Host/API variability +and timeout/cleanup overhead preclude production latency or hard-deadline +claims. The [full protocol and interpretation](assets/0048-agent-context.md#completed-agent-pilot-and-interpretation) +preserve these controls. + +The actionable lesson is to expose recipe semantics and source revision/status +context. Reading a real citation did not prevent unsupported conclusions. +This document-only pilot establishes neither a modality winner nor broader +analytical task quality, retrieval defaults or engine-defined confidence. + +### Research context and required qualification + +The broader motivation is consistent with +[Qdrant's staged hybrid queries](https://qdrant.tech/documentation/search/hybrid-queries/), +[Vespa's ranking phases](https://docs.vespa.ai/en/ranking/phased-ranking.html), +[role-specific encoding](https://www.sbert.net/examples/sentence_transformer/applications/semantic-search/README.html), +and [multivector retrieval](https://qdrant.tech/documentation/tutorials-search-engineering/using-multivector-representations/). +[Contextual retrieval](https://www.anthropic.com/engineering/contextual-retrieval) +motivates preserving original source identity separately from generated +indexing context. [Agentic-R](https://arxiv.org/abs/2601.11888), +[BRIGHT-PRO](https://aclanthology.org/2026.acl-long.1705/), and +[BrowseComp-Plus](https://arxiv.org/abs/2508.06600) motivate evaluating downstream +utility and complementary coverage on controlled tasks. These sources do not +prove a specific algorithm or default is optimal for OmniGraph. The stage and +identity laws also follow directly from the noncommuting operations in +Motivation; they do not depend on benchmark leadership claims. + +The [Vespa `rank` operator](https://docs.vespa.ai/en/reference/querying/yql.html#rank) +uses its first argument for matching while other arguments supply ranking +features. It is a concrete precedent for C3's separation of candidate +membership from scoring. [Cypher collection subqueries](https://neo4j.com/docs/cypher-manual/current/subqueries/collect/) +construct lists from correlated query results with defined variable scope; +they motivate C4's explicit correlation and typed result construction. +OmniGraph retains its own scope, null, duplicate and budget rules. These +references support specific design choices, not a claim of equivalent +capability, performance or agent utility. + +The [agent workload](#agent-workload-and-design-objective) also requires +analytical composition in both directions. C1–C4 are the required design +examples, while the matrix states which operators can be executed in the +initial release. The completed document-search pilot cannot establish those +broader capabilities or select universal workload defaults. + +Extend existing test owners rather than creating a parallel search harness: + +| Boundary | Required evidence and owner | +|---|---| +| Grammar / types / lowering | Compiler parser/typecheck/IR fixtures for schema default declarations and resolution, rank blocks, typed lexical queries, named metrics, invalid references, parameter bounds, removed syntax, and aggregate scope | +| Query semantics | `.gqt` cases for filter-before/after-rank, traversal-introduced targets, target/pair/binding-row counts, per-group multi-membership and null buckets, final limit independence, and exact verification | +| Search mechanisms | `search.rs`, `rrf_prefilter_gate.rs`, `ordering.rs`, `aggregation.rs`, and traversal owners for arm ranks through fan-out, missing arm versus rescore, common-identity deduplication, and graph populations | +| Substrate qualification | `lance_surface_guards.rs` and search owners for NFC/analyzer/index parity, native row-mask mapping, score statistics, vector metric/precision, tail coverage, complete boundaries and different partition layouts | +| Snapshot / policy / transport | `point_in_time.rs`, policy owners, server `data_routes`/`stored_queries`/`openapi`, and CLI parity for coherent follow-up, expiry/refusal, metadata, policy-safe counts and resolved query identity | +| Format | Existing schema/rebuild and cross-version owners for resolved-default persistence/export/reapplication, stamp refusal, rewrite idempotence, unresolved encoding refusal, and representation compatibility | +| Resource bounds | Checked-in cost instruments for NFC normalization, token construction, matching/scoring, coverage/statistics scans, graph fan-out, sort/spill, output bytes, cancellation, and shared fallback accounting | + +#### Mixed workload qualification + +Freeze tasks with exact graph facts and expected aggregate populations as +well as relevance judgments. Use the existing GQT, mechanism and benchmark +owners and extend their independent oracles at the boundary being tested. + +| Workload family | Qualification target | +|---|---| +| Exact graph analytics | Counts/reductions over the eligible graph, explicit binding multiplicity, selected properties and coherent follow-up | +| Graph-scoped lexical/semantic retrieval | Correct target population, exact/approximate distinction, representation coverage and source windows | +| Retrieval followed by graph aggregation | C2's initial terminal-aggregate subset; candidates and full-population totals stay distinguishable | +| Analytics followed by retrieval | C1's type/plan proof now; executable task qualification when intermediate operators land | +| Independent scoring | C3's type/plan proof now; feature/membership and cost qualification with the scorer implementation | +| Structured evidence with optional facts | C4's type/plan proof now; complete results, empty groups and cumulative resource qualification when implemented | + +For runnable families, record supported task correctness, query parse/type +failures and repair attempts, wrong-population answers, tool round trips, +latency, returned/context bytes and measured execution work. Preserve refusals, +timeouts, provider failures and unsupported requests in the report with +distinct dispositions; successful completed queries alone are not the task +denominator. Separate deterministic result oracles from judged relevance and +answer support, and record who supplied those judgments. + +Vary eligibility selectivity, graph fan-out, duplicate paths, number/skew of +groups, candidate windows, missing representations and payload width. Keep +agent/model/recipe, schema, snapshot and budgets fixed within a comparison. +Grammar evolution should reduce avoidable orchestration without forcing all +investigation into one query. A cheap small result and an expensive exact +summary are both legitimate workloads; measure their costs separately. +Deferred families stay visibly unimplemented until their operators land, and +must not count as proof of initial-release task coverage. + +#### Test harness integration + +Each `.gqt` case already exercises a real temporary graph through the compiler +and public engine API, with result-shape checks and ordered or unordered row +expectations. It is the preferred owner for the new language's observable +behavior. The runner's construct detection and refusal rules must evolve with +the AST: it currently prepares indexes for search cases and refuses some +embedding and ranked shapes. Do not infer absent-index, external-model, native +plan, or resource coverage from a passing golden query. Extend the runner only +where needed for a logical case; keep native index lifecycle, arithmetic near +ties, and execution-cost assertions in their existing Rust owners. Compiler +goldens and API/CLI contract tests still cover boundaries GQT does not invoke. + +The test-only [staged compiler prototype](../../crates/omnigraph-compiler/src/query/staged_probe.rs) +adds a separate Pest root using the existing grammar's tokens, graph patterns, +and scalar expressions. The extra grammar and checker are compiled only for +compiler unit tests; the production query root remains unchanged. Typed AST +prefixes reuse the real graph typechecker to validate bindings at each stage, +while the experimental plan retains the separate stages and incoming +population references. It exercises the GQ examples in this RFC, multi-stage +node and edge targets, non-leaking negation scopes, alias namespaces, metric +domains/origins, aggregate output identity, final order/window separation, +and rejection cases for inputs, bounds and references. Its `take` plan retains +the incoming stage, target, typed key tuple, comparator/reduction, quota bound +and proven binding dependencies. Tests cover composite and nullable keys, +non-null uniqueness, directed one-hop cardinality, explicit null ordering, +graph-only selection, zero quotas, and metric/order preservation. + +This is partial compiler evidence, not completed staged AST/IR or execution. +It does not qualify complete result-shape/metric-nullability inference, +every key type or comparator dependency, resolved analyzer/encoding identities, +runtime parameter admission, resource bounds, or graph/Lance lowering. +Its aggregate-order prototype uses +explicit projected aliases; the final compiler must also preserve valid +ordinary grouping-key expressions. Migrate or remove the experiment when the +production compiler owns these constructs; do not maintain a second compiler +or claim these fixtures replace executable `.gqt` cases. + +Default resolution requires its own compiler/schema and query fixtures: + +- Bare `@analyzed` and its explicit expansion resolve to the same field + fingerprint and permit matching and ranking. `scorer="none"` permits matching + while refusing a lexical ranking source. +- Missing source/dimension, omitted model without a default, unresolved explicit model, + incompatible dimensions, and distance omitted without a supplying recipe + all fail before publication. Explicit model overrides select a whole recipe; + explicit distance overrides take precedence and remain validated. A qualified + explicit model works without any schema default. +- Export/reload and no-op reapplication preserve resolved field bindings across + deployments and runtime provider/default changes. Changed schema defaults + expose affected field rebindings and required rebuilds in `schema plan`; + shorthand does not bypass existing migration refusals or rename identity. + +The exact lexical qualification matrix retains all preceding requirements: + +- Positive and negative matches; `all`/`any`; repeated/reordered terms; + null/token-empty values; empty and stop-word-only queries; validation even + on empty populations. Edit budgets reject negative, non-integer, oversized, + and narrowing-overflow inputs before execution. +- Case, stemming, folding, composed/decomposed text, multibyte characters, + and lengths around 40 UTF-8 bytes. Zero-edit equivalence, inclusion at + budgets 0/1/2, and transpositions costing two. +- NFC conformance for the pinned normalizer/Unicode version; canonically + equivalent inputs on both sides of matching and ranking through scan/index + paths; and long combining-mark sequences under bounded cancellation. + Preserve original stored values and exact String predicate behavior. + Reject or bypass artifacts that lack proof of the revised pipeline. +- Absent, empty, complete, partial, removed, and rebuilt indexes through + append/update/delete/overwrite/compaction. Identical values must match + identically under graph filters and supported negation. +- More than 50 expansion terms, vocabulary/segment changes, an exact term + crowded out by expansions, and one term consuming the budget before another. + Native overflow must fall back completely within the remaining budget or + fail; post-verifying a truncated set is insufficient. +- One huge token, automaton state failure, cancellation, and exhausted shared + budgets must produce typed failure without successful partial results. +- The same tolerant query through Boolean matching and fuzzy retrieval must + agree on membership before the ranking cutoff. Adding a predicate cannot + rewrite a neighboring exact source. Fuzzy/exact score and order parity need + independent fixtures, including corpus deletion/update and repeated terms. +- More tied targets than each native window, reversed stable-ID order, and + different physical partitions must preserve declared exact winners or fail + explicitly. Sorting an already truncated subset is not the assertion. + +Correctness and usefulness are separate evaluations. A fixed, checked-in +corpus should include names with typos, rare identifiers, semantic questions, +multi-hop investigation, duplicate-heavy candidates, missing representations, +and exhaustive verification tasks, alongside the analytical families in +[mixed workload qualification](#mixed-workload-qualification). Report +NDCG@10, MRR@10 and Recall@100 per modality, then task +answer/source-attribution correctness, complementary +coverage, tool calls, latency, and context consumption. Compare lexical, +dense and fused pipelines; add reranked variants when implemented. Hold corpus, +agent/model configuration and task budget fixed when attributing improvements. +Qualify `ann_default_v1` per index family against exact `knn` over the same +eligible population. No quality or cost result for the new pipeline is claimed +by this draft. + +## Rollout + +Acceptance requires the frontmatter design gates to have concrete dispositions +and owned evidence. Phase 0 supplies the decision work and prototypes needed +for acceptance; the later phases implement and qualify the accepted contracts. +Each phase can be reviewed in separate PRs. The public contract ships as one +coordinated pre-stable change with one necessary format rebuild, without +requiring users to adopt an interim language or rebuild after each phase. + +### Phase 0 acceptance versus release qualification + +An accepted design must name a feasible route, the interfaces it requires and +the tests that can disprove it. It does not assert that all production routes +already implement the contract. These obligations have separate completion +points: + +| Package | Phase 0 acceptance gate | Production release gate / owner | +|---|---|---| +| Language and composition | Checked syntax/types/plans for the initial stages and C1–C4; global-union compatibility proof; invalid-rewrite counterexamples; one actual compiler/engine/GQT slice with refusal | Phases 1/3: every enabled stage, walker, result type and optimized population boundary through GQT and mechanism owners. Deferred C1/C3/C4 execution remains outside this release. | +| Schema and identity | Default lifetime, space/binding separation, concrete `.pg` syntax and canonical serialization prototype; export/reapply/rename/drop-readd fixtures; coordinated format disposition | Phases 1/6: accepted SchemaIR, provider qualification, migration and predecessor/rebuild tests. No unresolved provider identity silently accepted. | +| Lexical/vector numerics | Complete numeric policies and independent oracles; discriminating native probes; explicit exact scan/fallback or upstream-dependency decisions | Phases 2/5: NFC/index parity, live statistics, fuzzy scoring, raw-vector rescoring, complete ties and all supported lifecycle states/targets. | +| Resource/read interfaces | Typed request/result/error contracts and charging units; minimal shared-budget refusal and snapshot/fingerprint round-trip proofs; every unaccounted native allocation/dispatch assigned a route decision | Phases 1/4/5: actual preallocation/dispatch gates, bounded cancellation/cleanup, every renderer and policy/retention path, whole-query and fallback accounting. | +| Retrieval evaluation | Frozen mixed tasks/corpus, judgment provenance, budgets, comparisons and acceptance criteria before tuning | Phase 5: measured correctness/quality/cost, source defaults and each enabled ANN family's effort mapping; retain failures and unsupported cases. | + +An unresolved design choice or a missing required prototype keeps Phase 0 +open. Production work scheduled above does not. An explicit upstream dependency +can close a route decision, but cannot qualify or enable that route; the release +still needs a working bounded implementation of every required capability. +No production gate is waived by this classification. The four recorded GQT +regressions remain release blockers even though they need not be repaired to +accept the design. + +### Implementation phases + +Phase 0 resolves the contracts; Phases 1–6 implement and qualify one coordinated +release. Fixtures and transport/migration tooling can proceed once their +required interfaces are fixed. Native experiments can begin against exact +references, but enabling a route requires its semantic and resource gates. + +These are required outcomes, not a claim that the proposed mechanisms work. +For each package, inspect the actual checkout/lockfile and complete relevant +[Lance documentation](../dev/lance.md), extend the [existing owner](../dev/testing.md), +and record the revision, toolchain/configuration, command, result and limits. +Use a test that would fail if the claim were false. Revalidate changed +dependencies; do not substitute historical pass counts for current evidence. + +| Evidence level | Meaning | +|---|---| +| Design argued | Coherent syntax/type/plan argument; execution and counterexamples still need proof | +| Prototype executed | Expected results for the recorded fixtures/configuration; production integration remains separate | +| Production qualified | Owned result, mechanism, resource and transport gates pass within the stated envelope | +| Open / unproved | Resolve through a discriminating test or explicit dependency/fallback/defer decision | + +Keep evidence in the [native matrix](#contract-to-code-qualification) and +[checkpoint](#implementation-handoff-and-validation-checkpoint), with production +status in frontmatter. A contradictory probe requires revising the proposed +mechanism, not weakening the promised result. + +#### Phase 0: resolve contracts and build the oracles + +**Input:** fresh production baseline, capability matrix, open gates and +archived experiments inspected at their recorded bases. + +**2026-09-13 decision checkpoint:** C3 retains partial term scores, distinguishes +zero from null and preserves membership. Shared expressions have explicit +precedence, typing, null and checked-arithmetic rules. Global execution is +deferred with compatibility obligations retained. On base `dd6959c8` plus this +change, Rust 1.97.1 passes all 367 compiler tests, the 13-case lexical oracle +and both `staged_composition` native tests. This settles the stated policies; +it does not complete any production evaluator or the phase's remaining gates. + +```bash +cargo +1.97.1 test --locked -p omnigraph-compiler +cargo +1.97.1 test --locked -p omnigraph-engine --test search lexical_scoring_v1_reference_oracle -- --exact +cargo +1.97.1 test --locked -p omnigraph-engine --test rrf_prefilter_gate staged_composition +``` + +| Decision package | Required disposition and proof | +|---|---| +| [Language](#language-evolution-and-compatibility) | Shared expressions, explicit output, namespaces, precedence, parameters and scope; parser/type/plan fixtures | +| [Composition](#required-composition-examples) | C1–C4 syntax, type derivation, golden plans, physical feasibility and invalid-rewrite counterexamples | +| [Selection/scoring](#target-identity-fan-out-grouping-and-metrics) | Target/binding/group multiplicity, total ties, live statistics, lexical/vector numeric policy and checked fusion arithmetic | +| [Representations](#representation-identity-and-source-attribution) | Schema defaults and overrides, resolved encoder/Unicode/analyzer identity, export/reapplication and format coordination with RFCs 0040/0043/0044 | +| [Global search](#graph-wide-discovery-across-entity-types) | Execution deferred by explicit decision; prove typed identity/source/table separation and future union/narrowing/projection compatibility before stabilization | +| [Execution/read contract](#result-metadata-coherent-continuation-and-budgets) | Resource units, admission/interfaces, error/result types, fingerprints, replay and retention | +| [Workload](#mixed-workload-qualification) | Fixed tasks/corpus, exact oracles, relevance judgments, recipes, budgets and acceptance criteria before tuning | + +**Exit:** concrete design dispositions for every frontmatter gate and +native-matrix row: a prototype-supported route, specified bounded fallback +with its feasibility proof, or explicit upstream dependency. Carry a minimal +query through the actual compiler/engine/GQT with success and shared-resource +refusal; include later-stage descriptors, column demand, GQT detection and +pre-scan parameter admission. Existing compiler, search, schema and read owners +supply the evidence. + +**Open:** logical goldens and native primitives do not establish integrated +C1–C4 lowering, numeric/null evaluation or whole-query bounds. The archived +lexical integration proves only its recorded slice. Full production C1/C3/C4 +and general C2 grouping are deferred, but their design proofs are due here. +Global execution is explicitly deferred; its compatibility proof is still due. +Do not start a dependent package before its interfaces are resolved. + +#### Phase 1: build representation, plan, and resource foundations + +**Input:** Phase 0's accepted interfaces and ordinary-query compatibility fixtures. + +Implement resolved per-field representations/defaults, shared expressions, +typed stage IR, metric/population identity and fingerprints. Analytical rows +need no score. All operators share one snapshot, admission, cancellation and +resource context. Extend compiler AST/IR/SchemaIR/descriptors, engine query +execution, sealed TableStore and schema/rebuild owners; every stage walker +must retain later reads and hidden columns. + +**Exit:** serialization, rename versus drop/re-add, parameter/refusal, +default/override and export/reapplication fixtures; no drift with runtime +defaults; one budget across stages/fallbacks. Close native allocation, +shared-buffer and dispatched-I/O gates before claiming whole-query bounds. +Exercise active reads/decode, retained plans/caches, output and encoder calls. + +**Open:** a shared DataFusion pool does not govern all native allocations or +tasks. Inspect ownership and prove preallocation refusal, cancellation and +cleanup. Typed extension points do not make deferred public operators available. + +#### Phase 2: implement complete lexical and vector retrieval + +**Input:** Phase 1 representations/resources and Phase 0 numeric oracles. + +Build bounded NFC/analysis, complete `Terms` matching, unified exact/fuzzy +scoring, exhaustive `knn` and declared `ann` with qualified fallback. +Charge statistics, coverage, scoring and selection to the shared context. +Extend search/substrate owners and the +[lexical qualification matrix](#test-harness-integration); observable rows, +shapes and errors belong in GQT. + +**Exit:** predicate/retriever membership agreement before cuts, independent +score and vector fixtures, total ties, all index/lifecycle states, and typed +budget/cancellation failures. Assert that intended index/compaction states +were reached; distinguish current from pinned answers and reopen snapshots. + +**Open:** native fuzzy analysis, float32 BM25, stale statistics and native cuts +are not substitutes for the accepted exact contract. Reproduce their +counterexamples before reuse. Numerical parity does not establish relevance +quality. Candidate-scoring/model operators and richer representations remain +deferred; Phase 5 chooses measured defaults. + +#### Phase 3: compose graph scope, fusion, and selection + +**Input:** Phase 2 reference retrievers and Phase 1 stage/resource interfaces. + +Connect graph-defined populations, named weighted RRF, inter-stage traversal, +group quotas, final projection/order and existing terminal aggregates. +Preserve distinct target identity, missing-arm metrics and every winning +binding row. Extend IR/query execution, traversal/projection/aggregation, +GQT and existing search/selection owners. + +**Exit:** executable counterexamples distinguish filter-before/after-cut, +source windows from final limits, target counts from path counts, and +missing membership from a new feature. C2's initial terminal subset returns +three binding rows from p1/p2 and excludes p3. Test null/multiple group +membership, metric reductions, inherited order and fan-out/sort refusal despite +a small final result. No successful truncated aggregate. + +**Open:** native relational probes do not prove GQ lowering, masks, metric +carriage, descriptor walkers or order after payload reads. Compare optimized +plans with independent oracles and retain the nullable dynamic-filter fence. +Reusable intermediate groups, general distinct aggregates and full per-parent +collection remain deferred. + +#### Phase 4: complete the agent-facing read path + +**Input:** Phase 0 read/error/fingerprint decisions and Phase 3 result semantics; +transport work can begin earlier against fixed interfaces. + +Expose inline/stored queries, inspectable plans, metrics, completion/coverage/ +selection metadata and coherent identity-based source reads. Reuse existing +snapshot carriers, descriptions/instructions and authorization. Extend API +types, server data/stored-query/policy/OpenAPI owners and CLI parity. + +**Exit:** discovery → pinned source read → expansion → verification, and +graph-scoped retrieval → terminal aggregate through embedded, HTTP, stored-query +and CLI paths. Include an entity without an application key. Change head so +pinned/current answers differ; test expired/unavailable snapshots, revoked +access, exact-coverage refusal, JSON/JSONL metadata and result types. + +**Open:** existing request fields do not prove renderer/client compatibility, +general system-ID lookup or retention. A graph snapshot does not freeze an +external encoder. Qualify identity and refusal boundaries; stable ranked +cursors and durable result storage remain deferred. + +#### Phase 5: qualify physical execution and measure retrieval + +**Input:** exact references from Phase 2; full-pipeline trials additionally need +Phases 3–4 and the fixed mixed-workload protocol. + +Compare each native route with the exact evaluator and composition oracle: +graph masks, analyzer certificates, expansions, live statistics, tails, raw +vectors, ties, partitions, fallback and cancellation. ANN is judged for recall +against exact `knn`, not exact candidate equality. Use existing search/ +substrate owners and benchmark instruments. + +**Exit:** every enabled path passes semantic/resource gates. Freeze +corpus/schema/snapshot, representations, model/recipes, windows, budgets, +expected facts, judgments and pass/refusal criteria before trials. Report +task/population errors, query repairs, round trips, context, quality and cost; +retain failed trials and judgment provenance. Choose source-window defaults +and each index family's `ann_default_v1` mapping only within that envelope. + +**Open:** the document pilot lacks independent human adjudication and broader +analytical coverage. No universal join strategy, candidate window or effort +mapping is established. Keep an unqualified native route disabled while a +qualified fallback serves the contract; future operators need fresh trials. + +#### Phase 6: ship the coordinated language and data migration + +**Input:** qualified Phases 1–5, final schema/format identity and wire decisions. + +Follow the [migration sequence](#migration-sequence): finalize identities; +prepare schema/query diagnostics and examples; export/init/load real predecessor +fixtures; regenerate incompatible representations and reconcile indexes; +verify ordinary and migrated queries, result types and follow-up; publish +coordinated docs/release notes and remove legacy syntax/scaffolding. + +**Exit:** existing schema/rebuild, cross-version, API/OpenAPI, CLI and query +owners prove mutual format refusal, compatible-value preservation, unresolved +encoding refusal, rewrite idempotence and successful migrated journeys. +Run the current [canonical checks](../dev/testing.md), including separate +GQT and required environments. Users encounter one supported cutover. + +**Open:** recheck format ownership as related RFCs land, actual client parsers, +and loss of old histories/snapshot references across rebuild. A skipped +predecessor test is not upgrade evidence. Deferred composition and any explicitly +deferred global-search capability must remain absent from release claims. + +### Release and completion criteria + +Phases 1–4 establish the full initial feature path; Phase 5 qualifies its +execution and defaults; Phase 6 makes it a supported release. Fuzzy ranked +retrieval, coherent follow-up, and whole-query resource bounds are required +for that release. They cannot be moved to the extension list to mark the +implementation complete. An acceleration path may remain disabled only when +the qualified fallback preserves the promised semantics and resource behavior. + +Frontmatter remains the implementation-status authority. Advance it when +implementation lands, and mark it complete only after the release criteria +and their owned evidence pass. Historical prototypes and current-code baseline +tests do not count as completion of a new phase. + +### Extensions after the initial release + +The [query capability matrix](#query-capability-matrix) owns the extension +inventory, including projection, grouping, optional matching, typed unions, +advanced retrieval and representation forms. Its deferred rows do not claim +support or require implementation for the initial release. +Every later extension retains the stated semantic and qualification boundary. +`Foundation` rows require their grammar/type/composition proofs before syntax +stabilization while their broader operators remain deferred. These design +proofs do not enlarge the advertised initial-release feature set. + +### Next delivery milestone: composed analytical answers + +After the initial release, implement C1, C4 and C2's general distinct/intermediate +group extensions using the accepted common grammar, IR and execution context. +C3 remains a separate scorer extension. Global search is also deferred; its +union, representation expansion and cross-type ranking require their own +implementation milestone after the compatibility proof. + +Build reusable computation/grouping first, then local selection and correlated +retrieval, then optional facts and typed collections. Preserve group identities, +computed facts, import/export scopes, empty results and one cumulative budget. +Extend existing compiler, traversal/projection/aggregation, GQT and transport +owners; retain compatibility with the initial release. + +**Exit:** full C1/C2/C4 journeys through public reads, with inferred/executed +types, deterministic selection, snapshot-bound follow-up and shared-budget +refusal. C1 selects A's increase of six over B's one; C2 counts three paths and +two distinct passages; C4 preserves B with a null owner and empty reports. + +**Open:** inspect integrated and optimized plans against independent +empty-group, duplicate-binding, null, wrong-group and skewed-fan-out controls. +API availability and the Phase 0 design argument do not prove safe correlation, +reuse or bounded materialization. Prove preservation of outer rows and budgets +before optimizing batching, and measure per-group rescan cost. + +## Unresolved questions + +The [acceptance/release matrix](#phase-0-acceptance-versus-release-qualification) +assigns the completion point for each proof. In particular, decided interfaces +still require prototypes; full production qualification belongs to its phase. + +1. Complete Phase 0 grammar/type/plan qualification for the stated stage + and metric scopes, per-group key types and tuple ordering, and any + user-defined selection tie keys. The partial compiler + prototype does not close the full result-schema and aggregation contract. + Null-bucket and multiple-membership semantics are specified above; prove + their lowering and retain RFC 0040 namespace coordination. Complete the + C1–C4 syntax/type/plan proofs, including the analytical population and + deferred correlation/scoring boundaries, before syntax stabilization. +2. Close the Phase 0 numeric policies/oracles and native route dispositions. + Phases 2/5 must qualify BM25 across supported targets, exact live-row + statistics and native/fallback score/winner parity. Polymorphic field-corpus + execution is deferred. Validate edit weights and maximum reduction on the + owned task corpus before release; numerical fixtures alone do not establish + a good relevance default. +3. Resolved representation serialization and immutable encoding revisions, + schema-wide default declaration syntax and migration integration, + normalizer/Unicode identity and analyzer/artifact fingerprint mapping, and + shared SchemaIR version assignment with RFCs 0040/0043/0044. Omission and + override semantics are specified above; their implementation must prove + that accepted bindings cannot drift with deployment defaults. +4. Prototype the decided resource units and shared reservation interfaces; + resolve native allocation/dispatch gaps for graph masks, scoring, coverage, + sort/spill and output. Actual default limits, complete enforcement and + cancellation qualification belong to Phases 1/4/5. +5. Prototype the decided read options, envelope, retention/refusal boundaries + and canonical definition/semantic/execution identities. Phase 4 qualifies + all transports and authorization paths. Stable ranked cursors remain deferred. +6. Freeze the mixed-workload corpus, judgments and acceptance protocol in + Phase 0. Phase 5 chooses ANN effort mappings and agent recipe defaults from + the resulting evaluation. The document pilot alone is insufficient. Further + multilingual profiles require matched-set evidence and versioned identities. +7. General all-node/type-union grammar, compatible representation expansion, + type narrowing and heterogeneous projection. Execution is explicitly + deferred; the compatibility proof remains due in Phase 0. The initial + same-binding fusion implementation cannot satisfy cross-type discovery. + +## Decision log + +- 2026-09-13 — separated Phase 0 acceptance from production release gates. + Decided default-binding lifetime, space versus field identity, read options + and replay/fingerprint boundaries, and shared resource units/reservations. + These are design dispositions; schema syntax/serialization and interface + prototypes remain open. Fresh checks on base `22d0472b` pass 367 compiler + and seven historical-read tests, establishing current owner baselines only. +- 2026-09-13 — decided membership-independent lexical features, null and checked + arithmetic policies, and deferred global execution. Extended the existing + Decimal, compiler and DataFusion probes; default native arithmetic is not + sufficient for the new numeric contract. Phase 0 remains incomplete. +- 2026-09-13 — retained the full long-form version as an + [agent context document](assets/0048-agent-context.md) alongside this concise + RFC. The context records its original revision; this RFC owns current decisions. +- 2026-09-13 — condensed the RFC around contracts, scope, evidence limits and + phase handoffs. Detailed experimental chronology remains in the linked + agent context and checked-in receipts; no semantic or release-scope + decision changes. +- 2026-09-11/12 — added explicit `yield` integration, C1–C4 logical plans and + native population/collection counterexamples. Phase 0 remains incomplete. +- 2026-09-10 — made mixed analytical/graph/retrieval tasks the agent objective; + added language-evolution and capability matrices, composition gates and + deferred analytical/nested delivery. +- 2026-09-09 — selected live field-corpus statistics, unified float64 + exact/fuzzy BM25, truthful unknown coverage, and a budgeted exact-coverage + option. Added numerical, native and isolated integration evidence. +- 2026-09-08 — chose one pre-stable staged-query/schema cutover; shared + `terms`, named windows/metrics, schema defaults and explicit NFC. Retained + ordinary graph identity, stored-query recipes and coherent follow-up; removed + proposed document/evidence wrappers and separate profile machinery. +- 2026-09-03 — published this draft alongside RFC 0047 after separating + plan-truth work from representation/search contracts. + +The [full prior decision log](assets/0048-agent-context.md#decision-log) +preserves the earlier revisions and their superseded proposals. + +## Appendix: implementation evidence (non-normative) + +The [upstream receipt](assets/0048-upstream-contract-checkpoint.json) and +[contract-to-code matrix](#contract-to-code-qualification) bind the source audit +to Lance 11.0.0, DataFusion 54.0.0 and Arrow 58.3.0. The recorded Lance commit +is `ab6b5bbe46009ed78746b444df8db59a8bc5d842`; later dependencies require +renewed qualification. The +[original source appendix](assets/0048-agent-context.md#appendix-implementation-evidence-non-normative) +retains exact implementation links for analyzer drift, fuzzy expansion, +flat scans, vector rescoring, lexical ties and native BM25. + +| Related RFC | Coordination boundary | +|---|---| +| [0047](0047-search-plan-truth.md) | Carry plan truth, projectable metrics, complete boundaries and read notices into stages; its interim retrieval field, projection spelling and T26 restriction do not constrain the final model | +| [0043](0043-full-text-index-compatibility.md) | Accepted-schema analyzer fingerprints become the authority for artifact certificates; index rebuilding preserves staging, recovery identity and single publication | +| 0040 | Use its system-identity and reserved `__` namespace; native `_score`/`_distance` are not public metric identities | +| 0044 | Coordinate accepted SchemaIR version assignment | +| 0046 | Reuse read-only index-status capabilities and its open `degraded` reasons rather than parallel status machinery | diff --git a/docs/rfcs/README.md b/docs/rfcs/README.md index 5ff4298a5..a548bb736 100644 --- a/docs/rfcs/README.md +++ b/docs/rfcs/README.md @@ -198,6 +198,8 @@ then dated RFCs by date. | [0044](0044-edge-keys.md) | Edge keys: derived edge identity | maintainer | draft | in-progress | | [0045](0045-gq-logic-tests.md) | GQ logic tests | maintainer | draft | partial | | [0046](0046-index-status.md) | Read-only index status | maintainer | draft | not-started | +| [0047](0047-search-plan-truth.md) | Search plan truth: projectable ranking, deterministic order, and loud search failures | public | draft | not-started | +| [0048](0048-search-contracts.md) | Search contracts and retrieval algebra | public | draft | not-started | | [0049](0049-control-plane-seams.md) | Control-plane seams: observe, readiness witness, bounded shutdown | maintainer | accepted | partial | | [0051](0051-json-output-via-arrow.md) | JSON output via Arrow | maintainer | draft | partial | | [0052](0052-managed-control-plane-cli.md) | Managed control-plane CLI | maintainer | accepted | complete | diff --git a/docs/rfcs/assets/0048-arrow-pool-probe.py b/docs/rfcs/assets/0048-arrow-pool-probe.py new file mode 100644 index 000000000..2e9ac4fb7 --- /dev/null +++ b/docs/rfcs/assets/0048-arrow-pool-probe.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Run the optional Arrow-pool qualification outside the production workspace. + +Requires Python 3.11+, Cargo, and the registry dependencies cached by the +workspace build. Offline resolution must retain versions/checksums from the +workspace lockfile. The temporary crate and output remain available for audit. +""" + +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import tempfile +import tomllib + + +def main(): + assets = Path(__file__).resolve().parent + root = assets.parents[2] + lock_bytes = (root / "Cargo.lock").read_bytes() + baseline = tomllib.loads(lock_bytes.decode()) + allowed = { + (p["name"], p["version"], p.get("source"), p.get("checksum")) + for p in baseline["package"] + } + directory = Path(tempfile.mkdtemp(prefix="omnigraph-0048-arrow-pool-")) + print(f"Probe directory: {directory}", flush=True) + (directory / "Cargo.toml").write_text('''[package] +name = "omnigraph-0048-arrow-pool-probe" +version = "0.0.0" +edition = "2024" + +[workspace] + +[dependencies] +arrow-array = { version = "=58.3.0", features = ["pool"] } +arrow-buffer = { version = "=58.3.0", features = ["pool"] } +datafusion-execution = { version = "=54.0.0", features = ["arrow_buffer_pool"] } + +[profile.dev] +debug = 0 +''') + (directory / "Cargo.lock").write_bytes(lock_bytes) + (directory / "src").mkdir() + source = assets / "0048-arrow-pool-probe.rs" + shutil.copyfile(source, directory / "src/main.rs") + subprocess.run( + ["cargo", "metadata", "--offline", "--format-version", "1"], + cwd=directory, check=True, stdout=subprocess.DEVNULL, + ) + resolved = tomllib.loads((directory / "Cargo.lock").read_text()) + for package in resolved["package"]: + if package["name"] == "omnigraph-0048-arrow-pool-probe": + continue + identity = (package["name"], package["version"], package.get("source"), package.get("checksum")) + if identity not in allowed: + raise RuntimeError(f"Dependency differs from workspace lock: {identity}") + output = subprocess.check_output( + ["cargo", "run", "--offline", "--locked", "--target-dir", str(directory / "target")], + cwd=directory, text=True, + ) + result = { + "workspace_lock_sha256": hashlib.sha256(lock_bytes).hexdigest(), + "probe_source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "resolved_lock_sha256": hashlib.sha256((directory / "Cargo.lock").read_bytes()).hexdigest(), + "registry_dependencies_match_workspace": True, + "observations": json.loads(output), + } + (directory / "result.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/docs/rfcs/assets/0048-arrow-pool-probe.rs b/docs/rfcs/assets/0048-arrow-pool-probe.rs new file mode 100644 index 000000000..2dadd14ac --- /dev/null +++ b/docs/rfcs/assets/0048-arrow-pool-probe.rs @@ -0,0 +1,55 @@ +// RFC 0048 qualification only. Run through 0048-arrow-pool-probe.py so the +// optional upstream features do not change OmniGraph's production feature set. +use std::sync::Arc; + +use arrow_array::{Array, Int32Array}; +use arrow_buffer::MemoryPool as ArrowPool; +use datafusion_execution::memory_pool::arrow::ArrowMemoryPool; +use datafusion_execution::memory_pool::{GreedyMemoryPool, MemoryConsumer, MemoryPool}; + +fn main() { + let first: Arc = Arc::new(GreedyMemoryPool::new(1)); + let second: Arc = Arc::new(GreedyMemoryPool::new(1)); + let first_arrow = ArrowMemoryPool::new(first.clone(), MemoryConsumer::new("query-a")); + let second_arrow = ArrowMemoryPool::new(second.clone(), MemoryConsumer::new("query-b")); + let original = Int32Array::from(vec![1, 2, 3, 4, 5]); + let bytes = original.get_buffer_memory_size(); + assert!(bytes > 1); + + // Positive refusal control: DataFusion's fallible operator reservation. + let admission = MemoryConsumer::new("fallible-control").register(&first); + assert!(admission.try_grow(bytes).is_err()); + assert_eq!(first.reserved(), 0); + + // Arrow's claim API tracks an already allocated buffer, infallibly. + original.claim(&first_arrow); + assert_eq!(first.reserved(), bytes); + assert!(first_arrow.available() < 0); + let over_limit_available = first_arrow.available(); + let shared = original.slice(0, 2); + shared.claim(&first_arrow); + assert_eq!( + first.reserved(), + bytes, + "same-owner slices are not double charged" + ); + + // The buffer has one reservation. Claiming it for another query transfers + // that reservation even though the first query still retains the array. + shared.claim(&second_arrow); + assert_eq!(first.reserved(), 0); + assert_eq!(second.reserved(), bytes); + drop(shared); + assert_eq!( + second.reserved(), + bytes, + "the first query still owns an alias" + ); + assert_eq!(original.value(4), 5); + drop(original); + assert_eq!(first.reserved(), 0); + assert_eq!(second.reserved(), 0); + println!( + "{{\"buffer_bytes\":{bytes},\"pool_capacity\":1,\"available_after_claim\":{over_limit_available},\"fallible_admission_refused\":true,\"same_owner_shared_once\":true,\"cross_owner_reservation_transferred\":true,\"released_after_last_alias\":true}}" + ); +} diff --git a/docs/rfcs/assets/0048-lexical-scoring-v1.json b/docs/rfcs/assets/0048-lexical-scoring-v1.json new file mode 100644 index 000000000..e22fb35f9 --- /dev/null +++ b/docs/rfcs/assets/0048-lexical-scoring-v1.json @@ -0,0 +1,412 @@ +[ + { + "name": "rare_variant", + "docs": [ + "beta", + "beta", + "beta", + "beto", + "gamma" + ], + "query": [ + "beta" + ], + "edits": 1, + "all_terms": false, + "expected_scores": [ + "0.28768207245178092743921900599382743150350971089776105650666568534929295072078044", + "0.28768207245178092743921900599382743150350971089776105650666568534929295072078044", + "0.28768207245178092743921900599382743150350971089776105650666568534929295072078044", + "0.14384103622589046371960950299691371575175485544888052825333284267464647536039022", + null + ], + "expected_features": [ + "0.28768207245178092743921900599382743150350971089776105650666568534929295072078044", + "0.28768207245178092743921900599382743150350971089776105650666568534929295072078044", + "0.28768207245178092743921900599382743150350971089776105650666568534929295072078044", + "0.14384103622589046371960950299691371575175485544888052825333284267464647536039022", + "0" + ], + "expected_order": [ + 0, + 1, + 2, + 3 + ] + }, + { + "name": "zero_edits", + "docs": [ + "beta", + "beta", + "beta", + "beto", + "gamma" + ], + "query": [ + "beta" + ], + "edits": 0, + "all_terms": false, + "expected_scores": [ + "0.53899650073268700512435673639569911116140609696139877151666420268670167440592912", + "0.53899650073268700512435673639569911116140609696139877151666420268670167440592912", + "0.53899650073268700512435673639569911116140609696139877151666420268670167440592912", + null, + null + ], + "expected_features": [ + "0.53899650073268700512435673639569911116140609696139877151666420268670167440592912", + "0.53899650073268700512435673639569911116140609696139877151666420268670167440592912", + "0.53899650073268700512435673639569911116140609696139877151666420268670167440592912", + "0", + "0" + ], + "expected_order": [ + 0, + 1, + 2 + ] + }, + { + "name": "zero_edits_multi_term", + "docs": [ + "beta gamma gamma", + "beta", + "gamma", + "zeta" + ], + "query": [ + "gamma", + "beta" + ], + "edits": 0, + "all_terms": false, + "expected_scores": [ + "1.2357761692752447451923272912857813558293338351379531123819519288056883929058444", + "0.80259147222730509511468982484630971040321068189082187319236632678182419385964649", + "0.80259147222730509511468982484630971040321068189082187319236632678182419385964649", + null + ], + "expected_features": [ + "1.2357761692752447451923272912857813558293338351379531123819519288056883929058444", + "0.80259147222730509511468982484630971040321068189082187319236632678182419385964649", + "0.80259147222730509511468982484630971040321068189082187319236632678182419385964649", + "0" + ], + "expected_order": [ + 0, + 1, + 2 + ] + }, + { + "name": "alternatives_do_not_add", + "docs": [ + "beta", + "beta beto", + "beta gamma", + "beto", + "beta beta", + "gamma" + ], + "query": [ + "beta" + ], + "edits": 1, + "all_terms": false, + "expected_scores": [ + "0.27924027631429145000705833224194705018943769579491494115542388982151562830461054", + "0.21222260999886150200536433250387975814397264880413535527812215626435187751150402", + "0.21222260999886150200536433250387975814397264880413535527812215626435187751150402", + "0.13962013815714572500352916612097352509471884789745747057771194491075781415230527", + "0.30317515714123071715052047500554251163424664114876479325446022323478839644500575", + null + ], + "expected_features": [ + "0.27924027631429145000705833224194705018943769579491494115542388982151562830461054", + "0.21222260999886150200536433250387975814397264880413535527812215626435187751150402", + "0.21222260999886150200536433250387975814397264880413535527812215626435187751150402", + "0.13962013815714572500352916612097352509471884789745747057771194491075781415230527", + "0.30317515714123071715052047500554251163424664114876479325446022323478839644500575", + "0" + ], + "expected_order": [ + 4, + 0, + 1, + 2, + 3 + ] + }, + { + "name": "one_term_satisfies_two_groups", + "docs": [ + "beto", + "beta", + "beta beto", + "gamma" + ], + "query": [ + "beta", + "beto" + ], + "edits": 1, + "all_terms": true, + "expected_scores": [ + "0.58268678960287962891668700351282612736695807171030143514554187049454260636308639", + "0.58268678960287962891668700351282612736695807171030143514554187049454260636308639", + "0.57276268369723447708598917133620865074513640138920141070267376077809054250775402", + null + ], + "expected_features": [ + "0.58268678960287962891668700351282612736695807171030143514554187049454260636308639", + "0.58268678960287962891668700351282612736695807171030143514554187049454260636308639", + "0.57276268369723447708598917133620865074513640138920141070267376077809054250775402", + "0" + ], + "expected_order": [ + 0, + 1, + 2 + ] + }, + { + "name": "all_counts_each_family_before_query_filter", + "docs": [ + "beta gamma", + "beto gamma", + "beta", + "gamma", + "gamma beta beta", + "other" + ], + "query": [ + "beta", + "gamma" + ], + "edits": 1, + "all_terms": true, + "expected_scores": [ + "0.81683365967553471412656105909568970967960356827946099667520948129677880355126166", + "0.61262524475665103559492079432176728225970267620959574750640711097258410266344625", + null, + null, + "0.82882241201910909838477868716460883457705371849882686257553055978939297385778704", + null + ], + "expected_features": [ + "0.81683365967553471412656105909568970967960356827946099667520948129677880355126166", + "0.61262524475665103559492079432176728225970267620959574750640711097258410266344625", + "0.52827829076841647272315633713253845354278709035465140545842352322998194360108769", + "0.52827829076841647272315633713253845354278709035465140545842352322998194360108769", + "0.82882241201910909838477868716460883457705371849882686257553055978939297385778704", + "0" + ], + "expected_order": [ + 4, + 0, + 1 + ] + }, + { + "name": "any_uses_the_same_statistics", + "docs": [ + "beta gamma", + "beto gamma", + "beta", + "gamma", + "gamma beta beta", + "other" + ], + "query": [ + "beta", + "gamma" + ], + "edits": 1, + "all_terms": false, + "expected_scores": [ + "0.81683365967553471412656105909568970967960356827946099667520948129677880355126166", + "0.61262524475665103559492079432176728225970267620959574750640711097258410266344625", + "0.52827829076841647272315633713253845354278709035465140545842352322998194360108769", + "0.52827829076841647272315633713253845354278709035465140545842352322998194360108769", + "0.82882241201910909838477868716460883457705371849882686257553055978939297385778704", + null + ], + "expected_features": [ + "0.81683365967553471412656105909568970967960356827946099667520948129677880355126166", + "0.61262524475665103559492079432176728225970267620959574750640711097258410266344625", + "0.52827829076841647272315633713253845354278709035465140545842352322998194360108769", + "0.52827829076841647272315633713253845354278709035465140545842352322998194360108769", + "0.82882241201910909838477868716460883457705371849882686257553055978939297385778704", + "0" + ], + "expected_order": [ + 4, + 0, + 1, + 2, + 3 + ] + }, + { + "name": "null_and_token_empty_are_outside_corpus", + "docs": [ + null, + "", + "beta", + "beto", + "beta gamma" + ], + "query": [ + "beta" + ], + "edits": 1, + "all_terms": false, + "expected_scores": [ + null, + null, + "0.14874382975896190932757264103745313625150100339118484181560999128796061111161147", + "0.074371914879480954663786320518726568125750501695592420907804995643980305555805734", + "0.11085625048073576261205885511281884682894886101795851418333197463914045545110666" + ], + "expected_features": [ + null, + "0", + "0.14874382975896190932757264103745313625150100339118484181560999128796061111161147", + "0.074371914879480954663786320518726568125750501695592420907804995643980305555805734", + "0.11085625048073576261205885511281884682894886101795851418333197463914045545110666" + ], + "expected_order": [ + 2, + 4, + 3 + ] + }, + { + "name": "two_edits_and_transposition", + "docs": [ + "abcd", + "abce", + "abef", + "ab", + "acbd", + "zxyz" + ], + "query": [ + "abcd" + ], + "edits": 2, + "all_terms": false, + "expected_scores": [ + "0.24116205681688807046064128693622699789087801000469926736150245030039986080852729", + "0.12058102840844403523032064346811349894543900500234963368075122515019993040426364", + "0.060290514204222017615160321734056749472719502501174816840375612575099965202131822", + "0.060290514204222017615160321734056749472719502501174816840375612575099965202131822", + "0.060290514204222017615160321734056749472719502501174816840375612575099965202131822", + null + ], + "expected_features": [ + "0.24116205681688807046064128693622699789087801000469926736150245030039986080852729", + "0.12058102840844403523032064346811349894543900500234963368075122515019993040426364", + "0.060290514204222017615160321734056749472719502501174816840375612575099965202131822", + "0.060290514204222017615160321734056749472719502501174816840375612575099965202131822", + "0.060290514204222017615160321734056749472719502501174816840375612575099965202131822", + "0" + ], + "expected_order": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + { + "name": "unicode_scalar_edit", + "docs": [ + "café", + "cafe", + "other" + ], + "query": [ + "cafe" + ], + "edits": 1, + "all_terms": false, + "expected_scores": [ + "0.23500181462286777682546851557417103235044952440612402022469606850300093910071319", + "0.47000362924573555365093703114834206470089904881224804044939213700600187820142638", + null + ], + "expected_features": [ + "0.23500181462286777682546851557417103235044952440612402022469606850300093910071319", + "0.47000362924573555365093703114834206470089904881224804044939213700600187820142638", + "0" + ], + "expected_order": [ + 1, + 0 + ] + }, + { + "name": "empty_corpus", + "docs": [], + "query": [ + "beta" + ], + "edits": 1, + "all_terms": false, + "expected_scores": [], + "expected_features": [], + "expected_order": [] + }, + { + "name": "only_missing_and_token_empty", + "docs": [ + null, + "", + " " + ], + "query": [ + "beta" + ], + "edits": 1, + "all_terms": false, + "expected_scores": [ + null, + null, + null + ], + "expected_features": [ + null, + "0", + "0" + ], + "expected_order": [] + }, + { + "name": "no_matching_family", + "docs": [ + null, + "", + "gamma" + ], + "query": [ + "beta" + ], + "edits": 1, + "all_terms": false, + "expected_scores": [ + null, + null, + null + ], + "expected_features": [ + null, + "0", + "0" + ], + "expected_order": [] + } +] diff --git a/docs/rfcs/assets/0048-lexical-scoring-v1.py b/docs/rfcs/assets/0048-lexical-scoring-v1.py new file mode 100644 index 000000000..f05a05cef --- /dev/null +++ b/docs/rfcs/assets/0048-lexical-scoring-v1.py @@ -0,0 +1,81 @@ +"""Regenerate lexical_scoring_v1.json after an intentional RFC 0048 change. + +Uses 80-digit Decimal arithmetic over already-analyzed fixture terms. This +is a numerical design oracle, not a tokenizer or retrieval benchmark. Cargo +does not run this generator: the checked-in expectations require review. +""" +from collections import Counter +from decimal import Decimal, localcontext +import json +from pathlib import Path + +def distance(a, b): + grid = [[0] * (len(b) + 1) for _ in range(len(a) + 1)] + for i in range(len(a) + 1): + grid[i][0] = i + for j in range(len(b) + 1): + grid[0][j] = j + for i, ca in enumerate(a, 1): + for j, cb in enumerate(b, 1): + grid[i][j] = min(grid[i-1][j] + 1, grid[i][j-1] + 1, + grid[i-1][j-1] + (ca != cb)) + return grid[-1][-1] + +cases = [ + ("rare_variant", ["beta", "beta", "beta", "beto", "gamma"], ["beta"], 1, False), + ("zero_edits", ["beta", "beta", "beta", "beto", "gamma"], ["beta"], 0, False), + ("zero_edits_multi_term", ["beta gamma gamma", "beta", "gamma", "zeta"], ["gamma", "beta"], 0, False), + ("alternatives_do_not_add", ["beta", "beta beto", "beta gamma", "beto", "beta beta", "gamma"], ["beta"], 1, False), + ("one_term_satisfies_two_groups", ["beto", "beta", "beta beto", "gamma"], ["beta", "beto"], 1, True), + ("all_counts_each_family_before_query_filter", ["beta gamma", "beto gamma", "beta", "gamma", "gamma beta beta", "other"], ["beta", "gamma"], 1, True), + ("any_uses_the_same_statistics", ["beta gamma", "beto gamma", "beta", "gamma", "gamma beta beta", "other"], ["beta", "gamma"], 1, False), + ("null_and_token_empty_are_outside_corpus", [None, "", "beta", "beto", "beta gamma"], ["beta"], 1, False), + ("two_edits_and_transposition", ["abcd", "abce", "abef", "ab", "acbd", "zxyz"], ["abcd"], 2, False), + ("unicode_scalar_edit", ["café", "cafe", "other"], ["cafe"], 1, False), + ("empty_corpus", [], ["beta"], 1, False), + ("only_missing_and_token_empty", [None, "", " "], ["beta"], 1, False), + ("no_matching_family", [None, "", "gamma"], ["beta"], 1, False), +] + +out = [] +with localcontext() as ctx: + ctx.prec = 80 + D = Decimal + for name, docs, query, edits, all_terms in cases: + terms = sorted(set(query)) + counts = [Counter((doc or "").split()) for doc in docs] + n = sum(bool(c) for c in counts) + total_length = sum(sum(c.values()) for c in counts) + scores = [None] * len(docs) + features = [None if doc is None else D(0) for doc in docs] + if n: + avg = D(total_length) / n + dfs = {q: sum(any(distance(q, t) <= edits for t in c) for c in counts) for q in terms} + for i, c in enumerate(counts): + contributions = [] + for q in terms: + weights = [] + for t, freq in c.items(): + edit = distance(q, t) + if edit <= edits: + norm = D("1.2") * (D("0.25") + D("0.75") * sum(c.values()) / avg) + weights.append((D(2) ** -edit) * D("2.2") * freq / (D(freq) + norm)) + if not weights: + contributions.append(D(0)) + else: + idf = (1 + (D(n - dfs[q]) + D("0.5")) / (D(dfs[q]) + D("0.5"))).ln() + contributions.append(idf * max(weights)) + matched = all(v > 0 for v in contributions) if all_terms else any(v > 0 for v in contributions) + if docs[i] is not None: + features[i] = sum(contributions) + if matched: + scores[i] = features[i] + rank = sorted((i for i, s in enumerate(scores) if s is not None), key=lambda i: (-scores[i], i)) + out.append(dict(name=name, docs=docs, query=query, edits=edits, all_terms=all_terms, + expected_scores=[str(s) if s is not None else None for s in scores], + expected_features=[str(s) if s is not None else None for s in features], + expected_order=rank)) + print(name, rank, [round(float(s), 9) if s is not None else None for s in scores]) + +path = Path(__file__).with_suffix(".json") +path.write_text(json.dumps(out, indent=2, ensure_ascii=False) + "\n") diff --git a/docs/rfcs/assets/0048-upstream-contract-checkpoint.json b/docs/rfcs/assets/0048-upstream-contract-checkpoint.json new file mode 100644 index 000000000..aaf8e954d --- /dev/null +++ b/docs/rfcs/assets/0048-upstream-contract-checkpoint.json @@ -0,0 +1,408 @@ +{ + "schema": "omnigraph-rfc0048-upstream-contract-checkpoint-v1", + "source_baseline_commit": "112507f183b0fc6e63a68c05e85ba56078edd13f", + "scope": "Pinned upstream mechanism qualification; not full-query resource, cloud cancellation, performance, or retrieval-quality evidence.", + "workspace_lock_sha256": "993d736f15af06b87c8ca1d6914db4283138ce30f8607e1a2632ee7dbc8d75f9", + "packages": [ + { + "name": "lance", + "version": "11.0.0", + "archive_sha256": "be781f40c7a75f9eae2188a2f71174acb7a360dca97163db40b041d0828dea48", + "matches_workspace_lock": true + }, + { + "name": "lance-index", + "version": "11.0.0", + "archive_sha256": "b0e0cb95f2c4f341c4dd04ac60f6a89ea26a6f75e09570225cbda4854c8b088e", + "matches_workspace_lock": true + }, + { + "name": "lance-io", + "version": "11.0.0", + "archive_sha256": "414d50997391b1ac83dc183c1612fdff88f58b959806078dc4c5e465154566de", + "matches_workspace_lock": true + }, + { + "name": "datafusion-catalog", + "version": "54.0.0", + "archive_sha256": "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139", + "matches_workspace_lock": true + }, + { + "name": "datafusion-expr", + "version": "54.0.0", + "archive_sha256": "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0", + "matches_workspace_lock": true + }, + { + "name": "datafusion-optimizer", + "version": "54.0.0", + "archive_sha256": "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281", + "matches_workspace_lock": true + }, + { + "name": "datafusion-physical-plan", + "version": "54.0.0", + "archive_sha256": "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc", + "matches_workspace_lock": true + }, + { + "name": "datafusion-execution", + "version": "54.0.0", + "archive_sha256": "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0", + "matches_workspace_lock": true + }, + { + "name": "arrow-buffer", + "version": "58.3.0", + "archive_sha256": "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0", + "matches_workspace_lock": true + }, + { + "name": "arrow-array", + "version": "58.3.0", + "archive_sha256": "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e", + "matches_workspace_lock": true + } + ], + "source_files": [ + { + "package": "lance-11.0.0", + "path": "src/dataset.rs", + "sha256": "e317e230733137ccb3323339a3c91ade50eb4fd06f46d9a460c42e34784077da", + "matches_archive": true, + "url": "https://github.com/lance-format/lance/blob/ab6b5bbe46009ed78746b444df8db59a8bc5d842/rust/lance/src/dataset.rs" + }, + { + "package": "lance-11.0.0", + "path": "src/dataset/scanner.rs", + "sha256": "5ccf3c5623d39fd5e68ede8139b270525486e60f8aa06acf7fcd2a2ed449295b", + "matches_archive": true, + "url": "https://github.com/lance-format/lance/blob/ab6b5bbe46009ed78746b444df8db59a8bc5d842/rust/lance/src/dataset/scanner.rs" + }, + { + "package": "lance-11.0.0", + "path": "src/datafusion/dataframe.rs", + "sha256": "b652ec61262a8231d26a66b180837a7c730f1b3e5bbb070dcdef4ccd637df811", + "matches_archive": true, + "url": "https://github.com/lance-format/lance/blob/ab6b5bbe46009ed78746b444df8db59a8bc5d842/rust/lance/src/datafusion/dataframe.rs" + }, + { + "package": "lance-11.0.0", + "path": "src/io/exec/fts.rs", + "sha256": "f690c3410371bae8a57e339a69b6d06233f21948d8d78fd33cdc97984800e683", + "matches_archive": true, + "url": "https://github.com/lance-format/lance/blob/ab6b5bbe46009ed78746b444df8db59a8bc5d842/rust/lance/src/io/exec/fts.rs" + }, + { + "package": "lance-11.0.0", + "path": "src/io/exec/take.rs", + "sha256": "31701ed49fd25a88b08bc6f77769f2d9b39476e6f881510cddd002f829a5ca50", + "matches_archive": true, + "url": "https://github.com/lance-format/lance/blob/ab6b5bbe46009ed78746b444df8db59a8bc5d842/rust/lance/src/io/exec/take.rs" + }, + { + "package": "lance-index-11.0.0", + "path": "src/scalar/inverted.rs", + "sha256": "d530ed8e801e118d3ad133242fe51a8c28866857114d98cd35046356db7c5b6d", + "matches_archive": true, + "url": "https://github.com/lance-format/lance/blob/ab6b5bbe46009ed78746b444df8db59a8bc5d842/rust/lance-index/src/scalar/inverted.rs" + }, + { + "package": "lance-index-11.0.0", + "path": "src/scalar/inverted/scorer.rs", + "sha256": "8936641fba2e67678abfe4867d88b1aec4c5ff684dfb31d1d2aff8d6f7dbe4e0", + "matches_archive": true, + "url": "https://github.com/lance-format/lance/blob/ab6b5bbe46009ed78746b444df8db59a8bc5d842/rust/lance-index/src/scalar/inverted/scorer.rs" + }, + { + "package": "lance-index-11.0.0", + "path": "src/scalar/inverted/tokenizer.rs", + "sha256": "35e21c43bcd339de52756f3bc5ef8e0a81b77673ab7c7c419f1ca6e98337ac26", + "matches_archive": true, + "url": "https://github.com/lance-format/lance/blob/ab6b5bbe46009ed78746b444df8db59a8bc5d842/rust/lance-index/src/scalar/inverted/tokenizer.rs" + }, + { + "package": "lance-index-11.0.0", + "path": "src/scalar/inverted/index/search.rs", + "sha256": "cf47f321ceda07b66d2df078c1e8a2a41ff34ed69cedc88e60b2e0436a39a5ea", + "matches_archive": true, + "url": "https://github.com/lance-format/lance/blob/ab6b5bbe46009ed78746b444df8db59a8bc5d842/rust/lance-index/src/scalar/inverted/index/search.rs" + }, + { + "package": "lance-index-11.0.0", + "path": "src/scalar/inverted/index/flat_search.rs", + "sha256": "75b8e07b3c4dd9b7292731a6161efac517986814694f2f51a1b8219f1ebf6d9a", + "matches_archive": true, + "url": "https://github.com/lance-format/lance/blob/ab6b5bbe46009ed78746b444df8db59a8bc5d842/rust/lance-index/src/scalar/inverted/index/flat_search.rs" + }, + { + "package": "lance-io-11.0.0", + "path": "src/scheduler.rs", + "sha256": "48478e4ddbfea3301934866d3143863420a5300809937758111b44d20e2f01e6", + "matches_archive": true, + "url": "https://github.com/lance-format/lance/blob/ab6b5bbe46009ed78746b444df8db59a8bc5d842/rust/lance-io/src/scheduler.rs" + }, + { + "package": "lance-io-11.0.0", + "path": "src/scheduler/lite.rs", + "sha256": "591b690d109802437c4de6144ae08183e87fbc6f90beb70993ef8eb6b5170191", + "matches_archive": true, + "url": "https://github.com/lance-format/lance/blob/ab6b5bbe46009ed78746b444df8db59a8bc5d842/rust/lance-io/src/scheduler/lite.rs" + }, + { + "package": "lance-io-11.0.0", + "path": "src/traits.rs", + "sha256": "d8344cdc3aa44d5459076ceaec3a480c28a396e7fd8d44383bf98097f0381c74", + "matches_archive": true, + "url": "https://github.com/lance-format/lance/blob/ab6b5bbe46009ed78746b444df8db59a8bc5d842/rust/lance-io/src/traits.rs" + }, + { + "package": "datafusion-catalog-54.0.0", + "path": "src/table.rs", + "sha256": "645adfdf201db4ac3e566f3e616a00ecff31acfebb1d8df560db2aa98e30a3af", + "matches_archive": true, + "url": "https://docs.rs/crate/datafusion-catalog/54.0.0/source/src/table.rs" + }, + { + "package": "datafusion-expr-54.0.0", + "path": "src/logical_plan/extension.rs", + "sha256": "9755a8ef93c3459feea8e3914e534acfc0a31b53ded5c821209e7bbe2e103256", + "matches_archive": true, + "url": "https://docs.rs/crate/datafusion-expr/54.0.0/source/src/logical_plan/extension.rs" + }, + { + "package": "datafusion-optimizer-54.0.0", + "path": "src/push_down_filter.rs", + "sha256": "13fc17abab2f379b0c13426ee6708d9b14c5146b4ef2d3a27a32db7905314c0c", + "matches_archive": true, + "url": "https://docs.rs/crate/datafusion-optimizer/54.0.0/source/src/push_down_filter.rs" + }, + { + "package": "datafusion-optimizer-54.0.0", + "path": "src/push_down_limit.rs", + "sha256": "cd1a33eb829a0bbfecf41af53dfa99d25ff74efdac4a7f770d0d8148c21c0c6b", + "matches_archive": true, + "url": "https://docs.rs/crate/datafusion-optimizer/54.0.0/source/src/push_down_limit.rs" + }, + { + "package": "datafusion-optimizer-54.0.0", + "path": "src/optimize_projections/mod.rs", + "sha256": "b5120f43adf11bac662ef4882b151baeb3df23ebedb2194b126c36f593c76698", + "matches_archive": true, + "url": "https://docs.rs/crate/datafusion-optimizer/54.0.0/source/src/optimize_projections/mod.rs" + }, + { + "package": "datafusion-physical-plan-54.0.0", + "path": "src/execution_plan.rs", + "sha256": "79db7478ca23047b0890082576d83df2283dc78e361e69048e9b352b79cb228b", + "matches_archive": true, + "url": "https://docs.rs/crate/datafusion-physical-plan/54.0.0/source/src/execution_plan.rs" + }, + { + "package": "datafusion-execution-54.0.0", + "path": "src/memory_pool/arrow.rs", + "sha256": "eadc063f12178d502debdbdcc9fa00e92003e775c819a302dbdc0e06b43ee9d1", + "matches_archive": true, + "url": "https://docs.rs/crate/datafusion-execution/54.0.0/source/src/memory_pool/arrow.rs" + }, + { + "package": "datafusion-execution-54.0.0", + "path": "src/memory_pool/mod.rs", + "sha256": "f306dc1800ce0512e5eff2bc31af215eed7ead1de6dfa7b6cc161c5a52c57f09", + "matches_archive": true, + "url": "https://docs.rs/crate/datafusion-execution/54.0.0/source/src/memory_pool/mod.rs" + }, + { + "package": "datafusion-execution-54.0.0", + "path": "src/disk_manager.rs", + "sha256": "c1bba907327a81bf264144322cf109ab91e8c8b9bcdb1a693d320e437c8898bb", + "matches_archive": true, + "url": "https://docs.rs/crate/datafusion-execution/54.0.0/source/src/disk_manager.rs" + }, + { + "package": "arrow-buffer-58.3.0", + "path": "src/pool.rs", + "sha256": "8a6a396005c6cd9c2f4970de004bb3a982eb4782373d9862eda0af3c038f996b", + "matches_archive": true, + "url": "https://docs.rs/crate/arrow-buffer/58.3.0/source/src/pool.rs" + }, + { + "package": "arrow-buffer-58.3.0", + "path": "src/bytes.rs", + "sha256": "77fa7219ae49504b95fe345047a3358d4f130b353628213b2f16d3378f4bbe46", + "matches_archive": true, + "url": "https://docs.rs/crate/arrow-buffer/58.3.0/source/src/bytes.rs" + }, + { + "package": "arrow-buffer-58.3.0", + "path": "src/buffer/immutable.rs", + "sha256": "3f10e81efa242387b11f4355ff7a509f9d0d672c1acbe682edac7142592a38ff", + "matches_archive": true, + "url": "https://docs.rs/crate/arrow-buffer/58.3.0/source/src/buffer/immutable.rs" + }, + { + "package": "arrow-array-58.3.0", + "path": "src/array/mod.rs", + "sha256": "64308d3f7ca0271d57cff5c407323a6315374bba8542b7dbf98392979539b83b", + "matches_archive": true, + "url": "https://docs.rs/crate/arrow-array/58.3.0/source/src/array/mod.rs" + } + ], + "additional_full_guides_read": [ + "https://lance.org/integrations/datafusion/", + "https://lance.org/guide/object_store/", + "https://lance.org/guide/observability/" + ], + "checked_at_utc": "2026-09-09T23:10:04.370399+00:00", + "local_date": "2026-09-10", + "local_timezone": "Europe/London", + "probe_source_files": [ + { + "path": "crates/omnigraph/tests/lance_surface_guards.rs", + "sha256": "4ef0ee1ad5c30330aaa9505d3dc63fafdc0be621e82d2fd4c982861c48f87505" + }, + { + "path": "crates/omnigraph/tests/rrf_prefilter_gate.rs", + "sha256": "52fd511e11a7d5b7f1036e2b22d4193d2ae112066783df815920bc702b4c5a24" + }, + { + "path": "crates/omnigraph/tests/benchmark_scenario_contract.rs", + "sha256": "14028cac7f691bdcfddb256d7f5a14802d138d6ad82813083ae7727b8155f39a" + }, + { + "path": "docs/rfcs/assets/0048-arrow-pool-probe.py", + "sha256": "4fee63ad32782ce3971d01b8ac0b73f788e09857ef4f7445ec974afb16acf1b2" + }, + { + "path": "docs/rfcs/assets/0048-arrow-pool-probe.rs", + "sha256": "840eb647e28ed66d0831a0ec0be4d5a646db022849f521ec14814ffcf4dc0ad5" + } + ], + "baseline": { + "commit": "112507f183b0fc6e63a68c05e85ba56078edd13f", + "lance_surface_guards": { + "passed": 42, + "failed": 0 + }, + "staged_target_selection_preserves_cutoffs_and_binding_rows": { + "passed": 1, + "failed": 0 + }, + "s3_configured": false + }, + "focused_suites": [ + { + "target": "lance_surface_guards", + "passed": 43, + "failed": 0, + "exit_code": 0 + }, + { + "target": "rrf_prefilter_gate", + "passed": 13, + "failed": 0, + "exit_code": 0 + }, + { + "target": "benchmark_scenario_contract", + "passed": 11, + "failed": 0, + "exit_code": 0 + } + ], + "environment_skips": [ + "public_physical_ref_token_rejects_s3_same_version_aba: OMNIGRAPH_S3_TEST_BUCKET absent; its early return is counted as passing by libtest. No S3 proof claimed." + ], + "stage_probe": { + "name": "staged_target_selection_preserves_cutoffs_and_binding_rows", + "configurations": 12, + "sources": [ + "memory", + "lance-ordered", + "lance-unordered" + ], + "target_partitions": [ + 1, + 4 + ], + "input_order": [ + "forward", + "reversed" + ], + "native_fragments": 3, + "input_batch_rows": 1, + "new_assertions": { + "hidden_columns_offset_then_null_filter": [ + "path-7" + ], + "binding_limit_then_path_filter": [], + "path_filter_then_binding_limit": [ + "path-7" + ], + "unprojected_score_filter_then_ordered_limit": [ + "path-4" + ], + "direct_provider_scan": { + "native_configurations": 8, + "projection": [ + "binding_id" + ], + "filter": "score < 10", + "limit": 1, + "returned_rows": 1, + "allowed_binding_ids": [ + "path-4", + "path-9" + ] + } + } + }, + "scheduler_probe": { + "name": "native_scheduler_drop_distinguishes_queued_and_dispatched_reads", + "boundary": "Real public ScanScheduler/FileScheduler with controlled Reader futures; no cloud transport or full-query claim.", + "io_buffer_size_bytes": 1, + "first_request_bytes": 16, + "standard": { + "started_reads": 1, + "active_after_all_public_owners_dropped": 1, + "completed_after_gate_released": 1, + "queued_read_started": false + }, + "lite": { + "started_reads": 1, + "active_after_all_public_owners_dropped": 0, + "completed_after_gate_released": 0, + "queued_read_started": false + } + }, + "arrow_probe": { + "workspace_lock_sha256": "993d736f15af06b87c8ca1d6914db4283138ce30f8607e1a2632ee7dbc8d75f9", + "probe_source_sha256": "840eb647e28ed66d0831a0ec0be4d5a646db022849f521ec14814ffcf4dc0ad5", + "resolved_lock_sha256": "eb75ceeae90329d1278d322ea6fd2ceaab047cd72fb10fa193a0d13136a5ea11", + "registry_dependencies_match_workspace": true, + "observations": { + "buffer_bytes": 20, + "pool_capacity": 1, + "available_after_claim": -19, + "fallible_admission_refused": true, + "same_owner_shared_once": true, + "cross_owner_reservation_transferred": true, + "released_after_last_alias": true + } + }, + "reproduction_commands": [ + "cargo test -p omnigraph-engine --test lance_surface_guards --test rrf_prefilter_gate --test benchmark_scenario_contract --locked", + "python3 docs/rfcs/assets/0048-arrow-pool-probe.py" + ], + "prior_evidence_unchanged": [ + "0048-staged-integration.patch", + "0048-validation-checkpoint.json", + "0048-agent-pilot-results.json" + ], + "open_release_gates": [ + "Four existing GQT search regressions recorded in RFC 0048 remain unresolved.", + "Whole-query preallocation, shared-buffer/cache ownership, dispatched I/O, decode/encoder cancellation and output accounting remain unqualified.", + "Native canonical fuzzy scoring/live statistics/ties and complete cross-type lowering remain unqualified." + ] +} From 961fa92e6a24820e9fed8e2a359d05859e080bc8 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 18 Sep 2026 10:41:38 +0200 Subject: [PATCH 02/16] rfc: split RFC 0048 and make RFC 0047 refuse unindexed search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RFC 0048 keeps the staged retrieval algebra. The lexical contract moves to 2026-09-18-analyzed-lexical-search (`@analyzed`, analyzer profiles, `terms`, `match_terms`, `bm25_v1`, the exact scan baseline, the regression cases); the grammar rules, capability matrix, C1–C4 and cross-type discovery move to 2026-09-18-gq-composition-and-language-evolution. Sections move verbatim; links are repointed and the evidence branch is cited by commit. Decisions recorded on the way: - The legacy spellings (`fuzzy`, `search`, `match_text`, `nearest`, retrieval in `order`, positional `rrf`) get a one-release deprecation window with a fixed mapping instead of a hard removal, following the compatibility-surfaces RFC; stored queries are re-parsed at boot. - The read options (`coverage`, `require_replay`, limits) are session settings after #742 landed `Session`, not a new request field. - The resource ledger and the DataFusion lowering sections are marked as requirements handed to the engine version 2 components named by RFC 0067 (PR #711); RFC 0048 no longer decides a planner or an admission mechanism. - RFC 0047 replaces the `full_text_search_unindexed` warning with a compile-time `T27` refusal of undeclared full-text targets and a plan-time `FullTextIndexRequired` refusal of unbuilt indexes (the RFC 0043 fence shape); a warning left invariant 7 open. --- ...5_statistics_ignore_eligibility_filter.gqt | 5 +- docs/rfcs/0047-search-plan-truth.md | 130 +- docs/rfcs/0048-search-contracts.md | 1581 +++-------------- .../2026-09-18-analyzed-lexical-search.md | 832 +++++++++ ...8-gq-composition-and-language-evolution.md | 791 +++++++++ docs/rfcs/README.md | 2 + 6 files changed, 1932 insertions(+), 1409 deletions(-) create mode 100644 docs/rfcs/2026-09-18-analyzed-lexical-search.md create mode 100644 docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md diff --git a/crates/omnigraph-gqt/cases/bm25_statistics_ignore_eligibility_filter.gqt b/crates/omnigraph-gqt/cases/bm25_statistics_ignore_eligibility_filter.gqt index 7c65266bf..b9f0b056d 100644 --- a/crates/omnigraph-gqt/cases/bm25_statistics_ignore_eligibility_filter.gqt +++ b/crates/omnigraph-gqt/cases/bm25_statistics_ignore_eligibility_filter.gqt @@ -3,8 +3,9 @@ # notes: group "o"; `Four` holds only the four group-"e" rows. The query "alpha beta" filtered to # notes: group "e" ranks an alpha row first: the filter is a prefilter and BM25 statistics come from # notes: the whole field (alpha is rare in ten). The same four rows as their own type rank the beta -# notes: row first (beta is rare in four). Same eligible rows, different winner; RFC 0048 chooses -# notes: the field corpus, so this pins the current answer as the contract. +# notes: row first (beta is rare in four). Same eligible rows, different winner; the analyzed +# notes: lexical search RFC (docs/rfcs/2026-09-18-analyzed-lexical-search.md) chooses the field +# notes: corpus, so this pins the current answer as the contract. --- runner timeout_ms: 10000 diff --git a/docs/rfcs/0047-search-plan-truth.md b/docs/rfcs/0047-search-plan-truth.md index d5af74a09..02486d2e3 100644 --- a/docs/rfcs/0047-search-plan-truth.md +++ b/docs/rfcs/0047-search-plan-truth.md @@ -7,7 +7,7 @@ implementation: not-started authors: - Ragnor Comerford (@ragnorc) created: 2026-09-01 -updated: 2026-09-09 +updated: 2026-09-18 discussion: "https://github.com/ModernRelay/omnigraph/pull/606" supersedes: [] superseded_by: [] @@ -17,12 +17,10 @@ blocked_on: # RFC 0047: Search plan truth: projectable ranking, deterministic order, and loud search failures -PR #606 reviews this correctness slice together with RFC 0048's final search -contract. Start with its [maintainer briefing](0048-search-contracts.md#maintainer-briefing) -for the combined design, migration tradeoffs and decisions still needed. -The [implementation handoff](0048-search-contracts.md#implementation-handoff-and-validation-checkpoint) -records the validated experiments and remaining gates; the interim mechanisms -in this RFC do not require a separate public release. +This is the correctness slice of the search work. [RFC 0048](0048-search-contracts.md) +owns the staged retrieval language and [Analyzed lexical search](2026-09-18-analyzed-lexical-search.md) +owns the analyzer, matching and scoring contract. The interim mechanisms in +this RFC do not require a separate public release. ## Summary @@ -41,12 +39,18 @@ that silently do nothing become errors or warnings: 4. Every ranked result has a total, deterministic order — including `rrf()` fusion (score, then trailing user keys applied inside score ties, then stable ids) and aggregated orderings. -5. The canonical read envelope gains three additive arrays: `warnings` - (first use: full-text search on a column with no FTS index serves through - the case-sensitive flat fallback — now loud), `metrics` (descriptors for - projected rank columns), and `retrievals` (every executed source, with - known/unknown representation coverage for `@embed`-backed vector retrievals - over the prefiltered population, with exact counts when requested). +5. Full-text search on a String property that the schema does not declare + `@index` is a stable `T27` compile diagnostic, and a declared index that + has not been built is a typed `FullTextIndexRequired` plan refusal, the + shape RFC 0043's `FullTextIndexRebuildRequired` already has. The + case-sensitive flat fallback that produced confident false negatives no + longer serves. +6. The canonical read envelope gains three additive arrays: `warnings` + (first uses: `search_order_ignored_by_aggregation` and + `embedding_coverage_pending`), `metrics` (descriptors for projected rank + columns), and `retrievals` (every executed source, with known/unknown + representation coverage for `@embed`-backed vector retrievals over the + prefiltered population, with exact counts when requested). Boundaries that do not change: no schema surface or storage-format change, no change to BM25/vector scoring math, the deprecated `POST /read` envelope stays @@ -55,11 +59,11 @@ were previously run-dependent). `fuzzy()` remains available in this slice. It has working exact and typo matches, but its analysis and matched set depend on index coverage. The -single breaking replacement of `fuzzy`, `search`, and `match_text` with -typed lexical matching and ranked retrieval belongs to -[RFC 0048](0048-search-contracts.md#user-and-operational-behavior), where the -schema-owned analyzer and exact lexical contract are defined. This RFC does -not introduce a `T25` retirement stage. +replacement of `fuzzy`, `search`, and `match_text` with typed lexical +matching and ranked retrieval, through a deprecation release, belongs to +[Analyzed lexical search](2026-09-18-analyzed-lexical-search.md), where the schema-owned analyzer and exact +lexical contract are defined. This RFC does not introduce a `T25` retirement +stage. This is the initial correctness slice. RFC 0048 owns the final staged query language: explicit ranking targets, graph-defined source populations, named @@ -120,6 +124,23 @@ once, coherently. This diagnostic protects the interim executor. RFC 0048 replaces the restriction for qualified graph-derived populations; unsupported shapes must continue to fail explicitly. +- `T27`: a full-text predicate or rank expression (`search`, `match_text`, + `fuzzy`, `bm25`) targets a String property the schema does not declare + `@index`: "declare `@index` on the property and reconcile indexes." + Searchability is a schema fact resolved through the catalog; the check + never reads physical index state. + +**Unbuilt indexes.** A declared index whose physical segments are absent at +the pinned snapshot refuses at plan time with the typed +`FullTextIndexRequired` outcome, naming the property and the remediation +(index reconciliation). This follows RFC 0043's `FullTextIndexRebuildRequired`, +which already refuses an uncertified index before execution; the two can +fire on one graph. Rows in the uncovered tail of an existing index keep +today's behavior and are scanned with the index analyzer. Today both the +undeclared and the unbuilt case silently plan the substrate's flat scan with +a bare, case-sensitive tokenizer; neither serves after this change. The +analyzer-equivalent scan that later lifts the plan-time refusal is the +lexical RFC's exact baseline. **Metric projection.** @@ -160,7 +181,7 @@ discovers the same candidates; RFC 0048 keeps stable ranked pagination separate. **Response envelope (canonical `/query` and stored-query reads; additive).** ```json -"warnings": [{ "code": "full_text_search_unindexed", "message": "…" }], +"warnings": [{ "code": "search_order_ignored_by_aggregation", "message": "…" }], "metrics": [{ "column": "score", "kind": "score", "source": "bm25", "variable": "d", "property": "body", "descending": true, "recall": "exact" }], @@ -171,8 +192,7 @@ discovers the same candidates; RFC 0048 keeps stable ranked pagination separate. ``` - `warnings` never change rows, membership, or order. Human CLI formats print - them to stderr; full-JSON output carries them in-band. Initial codes: - `full_text_search_unindexed`, `search_order_ignored_by_aggregation`, + them to stderr; full-JSON output carries them in-band. Initial codes: `search_order_ignored_by_aggregation` and `embedding_coverage_pending`. - `recall` reports the source *contract*: an index-accelerated `nearest` reports `approximate` even when execution happened to be exact, so clients @@ -191,9 +211,9 @@ discovers the same candidates; RFC 0048 keeps stable ranked pagination separate. - The deprecated `POST /read` envelope carries none of these fields, by construction. -**Operators** see `tracing` warnings for the unindexed-column condition and -remediate by declaring `@index` and running index reconciliation. No new -maintenance surface is added. +**Operators** see the `T27` diagnostic and the `FullTextIndexRequired` +refusal in query errors and `tracing`, and remediate by declaring `@index` +and running index reconciliation. No new maintenance surface is added. ## Design @@ -211,6 +231,11 @@ maintenance surface is added. cannot drift. Negation scopes check their own roots. This implementation guard does not establish a permanent first-declared-target rule for the language; RFC 0048's stage target validation replaces it as shapes qualify. +- **Searchability.** Typecheck resolves a full-text target through the + catalog and refuses an undeclared property (`T27`); the engine's scan + planner refuses an absent index for a declared property instead of falling + back to `default_text_tokenizer()`. Neither reads index *coverage*: a built + index with uncovered rows plans as today. - **Advisories.** Execution threads one explicit notice sink (deduplicating, so the bounded-bm25 retry and fusion's forked arms cannot double-report); results carry notices, metric descriptors, and retrieval descriptors as @@ -234,18 +259,20 @@ maintenance surface is added. ## Invariants - **Loud integrity failures (8):** strengthened for dropped search shapes: - `T26` refuses a predicate or ranking that cannot be attached to its target. - Warnings expose the absent-index fallback; they do not repair its matched - set. Exact lexical behavior, including typo tolerance, is RFC 0048's scope. + `T26` refuses a predicate or ranking that cannot be attached to its target; + `T27` and `FullTextIndexRequired` refuse the absent-index fallback instead + of serving it. Exact lexical behavior, including typo tolerance, is the + lexical RFC's scope. - **Query semantics are typed structures (9):** strengthened — retrieval moves from execution-time re-inference into the typed lowered plan; rank becomes an ordinary projected column, closing the recorded rank-carry gap. -- **Physical acceleration is derived (7):** the existing text-search - violation remains open in this slice: a warning does not make - index-dependent analysis correct. RFC 0048 closes it with schema-owned - analysis and exact evaluation across index states. This RFC's notices are - transitional visibility, not an exception to the invariant. Recall - reporting is contractual, not plan-derived. +- **Physical acceleration is derived (7):** the bare-tokenizer fallback no + longer serves, so exact-token `search`/`match_text` no longer change their + answer with index presence; a declared-but-unbuilt index is a loud refusal + (RFC 0043's accepted fence shape), not a different answer. Fuzzy analysis + on uncovered rows remains open here and is closed by the lexical RFC's + schema-owned analysis and exact evaluation. Recall reporting is + contractual, not plan-derived. - **Bounded, observable resource use (11):** requested coverage counts stream; retries, complete tie handling, and exact coverage share explicit query budgets. Streaming and finite tie plateaus alone do not prove bounded work or a @@ -262,10 +289,11 @@ maintenance surface is added. legacy `/read` envelope is untouched. OpenAPI regenerates with the new schemas. - **Language:** `T26` rejects queries whose search predicate or ranking was - previously dropped. Existing lexical spellings remain available until - RFC 0048 replaces them together at its breaking release boundary. No - compatibility alias or deprecation window is required for that lexical - replacement while the language is pre-stable. + previously dropped; `T27` rejects full-text search on undeclared + properties, and `FullTextIndexRequired` refuses declared-but-unbuilt + indexes; both previously served through the bare tokenizer. Existing + lexical spellings remain available until [Analyzed lexical search](2026-09-18-analyzed-lexical-search.md) + replaces them through its deprecation release. - **Order:** observable order changes only where scores tie; those orders were run-dependent before, so nothing reproducible is broken. - **Reverting** requires no storage or format work: the response fields are @@ -279,10 +307,12 @@ maintenance surface is added. repair requires an explicit analyzer and matching contract. RFC 0048 replaces the lexical surface in one breaking change; a standalone retirement neither defines that contract nor supplies typo tolerance. -- **Fail closed on unindexed text search** — deferred, not chosen now: the - flat fallback serves correct exact-token matches; warning preserves - service while removing silence. A future schema-owned analyzed-search - contract may revisit the posture. +- **Warn and keep serving unindexed text search through the bare + tokenizer** — rejected (2026-09-18): a warning does not repair the matched + set, and rows that differ by index presence are the deny-list's silent + partial results. Refusing an undeclared target at compile time and an + unbuilt index at plan time removes the false-negative class now; the + lexical RFC's analyzer-equivalent scan later lifts the plan-time refusal. - **Silently NULL (or best-effort match) mismatched metric projections** — rejected: a NULL column invites misreading; structural identity keeps the projection an observation of the executed retrieval. @@ -319,10 +349,15 @@ The prototype is historical evidence, not qualification of this revised proposal. The `fuzzy_does_not_match_under_default_tokenizer` characterization in [the existing search owner](../../crates/omnigraph/tests/search.rs) tests one capitalized query at one edit budget. The counterexamples in -[RFC 0048's evidence](0048-search-contracts.md#evidence-and-tests) disprove +[the lexical RFC's regression cases](2026-09-18-analyzed-lexical-search.md#regression-cases) disprove the universal-failure premise. That RFC owns the matching and index-lifecycle qualification required for the replacement. +Regression cases for the refusals: a `.gqt` case with a String property +without `@index` expecting `T27`, and a `search.rs` owner that drops the +physical index of a declared property and expects `FullTextIndexRequired` +(the PR #606 case `unindexed_search_is_case_sensitive` seeds both). + Revalidation against the pinned Lance source also leaves a separate determinism gate: its plain FTS collector can drop equal-score boundary rows before engine sorting. Extend `search.rs` with more tied rows than the native @@ -354,8 +389,8 @@ against the revised scope: 1. Substrate fences for the Lance 11 update→optimize stale-vector window (test-only; can land before acceptance as an ordinary change). -2. Warning carrier + `full_text_search_unindexed` (engine → API → CLI → - OpenAPI). +2. `T27`, the `FullTextIndexRequired` refusal and the warning carrier + (engine → API → CLI → OpenAPI). 3. Characterization goldens, then the retrieval-IR refactor (behavior- equivalent by construction; goldens prove it). 4. `T26` scan-rooted targets (compiler pass + engine backstops). @@ -415,6 +450,11 @@ language release or duplicate execution path. blanket claim that all RRF arm windows follow the output limit: BM25 arms are uncapped; vector arms inherit that limit. Distinguished historical prototype results from current evidence and total ordering from ANN replay. +- 2026-09-18 — replaced the `full_text_search_unindexed` warning with a + compile-time `T27` refusal of undeclared targets and a plan-time + `FullTextIndexRequired` refusal of unbuilt indexes (RFC 0043 precedent); a + warning left invariant 7 open. Linked the lexical RFC split out of + RFC 0048. - 2026-09-09 — aligned representation coverage with RFC 0048: known counts or explicit unknown by default, exact counts when explicitly requested. This keeps query completion separate from representation knowledge and avoids @@ -442,7 +482,7 @@ the pin, not the GitHub tag; the two diverge).** - The motivating case-sensitivity mechanism: `full_text_search` on a column with no FTS segments silently plans a flat scan with `default_text_tokenizer()` (bare `SimpleTokenizer`, no lowercasing). The - warning detects per-column via `TableStore::has_fts_index_on`. + plan-time refusal detects per-column via `TableStore::has_fts_index_on`. - The plain Match FTS path still compares score alone and leaf merges drop equal-score boundary candidates by arrival order; the adapter's own `.id`-column tie-breaks make the returned subset total, but cannot establish diff --git a/docs/rfcs/0048-search-contracts.md b/docs/rfcs/0048-search-contracts.md index 55df526be..728e26970 100644 --- a/docs/rfcs/0048-search-contracts.md +++ b/docs/rfcs/0048-search-contracts.md @@ -7,15 +7,17 @@ implementation: not-started authors: - Ragnor Comerford (@ragnorc) created: 2026-09-03 -updated: 2026-09-13 +updated: 2026-09-18 discussion: "https://github.com/ModernRelay/omnigraph/pull/606" supersedes: [] superseded_by: [] blocked_on: - - "Phase 0 language/type/plan proofs for staged scope, C1–C4, identity, grouping and future unions; reconcile RFC 0047 without freezing its interim IR restrictions" - - "Phase 0 resolved-schema syntax/serialization prototype, default/reapplication fixtures, qualified encoder identity and format coordination with RFCs 0040/0043/0044" - - "Phase 0 vector/fusion numeric policies and independent oracles; bounded NFC, live-statistics and exact-tie route dispositions against pinned upstream code" - - "Phase 0 shared resource/read interfaces and discriminating admission, fingerprint, follow-up and refusal prototypes" + - "RFC 0047 plan-truth guarantees carried into named stages" + - "Analyzed lexical search RFC (2026-09-18) accepted for the `lexical` source and `terms` contract" + - "Phase 0 language/type/plan proofs for staged scope, identity, grouping and per-group selection in the production compiler; the shared expression rules, C1–C4 and unions are owned by the GQ composition RFC (2026-09-18)" + - "Phase 0 resolved-schema syntax/serialization prototype for embedding recipes and defaults; qualified encoder identity and format coordination with RFCs 0040/0043/0044" + - "Phase 0 vector/fusion numeric policies and independent oracles; exact-tie route dispositions against pinned upstream code" + - "Read options expressed as session settings (RFC 2026-09-16); whole-query resource admission owned by the engine version 2 memory/admission component named by RFC 0067 (PR #711)" - "Phase 0 minimal compiler/engine/GQT vertical slice with later-stage discovery and shared-resource refusal" - "Phase 0 fixed mixed-workload corpus, judgments, budgets and acceptance protocol; production qualification belongs to Phases 1–6" --- @@ -24,10 +26,13 @@ blocked_on: ## Maintainer briefing -This is the concise decision document. The companion -[agent context](assets/0048-agent-context.md) preserves the full long-form -rationale, experiments and handoff detail. Use this RFC for current decisions -and the context document for the evidence behind them. +This RFC decides the staged retrieval algebra: rank stages, named sources and +metrics, vector retrieval, fusion, per-group selection, result metadata and +coherent follow-up reads. Two sibling documents were split out on 2026-09-18. +[Analyzed lexical search](2026-09-18-analyzed-lexical-search.md) owns `@analyzed`, analyzer profiles, `terms`, +`match_terms` and `bm25_v1`; [GQ composition and language evolution](2026-09-18-gq-composition-and-language-evolution.md) +owns the shared expression rules, the capability matrix, the C1–C4 +composition examples and cross-type discovery. The pre-split text, the test-only staged compiler, the DataFusion composition probes, the selection cost instrument, the lexical oracle test and the archived integration patches are retained at commit [`ce5a3012`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/docs/rfcs/0048-search-contracts.md) on the `rfc-0047-search-plan-truth` branch; they are evidence, not part of the tree. OmniGraph needs search that composes with exact lookup, graph traversal, aggregation and projection in one typed query language. The immediate defects @@ -56,13 +61,14 @@ agent-facing recipes. The initial release covers analyzed Boolean matching, exact/fuzzy lexical retrieval, exact `knn`, approximate `ann`, named fusion, graph-defined populations, per-group selection, terminal aggregates and coherent source -reads. The [capability matrix](#query-capability-matrix) distinguishes this +reads. The [capability matrix](2026-09-18-gq-composition-and-language-evolution.md#query-capability-matrix) distinguishes this scope from deferred operators. Cross-type, all-node and all-edge search are deferred; their typed union/projection and source contracts remain required language-evolution work, not an implied capability of same-type fusion. The pre-stable cutover deliberately breaks search queries and representation -declarations, with one coordinated format rebuild. The +declarations, with one coordinated format rebuild after a one-release +deprecation window for the legacy spellings. The [migration table](#user-facing-changes-and-migration) owns those consequences. **Phase 0 is incomplete.** There are checked logical plans, native probes and @@ -127,8 +133,8 @@ A later filter or reranker cannot recover a discarded candidate. Explicit stage boundaries expose this information loss and allow exhaustive analytics and bounded discovery to compose honestly. -Schema-owned analysis and an exact baseline also fix the current index-state -divergence: indexed `beto` can match `beta` while identical appended values +Schema-owned analysis and an exact baseline ([Analyzed lexical search](2026-09-18-analyzed-lexical-search.md)) +also fix the current index-state divergence: indexed `beto` can match `beta` while identical appended values are missed; `running` can disappear when indexing stores `run`. Boolean matching and fuzzy retrieval therefore consume one typed lexical description. @@ -150,12 +156,10 @@ graphs still cross the storage-format upgrade described below. | Existing usage or expectation | Proposed change | What users must do | |---|---|---| -| `fuzzy`, `search`, or `match_text` | These spellings are removed without compatibility aliases. Boolean matching and ranked lexical retrieval consume the same typed `terms` query. | Choose `match_terms` for filtering or `lexical` for retrieval; set term combination and edit tolerance deliberately. | -| `nearest`, retrieval expressions inside `order`, or positional RRF | Retrieval moves into explicit `rank` stages. Vector retrieval distinguishes exact `knn` from approximate `ann`; fusion names its inputs. | Rewrite inline and stored queries, choose exact versus approximate retrieval, and project named metrics instead of repeating retrieval expressions. | +| `fuzzy`, `search`, `match_text`, and implicit searchability through `@index` | Owned by [Analyzed lexical search](2026-09-18-analyzed-lexical-search.md#migration): one deprecation release, then removal with the GQ language major bump. | Follow that RFC's mapping to `match_terms` and `lexical`. | +| `nearest`, retrieval expressions inside `order`, or positional RRF | Retrieval moves into explicit `rank` stages. Vector retrieval distinguishes exact `knn` from approximate `ann`; fusion names its inputs. The old spellings compile to the staged IR with a deprecation diagnostic for one release before removal. | Rewrite inline and stored queries, choose exact versus approximate retrieval, and project named metrics instead of repeating retrieval expressions. | | Vector candidate depth inherited from final `limit`, while BM25 fusion arms scan uncapped | Each source and fusion stage has its own candidate window. Final `limit` counts output rows; graph fan-out can produce several rows per selected target. Stage comparators determine selection before final ordering. | Choose source/fusion windows explicitly and review tie keys and expected row counts. A migration cannot infer the intended recall/cost tradeoff from the old limit. | -| `@index` or `@key` implicitly makes a String searchable by analyzed text | Analyzed matching requires `@analyzed`, which enables BM25 ranking by default. Exact key/index annotations keep their separate meaning. | Declare `@analyzed` on searchable text, choose another analyzer when needed, or explicitly opt out of ranking with `scorer="none"`. An exact-only slug does not need an analyzer. | | Implicit vector geometry or an unresolved `@embed` model | A field's encoding recipe and geometry must resolve at schema acceptance. The model may inherit a schema-owned default recipe; distance may inherit that recipe's declared default. Raw vectors require explicit distance. | Declare the source and dimensions, then resolve a compatible recipe and geometry. A new default cannot identify how old vectors were produced; unresolved legacy vectors still need operator resolution or regeneration. | -| Existing search rows, scores, or ordering survive a spelling-only rewrite | `terms` defaults to all terms and zero edits. Analysis, complete fuzzy matching, and explicit selection boundaries can change results. Lexical ranking deduplicates query terms, groups fuzzy alternatives, and uses float64 scores with snapshot-visible field statistics independent of eligibility. | Review analyzer choices, fuzzy relevance, score thresholds, and tie fixtures. Repeating a query term no longer increases its weight. The rewrite does not promise equivalent results to legacy search. | | Queries relying on silently ignored search constructs or permissive parameter handling | Invalid shapes, incompatible representations, token-empty queries, and exhausted budgets produce typed failures. A successful partial candidate set cannot stand in for an exact result. | Handle the declared errors and size queries explicitly; do not interpret a failure as an empty successful search. | | Existing graph files open directly after the upgrade | The accepted-schema change requires an export/init/load rebuild. Compatible values and logical graph content are carried over; commit history, branches, and physical indexes are not preserved by that rebuild. | Plan the data upgrade even if no query uses search. Retain the predecessor graph if its history is needed, rebuild indexes explicitly, and obtain fresh snapshot references from the rebuilt graph. | @@ -222,7 +226,7 @@ The release must include the schema/query migration diagnostics, updated examples, user guides, and release notes. Detailed semantics and remaining acceptance gates follow; the migration tools must not guess unresolved choices. -### Schema and analyzers +### Schema declarations and vector defaults The annotations below are proposed syntax. The representation identity requirements in Design also apply; a provider/model label alone is not a @@ -239,6 +243,9 @@ node Organization { } ``` +`@analyzed`, its analyzer profiles and its scorer default are specified in +[Analyzed lexical search](2026-09-18-analyzed-lexical-search.md#schema-the-analyzed-capability); the rows +below are the vector side of the same resolution rule. #### Directive defaults and omission rules Defaults shorten the authored schema. They are resolved before acceptance @@ -247,9 +254,6 @@ content write. Unknown arguments and unresolved required choices are errors. | Surface | Omitted value | Explicit choice or failure | |---|---|---| -| `@analyzed` | No annotation means no analyzed capability; `@index` and `@key` do not supply it. | Exact predicates remain available without it. | -| `@analyzed.analyzer` | `standard_v1` | A named immutable profile such as `standard_folded_v1` or `english_v1`. | -| `@analyzed.scorer` | `bm25_v1` capability and default policy for exact or tolerant `terms` | `scorer="none"` permits analyzed filtering only; a lexical ranking source on that field is a typed error. | | `@embed` source | No default | The source property is mandatory and resolves to its stable identity. | | `Vector` dimensions | No default | A positive dimension is mandatory and must be compatible with the resolved encoding recipe when present. | | `@embed.model` | The schema's declared default embedding recipe | Without that default, require an explicit model that resolves to a complete qualified recipe. A model label alone is insufficient. | @@ -257,16 +261,6 @@ content write. Unknown arguments and unresolved required choices are errors. | `Vector.distance` without `@embed` | No default | Require explicit `l2`, `cosine`, or `dot`; the graph's embedding default does not assign geometry to raw vectors. | | Embedding normalization and query/document roles | The selected recipe's resolved choices | There is no global normalization fallback. An incomplete recipe fails acceptance. | -Thus bare `@analyzed` expands to -`@analyzed(analyzer="standard_v1", scorer="bm25_v1")`. -The initial scoring capability supplies one default policy, `bm25_v1`. -`lexical` may omit `scoring`; spelling `scoring: bm25_v1` explicitly is -equivalent. Exact and fuzzy search use that same policy, with edit tolerance -specified once in `terms`. There is no separate `fuzzy_bm25_v1` selector. -This default does not enable edit tolerance, execute ranking, create an -index, or remove the need for the policy's qualification. An explicit -`@analyzed(scorer="none")` retains the former matching-only use case. - The graph schema may declare one default embedding recipe in its accepted metadata. The recipe identifies the model revision, compatible query/record encoding behavior, dimension constraints, and any default distance. Its exact @@ -296,106 +290,17 @@ including previously inherited choices, so reinitialization does not require the old deployment's defaults. This rule requires accepted-state-aware schema resolution; parsing the desired source alone cannot implement it. -#### Analyzer profiles - -Initial immutable analyzer profiles resolve these settings explicitly; -unspecified Lance defaults are never part of the schema contract: - -| Setting | `standard_v1` (default) | `standard_folded_v1` | `english_v1` | -|---|---|---|---| -| Document / base tokenizer | Text / `simple` | Text / `simple` | Text / `simple` | -| Language setting | English (inactive) | English (inactive) | English | -| Lowercase | Yes | Yes | Yes | -| Stemming / stop words | Neither | Neither | English stemmer, then built-in English stop words | -| ASCII folding | No | Yes | Yes, after stemming and stop words | -| Token-length filter | Disabled (`max_token_length: None`) | Disabled | Disabled | -| Unicode normalization | NFC before tokenization; no NFKC | Same | Same | - -All three profiles first apply NFC to lexical analysis input, then run the -pinned `simple` tokenizer and Lance's filter order: lowercase, optional -stemmer, optional stop words, optional ASCII folding. Scalar String fields use -row document granularity; positions do not affect membership. These profiles -use no custom stop words, external dictionaries, n-grams, or code-tokenizer flags. -Posting positions and block layout remain derived index settings. - -NFC makes canonically equivalent spellings reach tokenization as the same -string while retaining distinctions that compatibility normalization can -erase. See the [Unicode normalization specification](https://www.unicode.org/reports/tr15/). -This preprocessing is part of the proposed analyzer, not a claim about the -unmodified Lance tokenizer. Query, scan, and index construction must use the -same qualified pipeline, with normalization work charged to the query/build -budget. A native path without that proof stays disabled. Stored String values -and exact predicates are unchanged; embedding input preprocessing remains -owned by its encoding recipe and does not inherit lexical normalization. - -Disabling the token-length filter is deliberate. Pinned Lance defaults to -`Some(40)`, whose filter retains only tokens shorter than 40 **UTF-8 bytes**, -before lowercasing or folding. Resource limits must reject excessive work, -not silently erase a long name or query term. Thus `english_v1` retains -the pinned linguistic filter choices but intentionally changes normalization -and long-token matching at this breaking boundary. - -`simple` splits at non-alphanumeric Unicode scalar values. Lowercasing is -not full case folding, and ASCII folding is distinct from Unicode -normalization. NFC preprocessing makes composed `résumé` and its canonically -equivalent decomposed spelling analyze identically. Accent removal is still -opt-in: with `english_v1`, `résumé` and `resume` become `resume` and -`resum`, respectively, because stemming precedes folding. These limitations -are explicit profile behavior, not promises of language-independent typo -equivalence. Once a profile is accepted, changing segmentation, normalization, -or filter order requires a new profile. Fingerprints include the normalizer -implementation and Unicode data identity as well as the Rust Unicode behavior -used by `simple` and lowercase. These are revised, unshipped `v1` definitions; -the earlier no-NFC probe does not qualify them. Adding a profile or scorer -version requires an RFC; accepted identities are never mutated. -Query-time analyzer or vector-distance overrides do not exist. A lexical -source resolves a versioned scoring policy compatible with the field's -declared capability, using its accepted default when omitted. That resolution -does not change analysis. The first release defines only `bm25_v1`; additional -policies need explicit semantics and qualification rather than runtime aliases. - -For spelling tolerance on names and titles, the non-stemming profiles keep -edit distance close to the spelling the user supplied. Under `english_v1`, -distance is measured after stemming; a one-character typo in the original -word need not remain one edit after analysis. The field's declared profile -decides this for every query and execution path. - -### One lexical query, two consumers - -A typed lexical query describes matching independently of its consumer. -The initial variant is `Terms { text, mode, max_edits }`; the grammar sketch -spells it `terms($q, mode: all, max_edits: 1)`. `mode` defaults to `all` and -`max_edits` to zero in both consumers. Candidate-oriented stored queries -usually choose `mode: any` explicitly. Phrase, prefix, and Boolean query -composition are future typed variants, not embedded vendor query strings. - -`match_terms(field, terms(...))` is a Boolean predicate on a scalar String -property with `@analyzed`. A `lexical` source consumes the same description -to select and rank matches with a resolved scoring policy. The Boolean -predicate introduces no score, ranking, candidate window, or implicit -retrieval. Its contract is: - -| Aspect | Rule | -|---|---| -| Query text | A non-null String literal or query parameter, constant for one execution; row-dependent query text is rejected. | -| Analysis | Apply the same resolved field analyzer to document and query text, at every edit budget including zero. | -| Distance | Minimum insertions, deletions, and substitutions over analyzed Unicode scalar values; each costs one. An adjacent transposition costs two. There is no implicit prefix restriction or length-based automatic tolerance. | -| `max_edits` | Optional integer literal or query parameter in `0..=2`, default `0`. Check literals at compile time and bound parameters before execution; reject negative, oversized, and non-integer values without narrowing casts. | -| `mode` | `all` by default: every analyzed query term has a document term within the budget. `any`: at least one does. | -| Term identity | Repeated query terms do not require repeated occurrences; a document term can satisfy multiple query terms. Reordering analyzed terms cannot change membership. Matching is neither phrase matching nor distance over the whole field value. | -| Empty text | A query yielding no searchable terms is a typed error. A null or token-empty document does not match. | -| Completeness | Every successful result satisfies the predicate exactly; an edit-tolerant predicate does not advertise approximate recall. | - -For a fixed schema, query, and document population, these are normative laws: +### Lexical sources -```text -matches(edits=0) ⊆ matches(edits=1) ⊆ matches(edits=2) -matches(indexed) = matches(unindexed) = matches(partially indexed) -``` - -Adding an analyzed query term cannot widen `mode: all`. Appending unrelated -documents cannot remove existing predicate matches. A resource failure is a -typed query failure, not an empty or truncated successful matched set. +A `lexical(field, terms(...), candidates: N)` source consumes the typed +`terms` description that [Analyzed lexical search](2026-09-18-analyzed-lexical-search.md#one-lexical-query-two-consumers) +defines, over a property declared `@analyzed`. Membership before the source's +cut is the shared `Terms` relation; ranking uses the field's resolved scoring +policy (`bm25_v1` by default) with the +[field-corpus statistics](2026-09-18-analyzed-lexical-search.md#lexical-scoring-and-shared-matching-semantics) +that RFC owns. A source cannot change analysis, edit budget or statistics +scope; those are field and query facts. `match_terms` in `match` is the same +description used as a Boolean predicate and introduces no rank or window. ### Clause composition and named stages @@ -497,289 +402,20 @@ A source cannot silently rank another binding or interpret one query vector as a batch of per-row queries. Correlated per-target retrieval requires a separate bounded operator contract; it is not implicit in repeated `match`. -Analyzed filtering alone remains an ordinary match predicate: - -```gq -query organization_names($q: String) { - match { - $o: Organization - match_terms($o.name, terms($q, mode: all, max_edits: 1)) - } - return { $o.slug, $o.name } - order { $o.slug asc } -} -``` - -No adjacent source inherits that edit budget. `lexical` with exact terms -continues to select only its own exact matches; fuzzy retrieval must consume -an explicitly tolerant query. Both resolve the same default scoring policy. - -### Language evolution and compatibility +Analyzed filtering alone remains an ordinary match predicate; the +[`match_terms` example](2026-09-18-analyzed-lexical-search.md#analyzed-filtering-in-match) and the rule that +no adjacent source inherits its edit budget are in the lexical RFC. -Extend one typed stage sequence followed by `return`, optional final `order` -and `limit`. Graph patterns remain declarative within `match`; source text -does not prescribe a physical scan/join schedule. The production single-match -grammar and test-only staged grammar are not the complete stable language. +### Language evolution -Use one expression model for properties, parameters, literals, aliases and -metrics wherever their types/scopes permit them. Keep graph patterns and stages -as dedicated nodes; a call-shaped retriever is not a scalar. Unknown -constructors/options fail. Fix precedence, associativity, null rules, name -resolution and positional/named arguments before stabilization. Scalar -negation must not inherit correlated graph `not` semantics; score arithmetic -retains explicit domain policies. The following Phase 0 expression decisions -apply when the corresponding operators are implemented: - -| Boundary | Decision | -|---|---| -| Precedence | Parentheses/calls, then `*` and `/`, then `+` and `-`, then one comparison, then scalar `not(...)`, then `and`, then `or`. Arithmetic at one level associates left; chained comparisons fail. Graph `not { ... }` retains its separate scope. | -| Numeric types | Arithmetic requires the same scalar numeric type; no implicit widening, integer/float mixing or conversion of metrics to numbers. Integer division remains unavailable until an explicit quotient/rounding contract; future casts must be named. | -| Null values | Arithmetic/comparison with null yields null. Boolean composition uses three-valued logic: false dominates `and`, true dominates `or`, otherwise an unknown operand yields null; `not(null)` is null. `is_null` returns non-null Bool; filters and `count_if` accept only true. | -| Numeric errors | Integer overflow/underflow is a typed failure, never wrapping, saturation or null. New floating arithmetic rejects non-finite operands/results and division by either signed zero. Null operands yield null without applying that arithmetic operation. These rules do not retroactively redefine stored scalar values or existing aggregate signatures. | -| Evaluation | Operands have no promised left-to-right or short-circuit evaluation. Boolean guards cannot make an invalid arithmetic expression safe; future conditional/try expressions need explicit evaluation rules. Constant folding must use the same types, null and error rules as execution. | -| Aliases | Sibling `let`/return expressions read the incoming scope; they cannot read an alias declared beside them. A later stage can read exported values. Final ordering can read output aliases. | - -The staged compiler checks precedence, same-type arithmetic and nullable -results. The native `staged_composition_numeric_and_null_contracts` probe -checks all nine Boolean pairs and exposes two adapter obligations: -DataFusion 54's default integer `BinaryExpr` wraps; its explicit -`with_fail_on_overflow(true)` path refuses the tested I64 overflows. That -switch still permits floating infinity/NaN, so finite arithmetic needs a -checked expression adapter. This is a physical-expression proof, not proof -that logical optimization, aggregate accumulators or GQ lowering preserve -these rules. Those remain qualification gates; a session default is -insufficient. See the pinned [binary evaluator](https://docs.rs/crate/datafusion-physical-expr/54.0.0/source/src/expressions/binary.rs) -and [Arrow arithmetic](https://docs.rs/crate/arrow-arith/58.3.0/source/src/numeric.rs). - -Every operator declares its scope and population effect: - -| Operation | Binding/identity rule | -|---|---| -| Predicate / ordinary projection | Preserve rows, duplicates, bindings and metric origins | -| Traversal | Extend bindings with existing endpoint/edge-instance semantics; bounded reachability does not become path enumeration | -| `rank` | Select distinct targets and retain their incoming binding rows | -| `take` | Select target/group pairs; it does not create reusable group features | -| Group/reduction | Export keys and reduced values, discard unreduced member bindings/metrics | -| Future optional/branch/collection | Explicit imports/exports, nullability, multiplicity and local scope | - -Terminal aggregate-return shorthand retains its existing implicit grouping: -adding a non-aggregate projection can change the groups. Future intermediate -grouping uses the same equality/null/duplicate rules with explicit exports; -ordinary intermediate projection must not acquire that implicit reduction. - -Each rank block ends with `yield `, resolving a declaration in that -block. Sources share the incoming population and explicit fusion dependencies. -Adding an unused declaration cannot select a different output or comparator; -every declaration is still validated and requested metrics consume resources. -Earlier metrics retain their original scope; reusing earlier candidates needs -explicit remapping/intersection. - -Keywords are contextual and token-bounded (`yield yield` can select an alias -named `yield`). Preserve property, system and source namespaces: -`$binding.property`, `$binding.@id` and explicit metric references. -New nested scopes must not capture unrelated bindings implicitly. Future -user-defined namespaces require collision rules. - -Admit count literals/parameters consistently while retaining their units: -positive source-target windows, nonnegative group-pair quotas and final -binding-row limits. Extending final-limit parameters is additive, not row-dependent -query batching. Physical effort, memory, I/O and output bytes stay separate. - -After the coordinated cutover, additive syntax must preserve valid queries' -parsing/name resolution, typing, populations, metric meaning and result schema -under the same accepted identities. Exact results stay exact; ANN preserves -its approximation/effort contract rather than a fixed candidate set. -All-node expansion after a schema change follows its explicit scope contract. - -Changing an analyzer, scorer, missing-value rule, default window, encoding -recipe or ANN mapping is a semantic change even if queries still parse. -Version policies/accepted bindings and preserve resolved choices. Unavoidable -future breaks require an explicit language/format boundary and migration. - -Before freezing syntax, qualify [C1–C4](#required-composition-examples) and a -compatibility corpus covering contextual identifiers, mixed/null expressions, -alias collisions, unused sources, projection/window independence, both sides -of cuts and discarded group/union metrics. Deferred operators need coherent -type/plan proofs now and executable GQT/resource tests when implemented. -Stored queries describe supported combinations; they cannot replace missing -language operators with unbounded client-side computation. - -### Query capability matrix - -This inventory covers the current public `.gq` surface, the initial release -described by this RFC, and the general graph/query/retrieval extensions the -grammar must leave room for. It separates language support from implementation -evidence; parser acceptance and native substrate support alone are insufficient. -It is a review checklist, not a promise to implement every conceivable query -operator or every future row in the initial release. - -**Current** describes the production path in this PR's source baseline, not -the test-only staged parser or the archived integration prototype. `Limited` -means a narrower capability or a known correctness gap, described in the cell. -The four [GQT search regressions](#ci-checkpoint-and-regression-disposition) -remain open. **RFC release** uses these dispositions: - -- `Keep`: preserve the existing operation and its meaning. -- `Deliver`: required for the coordinated release, subject to its stated gates. -- `Replace`: an existing surface migrates to the new contract at the cutover. -- `Foundation`: fix the shared grammar/type/scope rules now; the broader - user-facing operator remains deferred. -- `Defer`: outside the initial implementation; preserve the stated extension. -- `Separate`: an adjacent capability governed by its own contract, not a new - search feature. - -The last column states what can be added later and what the syntax must -preserve. Examples such as `yield`, `group`, `collect` and nested object -construction discussed during design are illustrative; this matrix does not -accept their spelling, introduce a general scalar-variable declaration, or -change the tested RFC query examples. - -#### Declarations, expressions and scope - -| Capability | Current | RFC release | Future extension / syntax constraint | -|---|---|---|---| -| Named queries; multiple query declarations per file | Yes | Keep | Query-body extensions retain the declaration and invocation model. | -| Typed required and nullable parameters | Yes; scalar, list and vector forms; nullable omission accepted | Keep | Relation/object parameters would need separate type and transport contracts; a list is not implicit query batching. | -| Stored-query tool metadata | Yes; `@description` and `@instruction` | Keep | Metadata describes the same query; it does not carry hidden execution semantics. | -| Literals, parameters, property values, `date`, `datetime`, `now` | Yes, with current context restrictions | Keep | New expression variants preserve existing types, time semantics and literal parsing. | -| One shared expression model across stages | Limited; fixed expression variants and context-specific handling | Foundation | Deliver shared parsing/typing for admitted expressions, including metrics; this does not deliver every scalar function below. | -| Scalar comparison, exact String prefix/substring and list membership | Yes | Keep | Exact String behavior remains case-sensitive; `contains` on a list remains membership. | -| General scalar Boolean composition and negation | Limited; match conditions conjoin and graph `not` tests pattern absence | Foundation | Add typed Boolean expressions with fixed precedence and null rules; graph absence keeps its correlation semantics. | -| Explicit null tests/replacement, conditionals, arithmetic, casts and general scalar functions | No general surface | Foundation | Extend shared expressions through typed signatures; preserve overflow, null and score-domain rules. | -| Intermediate named computations/projection | No | Foundation | Fix explicit value bindings and retained/exported scope through composition example C1; operator implementation remains deferred. | -| Result aliases used in final ordering | Yes | Keep | Result aliases remain distinct from graph parameters/bindings and source aliases. | -| Reusing an earlier result alias in another projection | No; production rejects `T36` | Foundation | Decide alias scope explicitly before relaxing this rule; shared expression parsing alone is not alias reuse. | -| Nested scopes, reusable query branches and bounded subqueries | Limited; correlated graph negation only | Defer | Declare imports, exports, shadowing, snapshot and shared budgets; distinguish reuse from re-execution. | -| User-defined functions or query-valued parameters | No | Separate | If justified later, define typed namespaces, versioning and execution limits; arbitrary evaluation is not an initial requirement. | - -#### Graph matching and identity - -| Capability | Current | RFC release | Future extension / syntax constraint | -|---|---|---|---| -| Concrete typed node scans and inline property constraints | Yes | Keep | New type selectors must not reinterpret an existing concrete type. | -| Directed traversal and binding joins within `match` | Yes | Keep | Preserve endpoint-pair versus edge-instance multiplicity and graph eligibility. | -| Undirected traversal | Yes, between the same endpoint type | Keep | Preserve orientation and duplicate rules when extending edge-pattern syntax. | -| Single-hop edge bindings and edge-property access | Yes | Keep | Node and edge kinds remain distinct even when names or ids coincide. | -| Bounded multi-hop reachability | Yes; shortest-distance semantics, no bound path/edge sequence | Keep | A future path value or path enumeration needs an explicit operator and bounds; it cannot redefine existing hop syntax. | -| Anonymous bindings and repeated-binding constraints | Yes | Keep | Anonymous bindings cannot name reusable rank/take targets; existing bindings cannot change type through rebinding. | -| Correlated pattern absence | Yes; `not { ... }` | Keep | Inner bindings do not escape; future scalar negation does not acquire this scope behavior. | -| Positive existence without outer-row fan-out | No dedicated positive-existence construct | Defer | Add a correlated pattern predicate; ordinary traversal continues to expose its matches. | -| Optional graph enrichment | No | Foundation | Example C4 must preserve unmatched inputs and nullable exports; distinguish inner predicates from later filters. Multiple matches may still fan out; implementation remains deferred. | -| Graph matching before, between and after retrieval cuts | Limited; one `match` and terminal search ordering | Deliver | Repeated stages retain their input population; later filters cannot move before an earlier selection cut. | -| Logical system identity projection and filtering | Limited; whole-node objects expose `id`, but `.id` is an ordinary declared property lookup | Deliver with RFC 0040 | Qualify `$binding.@id` and type/incarnation-aware follow-up; never expose native row IDs as graph identity. | -| Explicit unions of typed graph bindings | No; interfaces do not provide polymorphic query scans | Foundation | Execution deferred. Preserve typed branches, explicit exports, bag/set semantics and original entity identity. | -| All-node discovery, representation expansion and type narrowing | No | Foundation | Execution deferred. Resolve type scope separately from searchable representations at one snapshot; common fields and type-specific projections must typecheck. | -| Global all-edge or mixed node/edge discovery | No; a concrete edge can be reached through traversal | Defer | Make entity kind and selected scope explicit; an all-node selector must not silently start including edges. | -| Cross-graph federation | No combined `.gq` population; graph selection is external | Separate | Requires explicit authority, identity, snapshot and budget rules; global search in this RFC means one graph. | - -#### Projection and result shape - -| Capability | Current | RFC release | Future extension / syntax constraint | -|---|---|---|---| -| Flat projection of properties, parameters and literals | Yes | Keep | Ordinary projection preserves binding rows, duplicates and candidate windows. | -| Explicit column aliases and unique output names | Yes; duplicate names rejected; some unaliased inferred/executed names still differ | Keep; qualify schema agreement | Freeze naming and result-schema agreement; extra syntax must not capture existing aliases. | -| Whole-node object projection | Yes; id and properties except Blob/Vector | Keep | This remains schema-shaped shorthand; explicit fields provide stable compact results across schema additions. | -| Bare edge object projection | No; project edge properties explicitly | Defer | Define an edge object/reference type and endpoints deliberately; do not assume node-object behavior. | -| Existing list and vector property projection | Yes; each remains one column value | Keep | Projection does not unnest a list or execute vector retrieval. | -| Blob values in ordinary read projection | No; dedicated Blob API | Separate | Preserve the Blob access/resource contract; object shorthand must not fetch Blob contents. | -| Source score, distance, rank and fused-score projection | Limited; only a repeated leading `nearest`/`bm25` expression can expose its metric | Replace | Named `metric(source, field)` preserves domain, origin and missing membership through aliases and later stages. | -| Selected nested object construction | No; whole-node objects are the only node-object shorthand | Foundation | Example C4 fixes typed row reshaping separately from traversal/grouping/collection; constructor implementation remains deferred. | -| New list/object construction from expressions | Limited; list literals and existing list values | Defer | Define element/field types and nullability; constructing a value is separate from collecting rows. | -| List unnesting, mapping or comprehensions | No general surface | Defer | Unnest is an explicit population-changing operation; local value mapping needs its own typed scope and bounds. | -| Nested related-entity/evidence collections | No | Foundation | Example C4 fixes correlation, exports, duplicate and empty-result rules, local order, item limit and shared byte/work budgets; implementation remains deferred. | -| Snippets, highlights and source ranges | No dedicated query construct | Defer | Preserve source property/version, offset unit and bounds; generated text must remain distinguishable from stored source. | -| Token-budget result packing | No | Defer | Declare tokenizer, selection policy, attribution and completeness; row limits must not become token limits. | - -#### Aggregation, selection and ordering - -| Capability | Current | RFC release | Future extension / syntax constraint | -|---|---|---|---| -| Terminal `count`, `sum`, `avg`, `min`, `max` | Yes | Keep | Preserve admitted input types, null behavior and duplicates; `count($node)` counts binding rows. | -| Implicit grouping by non-aggregate return values | Yes, with current type restrictions | Keep | Adding a group projection can change row count; do not give ordinary intermediate projection this implicit effect. | -| Grouped metric projection and explicit metric reductions | No staged metrics; legacy search/aggregate combinations restricted | Deliver | Projected metrics can be group keys or explicit reductions; discarded source metrics cannot order a new group. | -| Reusable intermediate grouping/reduction | No | Foundation | Examples C1–C2 export group keys/entities and reduced values; drop unreduced member bindings and active ranks. Group-stage implementation remains deferred. | -| General distinct rows/entities and distinct aggregates | No dedicated syntax | Defer | Specify the identity/value tuple and duplicate equivalence; retrieval deduplication is not general `DISTINCT`. | -| Distinct-target ranking with retained binding rows | Limited; legacy search exists without the new stage contract | Deliver | Each target gets one source rank; selecting it retains its associated binding rows. | -| Quotas per explicit group, including ordinary non-search selection | No | Deliver through `take` | Select target/group pairs; use explicit reductions when order varies per pair; require local order without an active rank. | -| Parent selection using reduced child metrics | Limited; terminal aggregates cannot feed a quota | Deliver through `take` reductions | Selection does not create reusable group features or implicit parent-source ranks. | -| Top-N retrieval within every full group; row-dependent query inputs | No | Foundation | Examples C1/C4 require explicit correlation/partition scope, empty-group behavior and one total budget; implementation remains deferred. Global top-K plus `take` is different. | -| Final ordering by properties/aliases and identity tie-breaks | Yes, with documented legacy search boundary-tie gaps | Keep; repair retrieval ties | Final ordering cannot change a previous source cutoff; each selection owns its complete comparator. | -| Explicit `nulls first` / `nulls last` | No syntax; fixed ascending/descending defaults | Deliver | Apply consistently in local and final ordering; omission preserves existing defaults. | -| Final row limit, including zero | Yes; integer literal | Keep | Counts output rows; it does not resize retrieval or bound all intermediate work. | -| Parameterized final row limit | No | Foundation | An additive count-parameter extension remains possible; this matrix does not add it to the initial release. | -| Parameterized source windows and group quotas | No corresponding stage syntax | Deliver | Admit integers before execution; positive source windows and nonnegative quotas have different units. | -| General analytic windows, running aggregates and partition ranks | No public surface | Defer | Typed stage/expression rules must define partitions, frames and order; internal window use does not expose a language feature. | -| Stable ranked cursors/pagination | No replayable ranked-execution contract | Defer | Preserve actual candidate order or qualify reproducibility, retention and policy; snapshot identity alone is insufficient. | - -#### Retrieval and ranking - -| Capability | Current | RFC release | Future extension / syntax constraint | -|---|---|---|---| -| Analyzed lexical membership | Limited; `search`/`match_text` have known execution-path inconsistencies | Replace | One typed `terms` description consumed by `match_terms`; filtering introduces no rank/window. | -| Edit-tolerant lexical membership | Limited; `fuzzy` exists with analyzer/coverage defects | Replace | Shared analyzed edit semantics, explicit `mode` and `max_edits`, complete membership across index states. | -| Exact-term lexical ranking | Limited; legacy `bm25` exists | Replace | Explicit bounded `lexical` source with qualified `bm25_v1` statistics, numeric rules and ties. | -| Fuzzy lexical ranking | No unified ranked tolerant-query contract | Deliver | `lexical` consumes tolerant `terms`; no separate fuzzy scorer or inherited edit budget. | -| Exact vector top-K as an explicit user choice | No exactness selector; some physical paths scan exactly | Deliver through `knn` | Exact top-K over eligible valid vectors; an index is an acceleration choice. | -| Approximate vector top-K | Yes through legacy `nearest`, with implicit effort/window behavior | Replace with `ann` | Explicit approximation and bounded effort contract; exact rescoring does not imply complete recall. | -| String or raw-vector retrieval inputs | Yes | Keep with resolved encoding | Validate dimensions and compatible encoders; inputs are constant per execution, not implicit per-row batches. | -| Named sources, explicit candidate windows and explicit block output | No | Deliver | Sources share the incoming target population; select output independently of declaration order before freezing syntax. | -| Weighted multi-source rank fusion | Limited; legacy RRF has two inline arms and asymmetric windows | Replace | Named 2–16-arm RRF, explicit weights/windows, checked arithmetic and one vote per target per arm. | -| Search a traversal-introduced node or a bound edge | Limited; node search shapes have gaps and edge search is rejected | Deliver | Every source targets a property of the declared eligible binding; do not require a textual scan root. | -| Search, traverse and search again | No explicit staged surface | Deliver | Each stage retains population and metric origin; earlier candidates cannot be reopened implicitly. | -| Multiple fields on the same target as separate sources | Limited by legacy inline source forms | Deliver | Named sources may select different compatible field capabilities; fusion remains explicit. | -| One logical multi-field/cross-type lexical corpus | No unified contract | Defer | Specify field reduction and shared live statistics before candidate cuts; per-table BM25 values are not globally comparable by default. Multiple named fields of one target type can still use explicit fusion. | -| Phrase, token-prefix, proximity and Boolean lexical queries | No portable typed contract | Defer | Add `LexicalQuery` variants shared by matching/ranking consumers; avoid a vendor query-string sublanguage. | -| Geometric range retrieval and exact distance predicates | No dedicated public contract | Defer | Distinguish a complete eligible-population range query from a filter on ANN candidates. | -| Candidate rescoring, learned reranking and general feature combination | No general stage | Foundation | Example C3 fixes membership-preserving scoring, domains, statistics/normalization populations and resource behavior. Scoring/model/formula operators remain deferred. | -| General cross-identity fusion, such as passages with parent sources | No | Defer | Require graph mapping, reduction and fresh target ranks; matching id strings do not establish compatibility. | -| Semantic diversification / complementary evidence selection | No | Defer | Add a set-selection objective and qualification; per-group quotas alone do not supply it. | - -#### Representation and execution contracts - -These rows are part of a query's meaning or execution interface, even when -their authored syntax belongs to `.pg` or request options rather than `.gq`. - -| Capability | Current | RFC release | Future extension / syntax constraint | -|---|---|---|---| -| Analyzed String capability with immutable analyzer/scorer defaults | Limited; legacy FTS/index configuration and compatibility proofs | Deliver | Resolve `@analyzed` independently of `@index`; accepted semantics cannot follow mutable runtime defaults. | -| Dense vector fields and embedding-source declarations | Yes; existing `Vector`/`@embed` | Replace unresolved semantics | Resolve dimension, geometry, source mapping and compatible immutable encoding recipes; qualify providers. | -| Schema-owned default embedding recipe and reproducible exports | No complete resolved-default contract | Deliver | Final declaration spelling remains open; omission, explicit override and migration preserve accepted field bindings. | -| Sparse vectors, multivectors/late interaction and named analyzed views | No retrieval surface | Defer | Extend typed representation/capability variants; do not define every retriever input permanently as one dense vector. | -| Additional modalities or external retrieval sources | No general query source | Separate | Any later source needs explicit typing, authority, coherence and budget guarantees; no universal content object is introduced. | -| Coherent branch/snapshot reads | Yes through existing execution interfaces | Keep | Use the graph's accepted snapshot and existing authority; stage syntax must not reopen a fresh view. | -| Snapshot-coherent identity follow-up through all read transports | Limited; snapshot fields exist, system-id lookup and output paths have gaps | Deliver | Complete RFC 0040 lookup and JSON/JSONL continuation; unavailable/expired identity is explicit. | -| Inline/stored query invocation and typed result descriptors | Yes; result-schema agreement has known gaps | Keep and qualify staged reads | Derive reads, result shape and fingerprints from every stage; aliases and hidden demand must agree with execution. | -| Inspectable staged plans, metric origins and selection descriptors | No complete staged contract | Deliver | Expose resolved sources, inputs and semantics through the existing query interface; keep transport naming separate from graph properties. | -| Representation coverage, including explicitly requested exact counts | No complete source-level contract | Deliver | Known/unknown readiness differs from completion and index coverage; exact counts share the query budget. | -| Whole-query resource admission, cancellation and output accounting | Limited; existing local bounds do not establish the RFC guarantee | Deliver | All stages, encoders, fallbacks, nested work and projection share one execution context; no silent truncation. | -| Graph/branch and stored-query authorization | Yes | Keep | All source reads and graph expansions obey existing gates; a snapshot is not a retained-access capability. | -| Row/field-level security predicates | No policy engine for this scope | Separate | If added, constrain eligibility, statistics and metadata before selection; post-top-K filtering is insufficient. | - -#### Adjacent `.gq` operations - -| Capability | Current | RFC release | Future extension / syntax constraint | -|---|---|---|---| -| Named insert/update/delete query bodies | Yes, with current target and constructive/destructive restrictions | Keep | Read stages remain separate from mutation bodies; no implicit search-to-write pipeline. | -| Several mutation statements in one graph publication | Yes within admitted mutation forms | Keep | A future write extension must preserve the single publication and recovery contracts. | -| Standalone branch create/delete/merge/list statements | Yes; one per file, not beside query declarations | Keep | These control operations do not become data-query stages or typed unions. | -| Conditional writes after a coherent read | Yes through the execution interface | Keep | Preserve the returned graph commit precondition; do not infer a new transaction from projection syntax. | - -#### Syntax decisions needed now - -Every `Deliver`/`Replace` row needs accepted syntax and typing/lowering. -`Foundation` requires C1–C4 and global-extension design proofs now while its -broader implementation stays deferred. -All extensions follow the [language rules](#language-evolution-and-compatibility). - -Current-state evidence is the production -[grammar](../../crates/omnigraph-compiler/src/query/query.pest), -[typechecker](../../crates/omnigraph-compiler/src/query/typecheck.rs) and -[projection executor](../../crates/omnigraph/src/exec/projection.rs). -The query guide's earlier-alias-reuse claim is stale: `T36` and the -[score-projection case](../../crates/omnigraph-gqt/cases/issue_640_search_score_projection.gqt) -refuse it. `executed_column_name` also records unaliased inferred/executed -name drift. The [staged parser](../../crates/omnigraph-compiler/src/query/staged_probe.pest) -qualifies only its documented prototype subset. +The shared expression model, precedence, null and numeric rules, contextual +keywords, scope transitions, the +[query capability matrix](2026-09-18-gq-composition-and-language-evolution.md#query-capability-matrix) and the C1–C4 +composition examples are owned by +[GQ composition and language evolution](2026-09-18-gq-composition-and-language-evolution.md). This RFC's syntax must +satisfy those rules; the matrix's `Deliver` and `Replace` rows are this RFC's +initial release, and its `Foundation` rows are proofs due before the syntax +here stabilizes. ### Vector behavior and agent recipes @@ -863,6 +499,12 @@ All search-language replacements ship together. Diagnostics point callers from `fuzzy`, `search`, and `match_text` to typed lexical matching or retrieval; from retrieval in `order` and `nearest` to explicit rank stages and `knn`/`ann`; and from positional RRF to named inputs with declared candidate windows. +The legacy spellings are not removed in the same release: they compile to +the new typed IR with a deprecation diagnostic for one release and are +removed in the next, with the GQ language major bump the +[compatibility surfaces](2026-09-14-compatibility-surfaces.md) RFC defines. +The lexical RFC fixes the mapping for its spellings; Phase 6 fixes the +`nearest`, `order` and positional-`rrf` mapping. Rewritten application/stored queries and examples ship with the implementation. A rewrite must ask the caller to choose exact versus approximate retrieval and semantic windows when the old query never specified them. @@ -995,224 +637,14 @@ with an equivalence proof that preserves these populations, duplicate/null semantics and metric origins. Cost optimization cannot change which question the aggregate answers. -### Required composition examples - -C1–C4 are grammar-evolution acceptance cases. The test-only compiler parses and -checks these examples; the production parser rejects them. Their logical plans -and native probes are design evidence. Deferred operators require integrated -GQT/resource qualification when implemented. +### Composition examples -**C1 — Aggregate, select entities, then retrieve evidence.** A has prior/current -incident counts 2/8; B has 6/7. Select A's increase of six, irrespective of B's -report relevance. Missing periods count zero; incident-rooted input does not -invent services with no incidents. Including those needs a service population -and enrichment. - -```gq -query composition_c1($q: String) { - match { - $i: Incident - $s hasIncident $i - $i.period = "prior" or $i.period = "current" - } - group { - per { $s } - reduce { - count_if($i.period = "prior") as prior_count, - count_if($i.period = "current") as current_count - } - } - let { current_count - prior_count as increase } - select { order { increase desc, $s.@id asc } limit 1 } - match { $s hasReport $p } - rank $p { - lexical($p.text, terms($q), candidates: 2) as reports - yield reports - } - return { - $s as service, prior_count as prior_count, current_count as current_count, - increase as increase, $p.@id as report_id, $p.text as report_text - } -} -``` - -`group` exports entity keys under their binding names and named reductions, -discarding incident bindings. `let` adds row values whose sibling expressions -read the incoming scope; numeric results do not become identities. `select` -cuts rows, while `take` selects target/group pairs. Retain computed facts -through the service cut and later report reads. Multiple selected services -require C4's explicit correlation, not repeated global top-K plus `take`. - -**C2 — Retrieve, traverse, then aggregate the selected population.** Selected -p1/p2 have three bindings to project P (two from p1); eligible p3 falls outside -the cut. P must report three binding rows and two distinct passages. - -```gq -query composition_c2($q: String) { - match { $p: Passage } - rank $p { - lexical($p.text, terms($q), candidates: 2) as candidates - yield candidates - } - match { $p $membership:inProject $project } - group { - per { $project } - reduce { - count($p) as binding_rows, - count_distinct($p.@id) as passages, - min(metric(candidates, rank)) as best_rank - } - } - return { - $project as project, binding_rows as binding_rows, - passages as passages, best_rank as best_rank - } - order { $project.@id asc } -} -``` - -Group exports project identity/counts and explicitly reduced metrics; member -bindings and active rank disappear. In terminal aggregate shorthand, an -unreduced projected metric instead adds a grouping key and changes the question. -Keep the candidate barrier before traversal/aggregation. General distinct/ -intermediate grouping is deferred; the initial path must qualify terminal -binding counts and metric reductions. - -**C3 — Score existing candidates without changing membership.** Dense retrieval -selects d1/d2, both lexical matches, while the lexical arm's window contains only -d1. A fresh feature scores both without manufacturing lexical-arm membership. - -```gq -query composition_c3($q: String, $vector: Vector(3)) { - match { $p: Passage } - rank $p { - lexical($p.text, terms($q), candidates: 1) as words - knn($p.embedding, $vector, candidates: 2) as dense - yield dense - } - score $p { - lexical($p.text, terms($q), scoring: bm25_v1) as words_feature - } - return { - $p.@id as passage_id, metric(words, rank) as lexical_rank, - metric(words, score) as lexical_score, feature(words_feature) as lexical_feature - } -} -``` - -`score` preserves targets, bindings and comparator; it has no candidate window, -output selector, rank or fusion vote. `feature` names a scorer; `metric` -names retrieval membership. Reordering/cutting requires an explicit operation. -A hidden lexical top-K followed by a join can drop d2 and is invalid. -Scoring uses accepted corpus statistics and charges their work even for a -small target set. - -**Decision:** scoring evaluates the term contributions independently of -`all`/`any` membership. A partial `all` match can have a positive feature; -a present value with no matching terms, including a token-empty value, has -zero; a null field has a null feature. The -[lexical contract](#lexical-scoring-and-shared-matching-semantics) owns the -complete rules and numerical oracle. The native composition fixture still -uses precomputed features: it proves preservation, not their production. - -**C4 — Computed facts, optional graph facts and bounded evidence.** A has one -owner and three reports; B has neither. Both survive with their original -counts; A gets the first two reports under the local comparator and B gets a -null owner and empty typed list. - -```gq -query composition_c4($q: String) { - match { - $i: Incident - $s hasIncident $i - $i.period = "prior" or $i.period = "current" - } - group { - per { $s } - reduce { - count_if($i.period = "prior") as prior_count, - count_if($i.period = "current") as current_count - } - } - let { current_count - prior_count as increase } - select { order { increase desc, $s.@id asc } limit 2 } - optional ($s) as owner { - match { $s ownedBy $person } - return { $person as person } - } - collect ($s) as reports { - match { $s hasReport $p } - rank $p { - lexical($p.text, terms($q), candidates: 2) as relevant - yield relevant - } - return { $p.@id as id, $p.text as text } - order { metric(relevant, rank) asc, $p.@id asc } - limit 2 - } - return { - $s as service, prior_count as prior_count, current_count as current_count, - increase as increase, owner as owner, reports as reports - } -} -``` - -Imports explicitly distinguish `$entity` from computed row values; query -parameters stay available. Child bindings/sources do not escape. `optional` -returns a nullable object: zero rows produce null, one produces an object, -and more than one requires explicit selection/collection or a cardinality -error. `OwnedBy` is zero-or-one in this example. - -`collect` consumes ordered binding rows and requires a local comparator and -output limit. Unique reports need an explicit identity reduction; source -windows do not bound later fan-out or cumulative parent work. Both children -preserve the parent row, accepted snapshot and one whole-query budget. - -Correlation is per parent **row**, not merely entity ID: repeated service -bindings can carry different facts. Compiled source ID alone does not identify -all runtime per-parent rankings. Reuse requires equivalent imports, parameters, -representation and snapshot, preserving every parent row and charging actual -shared work. Edge imports/group exports, total row ties, nested-field access, -cardinality/resource enforcement and safe decorrelation remain unqualified. - -**Evidence and open boundaries.** The -[compiler](../../crates/omnigraph-compiler/src/query/staged_probe.rs) reads the -four examples directly. Its [derived logical views](../../crates/omnigraph-compiler/src/query/staged_probe/plan.rs) -retain relations, barriers, sources, group populations, projection types and -imports. They are neither executable IR nor a public serialization; their -snapshot/budget/statistics statements are requirements, not measured behavior. -Symbolic bindings and omitted options do not freeze defaults/fingerprints. - -| Example / checked logical plan | Prototype evidence | Still unproved / required falsifier | -|---|---|---| -| [C1](../../crates/omnigraph-compiler/src/query/staged_probe/composition_c1.json) | Group retains `$s`, exports counts and drops `$i`; selection precedes report retrieval. Native filtered counts produce A=2/8, B=6/7, C=1/0, selecting A. A report-driven prefilter selects B instead; `count(Boolean)` counts false; a duplicate path changes A's count. | Actual GQ lowering and optimized population barriers, entity rehydration, enforcement of the decided arithmetic/null/overflow rules and resource ownership. The native probe models the report filter; it does not retrieve reports. | -| [C2](../../crates/omnigraph-compiler/src/query/staged_probe/composition_c2.json) | Group retains project identity and reduced metric origin while dropping member bindings/order. Native selection of p1/p2 yields three binding rows and two distinct passages; removing the cut admits p3 and changes both counts. | Graph target-ID mapping, GQ aggregate lowering, general equality/null semantics, distinct-state memory and the complete optimized graph/retrieval plan. | -| [C3](../../crates/omnigraph-compiler/src/query/staged_probe/composition_c3.json) | The scorer creates a separate feature with no candidate window or rank; dense output remains the comparator. A native fixture preserves p2 with absent lexical-arm rank and a positive feature; filtering on lexical membership incorrectly drops it. Zero/null feature inputs also survive. The Decimal oracle separately checks partial/zero/null feature values. | Production scorer and enforcement of the decided nonmatch/empty/missing-field policy, fixed live statistics and numeric parity. Precomputed native features prove neither BM25 values nor scorer cost. | -| [C4](../../crates/omnigraph-compiler/src/query/staged_probe/composition_c4.json) | Checked imports/exports and separate child source IDs. Native ordered object lists preserve two rows for A and an empty B; presence markers distinguish absent objects from present null payloads. Duplicate paths consume collection rows, and fan-out exceeds the source window. An aggregate detects multiple owners. | Correlated GQ lowering, generation/preservation of parent-row identity, actual typed cardinality refusal, full entity-object projection, total row ties and shared budget/cancellation under many parents. Detection of ambiguous owners is not the query refusal path. | - -The [native probe](../../crates/omnigraph/tests/rrf_prefilter_gate/composition.rs) -uses DataFusion 54.0.0 / Arrow 58.3.0, memory inputs, one/four partitions, -forward/reversed one-row batches and two zero hash-join thresholds. It does -not qualify Lance providers, arbitrary optimizer configurations, snapshot -pinning, cancellation or allocations. See the -[checkpoint](#composition-plan-and-primitive-checkpoint) for reproduction; -GQT takes ownership when these operators become executable. - -C1/C2 use typed -[aggregation](https://docs.rs/crate/datafusion-expr/54.0.0/source/src/logical_plan/builder.rs), -[filtered expressions](https://docs.rs/crate/datafusion-expr/54.0.0/source/src/expr_fn.rs) -and [distinct count](https://docs.rs/crate/datafusion-functions-aggregate/54.0.0/source/src/count.rs). -Count matching rows: `count(predicate)` also counts false. Entity rehydration -and preservation of cuts remain OmniGraph lowering responsibilities. - -C4 must bridge [ArrayAgg's empty-input null](https://docs.rs/crate/datafusion-functions-aggregate/54.0.0/source/src/array_agg.rs) -to a typed empty list. Aggregate actual child rows by parent-row identity with -explicit order, then left-join to parents and construct empty lists for absent -groups. A left join before aggregation can create phantom `[null]` or `[{}]`. -Optional objects need a presence marker: null properties do not mean no row. -The native object-list probe exercises this distinction; the current JSON -writer omits null fields. These tests do not decide a new wire contract or -prove bounded nested execution. +C1–C4, the grammar-evolution acceptance cases, are in +[GQ composition and language evolution](2026-09-18-gq-composition-and-language-evolution.md#required-composition-examples). +Of them only C2's terminal subset is in this release: retrieve, traverse, +then terminally aggregate the selected population, with the candidate barrier +kept before traversal and aggregation. Intermediate grouping, `let`, +`select`, `score`, `optional` and `collect` are deferred there. ### Target identity, fan-out, grouping, and metrics @@ -1372,197 +804,13 @@ selection relative to already known information are extension requirements and evaluation concerns. Agent-side exclusions and follow-up queries remain ordinary graph operations. -### Lexical scoring and shared matching semantics - -Lower both lexical consumers through `LexicalQueryIR`, bound to the -rename-stable property identity and accepted analyzer fingerprint. Resolve -parameters and analyze once per compatible field/query instance per execution, -including on empty populations. Reuse resolved matching state across scan and -indexed paths. The consumers differ in role: matching returns a Boolean; -retrieval selects matches and computes ranking under its declared policy. -Removed spellings have no separate IR variants or compatibility evaluators. - -`bm25_v1` is one versioned term scorer for both exact and tolerant queries. -Zero edits reduces to BM25 with unit weight per distinct analyzed query term; -nonzero edits adds alternatives and an edit penalty within the same formula. -Candidate membership is always the shared `Terms` relation, evaluated before -the ranking cutoff. A score is not a second membership predicate. This removes -the earlier requirement to coordinate `terms.max_edits` with a separate exact -or fuzzy scorer selector. Query-term repetition and order carry no extra -weight; intentional source weighting belongs to explicit fusion or a separately -specified future scoring operator. - -The deferred membership-preserving `score` operator uses this same kernel -over its incoming targets, including targets that fail `Terms` membership: - -| Field/query outcome | Retrieval membership before its cut | Lexical feature | -|---|---|---| -| At least one matching term group, but not every group of an `all` query | False | Positive sum of the matching contributions | -| Nonempty field with no matching group | False | `0.0` | -| Present field with zero analyzed tokens | False | `0.0` | -| Null field | False | Null: no field value to score | -| Full query match | True | Same finite positive value as retrieval scoring | - -`mode` controls the Boolean consumer; it never masks term contributions. -Changing `all` to `any` alone cannot change a feature value. Changing the terms, -edit budget, analyzer or statistics can. A score of zero is observed absence -of term evidence; null remains missing representation. Neither adds a source -rank or fusion vote. Feature nullability follows field nullability, while a -retrieval metric can also be absent because its target missed that source's -window. A future explicit conditional can gate features by membership; no -second implicitly gated BM25 formula is introduced. - -An empty analyzed **query** is an error before any target evaluation, even for -an empty population or all-null fields. An empty corpus has no positive -contributions: return zero for present token-empty fields and null for null -fields without dividing by corpus length. Invalid representations/statistics, -non-finite arithmetic and resource failures remain errors. Scoring never -turns them into zero, null or dropped targets. - -The statistics population is part of source identity, independently of its -eligible population and candidate window. The initial contract uses the -snapshot-visible field corpus: distinct, policy-visible targets owning that -accepted type/property identity with nonempty analyzed values, before ordinary -graph or property filters and the source's lexical query. Null and token-empty -values contribute neither a document nor tokens. Graph fan-out contributes one -target, not repeated field values. A physical dataset, index segment, or shard -cannot choose a different logical corpus. Polymorphic fields must resolve -their corpus identities explicitly before that source shape is supported. - -At one snapshot, with the same query, analyzer, and scoring policy, narrowing -eligibility therefore leaves a surviving target's lexical score unchanged. -This makes scoring compose with graph filters and keeps it independent of -candidate windows. Selection still depends on eligibility: filtering before -top-k and filtering after top-k are different operations. Scores may change -across snapshots or policy-visible corpora; this is not a globally calibrated -score or a reason to compare unrelated source instances. - -The existing `fts_prefilter_does_not_change_covered_fragment_scores` guard -confirms that a native prefilter does not rescope covered-index statistics. -The new `fts_statistics_scope_can_reverse_ranking` probe goes further: with -the same four eligible targets, an alpha/beta query ranks an alpha target -first using the full ten-target corpus, but the beta target first using an -index containing only those four eligible targets. Eligibility and statistics -must remain separate typed plan facts. Implicitly recomputing statistics for -each match block would change score meaning with graph scope. An explicit -alternative corpus could be a future contract; it is not an initial query -option. - -This choice does not qualify native BM25 or make field statistics free. -Deleted/updated rows, unindexed tails, null/token-empty handling, arithmetic, -and complete boundary ties still require a checked-in score oracle and -scan/index parity. Reusable statistics must be derived from the accepted -snapshot and representation; a stale aggregate is not authority. Exact -statistics work consumes the query budget and may require a corpus scan. -The prefilter probe proves neither live-row statistics parity nor a cost -advantage for every query. Policy constrains the corpus before statistics; -ordinary query filters do not replace the access-control boundary. - -For each distinct analyzed query term `q`, treat all stored terms within the -declared edit budget as alternatives for that term. Let `N` be the number of -documents in the field corpus, `df_q` the number containing at least one such -alternative, `len_d` the total analyzed token count of document `d`, `avg_len` -the corpus mean length, and `tf(t,d)` the occurrence count of stored term `t`. -All analyzed occurrences contribute to length, including repetitions and -terms unrelated to the query. A field value is the document unit; graph paths -and index segments never multiply its frequency. - -The mathematical definition is: - -```text -idf(q) = log1p((N - df_q + 0.5) / (df_q + 0.5)) -norm(d) = 1.2 * (0.25 + 0.75 * len_d / avg_len) -weight(t,d) = 2.2 * tf(t,d) / (tf(t,d) + norm(d)) -part(q,d) = idf(q) * max [ 2^(-edit(q,t)) * weight(t,d) ] - over stored terms t within the edit budget -score(d) = sum part(q,d) over distinct query terms q -``` +### Lexical scoring -An empty maximum contributes zero. `df_q` counts a document once even when -several alternatives occur, before the full query's `all`/`any` membership -test and ordinary eligibility filters. Alternative-specific rarity cannot -give a misspelling extra weight: all alternatives share the term group's IDF. -Using the maximum or sum of individual term document frequencies would not -count the group's matched population: maximum misses disjoint occurrences, -while sum double-counts documents containing several alternatives. Union DF -also stays unchanged when spellings redistribute without changing which -documents match the group. This semantic benefit has a cost: native per-term -frequency metadata alone generally cannot produce it. -Within a document, maximum contribution prevents adding different spellings -from summing several pieces of evidence for one query term. Repeated occurrences -of a single stored term still affect its BM25 term frequency. A stored term -may contribute to several distinct query terms, matching the existing `Terms` -membership rule; the score is not a count of independent facts. - -Edit weights are `1`, `1/2`, and `1/4` for zero, one, and two edits. These are -versioned design choices, not values inferred from a backend or established -as globally optimal. Exact preference is a contribution boost at equal term -frequency and length, not a strict ordering tier above every fuzzy document. -Length, term frequency, and other query terms can still change the winner. -Changing `max_edits` may change group IDF and ranking, while membership retains -the declared monotonicity. At zero edits each group contains only the exact -term, so the same kernel supplies the exact-scorer reduction. - -The numeric profile uses float64 and `libm 0.2.16`'s scalar `log1p` -implementation, with its dependency identity recorded in the scorer -fingerprint. It does not delegate to the platform's logarithm. Count documents, -tokens, and frequencies with checked `u64` arithmetic; compute `N - df_q` -before conversion, reject inconsistent statistics, -and handle an empty corpus before division. Convert counts to float64 for -scoring, evaluate the parenthesized expressions above without algebraic -reassociation or fused operations, and sum contributions in ascending UTF-8 -query-term order. Zero-contribution terms add zero; matching rows must produce -a finite positive score or a typed numeric failure. Final ties use the source's -stable target comparator. One canonical numeric implementation must serve all -qualified paths; testing that pinned kernel across supported targets is -required before acceptance. The Decimal oracle verifies mathematical values -within its stated tolerance, not cross-platform bit identity. A future kernel -change needs a new scorer identity unless score/order equivalence is proved. - -Native Lance BM25 does not supply this contract. The -`native_fuzzy_bm25_rewards_a_rare_expansion` probe gives `beto` about 38 times -the score of exact `beta` in a 100-document fixture and doubles scores for a -repeated query term. The public float32 scorer also rounds a common term's -positive IDF to zero at `N = 2^24`; the -`native_bm25_idf_can_round_a_common_term_to_zero` guard reproduces this without -allocating that corpus. Native scores or top-k cuts require new qualification; -rescoring an incomplete native candidate set cannot establish exact winners. - -The [numerical fixtures](../../crates/omnigraph/tests/fixtures/lexical_scoring_v1.json) -and `lexical_scoring_v1_reference_oracle` in the existing search test owner -cover the chosen formula, zero-edit behavior, repeated/reordered terms, -alternative aggregation, overlapping groups, `all`/`any`, null/token-empty -values, and two-edit membership. They are design oracles over already-analyzed -terms, not a production evaluator or relevance benchmark. The initial complete -path must implement fuzzy ranking against these oracles; leaving it to a later -release is not completion. Scan-based correctness may ship before native -acceleration, with bounded failure when the work cannot complete. - -The same oracle now checks separate feature expectations, including partial -`all` matches, all-null/token-empty corpora, unchanged features under `all`/`any` -and eligibility changes, and identical retrieval/feature values for matches. -These settle C3's numeric meaning; they do not deliver the deferred scorer -operator or qualify analysis, live-row accounting and execution bounds. - -An exact physical baseline can compute field statistics in one snapshot-pinned -pass, then score eligible values in a second pass. For each query term, count -at most one family occurrence per document in the first pass; the second uses -that document's term frequencies and the same edit relation for its maximum. -Keep query-term statistics and bounded per-document state, not a retained -corpus or persistent expansion dictionary. N/mean-length metadata may be reused -when snapshot and analyzer identity qualify it; fuzzy family DF generally -depends on the query and edit budget. Both passes, normalization, matching, -and top-k selection share the execution budget. - -This is an implementation route to qualify, not a claim that two full scans -meet production latency goals. Indexed acceleration must supply complete -membership, live-row counts, term frequencies and group reductions, or prove -conservative bounds for every pruned candidate. Lance's public scorer trait -returns float32 additive term weights; merely substituting a scorer cannot -provide the group's maximum reduction or this float64 contract. Reuse Lance -postings through qualified upstream/adapter work and DataFusion group operators -where appropriate; do not introduce another stored index. A numeric or quota -post-pass over native top-k cannot restore discarded winners. +The `bm25_v1` definition, the unified exact/fuzzy formula, the field-corpus +statistics decision and the feature table for the deferred `score` operator +are owned by +[Analyzed lexical search](2026-09-18-analyzed-lexical-search.md#lexical-scoring-and-shared-matching-semantics). +Metrics produced by a `lexical` source carry the domain `Score`. ### Explicit ranking features and distance predicates @@ -1592,127 +840,15 @@ never be presented as a probability that an answer is correct. ### Graph-wide discovery across entity types **Phase 0 disposition:** defer cross-type union execution, all-node search, -all-edge search and mixed-kind search from the initial release. Concrete -node targets, and concrete edge targets reached through graph traversal, -retain their specified scope. A loop of independent client queries is not -the promised one-snapshot global query. This decision keeps the coordinated -release focused on the existing search correctness failures while the common -language acquires the union, narrowing and heterogeneous result machinery. - -Before stabilizing the initial grammar/IR, preserve these extension boundaries: -target identity carries entity kind and accepted type/incarnation as well as -entity ID; a logical source/window is independent of physical table count; -type scope and representation selection are separate typed plan facts. -Common-property projection requires compatible property types in every union -arm, with nullability widened when necessary. A property absent from an arm -requires explicit type narrowing or branch projection, never implicit null -or stringification. Union construction preserves binding rows; deduplication -and target selection remain explicit. An all-node expansion is the union of -the snapshot's concrete node scans, with each node identity once before any -graph fan-out. No wildcard or union punctuation is accepted by this decision; -its parser/type/plan compatibility proof remains a Phase 0 gate. - -Here, global search means discovery across eligible entity types in one -logical graph at one accepted snapshot. Its scope includes the target types, -searchable properties, representation policies and authorization rules. A -convenience stored query can own that scope; schema-driven expansion must -resolve it against the accepted catalog and include the resolved scope in its -query identity. Adding a searchable type can then change that explicitly -defined population. Ordinary unindexed or non-searchable fields do not acquire -an implicit text conversion or embedding recipe. - -An all-node selector must expand to every accepted node type without requiring -applications to enumerate the alternatives. Keep this type scope separate from -the representation selector: the latter resolves analyzed properties, field -reductions and compatible embedding spaces. A node type with no selected usable -representation remains in the declared scope but cannot contribute a ranked -hit. Coverage must distinguish this absence from an evaluated non-match; -unknown counts remain explicit unless exact coverage was requested. An index -is not what makes a property searchable, and missing indexes must not remove a -type from the logical scope. All-edge scope is a separate explicit choice. - -Resolve both selectors against the query's accepted snapshot and include their -expanded identities in execution identity. A newly added node type is therefore -included in a subsequent all-node query, while a read pinned to an older snapshot -retains that snapshot's type and representation scope. This automatic scope -expansion does not authorize implicit stringification, mixing vector spaces, or -resetting resource allowances per type. The wildcard spelling and typed -representation-selector grammar remain part of the general query-language -extension; a wildcard alone does not make `$hit.title` valid on every type. - -The current compiler binds a variable to one concrete node type. Schema -interfaces provide declarations and inheritance, but the query binder does -not yet execute polymorphic scans or a heterogeneous union. The initial -same-binding fusion contract alone therefore does not provide a single -cross-type global search query. This is an expressiveness gap to resolve -before advertising that capability. - -The preferred extension is a general typed union of graph bindings. Each -branch preserves the entity kind, accepted type/incarnation identity and -entity id; a hit remains its original node or edge. Common projections and -type-specific property access need ordinary query-language typing and type -narrowing. No synthetic stored `Document` entity or separate search registry -is needed. Source metrics and matched-property attribution remain attached to -their producing source. Two types with equal id strings are still different -targets. Passage-to-owner mapping remains the separate explicit graph mapping -and reduction problem described above. - -Ranking this union requires a declared cross-type policy: - -- A lexical source over a common logical text representation needs compatible - analysis, a defined treatment of multiple fields per target, and statistics - over that logical corpus. Per-table BM25 scores cannot be merged as if each - table had used those shared statistics. Multiple property hits must not - accidentally duplicate a target or let traversal fan-out add relevance. -- Compatible vector representations can share a distance comparator when - space/revision, query encoding, dimensions, metric and numeric rules agree. - Different spaces remain separate named sources, with their query encoding - calls included in the shared execution budget. -- Fusion can combine sources that now name this same typed target universe. - RRF over disjoint per-type lists is also a possible explicit policy, but it - gives each equally weighted type's first result the same contribution. It - is a source-balancing choice, not proof of comparable relevance. A common - scorer or qualified reranker is needed when the product requires a single - relevance ordering that those source ranks do not supply. - -Logical source count and physical table count are separate. Compatible tables -can be partitions of one logical source with one global candidate window. -Creating an independent fusion vote for every table would let physical schema -layout influence relevance. All partition scans and merges still consume the -same query budget; a small logical source count does not bound physical work. - -Physically, Lance remains responsible for each version-pinned dataset and -qualified index/scan path. DataFusion can union narrow candidate relations, -group by the full typed identity, rank/fuse and apply the global cut; hydrate -payloads from the corresponding pinned datasets after selection. For an exact -source split into disjoint physical partitions, local top-k followed by a -global top-k is valid only when every partition uses the same score and total -comparator as the global source. A partition cannot substitute local corpus -statistics or a different tie rule. Complete per-target field reduction and -deduplication before such a cut. If a target's score still depends on -contributions from several partitions, pruning those contributions requires a -valid global-score bound; ordinary local top-k is insufficient. ANN remains -explicitly approximate. - -Rescoring a shortlist selected with table-local BM25 statistics cannot repair -this error: the global winner may already have been discarded. Lance 11's -public `InvertedIndex::bm25_stats_for_terms` and `bm25_search` with a supplied -`MemBM25Scorer` provide a lower-level shared-statistics path to investigate. -They do not establish the complete contract: the scorer is native float32, -the RFC requires different fuzzy-group scoring, and immutable index statistics -can still count deleted rows. Reconcile live corpus statistics, uncovered -rows and invalidated coverage before using them for a global candidate cut. -Any cache remains derived from the accepted snapshot and representation -identities. Native index statistics alone are not a new source of truth. - -Before enabling that extension, qualification must cover two unrelated node -types, an explicitly selected edge type, equal id strings across types, multiple searchable fields, -incompatible vector spaces, empty/unavailable sources, snapshot changes, -policy and shared-resource refusal. Compare the physical fan-out plan with an -independent evaluator over the full logical union. Include disjoint-source -RRF behavior in the retrieval-task evaluation. The grammar, typed result and -global-corpus rules require an explicit extension; this section records its -design direction and does not claim implemented support. +all-edge search and mixed-kind search from the initial release. Concrete node +targets, and concrete edge targets reached through graph traversal, retain +their specified scope. Before stabilizing the initial grammar and IR, this +RFC preserves three boundaries: target identity carries entity kind and +accepted type/incarnation as well as entity id; a logical source and its +window are independent of physical table count; type scope and +representation selection are separate typed plan facts. The design direction, +the ranking policy for a typed union and the qualification list are in +[GQ composition and language evolution](2026-09-18-gq-composition-and-language-evolution.md#graph-wide-discovery-across-entity-types). ### Representation identity and source attribution @@ -1819,15 +955,22 @@ wire types and namespace choices must be checked before implementation: | Attribution | Graph snapshot context, graph binding identities, and selected properties; application-defined source relationships remain ordinary data | | Follow-up | A supported way to read/expand those bindings at that snapshot, or an explicit expired/unavailable outcome | -**Interface decision:** add one typed `execution` option to canonical inline -and stored-query requests, with the same options on the embedded entry point: -`coverage: report | exact` (default `report`), `require_replay: Bool` (default -false), and optional `limits`. This is request admission/metadata policy; -retrieval sources, windows and effort remain in the typed query. Unknown keys, -invalid units and unsupported requirements fail before query work. An omitted -limit inherits a finite operator-configured default; an explicit request may -tighten that cap; a value above the cap is an admission error. Zero means no -allowance for a counted resource, never unlimited. A deadline must be positive. +**Interface decision:** the read options are session settings, not a new +request field. [Session settings](2026-09-16-session-settings.md) and its +first implementation (#742) give every operation an explicit, typed +`SessionSettings` value with a declared scope: `Omnigraph::session(settings, +sources)` carries it into `query` and `query_with_head`, a GQ `set` prefix +applies to one statement, stored sources reject settings prefixes, and a +request-scoped value cannot widen an operator's cap. This RFC adds +request-scoped settings `coverage` (`report` | `exact`, default `report`), +`require_replay` (Bool, default false) and the resource limits named below, +each with a finite operator-configured default that an explicit value may +tighten but not exceed; the settings RFC decides whether they take a +`search.` namespace. Unknown names, invalid units and unsupported requirements +fail at settings validation before query work, as `ann_nprobes` already +does. Zero means no allowance for a counted resource, never unlimited; a +deadline must be positive. Retrieval sources, windows and effort remain in +the typed query. Retain `ReadOutput.graph_commit_id` as the replay identifier and `target` as the requested target. Add an `execution` result descriptor with its format @@ -1845,9 +988,10 @@ renderer/client fixtures must prove these additions; naming them here does not make them accepted request fields today. The current owners are [API types](../../crates/omnigraph-api-types/src/lib.rs), [engine query admission](../../crates/omnigraph/src/exec/query.rs) and -[CLI renderers](../../crates/omnigraph-cli/src/read_format.rs). The engine -currently captures its view before compiling and has no such shared execution -argument; implementing this decision requires changing that entry sequence. +[CLI renderers](../../crates/omnigraph-cli/src/read_format.rs). Since #742 +the engine's query entry points receive effective `SessionSettings` +explicitly through `Session`; the read options extend that table rather than +adding a parallel execution argument. Coverage knowledge is separate from query completion. By default, report exact counts only when required retrieval work or qualified snapshot-bound metadata @@ -1949,6 +1093,14 @@ Compact discovery and selective property reads support context control; token-budget packing and source snippets remain deferred, with their encoding/source attribution requirements retained. +#### Requirements handed to engine version 2 + +RFC 0067 (PR #711) names memory management and admission control as an +engine version 2 component with no owning RFC yet. This RFC does not decide +that mechanism. The interface decision and limit units below are the +requirements search execution places on it, retained here until the +component RFC exists; they are not this RFC's acceptance surface. + **Resource interface decision:** create one request-owned execution context before snapshot capture/compilation and retain it through serialization. It owns the accepted view once resolved, cancellation/deadline, resource ledger @@ -1997,103 +1149,20 @@ or boundedly spools the result before success; transport interruption still means the client did not receive a complete result. A future streaming-success protocol needs an explicit terminal completion record, not silent partial rows. -### Exact lexical execution and qualified acceleration - -**Analyzer parity without index coupling.** Instantiate the analyzer from -accepted SchemaIR, even when no FTS artifact exists. Eager empty indexes are -not the analyzer carrier: overwrite, index removal, or an incomplete rebuild -must not erase logical semantics. Use the pinned substrate tokenizer -implementation through one analyzer binding that includes the declared NFC -preprocessing step. Do not reimplement its filters or pass already analyzed -tokens into another path that analyzes them again. Normalization must precede -tokenization consistently in the query, scan, and index-builder paths; -normalizing only a query does not qualify an index built from raw spellings. -Creating or changing an analyzer profile continues to require the existing -schema publication and compatibility protocol; ordinary content writes do -not build indexes inline. - -**Exact scan baseline.** Evaluate the typed predicate over streamed document -tokens using that analyzer and the declared edit relation. Build bounded -query matching state once; honor cancellation and the query's execution -budgets while processing batches. This path is the correctness oracle and -remains available with full, partial, or absent index coverage. It composes -with typed graph/property filters at their declared stage. The first-declared -scan restriction is transitional; supported graph-derived populations must -retain their predicates. Unsupported placement fails validation rather than -losing the predicate. - -This is a new typed Boolean evaluator, not a wrapper around Lance's flat -BM25 scanner. `InvertedIndexParams::build()` already exposes the analyzer -without a dataset or index; its Text tokenizer shares query/document -tokenization. That construction alone does not supply the newly specified NFC -pipeline; its integration is an explicit qualification gate. Lance's flat -BM25 helper accepts a tokenizer but collects per-row scoring counts, does not -implement edit matching, and its fuzzy post-filter path rejects execution. -Reuse the public analyzer in a typed -engine/DataFusion filter over the sealed scan stream. Native index -acceleration additionally needs an analyzer-consistent query path and a -complete-expansion outcome; those scanner capabilities are not supplied by -the pin. Boolean evaluation introduces no dictionary or posting storage. - -The implementation must bound query bytes, distinct terms, individual -materialized values, matching state, and execution work. Enforce admission -before allocations that can exceed the budget, including Unicode normalization -and token construction. NFC can expand UTF-8 bytes: the pinned normalizer turns -the two-byte U+0344 into four bytes. It can also consume a long combining-mark -sequence before yielding its first output scalar. Bound source consumption, -internal normalization buffers/work, and output capacity; an input-byte limit -or cancellation checks only between emitted characters or tokens are -insufficient. A bounded implementation must enforce these limits at the -normalizer's input and internal work boundaries. -Use a bounded exact edit matcher, with cancellation checkpoints inside large -comparisons and across successive small comparisons. A work quantum that -resets for every token pair does not bound uninterrupted aggregate work. -Executor cooperation, cancellation propagation from the request boundary, -and cleanup of native tasks are separate qualification obligations. Marking a -function async or adding a yield does not establish end-to-end cancellation. -The pinned `fst` Levenshtein automaton agrees with the declared scalar-value distance, -but construction can consume substantial memory and hit its state limit. -Its default per-automaton cap is not a whole-query resource protocol. Numeric -limits, accounting units, and fallback charging must be specified and -qualified before shipping this evaluator. - -**Qualified index acceleration.** Lance continues to own dictionaries, -postings, and physical index state. A native path is eligible only when its -artifact passes RFC 0043's proof checks against the accepted profile and its -complete normalization/tokenization/filter pipeline is qualified for the -requested mode and edit budget. -Covered rows use complete term expansion and posting evaluation; uncovered -or rewritten rows use the same exact predicate on their accepted values. -Combine them at one snapshot with the existing visibility and row-identity -rules. Final limits and retrieval arm windows cannot change which terms or -rows satisfy the predicate; ordinary early termination is allowed when the -query plan proves the requested result complete. Post-verifying a truncated -candidate set cannot repair omitted matches. - -An indexed expansion must distinguish **complete** from **overflow**. The -current substrate shares a default budget of 50 expansions across query -terms within each segment, selecting lexically across that segment's -partitions. It returns tokens without a completeness flag. Earlier terms -can exhaust the budget before later terms are considered; changing the -vocabulary or segment layout can change the selected terms. This is not an -exact membership contract, and increasing the cap is not a completeness proof. -When complete acceleration cannot be established, use the exact scan within -the remaining query budget. If that budget is exhausted, fail the whole -query with a typed resource outcome; do not return partial success. Bound -token/automaton construction, memory, and execution work without silently -discarding analyzed terms or increasing budgets on fallback. This introduces -no query-level `max_expansions` knob or separate approximate predicate. - -The pinned Lance fuzzy scanner does not yet satisfy these requirements. -Until an upstream implementation and adapter pass the qualification matrix, -use the exact baseline for the affected shapes even when an index exists. -Keep native matching changes upstream where possible; do not build a second -index subsystem or a persistent shadow vocabulary in OmniGraph. BM25's -separate analyzer and corpus-statistics parity gate remains required; a -membership matcher alone does not qualify ranked scores. +### Exact lexical execution + +The exact scan baseline, analyzer parity without index coupling and the +qualified-acceleration rules for `Terms` are owned by +[Analyzed lexical search](2026-09-18-analyzed-lexical-search.md#exact-lexical-execution-and-qualified-acceleration). +A `lexical` source uses that path for membership and statistics; this RFC +adds only the candidate window and the source's total comparator. ### Lance, DataFusion, and graph execution +Planner and execution-engine mechanisms belong to the engine version 2 +component RFCs that RFC 0067 (PR #711) names. This section records what +staged retrieval needs from them and the substrate facts established so far; +it decides no planner. The audited pin is Lance 11.0.0, DataFusion 54.0.0 and Arrow 58.3.0. Current graph execution uses Arrow batches and engine-owned fusion/aggregation; this proposal does not assume a complete DataFusion logical plan already exists. @@ -2139,10 +1208,10 @@ tie key from `dense_rank` can exceed the quota. Reattaching selected pairs directly to incoming bindings can avoid repeating an already implied cutoff. These are qualified rewrite candidates, not a universal planner choice. -The existing -[selection owner](../../crates/omnigraph/tests/rrf_prefilter_gate.rs) and -[independent scenario oracle](../../crates/omnigraph/benches/scenarios/search_selection.rs) -cover duplicate paths, multiple memberships, null/composite keys, filter/cut +The retained +[selection probe](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph/tests/rrf_prefilter_gate.rs) and +[independent scenario oracle](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph/benches/scenarios/search_selection.rs) +on the evidence branch cover duplicate paths, multiple memberships, null/composite keys, filter/cut placement, memory and persisted Lance sources, input reversal and partition changes. Retain two native compatibility fences: @@ -2161,10 +1230,11 @@ checks payloads and final binding order, and retains both dense and distinct-pai routes because their costs differ with fan-out and payload width. Downstream reliance on Take's ordering metadata remains unqualified. -The [historical physical experiments](assets/0048-agent-context.md#lance-datafusion-and-graph-execution) +The [historical physical experiments](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/docs/rfcs/assets/0048-agent-context.md#lance-datafusion-and-graph-execution) record configurations, timing/I/O/spill tables and controls. They support these implementation candidates, not production latency or a universal winner. -Reproduce through the existing instrument: +Reproduce through the instrument on the evidence branch (it returns to the +tree with Phase 3): ```sh cargo bench -p omnigraph-engine --bench scenarios -- \ @@ -2244,18 +1314,6 @@ requires another explicit rebuild. - **Built-in document/evidence entities.** Applications already define nodes, properties, and relationships. Preserve graph identity and source version without imposing a second data model. -- **Fuzzy filtering as the complete typo-search feature.** An exact lexical - source can discard admitted typo matches. Shared lexical descriptions and - explicit fuzzy scoring close the ranked path. -- **Native fuzzy defaults or truncated expansion as exact semantics.** They - change results with analyzer/index/segment state. Complete evaluation or - typed failure is required; a larger cap is not a completeness proof. -- **Empty indexes as analyzer authority.** Artifact removal cannot erase - logical matching semantics. Accepted SchemaIR owns the analyzer. -- **Implicitly rescope lexical statistics with every graph filter.** This can - reverse the ordering of the same eligible targets and couples score meaning - to match-block placement. A snapshot-visible field corpus gives filtering - and scoring separate meanings; qualified statistics still have a cost. - **Implicit score blending or a blanket ban on all feature combination.** Named domains prevent accidental mixing while explicit normalized/model stages can define valid combinations. Geometric range predicates remain @@ -2282,7 +1340,7 @@ whole-query resource bounds. The [upstream receipt](assets/0048-upstream-contract-checkpoint.json) records checksum-matched crate sources and probe results. The -[agent context](assets/0048-agent-context.md#evidence-and-tests) +[pre-split text](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/docs/rfcs/assets/0048-agent-context.md#evidence-and-tests) retains the detailed validation history. The current obligations are summarized below; historical pass counts must not be reported as fresh evidence. @@ -2308,9 +1366,6 @@ determine what this pin can promise. | Search promise | Pinned interface and limit | Decision | Minimal falsifier and disposition | |---|---|---|---| -| Accepted analyzer, including fuzzy input | `InvertedIndexParams::build`; native fuzzy and flat paths choose different analysis/expansion behavior; NFC needs preprocessing. | Reuse tokenizer; adapter plus bounded fallback for complete membership. | Same composed/decomposed or capitalized typo before/after indexing must have identical membership. Native Unicode/analyzer guards expose the limits; the four current GQT regressions remain open. | -| One exact/fuzzy lexical score | `InvertedIndex::bm25_stats_for_terms`, `MemBM25Scorer`; native float32 BM25 does not implement the proposed grouped fuzzy formula or numeric policy. | Bounded exact scorer first; native acceleration only after parity qualification. | Rare expansion, repeated terms, common-term IDF and deletion change native scores. Existing native and independent Decimal fixtures exercise these counterexamples; full indexed parity remains open. | -| Cross-table lexical ranking | Public shared scorer can be supplied before native per-table cuts; immutable index statistics can include deleted rows. | Adapter owns live corpus aggregation, scoring policy and complete cut boundaries. | Two local top-1 cuts lose the global winner; the shared-statistics probe recovers its score band. Canonical ties and live multi-type statistics remain open. | | Exact `knn` and declared `ann` effort | `Scanner::nearest`, `use_index(false)`, metric/refinement controls and `scan_stats_callback`; refinement scores only retrieved candidates. | Reuse qualified scanner paths; adapter owns geometry, coverage, ties and effort. | ANN omits a true neighbor or returns an unreached `+inf` row. Existing vector guards fence the latter; family-specific recall, arithmetic and complete-tie qualification remain required. | | Graph-scoped candidate population | `Scanner::with_row_addr_prefilter` consumes the selected dataset's native `_rowid` domain. | Adapter resolves accepted graph identity to pinned native identity. | Delete/compact a target, then search an older snapshot with a graph mask. Native identity guards exist; the full graph-mask retriever remains open. | | Independent candidate windows and final output size | Scanner limits and FTS collectors can cut before OmniGraph sees candidates. Native row-ID or score-only ties are not the declared entity comparator. | Adapter owns every semantic cut; bounded fallback for an unqualified boundary. | More tied targets than the native window, with reversed graph/native identity order. Sorting a truncated subset cannot pass this gate; production qualification remains open. | @@ -2366,10 +1421,13 @@ Reproduce these native checks from the repository root: ```sh cargo test -p omnigraph-engine --test lance_surface_guards --locked -cargo test -p omnigraph-engine --test rrf_prefilter_gate --locked python3 docs/rfcs/assets/0048-arrow-pool-probe.py ``` +The six guard probes cited above are proposed in a separate test-only pull +request; the DataFusion selection probes run on the evidence branch +(`cargo test -p omnigraph-engine --test rrf_prefilter_gate --locked` there). + The Arrow runner creates an isolated temporary crate, checks all resolved registry versions/checksums against the workspace lockfile, and enables only there the optional Arrow pool features. Its [Rust probe](assets/0048-arrow-pool-probe.rs) @@ -2378,140 +1436,45 @@ These deterministic mechanism checks do not add latency, peak-memory or retrieval-quality claims. They supplement the frozen integrated prototype and agent evaluation rather than changing their inputs or results. -The recorded focused run reports 43 Lance surface guards, 13 RRF/prefilter tests and +At `ce5a3012` the recorded focused run reports 43 Lance surface guards, 13 RRF/prefilter tests and 11 benchmark contracts passing, plus the isolated Arrow assertions. The S3 same-version guard returns early without configured storage credentials, so its passing libtest entry is not remote-storage evidence. Both workspace Clippy feature graphs pass. The four previously recorded GQT failures remain open; these native checks neither repair them nor replace release validation. - ### Assumption audit -The pinned tokenizer/edit-distance probe passed 73,008 comparisons against an -independent Unicode-scalar evaluator at budgets zero through two. This covers -the primitive, not the revised NFC pipeline, indexed completeness or budgets. - -The [Decimal oracle](../../crates/omnigraph/tests/fixtures/lexical_scoring_v1.py) -generates thirteen 80-digit reference cases. The float64 evaluator uses pinned -`libm 0.2.16`, tolerance `2e-14 * max(1, expected)`, exact fixture order, -repeated-term invariance, eligibility-independent scores/features, all/any -feature invariance and edit-budget inclusion. Inputs already represent -analyzed tokens. Native/indexed numeric parity, complete ties and relevance -defaults remain unqualified. - -The [native matrix](#contract-to-code-qualification) and -[composition evidence](#required-composition-examples) own the remaining -assumptions and falsifiers; avoid duplicating their status here. - -#### CI checkpoint and regression disposition - -The 2026-09-09 GitHub Actions runs for PR head -`bf5a77e55bc1dcd8415f28d5900d0a9cd218d78e` confirm the same four failures in -both [GQ Logic Tests](https://github.com/ModernRelay/omnigraph/actions/runs/34371510788/job/102533700021) -and [Test Workspace](https://github.com/ModernRelay/omnigraph/actions/runs/34371511424/job/102534255348): -67 of 71 cases pass, and all 127 runner self-tests pass. The workspace job -tests GitHub's merge candidate `f4ce7e895eb5c29b34c70cb1dbfbf95fba2f8a27`, -which combines that PR head with base `ed3ea5006f55ef703a3331f29ac45a2262f13300`. -This identifies the tested source; it does not claim validation against a -later `main` revision. - -| Existing GQT owner | First observed failure | Required implementation and migration proof | -|---|---|---| -| [`fuzzy_query_bypasses_index_analyzer`](../../crates/omnigraph-gqt/cases/fuzzy_query_bypasses_index_analyzer.gqt) | Step 2: capitalized `Introductio` returns no rows; `intro` is expected. | Phase 2 must apply the accepted field analyzer at every edit budget. Retain the lowercase and zero-edit controls and reach the later assertions. | -| [`index_state_changes_text_matches`](../../crates/omnigraph-gqt/cases/index_state_changes_text_matches.gqt) | Step 5, including the two mutation steps: `running` finds only the appended row and loses the indexed row. | Phase 2 must use one matching definition for indexed and uncovered rows. Complete the later `beto` assertion as well; passing the first repaired step is insufficient. | -| [`search_on_traversal_target_is_dropped`](../../crates/omnigraph-gqt/cases/search_on_traversal_target_is_dropped.gqt) | Step 2: traversal returns B, C and D where only B and D match. | Phase 3 must retain the target predicate and rank the traversal target. The later ranking assertion must return D, rather than fail for a missing score column. | -| [`unindexed_search_is_case_sensitive`](../../crates/omnigraph-gqt/cases/unindexed_search_is_case_sensitive.gqt) | Step 3: unindexed `deep` finds only the lowercase row; both rows are expected. | Phase 2 must preserve matching with and without an index. Migrate both field declarations to the intended accepted analyzer, and run both query-case controls. | - -These cases were added in `b1df2041` and remain active regressions. During the -coordinated query migration, rewrite their legacy `search`, `fuzzy` and `bm25` -forms through the new typed terms and ranking constructs while preserving -their expected membership and ordering. Declare analyzer intent in the schema; -index presence cannot provide it. Removing the old syntax, blessing the -incorrect rows, or running only the isolated staged case does not close these -regressions. A passing implementation needs every step and the full corpus to -run through the production parser and engine. - -The same workspace run passes 361 compiler tests, 42 Lance surface guards, -56 search tests, 13 RRF/prefilter tests, 88 CLI data tests and 23 CLI parity -tests before failing at GQT. These counts describe the PR's checked-in code; -the archived integration experiment has different test counts and is outside -this CI build. Environment-specific skips inside native guards retain their -existing limitations. The workspace process stops on the GQT failure, so -later workspace test targets are not established by this run. Format and -Clippy checks pass independently. These historical CI failures require the -implementation above; later native probes do not close them. +The tokenizer/edit-distance probe and the Decimal scoring oracle are owned by +[Analyzed lexical search](2026-09-18-analyzed-lexical-search.md#assumption-audit). The native matrix above +and the [composition evidence](2026-09-18-gq-composition-and-language-evolution.md#required-composition-examples) own the +remaining assumptions and falsifiers. -### Implementation handoff and validation checkpoint +#### Regression disposition -Use the existing owners and archived experiments below. The checked-in staged -compiler is test-only; the production parser does not accept these examples. -Its shared expression root, precedence, contextual names, argument/refusal -rules, scope transitions and metric identities are partial design evidence. -Numeric/null evaluation, all result types, resolved identities and integrated -lowering still need qualification. - -#### Explicit-output integration checkpoint - -The [updated patch](assets/0048-phase0-integration.patch) applies directly at -`b87068cb452ac4c2af0c4056ed3a5cde0b678604`; its -[receipt](assets/0048-phase0-integration-checkpoint.json) records exact inputs, -commands, results and limitations. Do not apply the older patch first. - -This isolated experiment carries explicit `yield` through the real compiler, -IR, engine and GQT for multiple scalar-String lexical sources. It tests -independent windows, declaration reorder, unused sources, output choice, -inherited order and nullable missing metrics. Read descriptors, column demand -and GQT detection see later stages. Invalid parameters fail before scanning; -a suffix with 100,200 bindings refuses despite final `limit 1`. - -The pinned Rust 1.97.1 checks pass 368 compiler, 127 GQT unit, one staged GQT, -ten column-demand and four staged resource/scoring tests, plus the extended -admission owner; one diagnostic instrument remains ignored. The full 57-test -search owner predates the final admission assertions, whose owner was rerun. -Historical CLI/server counts do not qualify this patch revision. - -Limits remain explicit: alias-wrapper source IDs, experimental admission caps, -post-decode accounting and incomplete ownership of analyzed state, I/O and -output. Boolean matching, vector/fusion execution, full expressions, C1–C4, -global search, schema/default persistence and read/error contracts are not -implemented by this patch. - -#### Composition plan and primitive checkpoint - -Starting from `1fb0423d`, the checked-in C1–C4 extension passes 367 compiler -tests and adds four golden assertions to existing tests. One native composition -test passes four memory/partition/order configurations under Rust 1.97.1, -DataFusion 54.0.0 and Arrow 58.3.0. Focused Clippy, formatting and docs checks -pass. [C1–C4](#required-composition-examples) links the plans, probe and per-case -limits; those are the evidence authority. - -The logical goldens and hand-built native plans are separate. Materialized -ranks/features and fixture parent IDs do not prove retrieval, scoring, -correlation or generated identity. Ambiguity detection does not implement -typed cardinality refusal. This checkpoint neither completes Phase 0 nor -repairs the four GQT regressions. +The four GQT regressions recorded on PR #606 and their required +implementation proof are in +[Analyzed lexical search](2026-09-18-analyzed-lexical-search.md#regression-cases); they are held out of the +corpus until each fix lands under an `issue_N` name. -```bash -cargo +1.97.1 test --locked -p omnigraph-compiler -cargo +1.97.1 test --locked -p omnigraph-engine --test rrf_prefilter_gate staged_composition -``` +### Implementation handoff and validation checkpoint -#### Historical integration and diagnostic pilot +The experiments below ran on the PR #606 branch and are retained there; none +of them is in the tree. Their shared expression root, precedence, contextual +names, argument/refusal rules, scope transitions and metric identities are +partial design evidence. Numeric/null evaluation, all result types, resolved +identities and integrated lowering still need qualification. -The [historical patch](assets/0048-staged-integration.patch) applies at -`b1df2041c93e03aa13cee8308a0a574689baf210`. Its -[validation inventory](assets/0048-validation-checkpoint.json) records the -frozen sources and tested compiler, engine, GQT, stored-query and CLI boundaries. -It uses one lexical source per rank block and is outside this PR's production -build. Inspect it in an isolated checkout; prefer the later explicit-output -patch for ongoing integration work. +#### Retained experiments -Its evidence includes forty generated graph comparisons against an independent -evaluator, twelve Decimal/lifecycle fixtures, snapshot/current-read controls, -authorization refusals and resource counterexamples. These are finite -experimental proofs, not complete language, transport or resource qualification. -The [original checkpoint](assets/0048-agent-context.md#historical-integration-and-diagnostic-pilot) -retains exact commands and the distinction between fresh and earlier runs. +| Experiment | Where | What it established | Limits | +|---|---|---|---| +| Explicit-output integration patch and receipt | [`0048-phase0-integration.patch`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/docs/rfcs/assets/0048-phase0-integration.patch), [receipt](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/docs/rfcs/assets/0048-phase0-integration-checkpoint.json), applies at `b87068cb` | Explicit `yield` through the real compiler, IR, engine and GQT for multiple scalar-String lexical sources: independent windows, declaration reorder, unused sources, output choice, inherited order, nullable missing metrics; read descriptors, column demand and GQT detection see later stages; invalid parameters fail before scanning; a suffix with 100,200 bindings refuses despite final `limit 1`. | Alias-wrapper source IDs, experimental admission caps, post-decode accounting, incomplete ownership of analyzed state, I/O and output. Boolean matching, vector/fusion execution, full expressions, C1–C4, global search, schema/default persistence and read/error contracts are not implemented. | +| Historical integration patch and validation inventory | [`0048-staged-integration.patch`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/docs/rfcs/assets/0048-staged-integration.patch), [inventory](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/docs/rfcs/assets/0048-validation-checkpoint.json), applies at `b1df2041` | One lexical source per rank block through compiler, engine, GQT, stored queries and CLI: forty generated graph comparisons against an independent evaluator, twelve Decimal/lifecycle fixtures, snapshot/current-read controls, authorization refusals and resource counterexamples. | Finite experimental proofs, not language, transport or resource qualification; superseded by the explicit-output patch for ongoing work. | +| Test-only staged compiler and C1–C4 goldens | [`staged_probe.rs`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-compiler/src/query/staged_probe.rs) | See the [composition checkpoint](2026-09-18-gq-composition-and-language-evolution.md#composition-plan-and-primitive-checkpoint). | A second parser over compiler internals; deleted from the tree so that it is never maintained beside the production compiler. | +| DataFusion selection and composition probes | [`rrf_prefilter_gate.rs`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph/tests/rrf_prefilter_gate.rs), [`composition.rs`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph/tests/rrf_prefilter_gate/composition.rs) | Distinct-target windows, null-safe pair quotas, the nullable dynamic-filter defect (20 bindings become 18), C1–C4 population and collection contracts on memory tables and `LanceTableProvider::scan`. | Public DataFusion operators only; no GQ lowering, native retrieval or resource bounds. | +| Selection cost instrument | [`search_selection.rs`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph/benches/scenarios/search_selection.rs) | Dense versus distinct-pair group selection, late payload hydration, partition and memory variation against a scalar oracle. | Persisted binding relation, not a graph traversal; returns with Phase 3. | +| Lexical scoring oracle test | [`search.rs`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph/tests/search.rs) | Thirteen Decimal reference cases against a float64 evaluator with pinned `libm` `log1p`. | The evaluator lives in the test; returns with the Phase 2 scorer. Fixtures are in `assets/`. | +| Document-search agent pilot | [`0048-agent-pilot-results.json`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/docs/rfcs/assets/0048-agent-pilot-results.json) | 44 trials over 1,439 passages under four recipes; recipe semantics and source revision/status must be exposed to the agent. | Judged by the assistant without independent adjudication; establishes no modality winner, defaults or engine confidence. | #### Lessons the implementation must retain @@ -2531,39 +1494,6 @@ baseline recognizes versions 2 and 4 and burns 3. Export/reapplication must preserve resolved recipes. Keyed follow-up does not qualify general system-ID lookup; archived JSONL metadata changes need their actual transport owner. -#### Completed agent pilot and interpretation - -The frozen pilot uses 109 repository Markdown documents at -`bf1e5ca15868c9ce2444062e9d9d02b539d786e3`, split into 1,439 passages, with one -development question and eleven held-out questions across four recipes. -The [final record](assets/0048-agent-pilot-results.json) preserves all 44 trials, -model/configuration identities, answers, source reviews, refusals and failures. - -| Frozen recipe | Trials | Final answers, including abstentions | Answers with citations, all read | Strict supported tasks | -|---|---:|---:|---:|---:| -| Exact lexical, all terms | 11 | 11 | 4 | 3 | -| One-edit lexical, all terms | 11 | 10 | 5 | 3 | -| Dense | 11 | 11 | 11 | 8 | -| Exact lexical + dense, application RRF | 11 | 9 | 9 | 8 | - -Review was by the assistant, without independent human adjudication. Strict -success required complete supported claims, no draft mistaken for current -behavior, and full reads of citations; failures stayed in the denominator. -Independent arithmetic agreed on all 23 completed dense top-20 lists and ten -hybrid top-10 lists, but does not qualify near ties or ANN. - -The lexical recipes used `mode: all`, whose conjunction was not explained in -the shared tool description. Hybrid used application RRF over two engine -queries, not staged fusion or one shared native budget. Host/API variability -and timeout/cleanup overhead preclude production latency or hard-deadline -claims. The [full protocol and interpretation](assets/0048-agent-context.md#completed-agent-pilot-and-interpretation) -preserve these controls. - -The actionable lesson is to expose recipe semantics and source revision/status -context. Reading a real citation did not prevent unsupported conclusions. -This document-only pilot establishes neither a modality winner nor broader -analytical task quality, retrieval defaults or engine-defined confidence. - ### Research context and required qualification The broader motivation is consistent with @@ -2654,76 +1584,18 @@ where needed for a logical case; keep native index lifecycle, arithmetic near ties, and execution-cost assertions in their existing Rust owners. Compiler goldens and API/CLI contract tests still cover boundaries GQT does not invoke. -The test-only [staged compiler prototype](../../crates/omnigraph-compiler/src/query/staged_probe.rs) -adds a separate Pest root using the existing grammar's tokens, graph patterns, -and scalar expressions. The extra grammar and checker are compiled only for -compiler unit tests; the production query root remains unchanged. Typed AST -prefixes reuse the real graph typechecker to validate bindings at each stage, -while the experimental plan retains the separate stages and incoming -population references. It exercises the GQ examples in this RFC, multi-stage -node and edge targets, non-leaking negation scopes, alias namespaces, metric -domains/origins, aggregate output identity, final order/window separation, -and rejection cases for inputs, bounds and references. Its `take` plan retains -the incoming stage, target, typed key tuple, comparator/reduction, quota bound -and proven binding dependencies. Tests cover composite and nullable keys, -non-null uniqueness, directed one-hop cardinality, explicit null ordering, -graph-only selection, zero quotas, and metric/order preservation. - -This is partial compiler evidence, not completed staged AST/IR or execution. -It does not qualify complete result-shape/metric-nullability inference, -every key type or comparator dependency, resolved analyzer/encoding identities, -runtime parameter admission, resource bounds, or graph/Lance lowering. -Its aggregate-order prototype uses -explicit projected aliases; the final compiler must also preserve valid -ordinary grouping-key expressions. Migrate or remove the experiment when the -production compiler owns these constructs; do not maintain a second compiler -or claim these fixtures replace executable `.gqt` cases. - -Default resolution requires its own compiler/schema and query fixtures: - -- Bare `@analyzed` and its explicit expansion resolve to the same field - fingerprint and permit matching and ranking. `scorer="none"` permits matching - while refusing a lexical ranking source. -- Missing source/dimension, omitted model without a default, unresolved explicit model, - incompatible dimensions, and distance omitted without a supplying recipe - all fail before publication. Explicit model overrides select a whole recipe; - explicit distance overrides take precedence and remain validated. A qualified - explicit model works without any schema default. -- Export/reload and no-op reapplication preserve resolved field bindings across - deployments and runtime provider/default changes. Changed schema defaults - expose affected field rebindings and required rebuilds in `schema plan`; - shorthand does not bypass existing migration refusals or rename identity. - -The exact lexical qualification matrix retains all preceding requirements: - -- Positive and negative matches; `all`/`any`; repeated/reordered terms; - null/token-empty values; empty and stop-word-only queries; validation even - on empty populations. Edit budgets reject negative, non-integer, oversized, - and narrowing-overflow inputs before execution. -- Case, stemming, folding, composed/decomposed text, multibyte characters, - and lengths around 40 UTF-8 bytes. Zero-edit equivalence, inclusion at - budgets 0/1/2, and transpositions costing two. -- NFC conformance for the pinned normalizer/Unicode version; canonically - equivalent inputs on both sides of matching and ranking through scan/index - paths; and long combining-mark sequences under bounded cancellation. - Preserve original stored values and exact String predicate behavior. - Reject or bypass artifacts that lack proof of the revised pipeline. -- Absent, empty, complete, partial, removed, and rebuilt indexes through - append/update/delete/overwrite/compaction. Identical values must match - identically under graph filters and supported negation. -- More than 50 expansion terms, vocabulary/segment changes, an exact term - crowded out by expansions, and one term consuming the budget before another. - Native overflow must fall back completely within the remaining budget or - fail; post-verifying a truncated set is insufficient. -- One huge token, automaton state failure, cancellation, and exhausted shared - budgets must produce typed failure without successful partial results. -- The same tolerant query through Boolean matching and fuzzy retrieval must - agree on membership before the ranking cutoff. Adding a predicate cannot - rewrite a neighboring exact source. Fuzzy/exact score and order parity need - independent fixtures, including corpus deletion/update and repeated terms. -- More tied targets than each native window, reversed stable-ID order, and - different physical partitions must preserve declared exact winners or fail - explicitly. Sorting an already truncated subset is not the assertion. +The test-only staged compiler prototype that exercised the GQ examples in +this RFC, multi-stage node and edge targets, non-leaking negation scopes, +alias namespaces, metric domains and origins, aggregate output identity, +final order/window separation, `take` key and comparator rules and the +rejection cases is retained on the evidence branch +([`staged_probe.rs`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-compiler/src/query/staged_probe.rs)). +It is partial compiler evidence, not a staged AST/IR or execution, and it is +deliberately not in the tree: the production compiler owns these constructs +when Phase 1 lands, and `.gqt` cases own their behaviour. + +The default-resolution fixtures and the exact lexical qualification matrix +are in [Analyzed lexical search](2026-09-18-analyzed-lexical-search.md#qualification-matrix). Correctness and usefulness are separate evaluations. A fixed, checked-in corpus should include names with typos, rare identifiers, semantic questions, @@ -2811,6 +1683,7 @@ and both `staged_composition` native tests. This settles the stated policies; it does not complete any production evaluator or the phase's remaining gates. ```bash +# on the evidence branch at dd6959c8 cargo +1.97.1 test --locked -p omnigraph-compiler cargo +1.97.1 test --locked -p omnigraph-engine --test search lexical_scoring_v1_reference_oracle -- --exact cargo +1.97.1 test --locked -p omnigraph-engine --test rrf_prefilter_gate staged_composition @@ -2818,8 +1691,8 @@ cargo +1.97.1 test --locked -p omnigraph-engine --test rrf_prefilter_gate staged | Decision package | Required disposition and proof | |---|---| -| [Language](#language-evolution-and-compatibility) | Shared expressions, explicit output, namespaces, precedence, parameters and scope; parser/type/plan fixtures | -| [Composition](#required-composition-examples) | C1–C4 syntax, type derivation, golden plans, physical feasibility and invalid-rewrite counterexamples | +| [Language](2026-09-18-gq-composition-and-language-evolution.md#language-evolution-and-compatibility) | Shared expressions, explicit output, namespaces, precedence, parameters and scope; parser/type/plan fixtures | +| [Composition](2026-09-18-gq-composition-and-language-evolution.md#required-composition-examples) | C1–C4 syntax, type derivation, golden plans, physical feasibility and invalid-rewrite counterexamples | | [Selection/scoring](#target-identity-fan-out-grouping-and-metrics) | Target/binding/group multiplicity, total ties, live statistics, lexical/vector numeric policy and checked fusion arithmetic | | [Representations](#representation-identity-and-source-attribution) | Schema defaults and overrides, resolved encoder/Unicode/analyzer identity, export/reapplication and format coordination with RFCs 0040/0043/0044 | | [Global search](#graph-wide-discovery-across-entity-types) | Execution deferred by explicit decision; prove typed identity/source/table separation and future union/narrowing/projection compatibility before stabilization | @@ -2866,23 +1739,23 @@ cleanup. Typed extension points do not make deferred public operators available. **Input:** Phase 1 representations/resources and Phase 0 numeric oracles. -Build bounded NFC/analysis, complete `Terms` matching, unified exact/fuzzy -scoring, exhaustive `knn` and declared `ann` with qualified fallback. -Charge statistics, coverage, scoring and selection to the shared context. -Extend search/substrate owners and the -[lexical qualification matrix](#test-harness-integration); observable rows, -shapes and errors belong in GQT. +The lexical half of this phase (bounded NFC analysis, complete `Terms` +matching, unified exact/fuzzy scoring and qualified index acceleration) is +owned by [Analyzed lexical search](2026-09-18-analyzed-lexical-search.md#rollout). This RFC's half builds +exhaustive `knn` and declared `ann` with qualified fallback, charges +statistics, coverage, scoring and selection to the shared context, and +extends the search and substrate owners. Observable rows, shapes and errors +belong in GQT. **Exit:** predicate/retriever membership agreement before cuts, independent score and vector fixtures, total ties, all index/lifecycle states, and typed budget/cancellation failures. Assert that intended index/compaction states were reached; distinguish current from pinned answers and reopen snapshots. -**Open:** native fuzzy analysis, float32 BM25, stale statistics and native cuts -are not substitutes for the accepted exact contract. Reproduce their -counterexamples before reuse. Numerical parity does not establish relevance -quality. Candidate-scoring/model operators and richer representations remain -deferred; Phase 5 chooses measured defaults. +**Open:** native cuts are not substitutes for the accepted exact contract. +Numerical parity does not establish relevance quality. Candidate-scoring +operators and richer representations remain deferred; Phase 5 chooses +measured defaults. #### Phase 3: compose graph scope, fusion, and selection @@ -2988,7 +1861,7 @@ tests do not count as completion of a new phase. ### Extensions after the initial release -The [query capability matrix](#query-capability-matrix) owns the extension +The [query capability matrix](2026-09-18-gq-composition-and-language-evolution.md#query-capability-matrix) owns the extension inventory, including projection, grouping, optional matching, typed unions, advanced retrieval and representation forms. Its deferred rows do not claim support or require implementation for the initial release. @@ -2996,31 +1869,10 @@ Every later extension retains the stated semantic and qualification boundary. `Foundation` rows require their grammar/type/composition proofs before syntax stabilization while their broader operators remain deferred. These design proofs do not enlarge the advertised initial-release feature set. - ### Next delivery milestone: composed analytical answers -After the initial release, implement C1, C4 and C2's general distinct/intermediate -group extensions using the accepted common grammar, IR and execution context. -C3 remains a separate scorer extension. Global search is also deferred; its -union, representation expansion and cross-type ranking require their own -implementation milestone after the compatibility proof. - -Build reusable computation/grouping first, then local selection and correlated -retrieval, then optional facts and typed collections. Preserve group identities, -computed facts, import/export scopes, empty results and one cumulative budget. -Extend existing compiler, traversal/projection/aggregation, GQT and transport -owners; retain compatibility with the initial release. - -**Exit:** full C1/C2/C4 journeys through public reads, with inferred/executed -types, deterministic selection, snapshot-bound follow-up and shared-budget -refusal. C1 selects A's increase of six over B's one; C2 counts three paths and -two distinct passages; C4 preserves B with a null owner and empty reports. - -**Open:** inspect integrated and optimized plans against independent -empty-group, duplicate-binding, null, wrong-group and skewed-fan-out controls. -API availability and the Phase 0 design argument do not prove safe correlation, -reuse or bounded materialization. Prove preservation of outer rows and budgets -before optimizing batching, and measure per-group rescan cost. +Owned by [GQ composition and language evolution](2026-09-18-gq-composition-and-language-evolution.md#next-delivery-milestone-composed-analytical-answers): +C1, C4 and C2's general grouping extensions after this RFC's initial release. ## Unresolved questions @@ -3028,44 +1880,44 @@ The [acceptance/release matrix](#phase-0-acceptance-versus-release-qualification assigns the completion point for each proof. In particular, decided interfaces still require prototypes; full production qualification belongs to its phase. -1. Complete Phase 0 grammar/type/plan qualification for the stated stage - and metric scopes, per-group key types and tuple ordering, and any - user-defined selection tie keys. The partial compiler - prototype does not close the full result-schema and aggregation contract. - Null-bucket and multiple-membership semantics are specified above; prove - their lowering and retain RFC 0040 namespace coordination. Complete the - C1–C4 syntax/type/plan proofs, including the analytical population and - deferred correlation/scoring boundaries, before syntax stabilization. -2. Close the Phase 0 numeric policies/oracles and native route dispositions. - Phases 2/5 must qualify BM25 across supported targets, exact live-row - statistics and native/fallback score/winner parity. Polymorphic field-corpus - execution is deferred. Validate edit weights and maximum reduction on the - owned task corpus before release; numerical fixtures alone do not establish - a good relevance default. -3. Resolved representation serialization and immutable encoding revisions, - schema-wide default declaration syntax and migration integration, - normalizer/Unicode identity and analyzer/artifact fingerprint mapping, and - shared SchemaIR version assignment with RFCs 0040/0043/0044. Omission and - override semantics are specified above; their implementation must prove - that accepted bindings cannot drift with deployment defaults. -4. Prototype the decided resource units and shared reservation interfaces; - resolve native allocation/dispatch gaps for graph masks, scoring, coverage, - sort/spill and output. Actual default limits, complete enforcement and - cancellation qualification belong to Phases 1/4/5. -5. Prototype the decided read options, envelope, retention/refusal boundaries +1. Complete Phase 0 grammar/type/plan qualification for the stated stage and + metric scopes, per-group key types and tuple ordering, and any + user-defined selection tie keys, in the production compiler. Null-bucket + and multiple-membership semantics are specified above; prove their + lowering and retain RFC 0040 namespace coordination. The shared expression + rules and C1–C4 proofs are the [composition RFC](2026-09-18-gq-composition-and-language-evolution.md)'s questions. +2. Resolved representation serialization and immutable encoding revisions, + the schema-wide default embedding declaration syntax and its migration + integration, and the SchemaIR version assignment shared with RFCs 0040, + 0043 and 0044. Omission and override semantics are specified above; their + implementation must prove that accepted bindings cannot drift with + deployment defaults. +3. Prototype the read options as session settings and the requirements + handed to the engine version 2 memory/admission component; resolve native + allocation and dispatch gaps for graph masks, scoring, coverage, + sort/spill and output there. Actual default limits, complete enforcement + and cancellation qualification belong to Phases 1/4/5. +4. Prototype the decided read options, envelope, retention/refusal boundaries and canonical definition/semantic/execution identities. Phase 4 qualifies all transports and authorization paths. Stable ranked cursors remain deferred. -6. Freeze the mixed-workload corpus, judgments and acceptance protocol in +5. Freeze the mixed-workload corpus, judgments and acceptance protocol in Phase 0. Phase 5 chooses ANN effort mappings and agent recipe defaults from the resulting evaluation. The document pilot alone is insufficient. Further multilingual profiles require matched-set evidence and versioned identities. -7. General all-node/type-union grammar, compatible representation expansion, - type narrowing and heterogeneous projection. Execution is explicitly - deferred; the compatibility proof remains due in Phase 0. The initial - same-binding fusion implementation cannot satisfy cross-type discovery. ## Decision log +- 2026-09-18 — split the lexical contract into + [Analyzed lexical search](2026-09-18-analyzed-lexical-search.md) and the language rules, capability matrix, + C1–C4 and cross-type discovery into + [GQ composition and language evolution](2026-09-18-gq-composition-and-language-evolution.md); no semantic decision + changed. Replaced the hard removal of legacy spellings with a one-release + deprecation window (compatibility-surfaces RFC). Re-expressed the read + options as session settings after #742 landed `Session`. Marked the + resource-ledger and DataFusion sections as requirements handed to the + engine version 2 components named by RFC 0067. Removed the agent-context + copy, the archived patches, the pilot record and the prototypes from the + tree; all are retained on the evidence branch. - 2026-09-13 — separated Phase 0 acceptance from production release gates. Decided default-binding lifetime, space versus field identity, read options and replay/fingerprint boundaries, and shared resource units/reservations. @@ -3077,7 +1929,7 @@ still require prototypes; full production qualification belongs to its phase. Decimal, compiler and DataFusion probes; default native arithmetic is not sufficient for the new numeric contract. Phase 0 remains incomplete. - 2026-09-13 — retained the full long-form version as an - [agent context document](assets/0048-agent-context.md) alongside this concise + [agent context document](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/docs/rfcs/assets/0048-agent-context.md) alongside this concise RFC. The context records its original revision; this RFC owns current decisions. - 2026-09-13 — condensed the RFC around contracts, scope, evidence limits and phase handoffs. Detailed experimental chronology remains in the linked @@ -3098,7 +1950,7 @@ still require prototypes; full production qualification belongs to its phase. - 2026-09-03 — published this draft alongside RFC 0047 after separating plan-truth work from representation/search contracts. -The [full prior decision log](assets/0048-agent-context.md#decision-log) +The [full prior decision log](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/docs/rfcs/assets/0048-agent-context.md#decision-log) preserves the earlier revisions and their superseded proposals. ## Appendix: implementation evidence (non-normative) @@ -3108,7 +1960,7 @@ The [upstream receipt](assets/0048-upstream-contract-checkpoint.json) and to Lance 11.0.0, DataFusion 54.0.0 and Arrow 58.3.0. The recorded Lance commit is `ab6b5bbe46009ed78746b444df8db59a8bc5d842`; later dependencies require renewed qualification. The -[original source appendix](assets/0048-agent-context.md#appendix-implementation-evidence-non-normative) +[original source appendix](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/docs/rfcs/assets/0048-agent-context.md#appendix-implementation-evidence-non-normative) retains exact implementation links for analyzer drift, fuzzy expansion, flat scans, vector rescoring, lexical ties and native BM25. @@ -3119,3 +1971,8 @@ flat scans, vector rescoring, lexical ties and native BM25. | 0040 | Use its system-identity and reserved `__` namespace; native `_score`/`_distance` are not public metric identities | | 0044 | Coordinate accepted SchemaIR version assignment | | 0046 | Reuse read-only index-status capabilities and its open `degraded` reasons rather than parallel status machinery | +| [Analyzed lexical search](2026-09-18-analyzed-lexical-search.md) | Owns `@analyzed`, `terms`, `match_terms`, `bm25_v1`, the exact lexical baseline and the lexical deprecation window; this RFC consumes them through `lexical` sources | +| [GQ composition and language evolution](2026-09-18-gq-composition-and-language-evolution.md) | Owns shared expression rules, the capability matrix, C1–C4 and cross-type discovery; this RFC's syntax must satisfy them | +| [Session settings](2026-09-16-session-settings.md) | Carries the read options (`coverage`, `require_replay`, limits) as request-scoped settings | +| [Compatibility surfaces](2026-09-14-compatibility-surfaces.md) | Versions the GQ grammar; removals follow a deprecation release and a major bump | +| Engine version 2 (RFC 0067, PR #711) | Owns the planner, execution engine and memory/admission mechanisms this RFC states requirements on | diff --git a/docs/rfcs/2026-09-18-analyzed-lexical-search.md b/docs/rfcs/2026-09-18-analyzed-lexical-search.md new file mode 100644 index 000000000..0ecf297d9 --- /dev/null +++ b/docs/rfcs/2026-09-18-analyzed-lexical-search.md @@ -0,0 +1,832 @@ +--- +rfc: "2026-09-18-analyzed-lexical-search" +title: "Analyzed lexical search: schema-owned matching and ranking" +track: public +status: draft +implementation: not-started +authors: + - Ragnor Comerford (@ragnorc) +created: 2026-09-18 +updated: 2026-09-18 +discussion: "https://github.com/ModernRelay/omnigraph/pull/606" +supersedes: [] +superseded_by: [] +blocked_on: + - "NFC analyzer pipeline qualified through the query, scan and index-builder paths against the pinned Lance tokenizer" + - "bm25_v1 evaluator implemented against the checked-in Decimal oracle, with scan/index score and winner parity" + - "Deprecation route for fuzzy, search and match_text agreed with the compatibility-surfaces RFC" +--- + +# RFC: Analyzed lexical search: schema-owned matching and ranking + +## Summary + +A String property becomes searchable by declaring `@analyzed`, which binds +an immutable analyzer profile and a default scoring policy into the accepted +schema. One typed lexical description, `terms(text, mode, max_edits)`, is +consumed by two operators: `match_terms(field, terms(...))`, a Boolean +predicate in `match`, and the `lexical` ranking source that +[RFC 0048](0048-search-contracts.md#lexical-sources) places inside a `rank` stage. Both apply the +field's analyzer to document and query text at every edit budget, so the +same query has one matched set whether the field is indexed, partially +indexed or unindexed. An exact scan is the baseline and the correctness +oracle; a native index accelerates only after it proves the same membership. +Ranking uses one versioned scorer, `bm25_v1`, for exact and edit-tolerant +queries, with field-corpus statistics that graph filters do not rescope. + +This replaces `fuzzy`, `search` and `match_text`, whose analysis today +depends on index presence and coverage. The legacy spellings compile to the +new contract with a deprecation diagnostic for one release before removal. + +This document was split out of [RFC 0048](0048-search-contracts.md) on 2026-09-18 so that the +lexical contract can be reviewed and implemented on its own; RFC 0048 owns +the staged retrieval language and [RFC 0047](0047-search-plan-truth.md) the +plan-truth slice. The pre-split text, the test-only staged compiler, the DataFusion composition probes, the selection cost instrument, the lexical oracle test and the archived integration patches are retained at commit [`ce5a3012`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/docs/rfcs/0048-search-contracts.md) on the `rfc-0047-search-plan-truth` branch; they are evidence, not part of the tree. + +## Motivation + +Text search answers change with physical index state. On a String property +without an FTS index, `search` and `match_text` run the substrate's flat scan +with a bare, case-sensitive tokenizer, so `"Anthropic"` matches and +`"anthropic"` does not. With an index, `fuzzy` tokenizes the query bare at a +nonzero edit budget while the index stores lowercased stems, so a capitalized +typo spends its budget on letter case. Rows appended after an index build sit +in an uncovered fragment where `beto` misses `beta` and `running` misses the +stored stem `run`. Four checked-in GQT cases pin these divergences +([regression cases](#regression-cases)). Each violates invariant 7: +physical acceleration is derived state and may change cost, never meaning. + +The repair is a schema-owned analyzer and an exact evaluation path that does +not depend on the index, plus one scoring definition shared by exact and +fuzzy retrieval so that a filter and a ranking over the same query cannot +disagree on membership. + +## User and operational behavior + +### Schema: the `@analyzed` capability + +The annotations below are proposed syntax. `@analyzed` is independent of +`@index` and `@key`: an exact-only slug needs no analyzer, and an index does +not make a property searchable. + +```pg +node Organization { + slug: String @key // exact only + name: String @analyzed @index // matching and ranking + notes: String? @analyzed // matching and ranking +} +``` + +#### Directive defaults and omission rules + +Defaults shorten the authored schema. They are resolved before acceptance +and persisted as field semantics, never looked up afresh during a read or +content write. Unknown arguments and unresolved required choices are errors. + +| Surface | Omitted value | Explicit choice or failure | +|---|---|---| +| `@analyzed` | No annotation means no analyzed capability; `@index` and `@key` do not supply it. | Exact predicates remain available without it. | +| `@analyzed.analyzer` | `standard_v1` | A named immutable profile such as `standard_folded_v1` or `english_v1`. | +| `@analyzed.scorer` | `bm25_v1` capability and default policy for exact or tolerant `terms` | `scorer="none"` permits analyzed filtering only; a lexical ranking source on that field is a typed error. | + +Thus bare `@analyzed` expands to +`@analyzed(analyzer="standard_v1", scorer="bm25_v1")`. +The initial scoring capability supplies one default policy, `bm25_v1`. +`lexical` may omit `scoring`; spelling `scoring: bm25_v1` explicitly is +equivalent. Exact and fuzzy search use that same policy, with edit tolerance +specified once in `terms`. There is no separate `fuzzy_bm25_v1` selector. +This default does not enable edit tolerance, execute ranking, create an +index, or remove the need for the policy's qualification. An explicit +`@analyzed(scorer="none")` retains the former matching-only use case. + +Embedding and vector defaults follow the same resolution rule; their rows are +in [RFC 0048](0048-search-contracts.md#schema-declarations-and-vector-defaults). + +#### Analyzer profiles + +Initial immutable analyzer profiles resolve these settings explicitly; +unspecified Lance defaults are never part of the schema contract: + +| Setting | `standard_v1` (default) | `standard_folded_v1` | `english_v1` | +|---|---|---|---| +| Document / base tokenizer | Text / `simple` | Text / `simple` | Text / `simple` | +| Language setting | English (inactive) | English (inactive) | English | +| Lowercase | Yes | Yes | Yes | +| Stemming / stop words | Neither | Neither | English stemmer, then built-in English stop words | +| ASCII folding | No | Yes | Yes, after stemming and stop words | +| Token-length filter | Disabled (`max_token_length: None`) | Disabled | Disabled | +| Unicode normalization | NFC before tokenization; no NFKC | Same | Same | + +All three profiles first apply NFC to lexical analysis input, then run the +pinned `simple` tokenizer and Lance's filter order: lowercase, optional +stemmer, optional stop words, optional ASCII folding. Scalar String fields use +row document granularity; positions do not affect membership. These profiles +use no custom stop words, external dictionaries, n-grams, or code-tokenizer flags. +Posting positions and block layout remain derived index settings. + +NFC makes canonically equivalent spellings reach tokenization as the same +string while retaining distinctions that compatibility normalization can +erase. See the [Unicode normalization specification](https://www.unicode.org/reports/tr15/). +This preprocessing is part of the proposed analyzer, not a claim about the +unmodified Lance tokenizer. Query, scan, and index construction must use the +same qualified pipeline, with normalization work charged to the query/build +budget. A native path without that proof stays disabled. Stored String values +and exact predicates are unchanged; embedding input preprocessing remains +owned by its encoding recipe and does not inherit lexical normalization. + +Disabling the token-length filter is deliberate. Pinned Lance defaults to +`Some(40)`, whose filter retains only tokens shorter than 40 **UTF-8 bytes**, +before lowercasing or folding. Resource limits must reject excessive work, +not silently erase a long name or query term. Thus `english_v1` retains +the pinned linguistic filter choices but intentionally changes normalization +and long-token matching at this breaking boundary. + +`simple` splits at non-alphanumeric Unicode scalar values. Lowercasing is +not full case folding, and ASCII folding is distinct from Unicode +normalization. NFC preprocessing makes composed `résumé` and its canonically +equivalent decomposed spelling analyze identically. Accent removal is still +opt-in: with `english_v1`, `résumé` and `resume` become `resume` and +`resum`, respectively, because stemming precedes folding. These limitations +are explicit profile behavior, not promises of language-independent typo +equivalence. Once a profile is accepted, changing segmentation, normalization, +or filter order requires a new profile. Fingerprints include the normalizer +implementation and Unicode data identity as well as the Rust Unicode behavior +used by `simple` and lowercase. These are revised, unshipped `v1` definitions; +the earlier no-NFC probe does not qualify them. Adding a profile or scorer +version requires an RFC; accepted identities are never mutated. +Query-time analyzer or vector-distance overrides do not exist. A lexical +source resolves a versioned scoring policy compatible with the field's +declared capability, using its accepted default when omitted. That resolution +does not change analysis. The first release defines only `bm25_v1`; additional +policies need explicit semantics and qualification rather than runtime aliases. + +For spelling tolerance on names and titles, the non-stemming profiles keep +edit distance close to the spelling the user supplied. Under `english_v1`, +distance is measured after stemming; a one-character typo in the original +word need not remain one edit after analysis. The field's declared profile +decides this for every query and execution path. + +### One lexical query, two consumers + +A typed lexical query describes matching independently of its consumer. +The initial variant is `Terms { text, mode, max_edits }`; the grammar sketch +spells it `terms($q, mode: all, max_edits: 1)`. `mode` defaults to `all` and +`max_edits` to zero in both consumers. Candidate-oriented stored queries +usually choose `mode: any` explicitly. Phrase, prefix, and Boolean query +composition are future typed variants, not embedded vendor query strings. + +`match_terms(field, terms(...))` is a Boolean predicate on a scalar String +property with `@analyzed`. A `lexical` source consumes the same description +to select and rank matches with a resolved scoring policy. The Boolean +predicate introduces no score, ranking, candidate window, or implicit +retrieval. Its contract is: + +| Aspect | Rule | +|---|---| +| Query text | A non-null String literal or query parameter, constant for one execution; row-dependent query text is rejected. | +| Analysis | Apply the same resolved field analyzer to document and query text, at every edit budget including zero. | +| Distance | Minimum insertions, deletions, and substitutions over analyzed Unicode scalar values; each costs one. An adjacent transposition costs two. There is no implicit prefix restriction or length-based automatic tolerance. | +| `max_edits` | Optional integer literal or query parameter in `0..=2`, default `0`. Check literals at compile time and bound parameters before execution; reject negative, oversized, and non-integer values without narrowing casts. | +| `mode` | `all` by default: every analyzed query term has a document term within the budget. `any`: at least one does. | +| Term identity | Repeated query terms do not require repeated occurrences; a document term can satisfy multiple query terms. Reordering analyzed terms cannot change membership. Matching is neither phrase matching nor distance over the whole field value. | +| Empty text | A query yielding no searchable terms is a typed error. A null or token-empty document does not match. | +| Completeness | Every successful result satisfies the predicate exactly; an edit-tolerant predicate does not advertise approximate recall. | + +For a fixed schema, query, and document population, these are normative laws: + +```text +matches(edits=0) ⊆ matches(edits=1) ⊆ matches(edits=2) +matches(indexed) = matches(unindexed) = matches(partially indexed) +``` + +Adding an analyzed query term cannot widen `mode: all`. Appending unrelated +documents cannot remove existing predicate matches. A resource failure is a +typed query failure, not an empty or truncated successful matched set. + +### Analyzed filtering in `match` + +Analyzed filtering alone remains an ordinary match predicate: + +```gq +query organization_names($q: String) { + match { + $o: Organization + match_terms($o.name, terms($q, mode: all, max_edits: 1)) + } + return { $o.slug, $o.name } + order { $o.slug asc } +} +``` + +No adjacent source inherits that edit budget. `lexical` with exact terms +continues to select only its own exact matches; fuzzy retrieval must consume +an explicitly tolerant query. Both resolve the same default scoring policy. + +### Migration + +These rows are the lexical part of RFC 0048's +[migration table](0048-search-contracts.md#user-facing-changes-and-migration); the coordinated +release, the format rebuild and the `nearest`/`order`/positional-`rrf` +rewrites stay there. + +| Existing usage or expectation | Proposed change | What users must do | +|---|---|---| +| `fuzzy`, `search`, or `match_text` | Boolean matching and ranked lexical retrieval consume the same typed `terms` query. For one release the legacy spellings compile to that contract and carry a deprecation diagnostic naming the replacement; the following release removes them with the GQ language major bump the [compatibility surfaces](2026-09-14-compatibility-surfaces.md) RFC defines. | Choose `match_terms` for filtering or `lexical` for retrieval; set term combination and edit tolerance deliberately. Rewrite stored queries during the deprecation release: they persist as text and are re-parsed at boot, so a hard removal would fail them at the next start. | +| `@index` or `@key` implicitly makes a String searchable by analyzed text | Analyzed matching requires `@analyzed`, which enables BM25 ranking by default. Exact key/index annotations keep their separate meaning. | Declare `@analyzed` on searchable text, choose another analyzer when needed, or explicitly opt out of ranking with `scorer="none"`. An exact-only slug does not need an analyzer. | +| Existing search rows, scores, or ordering survive a spelling-only rewrite | `terms` defaults to all terms and zero edits. Analysis, complete fuzzy matching, and explicit selection boundaries can change results. Lexical ranking deduplicates query terms, groups fuzzy alternatives, and uses float64 scores with snapshot-visible field statistics independent of eligibility. | Review analyzer choices, fuzzy relevance, score thresholds, and tie fixtures. Repeating a query term no longer increases its weight. The rewrite does not promise equivalent results to legacy search. | + +The deprecation mapping is fixed here: `search(f, q)` and `match_text(f, q)` +become `match_terms(f, terms(q))` (all terms, zero edits) and +`fuzzy(f, q, n)` becomes `match_terms(f, terms(q, mode: all, max_edits: n))` +with `n` clamped to the admitted range or refused. The mapping changes +membership where today's answer depended on index state; the diagnostic says +so. The legacy spellings never gain new semantics during the window. + +### Errors and operations + +Typed errors include a missing analyzed capability, disabled or incompatible +scoring, a token-empty query, an invalid edit budget, and exhausted resources. +A resource failure is a typed query failure, never an empty or truncated +matched set. The coordinated diagnostics for the whole search cutover are in +[RFC 0048](0048-search-contracts.md#errors-and-operational-changes). + +Index reconciliation remains explicit and schema-profile-targeted, with +property selectors and RFC 0043's certification/publication discipline. +Generalizing rebuild support must retain its recovery ownership. Reads and +ordinary content writes never build indexes inline. + +The NFC profile change requires an index rebuild or proven parity; an old +RFC 0043 certificate does not establish the new analysis. + +## Design + +### Lexical scoring and shared matching semantics + +Lower both lexical consumers through `LexicalQueryIR`, bound to the +rename-stable property identity and accepted analyzer fingerprint. Resolve +parameters and analyze once per compatible field/query instance per execution, +including on empty populations. Reuse resolved matching state across scan and +indexed paths. The consumers differ in role: matching returns a Boolean; +retrieval selects matches and computes ranking under its declared policy. +Removed spellings have no separate IR variants or compatibility evaluators. + +`bm25_v1` is one versioned term scorer for both exact and tolerant queries. +Zero edits reduces to BM25 with unit weight per distinct analyzed query term; +nonzero edits adds alternatives and an edit penalty within the same formula. +Candidate membership is always the shared `Terms` relation, evaluated before +the ranking cutoff. A score is not a second membership predicate. This removes +the earlier requirement to coordinate `terms.max_edits` with a separate exact +or fuzzy scorer selector. Query-term repetition and order carry no extra +weight; intentional source weighting belongs to explicit fusion or a separately +specified future scoring operator. + +The deferred membership-preserving `score` operator uses this same kernel +over its incoming targets, including targets that fail `Terms` membership: + +| Field/query outcome | Retrieval membership before its cut | Lexical feature | +|---|---|---| +| At least one matching term group, but not every group of an `all` query | False | Positive sum of the matching contributions | +| Nonempty field with no matching group | False | `0.0` | +| Present field with zero analyzed tokens | False | `0.0` | +| Null field | False | Null: no field value to score | +| Full query match | True | Same finite positive value as retrieval scoring | + +`mode` controls the Boolean consumer; it never masks term contributions. +Changing `all` to `any` alone cannot change a feature value. Changing the terms, +edit budget, analyzer or statistics can. A score of zero is observed absence +of term evidence; null remains missing representation. Neither adds a source +rank or fusion vote. Feature nullability follows field nullability, while a +retrieval metric can also be absent because its target missed that source's +window. A future explicit conditional can gate features by membership; no +second implicitly gated BM25 formula is introduced. + +An empty analyzed **query** is an error before any target evaluation, even for +an empty population or all-null fields. An empty corpus has no positive +contributions: return zero for present token-empty fields and null for null +fields without dividing by corpus length. Invalid representations/statistics, +non-finite arithmetic and resource failures remain errors. Scoring never +turns them into zero, null or dropped targets. + +The statistics population is part of source identity, independently of its +eligible population and candidate window. The initial contract uses the +snapshot-visible field corpus: distinct, policy-visible targets owning that +accepted type/property identity with nonempty analyzed values, before ordinary +graph or property filters and the source's lexical query. Null and token-empty +values contribute neither a document nor tokens. Graph fan-out contributes one +target, not repeated field values. A physical dataset, index segment, or shard +cannot choose a different logical corpus. Polymorphic fields must resolve +their corpus identities explicitly before that source shape is supported. + +At one snapshot, with the same query, analyzer, and scoring policy, narrowing +eligibility therefore leaves a surviving target's lexical score unchanged. +This makes scoring compose with graph filters and keeps it independent of +candidate windows. Selection still depends on eligibility: filtering before +top-k and filtering after top-k are different operations. Scores may change +across snapshots or policy-visible corpora; this is not a globally calibrated +score or a reason to compare unrelated source instances. + +The existing `fts_prefilter_does_not_change_covered_fragment_scores` guard +confirms that a native prefilter does not rescope covered-index statistics. +The new `fts_statistics_scope_can_reverse_ranking` probe goes further: with +the same four eligible targets, an alpha/beta query ranks an alpha target +first using the full ten-target corpus, but the beta target first using an +index containing only those four eligible targets. Eligibility and statistics +must remain separate typed plan facts. Implicitly recomputing statistics for +each match block would change score meaning with graph scope. An explicit +alternative corpus could be a future contract; it is not an initial query +option. + +This choice does not qualify native BM25 or make field statistics free. +Deleted/updated rows, unindexed tails, null/token-empty handling, arithmetic, +and complete boundary ties still require a checked-in score oracle and +scan/index parity. Reusable statistics must be derived from the accepted +snapshot and representation; a stale aggregate is not authority. Exact +statistics work consumes the query budget and may require a corpus scan. +The prefilter probe proves neither live-row statistics parity nor a cost +advantage for every query. Policy constrains the corpus before statistics; +ordinary query filters do not replace the access-control boundary. + +For each distinct analyzed query term `q`, treat all stored terms within the +declared edit budget as alternatives for that term. Let `N` be the number of +documents in the field corpus, `df_q` the number containing at least one such +alternative, `len_d` the total analyzed token count of document `d`, `avg_len` +the corpus mean length, and `tf(t,d)` the occurrence count of stored term `t`. +All analyzed occurrences contribute to length, including repetitions and +terms unrelated to the query. A field value is the document unit; graph paths +and index segments never multiply its frequency. + +The mathematical definition is: + +```text +idf(q) = log1p((N - df_q + 0.5) / (df_q + 0.5)) +norm(d) = 1.2 * (0.25 + 0.75 * len_d / avg_len) +weight(t,d) = 2.2 * tf(t,d) / (tf(t,d) + norm(d)) +part(q,d) = idf(q) * max [ 2^(-edit(q,t)) * weight(t,d) ] + over stored terms t within the edit budget +score(d) = sum part(q,d) over distinct query terms q +``` + +An empty maximum contributes zero. `df_q` counts a document once even when +several alternatives occur, before the full query's `all`/`any` membership +test and ordinary eligibility filters. Alternative-specific rarity cannot +give a misspelling extra weight: all alternatives share the term group's IDF. +Using the maximum or sum of individual term document frequencies would not +count the group's matched population: maximum misses disjoint occurrences, +while sum double-counts documents containing several alternatives. Union DF +also stays unchanged when spellings redistribute without changing which +documents match the group. This semantic benefit has a cost: native per-term +frequency metadata alone generally cannot produce it. +Within a document, maximum contribution prevents adding different spellings +from summing several pieces of evidence for one query term. Repeated occurrences +of a single stored term still affect its BM25 term frequency. A stored term +may contribute to several distinct query terms, matching the existing `Terms` +membership rule; the score is not a count of independent facts. + +Edit weights are `1`, `1/2`, and `1/4` for zero, one, and two edits. These are +versioned design choices, not values inferred from a backend or established +as globally optimal. Exact preference is a contribution boost at equal term +frequency and length, not a strict ordering tier above every fuzzy document. +Length, term frequency, and other query terms can still change the winner. +Changing `max_edits` may change group IDF and ranking, while membership retains +the declared monotonicity. At zero edits each group contains only the exact +term, so the same kernel supplies the exact-scorer reduction. + +The numeric profile uses float64 and `libm 0.2.16`'s scalar `log1p` +implementation, with its dependency identity recorded in the scorer +fingerprint. It does not delegate to the platform's logarithm. Count documents, +tokens, and frequencies with checked `u64` arithmetic; compute `N - df_q` +before conversion, reject inconsistent statistics, +and handle an empty corpus before division. Convert counts to float64 for +scoring, evaluate the parenthesized expressions above without algebraic +reassociation or fused operations, and sum contributions in ascending UTF-8 +query-term order. Zero-contribution terms add zero; matching rows must produce +a finite positive score or a typed numeric failure. Final ties use the source's +stable target comparator. One canonical numeric implementation must serve all +qualified paths; testing that pinned kernel across supported targets is +required before acceptance. The Decimal oracle verifies mathematical values +within its stated tolerance, not cross-platform bit identity. A future kernel +change needs a new scorer identity unless score/order equivalence is proved. + +Native Lance BM25 does not supply this contract. The +`native_fuzzy_bm25_rewards_a_rare_expansion` probe gives `beto` about 38 times +the score of exact `beta` in a 100-document fixture and doubles scores for a +repeated query term. The public float32 scorer also rounds a common term's +positive IDF to zero at `N = 2^24`; the +`native_bm25_idf_can_round_a_common_term_to_zero` guard reproduces this without +allocating that corpus. Native scores or top-k cuts require new qualification; +rescoring an incomplete native candidate set cannot establish exact winners. + +The [numerical fixtures](assets/0048-lexical-scoring-v1.json) and the +[`lexical_scoring_v1_reference_oracle`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph/tests/search.rs) +probe on the evidence branch cover the chosen formula, zero-edit behavior, repeated/reordered terms, +alternative aggregation, overlapping groups, `all`/`any`, null/token-empty +values, and two-edit membership. They are design oracles over already-analyzed +terms, not a production evaluator or relevance benchmark. The initial complete +path must implement fuzzy ranking against these oracles; leaving it to a later +release is not completion. Scan-based correctness may ship before native +acceleration, with bounded failure when the work cannot complete. + +The same oracle now checks separate feature expectations, including partial +`all` matches, all-null/token-empty corpora, unchanged features under `all`/`any` +and eligibility changes, and identical retrieval/feature values for matches. +These settle C3's numeric meaning; they do not deliver the deferred scorer +operator or qualify analysis, live-row accounting and execution bounds. + +An exact physical baseline can compute field statistics in one snapshot-pinned +pass, then score eligible values in a second pass. For each query term, count +at most one family occurrence per document in the first pass; the second uses +that document's term frequencies and the same edit relation for its maximum. +Keep query-term statistics and bounded per-document state, not a retained +corpus or persistent expansion dictionary. N/mean-length metadata may be reused +when snapshot and analyzer identity qualify it; fuzzy family DF generally +depends on the query and edit budget. Both passes, normalization, matching, +and top-k selection share the execution budget. + +This is an implementation route to qualify, not a claim that two full scans +meet production latency goals. Indexed acceleration must supply complete +membership, live-row counts, term frequencies and group reductions, or prove +conservative bounds for every pruned candidate. Lance's public scorer trait +returns float32 additive term weights; merely substituting a scorer cannot +provide the group's maximum reduction or this float64 contract. Reuse Lance +postings through qualified upstream/adapter work and DataFusion group operators +where appropriate; do not introduce another stored index. A numeric or quota +post-pass over native top-k cannot restore discarded winners. + +### Exact lexical execution and qualified acceleration + +**Analyzer parity without index coupling.** Instantiate the analyzer from +accepted SchemaIR, even when no FTS artifact exists. Eager empty indexes are +not the analyzer carrier: overwrite, index removal, or an incomplete rebuild +must not erase logical semantics. Use the pinned substrate tokenizer +implementation through one analyzer binding that includes the declared NFC +preprocessing step. Do not reimplement its filters or pass already analyzed +tokens into another path that analyzes them again. Normalization must precede +tokenization consistently in the query, scan, and index-builder paths; +normalizing only a query does not qualify an index built from raw spellings. +Creating or changing an analyzer profile continues to require the existing +schema publication and compatibility protocol; ordinary content writes do +not build indexes inline. + +**Exact scan baseline.** Evaluate the typed predicate over streamed document +tokens using that analyzer and the declared edit relation. Build bounded +query matching state once; honor cancellation and the query's execution +budgets while processing batches. This path is the correctness oracle and +remains available with full, partial, or absent index coverage. It composes +with typed graph/property filters at their declared stage. The first-declared +scan restriction is transitional; supported graph-derived populations must +retain their predicates. Unsupported placement fails validation rather than +losing the predicate. + +This is a new typed Boolean evaluator, not a wrapper around Lance's flat +BM25 scanner. `InvertedIndexParams::build()` already exposes the analyzer +without a dataset or index; its Text tokenizer shares query/document +tokenization. That construction alone does not supply the newly specified NFC +pipeline; its integration is an explicit qualification gate. Lance's flat +BM25 helper accepts a tokenizer but collects per-row scoring counts, does not +implement edit matching, and its fuzzy post-filter path rejects execution. +Reuse the public analyzer in a typed +engine/DataFusion filter over the sealed scan stream. Native index +acceleration additionally needs an analyzer-consistent query path and a +complete-expansion outcome; those scanner capabilities are not supplied by +the pin. Boolean evaluation introduces no dictionary or posting storage. + +The implementation must bound query bytes, distinct terms, individual +materialized values, matching state, and execution work. Enforce admission +before allocations that can exceed the budget, including Unicode normalization +and token construction. NFC can expand UTF-8 bytes: the pinned normalizer turns +the two-byte U+0344 into four bytes. It can also consume a long combining-mark +sequence before yielding its first output scalar. Bound source consumption, +internal normalization buffers/work, and output capacity; an input-byte limit +or cancellation checks only between emitted characters or tokens are +insufficient. A bounded implementation must enforce these limits at the +normalizer's input and internal work boundaries. +Use a bounded exact edit matcher, with cancellation checkpoints inside large +comparisons and across successive small comparisons. A work quantum that +resets for every token pair does not bound uninterrupted aggregate work. +Executor cooperation, cancellation propagation from the request boundary, +and cleanup of native tasks are separate qualification obligations. Marking a +function async or adding a yield does not establish end-to-end cancellation. +The pinned `fst` Levenshtein automaton agrees with the declared scalar-value distance, +but construction can consume substantial memory and hit its state limit. +Its default per-automaton cap is not a whole-query resource protocol. Numeric +limits, accounting units, and fallback charging must be specified and +qualified before shipping this evaluator. + +**Qualified index acceleration.** Lance continues to own dictionaries, +postings, and physical index state. A native path is eligible only when its +artifact passes RFC 0043's proof checks against the accepted profile and its +complete normalization/tokenization/filter pipeline is qualified for the +requested mode and edit budget. +Covered rows use complete term expansion and posting evaluation; uncovered +or rewritten rows use the same exact predicate on their accepted values. +Combine them at one snapshot with the existing visibility and row-identity +rules. Final limits and retrieval arm windows cannot change which terms or +rows satisfy the predicate; ordinary early termination is allowed when the +query plan proves the requested result complete. Post-verifying a truncated +candidate set cannot repair omitted matches. + +An indexed expansion must distinguish **complete** from **overflow**. The +current substrate shares a default budget of 50 expansions across query +terms within each segment, selecting lexically across that segment's +partitions. It returns tokens without a completeness flag. Earlier terms +can exhaust the budget before later terms are considered; changing the +vocabulary or segment layout can change the selected terms. This is not an +exact membership contract, and increasing the cap is not a completeness proof. +When complete acceleration cannot be established, use the exact scan within +the remaining query budget. If that budget is exhausted, fail the whole +query with a typed resource outcome; do not return partial success. Bound +token/automaton construction, memory, and execution work without silently +discarding analyzed terms or increasing budgets on fallback. This introduces +no query-level `max_expansions` knob or separate approximate predicate. + +The pinned Lance fuzzy scanner does not yet satisfy these requirements. +Until an upstream implementation and adapter pass the qualification matrix, +use the exact baseline for the affected shapes even when an index exists. +Keep native matching changes upstream where possible; do not build a second +index subsystem or a persistent shadow vocabulary in OmniGraph. BM25's +separate analyzer and corpus-statistics parity gate remains required; a +membership matcher alone does not qualify ranked scores. + +### Analyzer identity + +Accepted SchemaIR owns the resolved analyzer and default-scorer fingerprints +of every `@analyzed` field. The fingerprint covers the profile name, the +normalizer implementation and Unicode data identity, the pinned tokenizer and +filter order, and the scoring policy version; RFC 0043's artifact-scoped +certificates bind a physical full-text index to that fingerprint. The version +and stamp assignment coordinates with RFCs 0040 and 0044 through the existing +`required_ir_version` owner, as [RFC 0048](0048-search-contracts.md#representation-identity-and-source-attribution) +records for every representation binding. Renames preserve the binding; +drop/re-add creates a new one. + +## Invariants + +- **Physical acceleration is derived (7):** index presence never chooses the + analyzer, the matched set or the score; the exact scan is the oracle at + every coverage state. +- **Integrity failures are loud (8):** an undeclared capability, a + token-empty query, an invalid edit budget, an incomplete native expansion + without budget for the exact scan, and an exhausted budget are typed + failures; none returns a plausible subset. +- **Query semantics are typed structures (9):** both consumers lower through + one `LexicalQueryIR` bound to rename-stable property identity and the + accepted analyzer fingerprint; there is no vendor query-string sublanguage. +- **Bounded, observable resource use (11):** normalization, token + construction, edit matching and statistics passes are admitted and charged + before allocation; the `fst` automaton's per-construction cap is not the + query's resource protocol. +- **One source of truth (12):** accepted SchemaIR owns the analyzer; Lance + owns dictionaries and postings; no shadow vocabulary or second index + subsystem is introduced. + +Deny-list: no synchronous index build on a content write, no logical +precondition on index coverage, no string-built predicates, no silent partial +results. + +## Compatibility and reversibility + +- **Language:** removing `fuzzy`, `search` and `match_text` is a major change + to the GQ language surface. The deprecation release keeps them compiling + with a diagnostic; the removal release bumps the GQ language major per the + compatibility-surfaces RFC. Stored queries and `cluster.yaml` inline + queries must be rewritten inside the window. +- **Schema and format:** `@analyzed` is a SchemaIR feature; incompatible + binaries refuse rather than reinterpret. The NFC profiles invalidate + existing full-text certificates; RFC 0043's rebuild procedure applies. +- **Results:** analyzed matching can change rows where the old answer + depended on index state. That is the correction this RFC exists for; it is + not a compatibility promise. +- **Reverting:** before implementation, delete this document. After the + cutover, reverting means restoring the removed spellings and their + index-dependent semantics, which needs another explicit language and + format boundary. + +## Alternatives + +- **Fuzzy filtering as the complete typo-search feature.** An exact lexical + source can discard admitted typo matches. Shared lexical descriptions and + explicit fuzzy scoring close the ranked path. +- **Native fuzzy defaults or truncated expansion as exact semantics.** They + change results with analyzer/index/segment state. Complete evaluation or + typed failure is required; a larger cap is not a completeness proof. +- **Empty indexes as analyzer authority.** Artifact removal cannot erase + logical matching semantics. Accepted SchemaIR owns the analyzer. +- **Implicitly rescope lexical statistics with every graph filter.** This can + reverse the ordering of the same eligible targets and couples score meaning + to match-block placement. A snapshot-visible field corpus gives filtering + and scoring separate meanings; qualified statistics still have a cost. +- **Warn on unindexed search and keep the bare tokenizer.** Rejected for the + same reason RFC 0047 now refuses instead of warning: a warning does not + repair the matched set. + +## Evidence and tests + +Evidence is tied to recorded source revisions and configurations. Historical +pass counts on the PR #606 branch are not fresh evidence; every claim below +names the owner that must reproduce it. + +### Contract-to-code qualification + +This matrix is the implementation decision record for Lance 11.0.0, +DataFusion 54.0.0 and Arrow 58.3.0. **Reuse** means use the public primitive +within its established semantics; **adapter** means OmniGraph must supply the +missing contract; **bounded fallback** means complete evaluation under the +same query budget or typed refusal. **Upstream** identifies a limitation that +needs a qualified workaround or upstream fix before enabling that route. +None of these labels means the full search feature has shipped. + +The [source and probe receipt](assets/0048-upstream-contract-checkpoint.json) +records the inspected source files, archive checksums, lockfile and executable +probe results. Every recorded source file was compared byte-for-byte with its +crate archive, whose checksum matched `Cargo.lock`. Full upstream guides were +read alongside the code, including [DataFusion integration](https://lance.org/integrations/datafusion/), +[object stores](https://lance.org/guide/object_store/) and +[observability](https://lance.org/guide/observability/). Live documentation +describes available concepts; the exact crate source and qualified tests +determine what this pin can promise. + +| Search promise | Pinned interface and limit | Decision | Minimal falsifier and disposition | +|---|---|---|---| +| Accepted analyzer, including fuzzy input | `InvertedIndexParams::build`; native fuzzy and flat paths choose different analysis/expansion behavior; NFC needs preprocessing. | Reuse tokenizer; adapter plus bounded fallback for complete membership. | Same composed/decomposed or capitalized typo before/after indexing must have identical membership. Native Unicode/analyzer guards expose the limits; the four current GQT regressions remain open. | +| One exact/fuzzy lexical score | `InvertedIndex::bm25_stats_for_terms`, `MemBM25Scorer`; native float32 BM25 does not implement the proposed grouped fuzzy formula or numeric policy. | Bounded exact scorer first; native acceleration only after parity qualification. | Rare expansion, repeated terms, common-term IDF and deletion change native scores. Existing native and independent Decimal fixtures exercise these counterexamples; full indexed parity remains open. | +| Cross-table lexical ranking | Public shared scorer can be supplied before native per-table cuts; immutable index statistics can include deleted rows. | Adapter owns live corpus aggregation, scoring policy and complete cut boundaries. | Two local top-1 cuts lose the global winner; the shared-statistics probe recovers its score band. Canonical ties and live multi-type statistics remain open. | + +The remaining rows of the matrix, covering vectors, graph masks and candidate +windows, are in [RFC 0048](0048-search-contracts.md#contract-to-code-qualification). + +### Assumption audit + +The pinned tokenizer/edit-distance probe passed 73,008 comparisons against an +independent Unicode-scalar evaluator at budgets zero through two. This covers +the primitive, not the revised NFC pipeline, indexed completeness or budgets. + +The [Decimal oracle](assets/0048-lexical-scoring-v1.py) +generates thirteen 80-digit reference cases. The float64 evaluator uses pinned +`libm 0.2.16`, tolerance `2e-14 * max(1, expected)`, exact fixture order, +repeated-term invariance, eligibility-independent scores/features, all/any +feature invariance and edit-budget inclusion. Inputs already represent +analyzed tokens. Native/indexed numeric parity, complete ties and relevance +defaults remain unqualified. + +The native probes that justify the decisions above +(`fts_statistics_scope_can_reverse_ranking`, +`nfc_preprocessing_requires_an_explicit_bounded_integration`, +`native_fuzzy_bm25_rewards_a_rare_expansion` and +`native_bm25_idf_can_round_a_common_term_to_zero`) are proposed for +`lance_surface_guards.rs` in a separate test-only pull request; the RFC's +acceptance does not depend on them and they do not depend on it. + +### Regression cases + +The 2026-09-09 GitHub Actions runs for PR head +`bf5a77e55bc1dcd8415f28d5900d0a9cd218d78e` confirm the same four failures in +both [GQ Logic Tests](https://github.com/ModernRelay/omnigraph/actions/runs/34371510788/job/102533700021) +and [Test Workspace](https://github.com/ModernRelay/omnigraph/actions/runs/34371511424/job/102534255348): +67 of 71 cases pass, and all 127 runner self-tests pass. The workspace job +tests GitHub's merge candidate `f4ce7e895eb5c29b34c70cb1dbfbf95fba2f8a27`, +which combines that PR head with base `ed3ea5006f55ef703a3331f29ac45a2262f13300`. +This identifies the tested source; it does not claim validation against a +later `main` revision. + +| Existing GQT owner | First observed failure | Required implementation and migration proof | +|---|---|---| +| [`fuzzy_query_bypasses_index_analyzer`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/fuzzy_query_bypasses_index_analyzer.gqt) | Step 2: capitalized `Introductio` returns no rows; `intro` is expected. | Phase 2 must apply the accepted field analyzer at every edit budget. Retain the lowercase and zero-edit controls and reach the later assertions. | +| [`index_state_changes_text_matches`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/index_state_changes_text_matches.gqt) | Step 5, including the two mutation steps: `running` finds only the appended row and loses the indexed row. | Phase 2 must use one matching definition for indexed and uncovered rows. Complete the later `beto` assertion as well; passing the first repaired step is insufficient. | +| [`search_on_traversal_target_is_dropped`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/search_on_traversal_target_is_dropped.gqt) | Step 2: traversal returns B, C and D where only B and D match. | Phase 3 must retain the target predicate and rank the traversal target. The later ranking assertion must return D, rather than fail for a missing score column. | +| [`unindexed_search_is_case_sensitive`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/unindexed_search_is_case_sensitive.gqt) | Step 3: unindexed `deep` finds only the lowercase row; both rows are expected. | Phase 2 must preserve matching with and without an index. Migrate both field declarations to the intended accepted analyzer, and run both query-case controls. | + +These cases were added in `b1df2041` and remain active regressions. During the +coordinated query migration, rewrite their legacy `search`, `fuzzy` and `bm25` +forms through the new typed terms and ranking constructs while preserving +their expected membership and ordering. Declare analyzer intent in the schema; +index presence cannot provide it. Removing the old syntax, blessing the +incorrect rows, or running only the isolated staged case does not close these +regressions. A passing implementation needs every step and the full corpus to +run through the production parser and engine. + +The same workspace run passes 361 compiler tests, 42 Lance surface guards, +56 search tests, 13 RRF/prefilter tests, 88 CLI data tests and 23 CLI parity +tests before failing at GQT. These counts describe the PR's checked-in code; +the archived integration experiment has different test counts and is outside +this CI build. Environment-specific skips inside native guards retain their +existing limitations. The workspace process stops on the GQT failure, so +later workspace test targets are not established by this run. Format and +Clippy checks pass independently. These historical CI failures require the +implementation above; later native probes do not close them. + +The four cases are held out of the corpus until each fix lands: a red case +in `main` blocks unrelated work, and the fix regression gate keys on an +`issue_N` name. Each will be filed as an issue and land as +`issue_N_.gqt` with the fix that turns it green. + +### Qualification matrix + +Default resolution requires its own compiler/schema and query fixtures: + +- Bare `@analyzed` and its explicit expansion resolve to the same field + fingerprint and permit matching and ranking. `scorer="none"` permits matching + while refusing a lexical ranking source. +- Missing source/dimension, omitted model without a default, unresolved explicit model, + incompatible dimensions, and distance omitted without a supplying recipe + all fail before publication. Explicit model overrides select a whole recipe; + explicit distance overrides take precedence and remain validated. A qualified + explicit model works without any schema default. +- Export/reload and no-op reapplication preserve resolved field bindings across + deployments and runtime provider/default changes. Changed schema defaults + expose affected field rebindings and required rebuilds in `schema plan`; + shorthand does not bypass existing migration refusals or rename identity. + +The exact lexical qualification matrix retains all preceding requirements: + +- Positive and negative matches; `all`/`any`; repeated/reordered terms; + null/token-empty values; empty and stop-word-only queries; validation even + on empty populations. Edit budgets reject negative, non-integer, oversized, + and narrowing-overflow inputs before execution. +- Case, stemming, folding, composed/decomposed text, multibyte characters, + and lengths around 40 UTF-8 bytes. Zero-edit equivalence, inclusion at + budgets 0/1/2, and transpositions costing two. +- NFC conformance for the pinned normalizer/Unicode version; canonically + equivalent inputs on both sides of matching and ranking through scan/index + paths; and long combining-mark sequences under bounded cancellation. + Preserve original stored values and exact String predicate behavior. + Reject or bypass artifacts that lack proof of the revised pipeline. +- Absent, empty, complete, partial, removed, and rebuilt indexes through + append/update/delete/overwrite/compaction. Identical values must match + identically under graph filters and supported negation. +- More than 50 expansion terms, vocabulary/segment changes, an exact term + crowded out by expansions, and one term consuming the budget before another. + Native overflow must fall back completely within the remaining budget or + fail; post-verifying a truncated set is insufficient. +- One huge token, automaton state failure, cancellation, and exhausted shared + budgets must produce typed failure without successful partial results. +- The same tolerant query through Boolean matching and fuzzy retrieval must + agree on membership before the ranking cutoff. Adding a predicate cannot + rewrite a neighboring exact source. Fuzzy/exact score and order parity need + independent fixtures, including corpus deletion/update and repeated terms. +- More tied targets than each native window, reversed stable-ID order, and + different physical partitions must preserve declared exact winners or fail + explicitly. Sorting an already truncated subset is not the assertion. + +## Rollout + +Phases are sequenced by [RFC 0048's implementation phases](0048-search-contracts.md#implementation-phases); +this RFC's `implementation` status advances with Phase 2 and completes when +the deprecation window has closed. The phase, restated here as the owner of +the lexical work: + +**Input:** Phase 1 representations/resources and Phase 0 numeric oracles. + +Build bounded NFC/analysis, complete `Terms` matching and unified +exact/fuzzy scoring; exhaustive `knn` and declared `ann` are the vector half +that RFC 0048 keeps. Charge statistics, coverage, scoring and selection to +the shared context. +Extend search/substrate owners and the +[lexical qualification matrix](#qualification-matrix); observable rows, +shapes and errors belong in GQT. + +**Exit:** predicate/retriever membership agreement before cuts, independent +score and vector fixtures, total ties, all index/lifecycle states, and typed +budget/cancellation failures. Assert that intended index/compaction states +were reached; distinguish current from pinned answers and reopen snapshots. + +**Open:** native fuzzy analysis, float32 BM25, stale statistics and native cuts +are not substitutes for the accepted exact contract. Reproduce their +counterexamples before reuse. Numerical parity does not establish relevance +quality. Candidate-scoring/model operators and richer representations remain +deferred; Phase 5 chooses measured defaults. + +## Unresolved questions + +1. Close the Phase 0 numeric policies/oracles and native route dispositions. + Phases 2/5 must qualify BM25 across supported targets, exact live-row + statistics and native/fallback score/winner parity. Polymorphic field-corpus + execution is deferred. Validate edit weights and maximum reduction on the + owned task corpus before release; numerical fixtures alone do not establish + a good relevance default. +2. Normalizer and Unicode data identity, the analyzer fingerprint's mapping + onto RFC 0043 artifact certificates, and the SchemaIR version assignment + shared with RFCs 0040 and 0044. +3. Whether the deprecation diagnostic should rewrite the query text for the + caller (a `--fix` style output) or only name the replacement. + +## Decision log + +- 2026-09-18 — split out of RFC 0048 at its 2026-09-13 revision with the + lexical decisions unchanged. Replaced the hard removal of `fuzzy`, `search` + and `match_text` with a one-release deprecation window and fixed the + mapping, following the compatibility-surfaces RFC. Moved the numerical + oracle fixtures to `assets/`; the oracle test and native probes stay on the + evidence branch until an evaluator exists. +- 2026-09-13 — decided membership-independent lexical features, null and checked + arithmetic policies, and deferred global execution. Extended the existing + Decimal, compiler and DataFusion probes; default native arithmetic is not + sufficient for the new numeric contract. Phase 0 remains incomplete. +- 2026-09-09 — selected live field-corpus statistics, unified float64 + exact/fuzzy BM25, truthful unknown coverage, and a budgeted exact-coverage + option. Added numerical, native and isolated integration evidence. +- 2026-09-08 — chose one pre-stable staged-query/schema cutover; shared + `terms`, named windows/metrics, schema defaults and explicit NFC. Retained + ordinary graph identity, stored-query recipes and coherent follow-up; removed + proposed document/evidence wrappers and separate profile machinery. +- 2026-09-03 — published this draft alongside RFC 0047 after separating + plan-truth work from representation/search contracts. diff --git a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md new file mode 100644 index 000000000..7e2fdab3a --- /dev/null +++ b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md @@ -0,0 +1,791 @@ +--- +rfc: "2026-09-18-gq-composition-and-language-evolution" +title: "GQ composition and language evolution" +track: maintainer +status: draft +implementation: not-started +authors: + - Ragnor Comerford (@ragnorc) +created: 2026-09-18 +updated: 2026-09-18 +discussion: "https://github.com/ModernRelay/omnigraph/pull/606" +supersedes: [] +superseded_by: [] +blocked_on: + - "Keyword rule reconciled with RFC 0056 (atomic keywords) and RFC 0055 before any staged syntax stabilizes" + - "C1–C4 golden plans re-established in the production compiler once RFC 0048's rank stages exist" +--- + +# RFC: GQ composition and language evolution + +## Summary + +This document fixes the grammar, typing, scope and null rules that the query +language must settle before staged syntax stabilizes, inventories every +current and proposed query capability with its release disposition, and +records four composition examples (C1–C4) whose logical plans the grammar must +leave room for. It also records the design direction for graph-wide discovery +across entity types, which is explicitly deferred. + +It decides no search semantics: [RFC 0048](0048-search-contracts.md) owns rank stages, sources, +fusion, selection and result metadata, and [Analyzed lexical search](2026-09-18-analyzed-lexical-search.md) +owns analyzers, matching and scoring. It was split out of RFC 0048 on +2026-09-18 so that search decisions are not blocked on language decisions +that RFC 0055, RFC 0056 and the session-settings RFC also touch. It is not +under review until RFC 0048's staged syntax is; the sections are the +verbatim 2026-09-13 text with links repointed. The pre-split text, the test-only staged compiler, the DataFusion composition probes, the selection cost instrument, the lexical oracle test and the archived integration patches are retained at commit [`ce5a3012`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/docs/rfcs/0048-search-contracts.md) on the `rfc-0047-search-plan-truth` branch; they are evidence, not part of the tree. + +## Motivation + +An agent's work mixes exact graph analytics, bounded discovery and evidence +assembly in one investigation, in both directions: aggregate then retrieve, +and retrieve then aggregate. The stage boundaries RFC 0048 introduces expose +information loss that a single `match`/`return` cannot express. Before that +syntax freezes, the shared expression model, contextual keywords, scope +transitions and null rules must be fixed once, or every later operator +(intermediate grouping, `let`, `select`, `optional`, `collect`, typed unions) +will be a special case. RFC 0048's [agent workload](0048-search-contracts.md#agent-workload-and-design-objective) +table states the requirements these rules serve. + +## User and operational behavior + +### Language evolution and compatibility + +Extend one typed stage sequence followed by `return`, optional final `order` +and `limit`. Graph patterns remain declarative within `match`; source text +does not prescribe a physical scan/join schedule. The production single-match +grammar and test-only staged grammar are not the complete stable language. + +Use one expression model for properties, parameters, literals, aliases and +metrics wherever their types/scopes permit them. Keep graph patterns and stages +as dedicated nodes; a call-shaped retriever is not a scalar. Unknown +constructors/options fail. Fix precedence, associativity, null rules, name +resolution and positional/named arguments before stabilization. Scalar +negation must not inherit correlated graph `not` semantics; score arithmetic +retains explicit domain policies. The following Phase 0 expression decisions +apply when the corresponding operators are implemented: + +| Boundary | Decision | +|---|---| +| Precedence | Parentheses/calls, then `*` and `/`, then `+` and `-`, then one comparison, then scalar `not(...)`, then `and`, then `or`. Arithmetic at one level associates left; chained comparisons fail. Graph `not { ... }` retains its separate scope. | +| Numeric types | Arithmetic requires the same scalar numeric type; no implicit widening, integer/float mixing or conversion of metrics to numbers. Integer division remains unavailable until an explicit quotient/rounding contract; future casts must be named. | +| Null values | Arithmetic/comparison with null yields null. Boolean composition uses three-valued logic: false dominates `and`, true dominates `or`, otherwise an unknown operand yields null; `not(null)` is null. `is_null` returns non-null Bool; filters and `count_if` accept only true. | +| Numeric errors | Integer overflow/underflow is a typed failure, never wrapping, saturation or null. New floating arithmetic rejects non-finite operands/results and division by either signed zero. Null operands yield null without applying that arithmetic operation. These rules do not retroactively redefine stored scalar values or existing aggregate signatures. | +| Evaluation | Operands have no promised left-to-right or short-circuit evaluation. Boolean guards cannot make an invalid arithmetic expression safe; future conditional/try expressions need explicit evaluation rules. Constant folding must use the same types, null and error rules as execution. | +| Aliases | Sibling `let`/return expressions read the incoming scope; they cannot read an alias declared beside them. A later stage can read exported values. Final ordering can read output aliases. | + +The staged compiler checks precedence, same-type arithmetic and nullable +results. The native `staged_composition_numeric_and_null_contracts` probe +checks all nine Boolean pairs and exposes two adapter obligations: +DataFusion 54's default integer `BinaryExpr` wraps; its explicit +`with_fail_on_overflow(true)` path refuses the tested I64 overflows. That +switch still permits floating infinity/NaN, so finite arithmetic needs a +checked expression adapter. This is a physical-expression proof, not proof +that logical optimization, aggregate accumulators or GQ lowering preserve +these rules. Those remain qualification gates; a session default is +insufficient. See the pinned [binary evaluator](https://docs.rs/crate/datafusion-physical-expr/54.0.0/source/src/expressions/binary.rs) +and [Arrow arithmetic](https://docs.rs/crate/arrow-arith/58.3.0/source/src/numeric.rs). + +Every operator declares its scope and population effect: + +| Operation | Binding/identity rule | +|---|---| +| Predicate / ordinary projection | Preserve rows, duplicates, bindings and metric origins | +| Traversal | Extend bindings with existing endpoint/edge-instance semantics; bounded reachability does not become path enumeration | +| `rank` | Select distinct targets and retain their incoming binding rows | +| `take` | Select target/group pairs; it does not create reusable group features | +| Group/reduction | Export keys and reduced values, discard unreduced member bindings/metrics | +| Future optional/branch/collection | Explicit imports/exports, nullability, multiplicity and local scope | + +Terminal aggregate-return shorthand retains its existing implicit grouping: +adding a non-aggregate projection can change the groups. Future intermediate +grouping uses the same equality/null/duplicate rules with explicit exports; +ordinary intermediate projection must not acquire that implicit reduction. + +Each rank block ends with `yield `, resolving a declaration in that +block. Sources share the incoming population and explicit fusion dependencies. +Adding an unused declaration cannot select a different output or comparator; +every declaration is still validated and requested metrics consume resources. +Earlier metrics retain their original scope; reusing earlier candidates needs +explicit remapping/intersection. + +Keywords are contextual and token-bounded (`yield yield` can select an alias +named `yield`). Preserve property, system and source namespaces: +`$binding.property`, `$binding.@id` and explicit metric references. +New nested scopes must not capture unrelated bindings implicitly. Future +user-defined namespaces require collision rules. + +Admit count literals/parameters consistently while retaining their units: +positive source-target windows, nonnegative group-pair quotas and final +binding-row limits. Extending final-limit parameters is additive, not row-dependent +query batching. Physical effort, memory, I/O and output bytes stay separate. + +After the coordinated cutover, additive syntax must preserve valid queries' +parsing/name resolution, typing, populations, metric meaning and result schema +under the same accepted identities. Exact results stay exact; ANN preserves +its approximation/effort contract rather than a fixed candidate set. +All-node expansion after a schema change follows its explicit scope contract. + +Changing an analyzer, scorer, missing-value rule, default window, encoding +recipe or ANN mapping is a semantic change even if queries still parse. +Version policies/accepted bindings and preserve resolved choices. Unavoidable +future breaks require an explicit language/format boundary and migration. + +Before freezing syntax, qualify [C1–C4](#required-composition-examples) and a +compatibility corpus covering contextual identifiers, mixed/null expressions, +alias collisions, unused sources, projection/window independence, both sides +of cuts and discarded group/union metrics. Deferred operators need coherent +type/plan proofs now and executable GQT/resource tests when implemented. +Stored queries describe supported combinations; they cannot replace missing +language operators with unbounded client-side computation. + +### Query capability matrix + +This inventory covers the current public `.gq` surface, the initial release +described by this RFC, and the general graph/query/retrieval extensions the +grammar must leave room for. It separates language support from implementation +evidence; parser acceptance and native substrate support alone are insufficient. +It is a review checklist, not a promise to implement every conceivable query +operator or every future row in the initial release. + +**Current** describes the production path in this PR's source baseline, not +the test-only staged parser or the archived integration prototype. `Limited` +means a narrower capability or a known correctness gap, described in the cell. +The four [GQT search regressions](2026-09-18-analyzed-lexical-search.md#regression-cases) +remain open. **RFC release** uses these dispositions: + +- `Keep`: preserve the existing operation and its meaning. +- `Deliver`: required for the coordinated release, subject to its stated gates. +- `Replace`: an existing surface migrates to the new contract at the cutover. +- `Foundation`: fix the shared grammar/type/scope rules now; the broader + user-facing operator remains deferred. +- `Defer`: outside the initial implementation; preserve the stated extension. +- `Separate`: an adjacent capability governed by its own contract, not a new + search feature. + +The last column states what can be added later and what the syntax must +preserve. Examples such as `yield`, `group`, `collect` and nested object +construction discussed during design are illustrative; this matrix does not +accept their spelling, introduce a general scalar-variable declaration, or +change the tested RFC query examples. + +#### Declarations, expressions and scope + +| Capability | Current | RFC release | Future extension / syntax constraint | +|---|---|---|---| +| Named queries; multiple query declarations per file | Yes | Keep | Query-body extensions retain the declaration and invocation model. | +| Typed required and nullable parameters | Yes; scalar, list and vector forms; nullable omission accepted | Keep | Relation/object parameters would need separate type and transport contracts; a list is not implicit query batching. | +| Stored-query tool metadata | Yes; `@description` and `@instruction` | Keep | Metadata describes the same query; it does not carry hidden execution semantics. | +| Literals, parameters, property values, `date`, `datetime`, `now` | Yes, with current context restrictions | Keep | New expression variants preserve existing types, time semantics and literal parsing. | +| One shared expression model across stages | Limited; fixed expression variants and context-specific handling | Foundation | Deliver shared parsing/typing for admitted expressions, including metrics; this does not deliver every scalar function below. | +| Scalar comparison, exact String prefix/substring and list membership | Yes | Keep | Exact String behavior remains case-sensitive; `contains` on a list remains membership. | +| General scalar Boolean composition and negation | Limited; match conditions conjoin and graph `not` tests pattern absence | Foundation | Add typed Boolean expressions with fixed precedence and null rules; graph absence keeps its correlation semantics. | +| Explicit null tests/replacement, conditionals, arithmetic, casts and general scalar functions | No general surface | Foundation | Extend shared expressions through typed signatures; preserve overflow, null and score-domain rules. | +| Intermediate named computations/projection | No | Foundation | Fix explicit value bindings and retained/exported scope through composition example C1; operator implementation remains deferred. | +| Result aliases used in final ordering | Yes | Keep | Result aliases remain distinct from graph parameters/bindings and source aliases. | +| Reusing an earlier result alias in another projection | No; production rejects `T36` | Foundation | Decide alias scope explicitly before relaxing this rule; shared expression parsing alone is not alias reuse. | +| Nested scopes, reusable query branches and bounded subqueries | Limited; correlated graph negation only | Defer | Declare imports, exports, shadowing, snapshot and shared budgets; distinguish reuse from re-execution. | +| User-defined functions or query-valued parameters | No | Separate | If justified later, define typed namespaces, versioning and execution limits; arbitrary evaluation is not an initial requirement. | + +#### Graph matching and identity + +| Capability | Current | RFC release | Future extension / syntax constraint | +|---|---|---|---| +| Concrete typed node scans and inline property constraints | Yes | Keep | New type selectors must not reinterpret an existing concrete type. | +| Directed traversal and binding joins within `match` | Yes | Keep | Preserve endpoint-pair versus edge-instance multiplicity and graph eligibility. | +| Undirected traversal | Yes, between the same endpoint type | Keep | Preserve orientation and duplicate rules when extending edge-pattern syntax. | +| Single-hop edge bindings and edge-property access | Yes | Keep | Node and edge kinds remain distinct even when names or ids coincide. | +| Bounded multi-hop reachability | Yes; shortest-distance semantics, no bound path/edge sequence | Keep | A future path value or path enumeration needs an explicit operator and bounds; it cannot redefine existing hop syntax. | +| Anonymous bindings and repeated-binding constraints | Yes | Keep | Anonymous bindings cannot name reusable rank/take targets; existing bindings cannot change type through rebinding. | +| Correlated pattern absence | Yes; `not { ... }` | Keep | Inner bindings do not escape; future scalar negation does not acquire this scope behavior. | +| Positive existence without outer-row fan-out | No dedicated positive-existence construct | Defer | Add a correlated pattern predicate; ordinary traversal continues to expose its matches. | +| Optional graph enrichment | No | Foundation | Example C4 must preserve unmatched inputs and nullable exports; distinguish inner predicates from later filters. Multiple matches may still fan out; implementation remains deferred. | +| Graph matching before, between and after retrieval cuts | Limited; one `match` and terminal search ordering | Deliver | Repeated stages retain their input population; later filters cannot move before an earlier selection cut. | +| Logical system identity projection and filtering | Limited; whole-node objects expose `id`, but `.id` is an ordinary declared property lookup | Deliver with RFC 0040 | Qualify `$binding.@id` and type/incarnation-aware follow-up; never expose native row IDs as graph identity. | +| Explicit unions of typed graph bindings | No; interfaces do not provide polymorphic query scans | Foundation | Execution deferred. Preserve typed branches, explicit exports, bag/set semantics and original entity identity. | +| All-node discovery, representation expansion and type narrowing | No | Foundation | Execution deferred. Resolve type scope separately from searchable representations at one snapshot; common fields and type-specific projections must typecheck. | +| Global all-edge or mixed node/edge discovery | No; a concrete edge can be reached through traversal | Defer | Make entity kind and selected scope explicit; an all-node selector must not silently start including edges. | +| Cross-graph federation | No combined `.gq` population; graph selection is external | Separate | Requires explicit authority, identity, snapshot and budget rules; global search in this RFC means one graph. | + +#### Projection and result shape + +| Capability | Current | RFC release | Future extension / syntax constraint | +|---|---|---|---| +| Flat projection of properties, parameters and literals | Yes | Keep | Ordinary projection preserves binding rows, duplicates and candidate windows. | +| Explicit column aliases and unique output names | Yes; duplicate names rejected; some unaliased inferred/executed names still differ | Keep; qualify schema agreement | Freeze naming and result-schema agreement; extra syntax must not capture existing aliases. | +| Whole-node object projection | Yes; id and properties except Blob/Vector | Keep | This remains schema-shaped shorthand; explicit fields provide stable compact results across schema additions. | +| Bare edge object projection | No; project edge properties explicitly | Defer | Define an edge object/reference type and endpoints deliberately; do not assume node-object behavior. | +| Existing list and vector property projection | Yes; each remains one column value | Keep | Projection does not unnest a list or execute vector retrieval. | +| Blob values in ordinary read projection | No; dedicated Blob API | Separate | Preserve the Blob access/resource contract; object shorthand must not fetch Blob contents. | +| Source score, distance, rank and fused-score projection | Limited; only a repeated leading `nearest`/`bm25` expression can expose its metric | Replace | Named `metric(source, field)` preserves domain, origin and missing membership through aliases and later stages. | +| Selected nested object construction | No; whole-node objects are the only node-object shorthand | Foundation | Example C4 fixes typed row reshaping separately from traversal/grouping/collection; constructor implementation remains deferred. | +| New list/object construction from expressions | Limited; list literals and existing list values | Defer | Define element/field types and nullability; constructing a value is separate from collecting rows. | +| List unnesting, mapping or comprehensions | No general surface | Defer | Unnest is an explicit population-changing operation; local value mapping needs its own typed scope and bounds. | +| Nested related-entity/evidence collections | No | Foundation | Example C4 fixes correlation, exports, duplicate and empty-result rules, local order, item limit and shared byte/work budgets; implementation remains deferred. | +| Snippets, highlights and source ranges | No dedicated query construct | Defer | Preserve source property/version, offset unit and bounds; generated text must remain distinguishable from stored source. | +| Token-budget result packing | No | Defer | Declare tokenizer, selection policy, attribution and completeness; row limits must not become token limits. | + +#### Aggregation, selection and ordering + +| Capability | Current | RFC release | Future extension / syntax constraint | +|---|---|---|---| +| Terminal `count`, `sum`, `avg`, `min`, `max` | Yes | Keep | Preserve admitted input types, null behavior and duplicates; `count($node)` counts binding rows. | +| Implicit grouping by non-aggregate return values | Yes, with current type restrictions | Keep | Adding a group projection can change row count; do not give ordinary intermediate projection this implicit effect. | +| Grouped metric projection and explicit metric reductions | No staged metrics; legacy search/aggregate combinations restricted | Deliver | Projected metrics can be group keys or explicit reductions; discarded source metrics cannot order a new group. | +| Reusable intermediate grouping/reduction | No | Foundation | Examples C1–C2 export group keys/entities and reduced values; drop unreduced member bindings and active ranks. Group-stage implementation remains deferred. | +| General distinct rows/entities and distinct aggregates | No dedicated syntax | Defer | Specify the identity/value tuple and duplicate equivalence; retrieval deduplication is not general `DISTINCT`. | +| Distinct-target ranking with retained binding rows | Limited; legacy search exists without the new stage contract | Deliver | Each target gets one source rank; selecting it retains its associated binding rows. | +| Quotas per explicit group, including ordinary non-search selection | No | Deliver through `take` | Select target/group pairs; use explicit reductions when order varies per pair; require local order without an active rank. | +| Parent selection using reduced child metrics | Limited; terminal aggregates cannot feed a quota | Deliver through `take` reductions | Selection does not create reusable group features or implicit parent-source ranks. | +| Top-N retrieval within every full group; row-dependent query inputs | No | Foundation | Examples C1/C4 require explicit correlation/partition scope, empty-group behavior and one total budget; implementation remains deferred. Global top-K plus `take` is different. | +| Final ordering by properties/aliases and identity tie-breaks | Yes, with documented legacy search boundary-tie gaps | Keep; repair retrieval ties | Final ordering cannot change a previous source cutoff; each selection owns its complete comparator. | +| Explicit `nulls first` / `nulls last` | No syntax; fixed ascending/descending defaults | Deliver | Apply consistently in local and final ordering; omission preserves existing defaults. | +| Final row limit, including zero | Yes; integer literal | Keep | Counts output rows; it does not resize retrieval or bound all intermediate work. | +| Parameterized final row limit | No | Foundation | An additive count-parameter extension remains possible; this matrix does not add it to the initial release. | +| Parameterized source windows and group quotas | No corresponding stage syntax | Deliver | Admit integers before execution; positive source windows and nonnegative quotas have different units. | +| General analytic windows, running aggregates and partition ranks | No public surface | Defer | Typed stage/expression rules must define partitions, frames and order; internal window use does not expose a language feature. | +| Stable ranked cursors/pagination | No replayable ranked-execution contract | Defer | Preserve actual candidate order or qualify reproducibility, retention and policy; snapshot identity alone is insufficient. | + +#### Retrieval and ranking + +| Capability | Current | RFC release | Future extension / syntax constraint | +|---|---|---|---| +| Analyzed lexical membership | Limited; `search`/`match_text` have known execution-path inconsistencies | Replace | One typed `terms` description consumed by `match_terms`; filtering introduces no rank/window. | +| Edit-tolerant lexical membership | Limited; `fuzzy` exists with analyzer/coverage defects | Replace | Shared analyzed edit semantics, explicit `mode` and `max_edits`, complete membership across index states. | +| Exact-term lexical ranking | Limited; legacy `bm25` exists | Replace | Explicit bounded `lexical` source with qualified `bm25_v1` statistics, numeric rules and ties. | +| Fuzzy lexical ranking | No unified ranked tolerant-query contract | Deliver | `lexical` consumes tolerant `terms`; no separate fuzzy scorer or inherited edit budget. | +| Exact vector top-K as an explicit user choice | No exactness selector; some physical paths scan exactly | Deliver through `knn` | Exact top-K over eligible valid vectors; an index is an acceleration choice. | +| Approximate vector top-K | Yes through legacy `nearest`, with implicit effort/window behavior | Replace with `ann` | Explicit approximation and bounded effort contract; exact rescoring does not imply complete recall. | +| String or raw-vector retrieval inputs | Yes | Keep with resolved encoding | Validate dimensions and compatible encoders; inputs are constant per execution, not implicit per-row batches. | +| Named sources, explicit candidate windows and explicit block output | No | Deliver | Sources share the incoming target population; select output independently of declaration order before freezing syntax. | +| Weighted multi-source rank fusion | Limited; legacy RRF has two inline arms and asymmetric windows | Replace | Named 2–16-arm RRF, explicit weights/windows, checked arithmetic and one vote per target per arm. | +| Search a traversal-introduced node or a bound edge | Limited; node search shapes have gaps and edge search is rejected | Deliver | Every source targets a property of the declared eligible binding; do not require a textual scan root. | +| Search, traverse and search again | No explicit staged surface | Deliver | Each stage retains population and metric origin; earlier candidates cannot be reopened implicitly. | +| Multiple fields on the same target as separate sources | Limited by legacy inline source forms | Deliver | Named sources may select different compatible field capabilities; fusion remains explicit. | +| One logical multi-field/cross-type lexical corpus | No unified contract | Defer | Specify field reduction and shared live statistics before candidate cuts; per-table BM25 values are not globally comparable by default. Multiple named fields of one target type can still use explicit fusion. | +| Phrase, token-prefix, proximity and Boolean lexical queries | No portable typed contract | Defer | Add `LexicalQuery` variants shared by matching/ranking consumers; avoid a vendor query-string sublanguage. | +| Geometric range retrieval and exact distance predicates | No dedicated public contract | Defer | Distinguish a complete eligible-population range query from a filter on ANN candidates. | +| Candidate rescoring, learned reranking and general feature combination | No general stage | Foundation | Example C3 fixes membership-preserving scoring, domains, statistics/normalization populations and resource behavior. Scoring/model/formula operators remain deferred. | +| General cross-identity fusion, such as passages with parent sources | No | Defer | Require graph mapping, reduction and fresh target ranks; matching id strings do not establish compatibility. | +| Semantic diversification / complementary evidence selection | No | Defer | Add a set-selection objective and qualification; per-group quotas alone do not supply it. | + +#### Representation and execution contracts + +These rows are part of a query's meaning or execution interface, even when +their authored syntax belongs to `.pg` or request options rather than `.gq`. + +| Capability | Current | RFC release | Future extension / syntax constraint | +|---|---|---|---| +| Analyzed String capability with immutable analyzer/scorer defaults | Limited; legacy FTS/index configuration and compatibility proofs | Deliver | Resolve `@analyzed` independently of `@index`; accepted semantics cannot follow mutable runtime defaults. | +| Dense vector fields and embedding-source declarations | Yes; existing `Vector`/`@embed` | Replace unresolved semantics | Resolve dimension, geometry, source mapping and compatible immutable encoding recipes; qualify providers. | +| Schema-owned default embedding recipe and reproducible exports | No complete resolved-default contract | Deliver | Final declaration spelling remains open; omission, explicit override and migration preserve accepted field bindings. | +| Sparse vectors, multivectors/late interaction and named analyzed views | No retrieval surface | Defer | Extend typed representation/capability variants; do not define every retriever input permanently as one dense vector. | +| Additional modalities or external retrieval sources | No general query source | Separate | Any later source needs explicit typing, authority, coherence and budget guarantees; no universal content object is introduced. | +| Coherent branch/snapshot reads | Yes through existing execution interfaces | Keep | Use the graph's accepted snapshot and existing authority; stage syntax must not reopen a fresh view. | +| Snapshot-coherent identity follow-up through all read transports | Limited; snapshot fields exist, system-id lookup and output paths have gaps | Deliver | Complete RFC 0040 lookup and JSON/JSONL continuation; unavailable/expired identity is explicit. | +| Inline/stored query invocation and typed result descriptors | Yes; result-schema agreement has known gaps | Keep and qualify staged reads | Derive reads, result shape and fingerprints from every stage; aliases and hidden demand must agree with execution. | +| Inspectable staged plans, metric origins and selection descriptors | No complete staged contract | Deliver | Expose resolved sources, inputs and semantics through the existing query interface; keep transport naming separate from graph properties. | +| Representation coverage, including explicitly requested exact counts | No complete source-level contract | Deliver | Known/unknown readiness differs from completion and index coverage; exact counts share the query budget. | +| Whole-query resource admission, cancellation and output accounting | Limited; existing local bounds do not establish the RFC guarantee | Deliver | All stages, encoders, fallbacks, nested work and projection share one execution context; no silent truncation. | +| Graph/branch and stored-query authorization | Yes | Keep | All source reads and graph expansions obey existing gates; a snapshot is not a retained-access capability. | +| Row/field-level security predicates | No policy engine for this scope | Separate | If added, constrain eligibility, statistics and metadata before selection; post-top-K filtering is insufficient. | + +#### Adjacent `.gq` operations + +| Capability | Current | RFC release | Future extension / syntax constraint | +|---|---|---|---| +| Named insert/update/delete query bodies | Yes, with current target and constructive/destructive restrictions | Keep | Read stages remain separate from mutation bodies; no implicit search-to-write pipeline. | +| Several mutation statements in one graph publication | Yes within admitted mutation forms | Keep | A future write extension must preserve the single publication and recovery contracts. | +| Standalone branch create/delete/merge/list statements | Yes; one per file, not beside query declarations | Keep | These control operations do not become data-query stages or typed unions. | +| Conditional writes after a coherent read | Yes through the execution interface | Keep | Preserve the returned graph commit precondition; do not infer a new transaction from projection syntax. | + +#### Syntax decisions needed now + +Every `Deliver`/`Replace` row needs accepted syntax and typing/lowering. +`Foundation` requires C1–C4 and global-extension design proofs now while its +broader implementation stays deferred. +All extensions follow the [language rules](#language-evolution-and-compatibility). + +Current-state evidence is the production +[grammar](../../crates/omnigraph-compiler/src/query/query.pest), +[typechecker](../../crates/omnigraph-compiler/src/query/typecheck.rs) and +[projection executor](../../crates/omnigraph/src/exec/projection.rs). +The query guide's earlier-alias-reuse claim is stale: `T36` and the +[score-projection case](../../crates/omnigraph-gqt/cases/issue_640_search_score_projection.gqt) +refuse it. `executed_column_name` also records unaliased inferred/executed +name drift. The [staged parser](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-compiler/src/query/staged_probe.pest) +qualifies only its documented prototype subset. + +## Design + +### Required composition examples + +C1–C4 are grammar-evolution acceptance cases. The test-only compiler parses and +checks these examples; the production parser rejects them. Their logical plans +and native probes are design evidence. Deferred operators require integrated +GQT/resource qualification when implemented. + +**C1 — Aggregate, select entities, then retrieve evidence.** A has prior/current +incident counts 2/8; B has 6/7. Select A's increase of six, irrespective of B's +report relevance. Missing periods count zero; incident-rooted input does not +invent services with no incidents. Including those needs a service population +and enrichment. + +```gq +query composition_c1($q: String) { + match { + $i: Incident + $s hasIncident $i + $i.period = "prior" or $i.period = "current" + } + group { + per { $s } + reduce { + count_if($i.period = "prior") as prior_count, + count_if($i.period = "current") as current_count + } + } + let { current_count - prior_count as increase } + select { order { increase desc, $s.@id asc } limit 1 } + match { $s hasReport $p } + rank $p { + lexical($p.text, terms($q), candidates: 2) as reports + yield reports + } + return { + $s as service, prior_count as prior_count, current_count as current_count, + increase as increase, $p.@id as report_id, $p.text as report_text + } +} +``` + +`group` exports entity keys under their binding names and named reductions, +discarding incident bindings. `let` adds row values whose sibling expressions +read the incoming scope; numeric results do not become identities. `select` +cuts rows, while `take` selects target/group pairs. Retain computed facts +through the service cut and later report reads. Multiple selected services +require C4's explicit correlation, not repeated global top-K plus `take`. + +**C2 — Retrieve, traverse, then aggregate the selected population.** Selected +p1/p2 have three bindings to project P (two from p1); eligible p3 falls outside +the cut. P must report three binding rows and two distinct passages. + +```gq +query composition_c2($q: String) { + match { $p: Passage } + rank $p { + lexical($p.text, terms($q), candidates: 2) as candidates + yield candidates + } + match { $p $membership:inProject $project } + group { + per { $project } + reduce { + count($p) as binding_rows, + count_distinct($p.@id) as passages, + min(metric(candidates, rank)) as best_rank + } + } + return { + $project as project, binding_rows as binding_rows, + passages as passages, best_rank as best_rank + } + order { $project.@id asc } +} +``` + +Group exports project identity/counts and explicitly reduced metrics; member +bindings and active rank disappear. In terminal aggregate shorthand, an +unreduced projected metric instead adds a grouping key and changes the question. +Keep the candidate barrier before traversal/aggregation. General distinct/ +intermediate grouping is deferred; the initial path must qualify terminal +binding counts and metric reductions. + +**C3 — Score existing candidates without changing membership.** Dense retrieval +selects d1/d2, both lexical matches, while the lexical arm's window contains only +d1. A fresh feature scores both without manufacturing lexical-arm membership. + +```gq +query composition_c3($q: String, $vector: Vector(3)) { + match { $p: Passage } + rank $p { + lexical($p.text, terms($q), candidates: 1) as words + knn($p.embedding, $vector, candidates: 2) as dense + yield dense + } + score $p { + lexical($p.text, terms($q), scoring: bm25_v1) as words_feature + } + return { + $p.@id as passage_id, metric(words, rank) as lexical_rank, + metric(words, score) as lexical_score, feature(words_feature) as lexical_feature + } +} +``` + +`score` preserves targets, bindings and comparator; it has no candidate window, +output selector, rank or fusion vote. `feature` names a scorer; `metric` +names retrieval membership. Reordering/cutting requires an explicit operation. +A hidden lexical top-K followed by a join can drop d2 and is invalid. +Scoring uses accepted corpus statistics and charges their work even for a +small target set. + +**Decision:** scoring evaluates the term contributions independently of +`all`/`any` membership. A partial `all` match can have a positive feature; +a present value with no matching terms, including a token-empty value, has +zero; a null field has a null feature. The +[lexical contract](2026-09-18-analyzed-lexical-search.md#lexical-scoring-and-shared-matching-semantics) owns the +complete rules and numerical oracle. The native composition fixture still +uses precomputed features: it proves preservation, not their production. + +**C4 — Computed facts, optional graph facts and bounded evidence.** A has one +owner and three reports; B has neither. Both survive with their original +counts; A gets the first two reports under the local comparator and B gets a +null owner and empty typed list. + +```gq +query composition_c4($q: String) { + match { + $i: Incident + $s hasIncident $i + $i.period = "prior" or $i.period = "current" + } + group { + per { $s } + reduce { + count_if($i.period = "prior") as prior_count, + count_if($i.period = "current") as current_count + } + } + let { current_count - prior_count as increase } + select { order { increase desc, $s.@id asc } limit 2 } + optional ($s) as owner { + match { $s ownedBy $person } + return { $person as person } + } + collect ($s) as reports { + match { $s hasReport $p } + rank $p { + lexical($p.text, terms($q), candidates: 2) as relevant + yield relevant + } + return { $p.@id as id, $p.text as text } + order { metric(relevant, rank) asc, $p.@id asc } + limit 2 + } + return { + $s as service, prior_count as prior_count, current_count as current_count, + increase as increase, owner as owner, reports as reports + } +} +``` + +Imports explicitly distinguish `$entity` from computed row values; query +parameters stay available. Child bindings/sources do not escape. `optional` +returns a nullable object: zero rows produce null, one produces an object, +and more than one requires explicit selection/collection or a cardinality +error. `OwnedBy` is zero-or-one in this example. + +`collect` consumes ordered binding rows and requires a local comparator and +output limit. Unique reports need an explicit identity reduction; source +windows do not bound later fan-out or cumulative parent work. Both children +preserve the parent row, accepted snapshot and one whole-query budget. + +Correlation is per parent **row**, not merely entity ID: repeated service +bindings can carry different facts. Compiled source ID alone does not identify +all runtime per-parent rankings. Reuse requires equivalent imports, parameters, +representation and snapshot, preserving every parent row and charging actual +shared work. Edge imports/group exports, total row ties, nested-field access, +cardinality/resource enforcement and safe decorrelation remain unqualified. + +**Evidence and open boundaries.** The +[compiler](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-compiler/src/query/staged_probe.rs) reads the +four examples directly. Its [derived logical views](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-compiler/src/query/staged_probe/plan.rs) +retain relations, barriers, sources, group populations, projection types and +imports. They are neither executable IR nor a public serialization; their +snapshot/budget/statistics statements are requirements, not measured behavior. +Symbolic bindings and omitted options do not freeze defaults/fingerprints. + +| Example / checked logical plan | Prototype evidence | Still unproved / required falsifier | +|---|---|---| +| [C1](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-compiler/src/query/staged_probe/composition_c1.json) | Group retains `$s`, exports counts and drops `$i`; selection precedes report retrieval. Native filtered counts produce A=2/8, B=6/7, C=1/0, selecting A. A report-driven prefilter selects B instead; `count(Boolean)` counts false; a duplicate path changes A's count. | Actual GQ lowering and optimized population barriers, entity rehydration, enforcement of the decided arithmetic/null/overflow rules and resource ownership. The native probe models the report filter; it does not retrieve reports. | +| [C2](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-compiler/src/query/staged_probe/composition_c2.json) | Group retains project identity and reduced metric origin while dropping member bindings/order. Native selection of p1/p2 yields three binding rows and two distinct passages; removing the cut admits p3 and changes both counts. | Graph target-ID mapping, GQ aggregate lowering, general equality/null semantics, distinct-state memory and the complete optimized graph/retrieval plan. | +| [C3](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-compiler/src/query/staged_probe/composition_c3.json) | The scorer creates a separate feature with no candidate window or rank; dense output remains the comparator. A native fixture preserves p2 with absent lexical-arm rank and a positive feature; filtering on lexical membership incorrectly drops it. Zero/null feature inputs also survive. The Decimal oracle separately checks partial/zero/null feature values. | Production scorer and enforcement of the decided nonmatch/empty/missing-field policy, fixed live statistics and numeric parity. Precomputed native features prove neither BM25 values nor scorer cost. | +| [C4](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-compiler/src/query/staged_probe/composition_c4.json) | Checked imports/exports and separate child source IDs. Native ordered object lists preserve two rows for A and an empty B; presence markers distinguish absent objects from present null payloads. Duplicate paths consume collection rows, and fan-out exceeds the source window. An aggregate detects multiple owners. | Correlated GQ lowering, generation/preservation of parent-row identity, actual typed cardinality refusal, full entity-object projection, total row ties and shared budget/cancellation under many parents. Detection of ambiguous owners is not the query refusal path. | + +The [native probe](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph/tests/rrf_prefilter_gate/composition.rs) +uses DataFusion 54.0.0 / Arrow 58.3.0, memory inputs, one/four partitions, +forward/reversed one-row batches and two zero hash-join thresholds. It does +not qualify Lance providers, arbitrary optimizer configurations, snapshot +pinning, cancellation or allocations. See the +[checkpoint](#composition-plan-and-primitive-checkpoint) for reproduction; +GQT takes ownership when these operators become executable. + +C1/C2 use typed +[aggregation](https://docs.rs/crate/datafusion-expr/54.0.0/source/src/logical_plan/builder.rs), +[filtered expressions](https://docs.rs/crate/datafusion-expr/54.0.0/source/src/expr_fn.rs) +and [distinct count](https://docs.rs/crate/datafusion-functions-aggregate/54.0.0/source/src/count.rs). +Count matching rows: `count(predicate)` also counts false. Entity rehydration +and preservation of cuts remain OmniGraph lowering responsibilities. + +C4 must bridge [ArrayAgg's empty-input null](https://docs.rs/crate/datafusion-functions-aggregate/54.0.0/source/src/array_agg.rs) +to a typed empty list. Aggregate actual child rows by parent-row identity with +explicit order, then left-join to parents and construct empty lists for absent +groups. A left join before aggregation can create phantom `[null]` or `[{}]`. +Optional objects need a presence marker: null properties do not mean no row. +The native object-list probe exercises this distinction; the current JSON +writer omits null fields. These tests do not decide a new wire contract or +prove bounded nested execution. + +### Graph-wide discovery across entity types + +**Phase 0 disposition:** defer cross-type union execution, all-node search, +all-edge search and mixed-kind search from the initial release. Concrete +node targets, and concrete edge targets reached through graph traversal, +retain their specified scope. A loop of independent client queries is not +the promised one-snapshot global query. This decision keeps the coordinated +release focused on the existing search correctness failures while the common +language acquires the union, narrowing and heterogeneous result machinery. + +Before stabilizing the initial grammar/IR, preserve these extension boundaries: +target identity carries entity kind and accepted type/incarnation as well as +entity ID; a logical source/window is independent of physical table count; +type scope and representation selection are separate typed plan facts. +Common-property projection requires compatible property types in every union +arm, with nullability widened when necessary. A property absent from an arm +requires explicit type narrowing or branch projection, never implicit null +or stringification. Union construction preserves binding rows; deduplication +and target selection remain explicit. An all-node expansion is the union of +the snapshot's concrete node scans, with each node identity once before any +graph fan-out. No wildcard or union punctuation is accepted by this decision; +its parser/type/plan compatibility proof remains a Phase 0 gate. + +Here, global search means discovery across eligible entity types in one +logical graph at one accepted snapshot. Its scope includes the target types, +searchable properties, representation policies and authorization rules. A +convenience stored query can own that scope; schema-driven expansion must +resolve it against the accepted catalog and include the resolved scope in its +query identity. Adding a searchable type can then change that explicitly +defined population. Ordinary unindexed or non-searchable fields do not acquire +an implicit text conversion or embedding recipe. + +An all-node selector must expand to every accepted node type without requiring +applications to enumerate the alternatives. Keep this type scope separate from +the representation selector: the latter resolves analyzed properties, field +reductions and compatible embedding spaces. A node type with no selected usable +representation remains in the declared scope but cannot contribute a ranked +hit. Coverage must distinguish this absence from an evaluated non-match; +unknown counts remain explicit unless exact coverage was requested. An index +is not what makes a property searchable, and missing indexes must not remove a +type from the logical scope. All-edge scope is a separate explicit choice. + +Resolve both selectors against the query's accepted snapshot and include their +expanded identities in execution identity. A newly added node type is therefore +included in a subsequent all-node query, while a read pinned to an older snapshot +retains that snapshot's type and representation scope. This automatic scope +expansion does not authorize implicit stringification, mixing vector spaces, or +resetting resource allowances per type. The wildcard spelling and typed +representation-selector grammar remain part of the general query-language +extension; a wildcard alone does not make `$hit.title` valid on every type. + +The current compiler binds a variable to one concrete node type. Schema +interfaces provide declarations and inheritance, but the query binder does +not yet execute polymorphic scans or a heterogeneous union. The initial +same-binding fusion contract alone therefore does not provide a single +cross-type global search query. This is an expressiveness gap to resolve +before advertising that capability. + +The preferred extension is a general typed union of graph bindings. Each +branch preserves the entity kind, accepted type/incarnation identity and +entity id; a hit remains its original node or edge. Common projections and +type-specific property access need ordinary query-language typing and type +narrowing. No synthetic stored `Document` entity or separate search registry +is needed. Source metrics and matched-property attribution remain attached to +their producing source. Two types with equal id strings are still different +targets. Passage-to-owner mapping remains the separate explicit graph mapping +and reduction problem described above. + +Ranking this union requires a declared cross-type policy: + +- A lexical source over a common logical text representation needs compatible + analysis, a defined treatment of multiple fields per target, and statistics + over that logical corpus. Per-table BM25 scores cannot be merged as if each + table had used those shared statistics. Multiple property hits must not + accidentally duplicate a target or let traversal fan-out add relevance. +- Compatible vector representations can share a distance comparator when + space/revision, query encoding, dimensions, metric and numeric rules agree. + Different spaces remain separate named sources, with their query encoding + calls included in the shared execution budget. +- Fusion can combine sources that now name this same typed target universe. + RRF over disjoint per-type lists is also a possible explicit policy, but it + gives each equally weighted type's first result the same contribution. It + is a source-balancing choice, not proof of comparable relevance. A common + scorer or qualified reranker is needed when the product requires a single + relevance ordering that those source ranks do not supply. + +Logical source count and physical table count are separate. Compatible tables +can be partitions of one logical source with one global candidate window. +Creating an independent fusion vote for every table would let physical schema +layout influence relevance. All partition scans and merges still consume the +same query budget; a small logical source count does not bound physical work. + +Physically, Lance remains responsible for each version-pinned dataset and +qualified index/scan path. DataFusion can union narrow candidate relations, +group by the full typed identity, rank/fuse and apply the global cut; hydrate +payloads from the corresponding pinned datasets after selection. For an exact +source split into disjoint physical partitions, local top-k followed by a +global top-k is valid only when every partition uses the same score and total +comparator as the global source. A partition cannot substitute local corpus +statistics or a different tie rule. Complete per-target field reduction and +deduplication before such a cut. If a target's score still depends on +contributions from several partitions, pruning those contributions requires a +valid global-score bound; ordinary local top-k is insufficient. ANN remains +explicitly approximate. + +Rescoring a shortlist selected with table-local BM25 statistics cannot repair +this error: the global winner may already have been discarded. Lance 11's +public `InvertedIndex::bm25_stats_for_terms` and `bm25_search` with a supplied +`MemBM25Scorer` provide a lower-level shared-statistics path to investigate. +They do not establish the complete contract: the scorer is native float32, +the RFC requires different fuzzy-group scoring, and immutable index statistics +can still count deleted rows. Reconcile live corpus statistics, uncovered +rows and invalidated coverage before using them for a global candidate cut. +Any cache remains derived from the accepted snapshot and representation +identities. Native index statistics alone are not a new source of truth. + +Before enabling that extension, qualification must cover two unrelated node +types, an explicitly selected edge type, equal id strings across types, multiple searchable fields, +incompatible vector spaces, empty/unavailable sources, snapshot changes, +policy and shared-resource refusal. Compare the physical fan-out plan with an +independent evaluator over the full logical union. Include disjoint-source +RRF behavior in the retrieval-task evaluation. The grammar, typed result and +global-corpus rules require an explicit extension; this section records its +design direction and does not claim implemented support. + +## Invariants + +- **Query semantics are typed structures (9):** every rule here is a + parser, type or plan rule; no semantics ride in strings, settings or side + tables. Graph `not { ... }` keeps its correlated scope; scalar `not(...)` + is a separate typed expression. +- **Integrity failures are loud (8):** integer overflow, non-finite results, + chained comparisons, unknown constructors and unresolved aliases are typed + failures. +- **Bounded resource use (11):** deferred operators (`collect`, `optional`, + intermediate grouping) enter the language only with one whole-query budget + and explicit empty-group, duplicate and cardinality rules. + +No invariant is weakened; no deny-list item is touched. + +## Compatibility and reversibility + +The rules under [Language evolution and compatibility](#language-evolution-and-compatibility) +are the compatibility contract: after the coordinated cutover, additive syntax +preserves parsing, name resolution, typing, populations and result schema of +every valid query. The GQ grammar is a versioned compatibility surface under +the [compatibility surfaces](2026-09-14-compatibility-surfaces.md) RFC; a +removed rule is a major bump after a deprecation release. RFC 0056 treats +every keyword as an atomic rule disambiguated by position, while this +document proposes contextual, token-bounded keywords (`yield yield`); one +rule must be chosen for both before either lands. Reverting this document +before any operator ships is deleting it. + +## Alternatives + +- **Freeze the staged retrieval syntax first and fit analytical composition + around it later.** Rejected: C1–C4 show noncommuting operations (aggregate + before/after a cut, quota versus global top-K) that a grammar designed for + one `rank` shape would have to special-case afterwards. +- **A general relation-valued variable model.** RFC 0048's alternatives + reject it for search; the same holds here: explicit stages with declared + exports are cheaper than a new value category. +- **Ship an all-node wildcard now over per-type fusion.** Rejected until the + typed-union, narrowing and shared-statistics rules above are proved; a + wildcard that silently strings together per-table scores would make + physical layout influence relevance. + +## Evidence and tests + +### Composition plan and primitive checkpoint + +On the evidence branch, starting from `1fb0423d`, the C1–C4 extension passes 367 compiler +tests and adds four golden assertions to existing tests. One native composition +test passes four memory/partition/order configurations under Rust 1.97.1, +DataFusion 54.0.0 and Arrow 58.3.0. Focused Clippy, formatting and docs checks +pass. [C1–C4](#required-composition-examples) links the plans, probe and per-case +limits; those are the evidence authority. + +The logical goldens and hand-built native plans are separate. Materialized +ranks/features and fixture parent IDs do not prove retrieval, scoring, +correlation or generated identity. Ambiguity detection does not implement +typed cardinality refusal. This checkpoint neither completes Phase 0 nor +repairs the four GQT regressions. + +```bash +cargo +1.97.1 test --locked -p omnigraph-compiler +cargo +1.97.1 test --locked -p omnigraph-engine --test rrf_prefilter_gate staged_composition +``` + +The staged compiler, its golden plans and the DataFusion composition probes +live on the evidence branch; when the production compiler owns these +constructs, `.gqt` cases own their observable behaviour and the prototype is +deleted, never maintained beside the compiler. + +## Rollout + +No production work starts from this document. Its rules gate RFC 0048's +syntax stabilization (the `Foundation` rows of the matrix) and the milestone +below follows RFC 0048's initial release. + +### Next delivery milestone: composed analytical answers + +After the initial release, implement C1, C4 and C2's general distinct/intermediate +group extensions using the accepted common grammar, IR and execution context. +C3 remains a separate scorer extension. Global search is also deferred; its +union, representation expansion and cross-type ranking require their own +implementation milestone after the compatibility proof. + +Build reusable computation/grouping first, then local selection and correlated +retrieval, then optional facts and typed collections. Preserve group identities, +computed facts, import/export scopes, empty results and one cumulative budget. +Extend existing compiler, traversal/projection/aggregation, GQT and transport +owners; retain compatibility with the initial release. + +**Exit:** full C1/C2/C4 journeys through public reads, with inferred/executed +types, deterministic selection, snapshot-bound follow-up and shared-budget +refusal. C1 selects A's increase of six over B's one; C2 counts three paths and +two distinct passages; C4 preserves B with a null owner and empty reports. + +**Open:** inspect integrated and optimized plans against independent +empty-group, duplicate-binding, null, wrong-group and skewed-fan-out controls. +API availability and the Phase 0 design argument do not prove safe correlation, +reuse or bounded materialization. Prove preservation of outer rows and budgets +before optimizing batching, and measure per-group rescan cost. + +## Unresolved questions + +1. Complete Phase 0 grammar/type/plan qualification for the stated stage + and metric scopes, per-group key types and tuple ordering, and any + user-defined selection tie keys. The partial compiler + prototype does not close the full result-schema and aggregation contract. + Null-bucket and multiple-membership semantics are specified above; prove + their lowering and retain RFC 0040 namespace coordination. Complete the + C1–C4 syntax/type/plan proofs, including the analytical population and + deferred correlation/scoring boundaries, before syntax stabilization. +2. General all-node/type-union grammar, compatible representation expansion, + type narrowing and heterogeneous projection. Execution is explicitly + deferred; the compatibility proof remains due in Phase 0. The initial + same-binding fusion implementation cannot satisfy cross-type discovery. +3. The keyword rule shared with RFC 0055 and RFC 0056 (atomic versus + contextual), and where the shared expression grammar lives so that + control statements and query bodies parse scalars identically. + +## Decision log + +- 2026-09-18 — split out of RFC 0048 at its 2026-09-13 revision; text moved + verbatim, links repointed, no rule changed. +- 2026-09-11/12 — added explicit `yield` integration, C1–C4 logical plans and + native population/collection counterexamples. Phase 0 remains incomplete. +- 2026-09-10 — made mixed analytical/graph/retrieval tasks the agent objective; + added language-evolution and capability matrices, composition gates and + deferred analytical/nested delivery. diff --git a/docs/rfcs/README.md b/docs/rfcs/README.md index a548bb736..6ef253802 100644 --- a/docs/rfcs/README.md +++ b/docs/rfcs/README.md @@ -219,4 +219,6 @@ then dated RFCs by date. | [2026-09-09](2026-09-09-identity-credentials-and-applied-policy.md) | Identity credentials and applied policy authorization | maintainer | accepted | complete | | [2026-09-14](2026-09-14-compatibility-surfaces.md) | Compatibility surfaces | maintainer | draft | not-started | | [2026-09-16](2026-09-16-session-settings.md) | Session settings | maintainer | draft | in-progress | +| [2026-09-18](2026-09-18-analyzed-lexical-search.md) | Analyzed lexical search: schema-owned matching and ranking | public | draft | not-started | +| [2026-09-18](2026-09-18-gq-composition-and-language-evolution.md) | GQ composition and language evolution | maintainer | draft | not-started | | [2026-09-24](2026-09-24-shared-expression-model.md) | Shared expression model | maintainer | draft | in-progress | From 765b89bee5762fe88497eac9309720be1603f6bb Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 18 Sep 2026 11:18:42 +0200 Subject: [PATCH 03/16] rfc: propose the kernel grammar, one stage per job Adds the kernel section to the GQ composition RFC: eight stage kinds (match, filter, let, rank, group, order, limit, return), exactly two cuts (a retriever's `candidates:` window and `limit`), subqueries as correlated expressions under a reduction (collect, one, exists, count), three open sets the typechecker resolves (pattern items, expression functions, retrieval sources) so extensions add no grammar rule, and atomic-positional keywords (the RFC 0056 rule). It collapses RFC 0048's `select`, `take`, `score`, `collect` and `optional` spellings, moves scalar predicates out of `match` (the mechanism behind the dropped-predicate class RFC 0047's T26 refuses), and gives sources a `ties:` option so trailing order keys no longer decide a cut. Cypher/GQL constructs and the agent retrieval needs are mapped onto it; the RFC examples are shown in kernel form. RFC 0048 cross-references the proposal as a pending decision; its own semantics and examples are unchanged. --- docs/rfcs/0048-search-contracts.md | 14 ++ ...8-gq-composition-and-language-evolution.md | 222 +++++++++++++++++- 2 files changed, 232 insertions(+), 4 deletions(-) diff --git a/docs/rfcs/0048-search-contracts.md b/docs/rfcs/0048-search-contracts.md index 728e26970..50e773def 100644 --- a/docs/rfcs/0048-search-contracts.md +++ b/docs/rfcs/0048-search-contracts.md @@ -304,6 +304,17 @@ description used as a Boolean predicate and introduces no rank or window. ### Clause composition and named stages +**Kernel alignment (proposed 2026-09-18).** The +[kernel](2026-09-18-gq-composition-and-language-evolution.md#kernel-one-stage-per-job) +in the composition RFC keeps this section's `rank` block, `yield`, named +arms and `metric()` unchanged and collapses the rest of this RFC's sketch: +`select` becomes `order` + `limit`, `take` becomes `limit n of $x per { … }`, +`score` becomes `let`, `collect`/`optional` blocks become `sub(…)` +expressions under a reduction, scalar predicates leave `match` for +`filter`, and stage tie keys are a `ties:` option on a source. The +semantics below are unchanged by that proposal; the spellings in the +examples are not yet rewritten to it. + The current grammar has one `match`, followed by `return`, optional `order`, and optional `limit`. The proposed extension admits explicit rank boundaries between graph blocks. Each `match` retains graph-pattern semantics. Stage @@ -1907,6 +1918,9 @@ still require prototypes; full production qualification belongs to its phase. ## Decision log +- 2026-09-18 — recorded the composition RFC's kernel proposal as a pending + decision for this RFC's syntax (one spelling per operation; `select`, + `take`, `score`, `collect`, `optional` collapse; predicates leave `match`). - 2026-09-18 — split the lexical contract into [Analyzed lexical search](2026-09-18-analyzed-lexical-search.md) and the language rules, capability matrix, C1–C4 and cross-type discovery into diff --git a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md index 7e2fdab3a..aefeab96e 100644 --- a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md +++ b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md @@ -27,7 +27,10 @@ records four composition examples (C1–C4) whose logical plans the grammar must leave room for. It also records the design direction for graph-wide discovery across entity types, which is explicitly deferred. -It decides no search semantics: [RFC 0048](0048-search-contracts.md) owns rank stages, sources, +The [kernel](#kernel-one-stage-per-job) section proposes the closed set of +stage kinds every query composes from, the open sets extension flows +through, and the spellings in RFC 0048's sketch it collapses so that each +operation has one form. It decides no search semantics: [RFC 0048](0048-search-contracts.md) owns rank stages, sources, fusion, selection and result metadata, and [Analyzed lexical search](2026-09-18-analyzed-lexical-search.md) owns analyzers, matching and scoring. It was split out of RFC 0048 on 2026-09-18 so that search decisions are not blocked on language decisions @@ -662,6 +665,207 @@ RRF behavior in the retrieval-task evaluation. The grammar, typed result and global-corpus rules require an explicit extension; this section records its design direction and does not claim implemented support. +### Kernel: one stage per job + +**Principle.** The language is a small kernel of stage kinds over binding +tables plus one expression grammar. Each kernel stage has one job and one +spelling. A convenience spelling is admitted only when it desugars to exactly +one kernel form, `explain` shows that form, and the RFC that adds the spelling +names the desugaring. A construct that cannot be desugared is a new kernel +stage and needs its own RFC. Extensibility comes from three open sets that +the typechecker resolves — pattern items, expression functions and retrieval +sources — never from new clause shapes, so a new retriever, scoring feature, +path selector or subquery reduction adds no grammar rule. + +**Stages.** A query is `match` followed by any sequence of stages, then +`return`; `order` and `limit` may also follow `return`, where they read the +projected aliases (the GQL result-statement convention). Every stage is a +function from a working table to a working table under one accepted snapshot +and one budget. + +| Stage | Job | Rows in → out | Rule | +|---|---|---|---| +| `match { pattern }` | Extend bindings by a graph pattern | Join; may fan out | Pattern items only: typed bindings with inline constraints (`{ name: "x" }`, `{ @id: $id }`), traversals and paths, `not { }`, `optional { }`. No scalar predicates. | +| `filter { predicate }` | Keep rows | Subset | The only home for scalar predicates, `match_terms` included. A `filter` after a cut is the "different question" the composition laws describe. | +| `let { expr as name, … }` | Add columns | Same rows | Scalar expressions, subquery reductions and membership-preserving scoring features (`lexical_score($p.text, terms($q))`), which replaces RFC 0048's `score` stage. Siblings read the incoming scope. | +| `rank $x { source … yield s }` | Select and order distinct targets | Ranked rows; losing targets drop | Sources are an open set; `candidates:` is the retriever's window; `ties: [key, …]` extends the stage comparator before stable identity; `yield` names the output. One form even for one source. | +| `group { per { … } reduce { … } }` | Aggregate | Group rows | The only aggregation. Exports keys and reductions; drops member bindings and the active order. | +| `order { key [asc\|desc] [nulls first\|last], … }` | Order rows | Same rows | Usable at any position; replaces the inner order of RFC 0048's `select`. | +| `limit n [of $x] [per { key, … }]` | Cut | Subset | The only cut. Unit is rows, or with `of` the distinct targets of `$x`, whose winning targets keep every binding row; `per` partitions by keys with the target/group-pair semantics RFC 0048 specifies for `take`. `of` and `per` require an active order (the latest `rank`, or a preceding `order`). Replaces `select`, `take` and the terminal `limit`. | +| `return { expr as name, … }` | Project | Result rows | Pure projection. Aggregates inside `return` are sugar for `group` with the non-aggregate projections as keys followed by `return`; `explain` shows the `group`. This is the one admitted sugar, kept because it is the Cypher/GQL convention agents already know. | + +**Cuts.** There are exactly two. A retrieval window (`candidates:` on a +source or a fusion) defines a candidate set inside `rank`; `limit` cuts a +working table. Nothing else discards rows, and neither knows about the other: +a small `limit` never resizes a window, a window never bounds output rows. + +**Subqueries are expressions.** `sub($import, …) { stages … return { … } }` +is a correlated subquery evaluated once per incoming row over the imported +bindings and the query parameters; its own bindings never escape. It is not a +value by itself and appears only under a reduction in `let` or `return`: +`collect(sub …)` gives a typed list (empty for no rows), `one(sub …)` a +nullable object (a cardinality error above one row), `exists(sub …)` a Bool +and `count(sub …)` an integer. Local `order` and `limit` inside the subquery +bound its rows. This one construct replaces RFC 0048's `collect (…) as …` +and `optional (…) as …` blocks and maps Cypher's `CALL { }`, `EXISTS { }`, +`COUNT { }` and `COLLECT { }` subqueries. `match optional { }` remains the +row-producing outer join (Cypher `OPTIONAL MATCH`), which fans out; the two +are different operations, not two spellings. + +**Spellings this removes from RFC 0048's sketch.** + +| RFC 0048 sketch | Kernel form | +|---|---| +| `select { order { … } limit n }` | `order { … }` then `limit n` | +| `take $i { per { $o.slug } order { … } limit 2 }` | `order { … }` (or the active rank) then `limit 2 of $i per { $o.slug }` | +| `score $p { lexical(…) as f }` | `let { lexical_score(…) as f }` | +| `collect ($s) as reports { … }` / `optional ($s) as owner { … }` | `let { collect(sub($s) { … }) as reports, one(sub($s) { … }) as owner }` | +| scalar predicates inside `match { }` | `filter { … }` after the `match` | +| `nearest`, `bm25` or `rrf` inside `order` | `rank` with one source; no sugar | +| a stage's own tie keys expressed as a later `order` | `ties:` on the source or fusion | + +Mixing predicates into `match` is not only a second spelling: it is the +mechanism behind the dropped traversal-target predicate that RFC 0047's `T26` +refuses. Separating pattern from predicate removes the class. + +**Open sets.** + +- *Pattern items*: typed bindings, including a type union (`$x: Person | + Organization`) whose common properties are accessible and whose arms are + narrowed by an `is` predicate; inline property and `@id` constraints; + directed and undirected traversals with edge bindings; paths with a + quantifier and a named selector (`shortest k`, `all`, with acyclic/trail + modes) bound to a path variable. Today's `{n,m}` bound with its implicit + shortest-distance semantics becomes the explicit `shortest` selector so the + semantics is named. A type union in a binding is also the union machinery + for graph-wide search: `rank $x { … }` over a union target ranks every arm + under the [cross-type rules](#graph-wide-discovery-across-entity-types), + and no query-level `union` statement is needed for it. +- *Expression functions*: scalar functions and casts, aggregates, subquery + reductions, scoring features, `metric(alias, rank | score | distance)`. +- *Retrieval sources*: `lexical`, `knn`, `ann`, `rrf`; later `rerank(arm(x), + model: …)`, sparse and multivector representations, geometric range. All + share the shape `kind(args, option: value, …) as alias`. +- *Value types*: lists, nullable objects, paths. Projection and typing rules + for each are the extension's obligation. + +Reserved extension slots that are stage kinds, not open-set members, and are +therefore future RFCs: `unnest { list as $item }` (Cypher `UNWIND`), +`distinct { keys }` (or `group` with keys only), a query-level `union`, and +`offset` on `limit` (Cypher `SKIP`). Each is one new row in the stage table. + +**Keyword rule.** Stage keywords are atomic and positional, as RFC 0056 +treats control statements: `match`, `filter`, `let`, `rank`, `group`, +`order`, `limit`, `return`, `yield` (inside `rank`), `per`, `reduce`, `of`, +`not`, `optional`. Everything else — source kinds, `terms`, `arm`, `metric`, +`sub`, reductions, aggregates — is an identifier the typechecker resolves as a +function, so adding one never touches the grammar and never collides with a +property name. This replaces the contextual, token-bounded keyword rule +proposed above and is the one rule for RFCs 0055, 0056 and 0048. + +**Cypher and GQL constructs on the kernel.** + +| Construct | Kernel | +|---|---| +| `MATCH` / `OPTIONAL MATCH` | `match { }` / `match optional { }` | +| `WHERE` | `filter { }` | +| `WITH` projection / aggregation / `ORDER BY … LIMIT` | `let` / `group` / `order`, `limit` — `WITH` is three jobs, spelled as three stages | +| `RETURN DISTINCT` | `distinct` extension slot, or `group` with keys only | +| `UNWIND` | `unnest` extension slot | +| `CALL { }`, `EXISTS { }`, `COUNT { }`, `COLLECT { }` subqueries | `sub(…)` under `one`/`exists`/`count`/`collect` | +| Path patterns, quantifiers, `SHORTEST`, path variables, path modes | Pattern items | +| Label disjunction, multiple labels | Type union in a binding, `is` narrowing | +| `CASE`, `coalesce`, functions | Expression grammar | +| `UNION` | Query-level `union` extension slot | +| `SKIP` | `offset` extension of `limit` | +| `CREATE` / `MERGE` / `SET` / `DELETE` | Mutation bodies; unchanged | +| `LOAD CSV`, procedures | RFC 0056 control statements | + +**Agent retrieval semantics on the kernel.** + +| Need | Kernel | +|---|---| +| Vector, lexical, typo-tolerant retrieval | `rank` with `knn`/`ann`/`lexical` | +| Hybrid retrieval | `rank` with named arms and `rrf` | +| Reranking | A `rerank` source consuming an arm | +| Typo-tolerant eligibility without ranking | `filter { match_terms(…) }` | +| A relevance feature on already selected rows | `let { lexical_score(…) as f }` | +| Graph-scoped candidates | `match`/`filter` before `rank` | +| Search, expand, search again | `rank`, `match`, `rank` | +| Diversity across owners | `limit n of $x per { … }` | +| Global search across types | Type-union binding as the rank target | +| Neighbourhood, connection explanation | Traversal bounds; path variables with `shortest` | +| Identity follow-up at a snapshot | `match { $x: Person { @id: $id } }` under the session's snapshot setting | +| Exhaustive verification beside a bounded search | `group`/`filter` over the eligible population in the same query | +| Budgets, coverage, replay | Session settings, outside the body | + +**The RFC examples in kernel form.** + +```gq +query find_organizations($q: String) { + match { $o: Organization } + rank $o { + lexical($o.name, terms($q, mode: any, max_edits: 1), candidates: 100) as words + ann($o.embedding, $q, oversample: 4, candidates: 100) as meaning + rrf(arm(words), arm(meaning, weight: 1.5), k: 60, candidates: 20) as combined + yield combined + } + return { $o.slug, $o.name, metric(combined, score) as score } + order { score desc, $o.@id asc } + limit 8 +} + +query incidents_per_organization($q: String) { + match { $o: Organization $o hasIncident $i } + rank $i { + lexical($i.title, terms($q), candidates: 100, ties: [$i.opened_at desc]) as incidents + yield incidents + } + limit 2 of $i per { $o.slug } + return { $o.slug, $i.slug, metric(incidents, rank) as rank } + order { $o.slug asc, rank asc, $i.@id asc } + limit 20 +} + +query composition_c4($q: String) { + match { $i: Incident $s hasIncident $i } + filter { $i.period = "prior" or $i.period = "current" } + group { + per { $s } + reduce { + count_if($i.period = "prior") as prior_count, + count_if($i.period = "current") as current_count + } + } + let { current_count - prior_count as increase } + order { increase desc, $s.@id asc } + limit 2 + let { + one(sub($s) { match { $s ownedBy $person } return { $person as person } }) as owner, + collect(sub($s, $q) { + match { $s hasReport $p } + rank $p { lexical($p.text, terms($q), candidates: 2) as relevant yield relevant } + return { $p.@id as id, $p.text as text } + order { metric(relevant, rank) asc, $p.@id asc } + limit 2 + }) as reports + } + return { + $s as service, prior_count, current_count, increase, owner, reports + } +} +``` + +Each example has one spelling per operation; the second no longer needs a +`take` block, and the third no longer needs `select`, `optional` or +`collect` stages. The semantics RFC 0048 specifies for pair selection, +missing arms and nested budgets are unchanged; only the spellings moved. + +**Decision needed.** Adopt the kernel as the grammar RFC 0048's Phase 0 +stabilizes against, and rewrite its sketch and the `take`/`score`/`select` +sections accordingly, or keep the separate stage kinds and record why each +extra spelling earns its place. + ## Invariants - **Query semantics are typed structures (9):** every rule here is a @@ -776,14 +980,24 @@ before optimizing batching, and measure per-group rescan cost. type narrowing and heterogeneous projection. Execution is explicitly deferred; the compatibility proof remains due in Phase 0. The initial same-binding fusion implementation cannot satisfy cross-type discovery. -3. The keyword rule shared with RFC 0055 and RFC 0056 (atomic versus - contextual), and where the shared expression grammar lives so that - control statements and query bodies parse scalars identically. +3. Whether the kernel's atomic-positional keyword rule (which also settles + the RFC 0056 tension) is adopted, and where the shared expression + grammar lives so that control statements and query bodies parse + scalars identically. +4. Whether `return` keeps its aggregate sugar as the one admitted + convenience form, and which of the reserved extension slots + (`unnest`, `distinct`, `union`, `offset`) the initial release must + already parse-and-refuse so that later additions are non-breaking. ## Decision log - 2026-09-18 — split out of RFC 0048 at its 2026-09-13 revision; text moved verbatim, links repointed, no rule changed. +- 2026-09-18 — proposed the kernel: eight stage kinds, two cuts, subqueries + as reduced expressions, three open sets, atomic-positional keywords; it + collapses `select`, `take`, `score`, `collect` and `optional` and moves + scalar predicates out of `match`. Recorded as a decision RFC 0048 must + take before its syntax stabilizes; the moved-in text above is unchanged. - 2026-09-11/12 — added explicit `yield` integration, C1–C4 logical plans and native population/collection counterexamples. Phase 0 remains incomplete. - 2026-09-10 — made mixed analytical/graph/retrieval tasks the agent objective; From 7ca000758f71f47b65a1bf31bd9eef463346d52e Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 18 Sep 2026 11:24:21 +0200 Subject: [PATCH 04/16] rfc: fix the pattern/predicate rule in the kernel (SQL ON/WHERE rule) --- ...8-gq-composition-and-language-evolution.md | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md index aefeab96e..401baebab 100644 --- a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md +++ b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md @@ -685,7 +685,7 @@ and one budget. | Stage | Job | Rows in → out | Rule | |---|---|---|---| -| `match { pattern }` | Extend bindings by a graph pattern | Join; may fan out | Pattern items only: typed bindings with inline constraints (`{ name: "x" }`, `{ @id: $id }`), traversals and paths, `not { }`, `optional { }`. No scalar predicates. | +| `match { pattern }` | Extend bindings by a graph pattern | Join; may fan out | Pattern items only: typed bindings with inline constraints (`{ name: "x" }`, `{ @id: $id }`), traversals and paths, `not { }`, `optional { }`. A scalar predicate is admitted only inside `optional { }` and `not { }`, where its placement changes the answer; nowhere else. | | `filter { predicate }` | Keep rows | Subset | The only home for scalar predicates, `match_terms` included. A `filter` after a cut is the "different question" the composition laws describe. | | `let { expr as name, … }` | Add columns | Same rows | Scalar expressions, subquery reductions and membership-preserving scoring features (`lexical_score($p.text, terms($q))`), which replaces RFC 0048's `score` stage. Siblings read the incoming scope. | | `rank $x { source … yield s }` | Select and order distinct targets | Ranked rows; losing targets drop | Sources are an open set; `candidates:` is the retriever's window; `ties: [key, …]` extends the stage comparator before stable identity; `yield` names the output. One form even for one source. | @@ -728,6 +728,24 @@ Mixing predicates into `match` is not only a second spelling: it is the mechanism behind the dropped traversal-target predicate that RFC 0047's `T26` refuses. Separating pattern from predicate removes the class. +**Pattern versus predicate.** A pattern item creates rows: it introduces or +joins bindings and its output is every way the shape matches, unbounded by +its input. A predicate removes rows: it is a Boolean over bindings that +already exist, evaluated row by row, and a predicate over an unbound +variable is a type error (Datalog's safety condition). Every mature +language keeps both a pattern-scoped and a table-scoped position, because +the two differ exactly across outer joins, negation, aggregation and cuts: +SQL's `ON` versus `WHERE` on a `LEFT JOIN` and `WHERE` versus `HAVING`, +Cypher's `OPTIONAL MATCH … WHERE` versus a later `WITH … WHERE`, GQL's +pattern `WHERE` versus its `FILTER` statement, SQL/PGQ's `GRAPH_TABLE` +`WHERE` versus the enclosing `WHERE`, SPARQL's group-scoped `FILTER`. The +rule here is the SQL `ON`/`WHERE` rule: a predicate may sit inside a pattern +block only in the sub-patterns where moving it outside would change the +answer, `optional { }` and `not { }`; inline constraints remain pattern +properties; every other predicate is a `filter` stage. Gremlin, which makes +no distinction and lets step order carry the meaning, is the counterexample: +the planner cannot tell a shape from a selection. + **Open sets.** - *Pattern items*: typed bindings, including a type union (`$x: Person | @@ -998,6 +1016,10 @@ before optimizing batching, and measure per-group rescan cost. collapses `select`, `take`, `score`, `collect` and `optional` and moves scalar predicates out of `match`. Recorded as a decision RFC 0048 must take before its syntax stabilizes; the moved-in text above is unchanged. +- 2026-09-18 — fixed the pattern/predicate rule as SQL's `ON`/`WHERE` rule: + predicates inside a pattern block only within `optional { }` and + `not { }`, inline constraints as pattern properties, `filter` everywhere + else. - 2026-09-11/12 — added explicit `yield` integration, C1–C4 logical plans and native population/collection counterexamples. Phase 0 remains incomplete. - 2026-09-10 — made mixed analytical/graph/retrieval tasks the agent objective; From c35a97a2f198ac81f6da84ee82e3ecb323143d37 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 18 Sep 2026 11:44:26 +0200 Subject: [PATCH 05/16] rfc: design the search surface for an in-context agent caller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The caller is a general agent trained with RL elsewhere, using the language in context; nothing here assumes training on it or proximity to a prior language. - RFC 0047: a diagnostics contract for every compile diagnostic and typed read failure — stable code, position or stage, expectation, one fix; unknown names enumerate their set. No catalogue of foreign idioms. - RFC 0048: an agent-facing surface (stored queries as the typed door, `@description` on schema declarations surfaced by `schema show`, named defaults); a reader decision rule putting completion, coverage, selection and usage in the response body, with usage as contract; the in-context competence metric in the mixed-workload qualification. - Composition RFC: the in-context consumer as the kernel's design target (card-sized, regular, one-turn repair), case-insensitive atomic keywords, and an empirical instrument — first-try validity, turns to a correct query, repairs by code, over ground truth the graph computes itself — as the tie-break for spellings and the gate for new stages. --- docs/rfcs/0047-search-plan-truth.md | 21 +++++++ docs/rfcs/0048-search-contracts.md | 60 ++++++++++++++++++- ...8-gq-composition-and-language-evolution.md | 33 +++++++++- 3 files changed, 112 insertions(+), 2 deletions(-) diff --git a/docs/rfcs/0047-search-plan-truth.md b/docs/rfcs/0047-search-plan-truth.md index 02486d2e3..78fef8a2b 100644 --- a/docs/rfcs/0047-search-plan-truth.md +++ b/docs/rfcs/0047-search-plan-truth.md @@ -142,6 +142,24 @@ a bare, case-sensitive tokenizer; neither serves after this change. The analyzer-equivalent scan that later lifts the plan-time refusal is the lexical RFC's exact baseline. +**Diagnostics contract.** Every compile diagnostic (`T…` code) and every +typed execution failure a read can produce carries four fields: a stable +code; a source position, or the stage and expression when the failure is +post-parse; what was expected or violated; and one concrete fix. The fix +names the construct to use, not the rule that was broken: "`search` requires +`@index` on `Doc.title`; declare it and run index reconciliation"; +"`binding_rows` exhausted at stage 3 (`match`); narrow the population before +`rank` or raise the limit". A failure with no fix names the decision instead +("a retry will not help; the snapshot is unavailable"). An unknown name +enumerates the set it failed against: functions, sources, metric fields, +settings, or the properties of the bound type. The reader is an agent that +treats an error as the documentation it acts on and a retry as its default +response; the contract makes one repair turn the norm and a blind retry the +exception. The human CLI prints the same four fields. The contract does not +depend on recognizing other languages' idioms and no catalogue of them is +maintained; a reader that generalizes from the schema and the card is what +the design leans on. + **Metric projection.** ```gq @@ -450,6 +468,9 @@ language release or duplicate execution path. blanket claim that all RRF arm windows follow the output limit: BM25 arms are uncapped; vector arms inherit that limit. Distinguished historical prototype results from current evidence and total ordering from ANN replay. +- 2026-09-18 — added the diagnostics contract (code, position or stage, + expectation, one fix; unknown names enumerate their set) for every + compile diagnostic and typed read failure. - 2026-09-18 — replaced the `full_text_search_unindexed` warning with a compile-time `T27` refusal of undeclared targets and a plan-time `FullTextIndexRequired` refusal of unbuilt indexes (RFC 0043 precedent); a diff --git a/docs/rfcs/0048-search-contracts.md b/docs/rfcs/0048-search-contracts.md index 50e773def..8750dd4c8 100644 --- a/docs/rfcs/0048-search-contracts.md +++ b/docs/rfcs/0048-search-contracts.md @@ -497,6 +497,38 @@ fact verification uses ordinary predicates, negation, and aggregates over the intended population. A completed search with no hits cannot prove that population contains no relevant fact. +### Agent-facing surface + +The primary caller is a general agent that learned tool use elsewhere and +meets this language in context, through a schema, a one-page card and its +own errors; it is not a model trained on the language. That fixes three +priorities. + +- **Stored queries are the door; the language is the authoring surface.** A + stored query is a typed tool: name, parameters, `@description` and + `@instruction`. Its description states the population it searches, what it + returns and the one trade-off the caller can turn ("`candidates` trades + recall for cost; raise it when the answer must be exhaustive"). Recipes + bake in windows, modes and edit budgets; the raw language exposes them for + the long tail and for authoring. +- **The schema is the prompt.** Node, edge and property declarations in + `.pg` accept `@description("…")`, and `schema show` prints it beside the + resolved analyzer, scorer and embedding recipe, so a caller grounds a query + in one read. The annotation is metadata: it changes no accepted semantics + and no field binding, and it is a PG-language surface change under the + [compatibility surfaces](2026-09-14-compatibility-surfaces.md) RFC. +- **Defaults carry the common case.** Every knob has a default the + description names; a caller that never touches `candidates`, `oversample` + or `max_edits` gets a sound query, not a surprising one. + +The language is designed for that use: the +[kernel](2026-09-18-gq-composition-and-language-evolution.md#kernel-one-stage-per-job) fits on a card, every stage has +one shape, keywords are atomic and case-insensitive, and diagnostics follow +RFC 0047's contract so that one repair turn is the norm. None of it assumes +a particular prior language; the caller's ability to generalize from a card +and a schema is what the design leans on, and that ability improves faster +than any catalogue of foreign idioms could be maintained. + ### Errors and operational changes Typed errors include missing analyzed capability, disabled or incompatible @@ -506,6 +538,13 @@ mappings or inherited metrics, and exhausted resources. Unsupported plan shapes results are presented as complete. Ordinary absence of a hit in one fusion arm is represented as missing membership, not a query failure. +Every such failure follows [RFC 0047's diagnostics contract](0047-search-plan-truth.md#user-and-operational-behavior): +a stable code, a position or stage, the expectation, and one fix. An unknown +source kind, function, metric field or setting enumerates the admitted set; +a resource failure names the stage and the remedy (narrow the population, +lower a window, raise a limit) so that a retry without change is never the +implied response. + All search-language replacements ship together. Diagnostics point callers from `fuzzy`, `search`, and `match_text` to typed lexical matching or retrieval; from retrieval in `order` and `nearest` to explicit rank stages and `knn`/`ann`; @@ -966,6 +1005,17 @@ wire types and namespace choices must be checked before implementation: | Attribution | Graph snapshot context, graph binding identities, and selected properties; application-defined source relationships remain ordinary data | | Follow-up | A supported way to read/expand those bindings at that snapshot, or an explicit expired/unavailable outcome | +**Reader decision rule.** Every descriptor is in the response body — JSON, +the JSONL metadata record and Arrow response metadata alike, never only a +transport header — and is phrased for the decision its reader takes next. +Completion answers "trust or fail"; coverage answers "rephrase, narrow, or +stop" (no match over a fully represented population is a different fact +from an unrepresented one, and the reader must not learn to rephrase on +both); selection answers "widen the window or not"; usage answers "can I +afford another call". Usage — elapsed time, rows produced, storage reads, +provider calls, output bytes — is part of the read contract, not optional +diagnostics, because the reader budgets its next call from it. + **Interface decision:** the read options are session settings, not a new request field. [Session settings](2026-09-16-session-settings.md) and its first implementation (#742) give every operation an explicit, typed @@ -1567,7 +1617,11 @@ owners and extend their independent oracles at the boundary being tested. For runnable families, record supported task correctness, query parse/type failures and repair attempts, wrong-population answers, tool round trips, -latency, returned/context bytes and measured execution work. Preserve refusals, +latency, returned/context bytes and measured execution work. Record +in-context competence per model with the composition RFC's +[instrument](2026-09-18-gq-composition-and-language-evolution.md#in-context-competence): first-try parse validity, turns +to the first correct query and repairs by diagnostic code, beside task +correctness. Preserve refusals, timeouts, provider failures and unsupported requests in the report with distinct dispositions; successful completed queries alone are not the task denominator. Separate deterministic result oracles from judged relevance and @@ -1918,6 +1972,10 @@ still require prototypes; full production qualification belongs to its phase. ## Decision log +- 2026-09-18 — added the agent-facing surface (stored queries as the door, + `@description` on schema declarations, named defaults), the reader + decision rule for result descriptors with usage as contract, and the + in-context competence metric in the mixed-workload qualification. - 2026-09-18 — recorded the composition RFC's kernel proposal as a pending decision for this RFC's syntax (one spelling per operation; `select`, `take`, `score`, `collect`, `optional` collapse; predicates leave `match`). diff --git a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md index 401baebab..42f15d997 100644 --- a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md +++ b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md @@ -677,6 +677,17 @@ the typechecker resolves — pattern items, expression functions and retrieval sources — never from new clause shapes, so a new retriever, scoring feature, path selector or subquery reduction adds no grammar rule. +**Designing for an in-context consumer.** The caller is a general agent +that learned tool use elsewhere and meets this language through a schema, a +one-page card and its own errors. The kernel is therefore judged by three +measurable properties: it fits on the card (eight stage kinds, three open +sets); it is regular (one shape per stage, atomic case-insensitive keywords, +no contextual spellings); and its diagnostics repair in one turn (RFC 0047's +contract). Proximity to any particular existing language is not a goal. +Where two spellings are otherwise equal, the one a fresh reader guesses +right more often wins, and that is measured, not argued (see +[in-context competence](#in-context-competence)). + **Stages.** A query is `match` followed by any sequence of stages, then `return`; `order` and `limit` may also follow `return`, where they read the projected aliases (the GQL result-statement convention). Every stage is a @@ -779,7 +790,10 @@ treats control statements: `match`, `filter`, `let`, `rank`, `group`, `sub`, reductions, aggregates — is an identifier the typechecker resolves as a function, so adding one never touches the grammar and never collides with a property name. This replaces the contextual, token-bounded keyword rule -proposed above and is the one rule for RFCs 0055, 0056 and 0048. +proposed above and is the one rule for RFCs 0055, 0056 and 0048. Keywords +are case-insensitive (`MATCH` and `match` are one token): letter case +carries no meaning, so accepting both removes an error class without adding +a spelling. Identifiers, property names and string literals keep their case. **Cypher and GQL constructs on the kernel.** @@ -953,6 +967,20 @@ live on the evidence branch; when the production compiler owns these constructs, `.gqt` cases own their observable behaviour and the prototype is deleted, never maintained beside the compiler. +### In-context competence + +Because the consumer is fixed and available, syntax choices are decided +empirically. The instrument is a schema with descriptions, the one-page +card, and a task set whose ground truth is computed by exact queries at a +pinned snapshot: the graph is its own verifier. For each candidate spelling +and each model under test, record first-try parse validity, turns to the +first correct query, task success, tokens consumed, and the diagnostic code +behind each repair. A spelling wins on those numbers, not on analogy. The +same instrument is the acceptance gate for a new stage or source and feeds +RFC 0048's [mixed workload qualification](0048-search-contracts.md#mixed-workload-qualification). +Hold the model, card, schema and snapshot fixed within a comparison and +retain failed trajectories; a model revision reopens the measurement. + ## Rollout No production work starts from this document. Its rules gate RFC 0048's @@ -1016,6 +1044,9 @@ before optimizing batching, and measure per-group rescan cost. collapses `select`, `take`, `score`, `collect` and `optional` and moves scalar predicates out of `match`. Recorded as a decision RFC 0048 must take before its syntax stabilizes; the moved-in text above is unchanged. +- 2026-09-18 — made the in-context consumer the design target: card-sized + kernel, case-insensitive atomic keywords, spellings decided by measured + first-try validity rather than analogy to any prior language. - 2026-09-18 — fixed the pattern/predicate rule as SQL's `ON`/`WHERE` rule: predicates inside a pattern block only within `optional { }` and `not { }`, inline constraints as pattern properties, `filter` everywhere From 98d49d896b455843af09365dcda11214eda853eb Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 18 Sep 2026 11:55:40 +0200 Subject: [PATCH 06/16] rfc: expressions over named retrieval, and the planner's rewrite catalogue RFC 0048: combination is an expression, retrieval is named. `fuse(expr, candidates:)` is the general fusion source, evaluated over the union of named arms; `rrf` is its named, qualified policy `rrf_v1`. Explicit mixing only, missing arms explicit (coalesce or nulls placement), structural identity for inline formulas, models enter as budgeted sources, control flow stays outside the query. A rewrite catalogue (always-legal rewrites, barriers, cost-based choices with visible statistics and differential oracles, the `ann`-only approximation boundary, in-attempt adaptivity, plan cache) and an `explain` contract are handed to the engine version 2 planner. Composition RFC: programmability through transparent `define`, a typed plan input surface and multi-statement requests at one snapshot; procedural loops, opaque functions and free inference calls excluded. --- docs/rfcs/0048-search-contracts.md | 98 +++++++++++++++++++ ...8-gq-composition-and-language-evolution.md | 36 ++++++- 2 files changed, 132 insertions(+), 2 deletions(-) diff --git a/docs/rfcs/0048-search-contracts.md b/docs/rfcs/0048-search-contracts.md index 8750dd4c8..6de4985df 100644 --- a/docs/rfcs/0048-search-contracts.md +++ b/docs/rfcs/0048-search-contracts.md @@ -877,6 +877,65 @@ changing it can legitimately change scores. It must not silently normalize against the entire corpus. Learned model execution and general feature combination syntax are deferred until these contracts are qualified. +**Decision (2026-09-18): combination is an expression, retrieval is named.** +The asymmetry the algebra needs is between retrievers and scorers. A +retriever produces candidates and needs a qualified physical path (index +parity, coverage, budgets), so retrievers are a closed, named set. A scorer +or combiner runs in-engine over candidates that are already bounded, so it +can be any expression of the kernel's expression grammar: pure, total, +typed, linear in its inputs. The general fusion source is therefore + +```gq +fuse(, candidates: N) as alias +``` + +evaluated over the union of the named arms' candidate sets, ordered by the +expression's value with stable identity as the final key, cut at +`candidates`. `rrf(arm(a), arm(b, weight: w), k: 60, candidates: N)` is the +named policy `rrf_v1` of that form: qualified, accelerable and fingerprinted +as a policy, and defined as the `fuse` expression `sum over arms of +weight / (k + metric(arm, rank))` with a missing arm contributing zero. The +initial release ships `rrf`; `fuse` is the extension whose semantics are +fixed here so that `rrf` is an instance of it rather than a special case, +and so that a caller can write a fusion formula on the fly without a new +operator. + +Rules that make an open expression safe in that position: + +- *Explicit mixing only.* `metric(a, score)` and `metric(b, distance)` carry + their domains; an arithmetic expression over metrics yields a plain `F64` + by the author's act of writing it. The typed domains exist to refuse + *implicit* mixing: `order { metric(a, score) desc, metric(b, distance) asc }` + is two orderings, not one combined score. +- *Missing arms are explicit.* A target absent from an arm has a null metric + there; an expression that can be null is refused as a `fuse` argument or + an `order` key unless wrapped in `coalesce` or the ordering states `nulls + first`/`nulls last`. A missing arm can never silently drop or demote a + target. +- *Identity is structural.* The fingerprint of an inline expression is the + hash of its normalized typed AST, so `explain`, replay descriptors and the + semantic digest treat an on-the-fly formula exactly as a named policy; a + named policy additionally records its qualification. +- *Bounded by construction.* No recursion, no loops, total functions under + the [numeric rules](2026-09-18-gq-composition-and-language-evolution.md#language-evolution-and-compatibility): checked + integer arithmetic, non-finite results as typed failures. Cost is the + candidate union times the expression size, charged to the stage. +- *Models are sources, not calls.* A learned reranker enters as a source + (`rerank(arm(x), model: …, candidates: N)`) with a declared model identity, + a budget and a window, never as a free function inside an expression; the + same holds for any inference feature. Nondeterministic and remote work is + retrieval-side work under retrieval-side contracts. + +Control flow stays outside the query. "Widen and retry when recall looks +low" is the caller's decision; the engine's part is to make each step cheap, +coherent and replayable: pinned snapshots, usage and coverage in every +result, and a multi-statement request at one snapshot so that adapting costs +one round trip. A loop inside a query would break the one-snapshot, +one-budget, one-publication contract that makes those steps safe. Reuse +comes from transparent definitions +([`define`](2026-09-18-gq-composition-and-language-evolution.md#kernel-one-stage-per-job)), not from opaque user-defined +functions. + A threshold on a declared geometric distance is legitimate and need not be called confidence. An exact range query evaluates the distance predicate over the specified eligible population under defined inclusive/exclusive bounds. @@ -1336,6 +1395,39 @@ fence, not justification to switch every backend. Materializing everything in a `MemTable` or replacing graph traversal with eager cross products does not satisfy these resource obligations. +#### Rewrite catalogue and `explain` contract + +The composition laws are the planner's specification. They are handed to +the engine version 2 planner component as a rewrite catalogue: each rewrite +with the precondition that proves equivalence, each cost-based choice with a +differential oracle. Nothing here decides a planner. + +| Class | Rewrites | Precondition | +|---|---|---| +| Always legal | Predicate pushdown through pattern joins; predicate pushdown into a retriever as a prefilter; projection pruning; join order and traversal direction inside one `match`; fusing adjacent `filter`/`let`; top-k pushdown of `order` + `limit` over one scan | The predicate precedes the `rank` it is pushed into; pruning retains hidden metric and identity columns; the scan's order is known and the cut unit is rows | +| Barriers | None across `rank`, `group`, `limit` | A rewrite that crosses one changes the question; it is refused, and `explain` names the law that refused it | +| Cost-based | Prefilter versus postfilter for a graph-scoped `rank` (the #587 gate; `rrf_plan` overrides); index versus exact scan; qualified fallback; late payload hydration; dense versus distinct-pair group selection | Statistics are visible, derived, explicitly maintained state (row counts, index coverage, selectivity estimates), never hidden; every route pair has a checked-in differential oracle proving identical results, `rrf_prefilter_gate.rs` being the pattern | +| Approximation | Recall may be traded inside `ann` only, through the `oversample` mapping | That is the declared contract; `knn`, `lexical` and every exact operator admit no recall trade | +| Adaptive, in-attempt | Probe-widening ladders, the bounded-BM25 retry, spill | Bounded, charged to the budget, one snapshot; re-planning across snapshots is a new attempt | +| Cache | Compiled plans keyed by the semantic fingerprint; parameters bind at execution | Derived state under invariant 12, never commit authority | + +`explain` is a contract, not a debugging aid. It prints the kernel program +(the logical stages after desugaring), the physical route chosen for each +cost-based class with the statistics that chose it, estimated usage in the +same units the result's usage descriptor reports, and every refused rewrite +with the law that refused it. A caller rewrites queries too, and its +rewrites change meaning when they move a stage; `explain` keeps the +engine's meaning-preserving rewrites and the caller's meaning-changing ones +distinguishable. + +One measurement outranks the catalogue. For the agent workload of many +small calls, the fixed per-request cost of reloading version-pinned control +state (on a fragmented graph, measured in hundreds of object-store reads +per request for the manifest alone) dwarfs any plan choice. Session-scoped +caching of immutable version-pinned state and multi-statement requests at +one snapshot are worth more than every rewrite above, and both are +invariant-12-clean because the cached state is immutable. + ## Invariants The design strengthens schema authority, coherent snapshots, typed query @@ -1972,6 +2064,12 @@ still require prototypes; full production qualification belongs to its phase. ## Decision log +- 2026-09-18 — decided that combination is an expression and retrieval is + named: `fuse(expr)` as the general fusion source with `rrf_v1` as its named + policy, explicit-mixing and missing-arm rules, structural identity, + models as sources; control flow stays outside the query. Added the + rewrite catalogue and `explain` contract handed to the engine version 2 + planner. - 2026-09-18 — added the agent-facing surface (stored queries as the door, `@description` on schema declarations, named defaults), the reader decision rule for result descriptors with usage as contract, and the diff --git a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md index 42f15d997..27fc870d5 100644 --- a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md +++ b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md @@ -773,11 +773,40 @@ the planner cannot tell a shape from a selection. - *Expression functions*: scalar functions and casts, aggregates, subquery reductions, scoring features, `metric(alias, rank | score | distance)`. - *Retrieval sources*: `lexical`, `knn`, `ann`, `rrf`; later `rerank(arm(x), - model: …)`, sparse and multivector representations, geometric range. All - share the shape `kind(args, option: value, …) as alias`. + model: …)`, sparse and multivector representations, geometric range, and + `fuse(expr)` as the general combiner of which `rrf` is a named policy + ([RFC 0048](0048-search-contracts.md#explicit-ranking-features-and-distance-predicates)). + All share the shape `kind(args, option: value, …) as alias`. - *Value types*: lists, nullable objects, paths. Projection and typing rules for each are the extension's obligation. +**Programmability.** Three constructs give a caller that writes code a +place to put it without making the query procedural: + +- `define`: a transparent, content-hashed definition — + `define scorer my_fuse(a, b) = 1.0 / (60 + metric(a, rank)) + 1.5 / (60 + coalesce(metric(b, rank), 1000))`, + `define pattern churned($p) = $p: Person not { $p placed $o }` — stored + beside stored queries and inlined at compile time. It is a SQL-language + macro, not a UDF: the optimizer sees through it, its identity is its hash, + and it can introduce neither recursion nor side effects. Datalog's named + rules and Vespa's rank profiles are the precedents. +- A typed plan surface: the kernel program as a JSON document compiled to + the same IR as `.gq` text, so that caller-written code constructs queries + structurally and constrained decoding can target it. Query semantics are + already typed structures; this exposes them as an input. +- A multi-statement request at one snapshot: several queries in one call, + sharing the snapshot and the budget, with no control flow between them. + Adaptation ("widen, then verify") is the caller's loop over cheap coherent + calls, not a loop inside the language. + +Excluded, and why: procedural loops and branches inside a query (unbounded +work and hidden re-planning); opaque functions (they block every rewrite and +hide cost); inference as a free expression (nondeterministic and remote; it +enters as a budgeted source). As models get better at code they write +orchestration in their own sandbox and compile intent into declarative +queries; the language's value is the guarantees code cannot give itself, so +it grows more inspectable and more programmable, not more procedural. + Reserved extension slots that are stage kinds, not open-set members, and are therefore future RFCs: `unnest { list as $item }` (Cypher `UNWIND`), `distinct { keys }` (or `group` with keys only), a query-level `union`, and @@ -1044,6 +1073,9 @@ before optimizing batching, and measure per-group rescan cost. collapses `select`, `take`, `score`, `collect` and `optional` and moves scalar predicates out of `match`. Recorded as a decision RFC 0048 must take before its syntax stabilizes; the moved-in text above is unchanged. +- 2026-09-18 — added programmability: transparent `define`, a typed plan + input surface, multi-statement requests at one snapshot; procedural + control flow, opaque functions and free inference calls excluded. - 2026-09-18 — made the in-context consumer the design target: card-sized kernel, case-insensitive atomic keywords, spellings decided by measured first-try validity rather than analogy to any prior language. From db1a170593a44cc3e92c5ca0a78492a0a6678579 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 18 Sep 2026 12:11:36 +0200 Subject: [PATCH 07/16] =?UTF-8?q?rfc:=20phase=20the=20search=20work=20A?= =?UTF-8?q?=E2=80=93G=20by=20risk=20instead=20of=20one=20coordinated=20cut?= =?UTF-8?q?over?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RFC 0048's rollout no longer promises a single pre-stable change with a mandatory format rebuild. Phases A–G ship one reversible step at a time: truth first (RFC 0047 plus session-scoped caching), the agent door (stored queries, `@description`, the in-context instrument and its baseline), representations opt-in per field (no rebuild for non-adopters), the kernel stages additive behind a session setting and A/B-measured against the old grammar, a deprecate-then-remove cutover, acceleration behind parity oracles, and programmability. Each phase names its owner RFC, gate, exit and open items; a risk register says where each assumption is tested and what happens if it is wrong. The lexical RFC becomes Phase C, RFC 0047 Phase A, the composition RFC Phase G; the Phase 0 design gates keep their name. --- docs/rfcs/0047-search-plan-truth.md | 12 +- docs/rfcs/0048-search-contracts.md | 437 +++++++++--------- .../2026-09-18-analyzed-lexical-search.md | 60 +-- ...8-gq-composition-and-language-evolution.md | 7 +- 4 files changed, 262 insertions(+), 254 deletions(-) diff --git a/docs/rfcs/0047-search-plan-truth.md b/docs/rfcs/0047-search-plan-truth.md index 78fef8a2b..4fec3cd0a 100644 --- a/docs/rfcs/0047-search-plan-truth.md +++ b/docs/rfcs/0047-search-plan-truth.md @@ -394,13 +394,10 @@ prototype's retirement label must not be reused as a current-code fact. ## Rollout -For a standalone RFC 0047 implementation, use the ordered stages below after -acceptance. For the combined release, use -[RFC 0048's implementation phases](0048-search-contracts.md#implementation-phases): -plan identity and target validation belong to foundations/composition; -projectable metrics and complete ties to retrieval/composition qualification; -and warnings, retrieval descriptors, and coverage to the agent-facing read -path. These are the same correctness obligations, implemented once. +This RFC is Phase A of [RFC 0048's rollout](0048-search-contracts.md#implementation-phases): +it ships first and alone, with no syntax or format change, because every +item is a refusal of a wrong answer or an additive field. The ordered stages +below are its internal order. The retained prototype supplies starting points; each stage must be checked against the revised scope: @@ -468,6 +465,7 @@ language release or duplicate execution path. blanket claim that all RRF arm windows follow the output limit: BM25 arms are uncapped; vector arms inherit that limit. Distinguished historical prototype results from current evidence and total ordering from ANN replay. +- 2026-09-18 — became Phase A of RFC 0048's rollout. - 2026-09-18 — added the diagnostics contract (code, position or stage, expectation, one fix; unknown names enumerate their set) for every compile diagnostic and typed read failure. diff --git a/docs/rfcs/0048-search-contracts.md b/docs/rfcs/0048-search-contracts.md index 6de4985df..4a08ae40b 100644 --- a/docs/rfcs/0048-search-contracts.md +++ b/docs/rfcs/0048-search-contracts.md @@ -19,7 +19,7 @@ blocked_on: - "Phase 0 vector/fusion numeric policies and independent oracles; exact-tie route dispositions against pinned upstream code" - "Read options expressed as session settings (RFC 2026-09-16); whole-query resource admission owned by the engine version 2 memory/admission component named by RFC 0067 (PR #711)" - "Phase 0 minimal compiler/engine/GQT vertical slice with later-stage discovery and shared-resource refusal" - - "Phase 0 fixed mixed-workload corpus, judgments, budgets and acceptance protocol; production qualification belongs to Phases 1–6" + - "Phase 0 fixed mixed-workload corpus, judgments, budgets and acceptance protocol; production qualification belongs to Phases A–G" --- # RFC 0048: Search contracts and retrieval algebra @@ -79,7 +79,7 @@ those boundaries. The [phase handoffs](#implementation-phases) name the remaining decisions, owners and required proof. Implementers must investigate open claims rather than treating proposed syntax or passing probes as truth. Frontmatter lists remaining design-acceptance gates; the -[release gates](#phase-0-acceptance-versus-release-qualification) retain full +[release gates](#acceptance-versus-release-qualification) retain full production obligations. Moving a check to its implementation phase does not waive it or make the current code correct. @@ -553,7 +553,7 @@ The legacy spellings are not removed in the same release: they compile to the new typed IR with a deprecation diagnostic for one release and are removed in the next, with the GQ language major bump the [compatibility surfaces](2026-09-14-compatibility-surfaces.md) RFC defines. -The lexical RFC fixes the mapping for its spellings; Phase 6 fixes the +The lexical RFC fixes the mapping for its spellings; Phase E fixes the `nearest`, `order` and positional-`rrf` mapping. Rewritten application/stored queries and examples ship with the implementation. A rewrite must ask the caller to choose exact versus approximate retrieval @@ -1354,7 +1354,7 @@ The [historical physical experiments](https://github.com/ModernRelay/omnigraph/b record configurations, timing/I/O/spill tables and controls. They support these implementation candidates, not production latency or a universal winner. Reproduce through the instrument on the evidence branch (it returns to the -tree with Phase 3): +tree with Phase F): ```sh cargo bench -p omnigraph-engine --bench scenarios -- \ @@ -1749,7 +1749,7 @@ rejection cases is retained on the evidence branch ([`staged_probe.rs`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-compiler/src/query/staged_probe.rs)). It is partial compiler evidence, not a staged AST/IR or execution, and it is deliberately not in the tree: the production compiler owns these constructs -when Phase 1 lands, and `.gqt` cases own their behaviour. +when Phase D lands, and `.gqt` cases own their behaviour. The default-resolution fixtures and the exact lexical qualification matrix are in [Analyzed lexical search](2026-09-18-analyzed-lexical-search.md#qualification-matrix). @@ -1770,45 +1770,47 @@ by this draft. ## Rollout -Acceptance requires the frontmatter design gates to have concrete dispositions -and owned evidence. Phase 0 supplies the decision work and prototypes needed -for acceptance; the later phases implement and qualify the accepted contracts. -Each phase can be reviewed in separate PRs. The public contract ships as one -coordinated pre-stable change with one necessary format rebuild, without -requiring users to adopt an interim language or rebuild after each phase. - -### Phase 0 acceptance versus release qualification - -An accepted design must name a feasible route, the interfaces it requires and -the tests that can disprove it. It does not assert that all production routes -already implement the contract. These obligations have separate completion -points: - -| Package | Phase 0 acceptance gate | Production release gate / owner | +Three principles order the work. Irreversible commitments — SchemaIR +features, `@analyzed` bindings, embedding recipes, index certificates, and +language removals — ship last and opt-in; additive syntax, metadata, +diagnostics, settings, stored queries and `explain` ship first. Information +is front-loaded and commitment back-loaded: the in-context assumption is +measured on a prototype surface before syntax freezes, and correctness ships +before capability before performance. Every phase is worth shipping alone and +revertible alone; no phase depends on a coordinated cutover with a mandatory +format rebuild. Each phase can be reviewed in separate PRs. + +### Acceptance versus release qualification + +Acceptance of this RFC requires the frontmatter design gates (Phase 0: the +decision work and prototypes) to have concrete dispositions and owned +evidence. An accepted design names a feasible route, the interfaces it +requires and the tests that can disprove it; it does not assert that +production routes implement the contract. Production status advances phase +by phase in the frontmatter of the RFC that owns the phase. An explicit +upstream dependency can close a route decision but cannot qualify or enable +the route. No production gate is waived by this classification, and the four +recorded GQT regressions remain release blockers for the phases that own them +even though they need not be repaired to accept the design. + +| Package | Phase 0 acceptance gate | Production gate and phase | |---|---|---| -| Language and composition | Checked syntax/types/plans for the initial stages and C1–C4; global-union compatibility proof; invalid-rewrite counterexamples; one actual compiler/engine/GQT slice with refusal | Phases 1/3: every enabled stage, walker, result type and optimized population boundary through GQT and mechanism owners. Deferred C1/C3/C4 execution remains outside this release. | -| Schema and identity | Default lifetime, space/binding separation, concrete `.pg` syntax and canonical serialization prototype; export/reapply/rename/drop-readd fixtures; coordinated format disposition | Phases 1/6: accepted SchemaIR, provider qualification, migration and predecessor/rebuild tests. No unresolved provider identity silently accepted. | -| Lexical/vector numerics | Complete numeric policies and independent oracles; discriminating native probes; explicit exact scan/fallback or upstream-dependency decisions | Phases 2/5: NFC/index parity, live statistics, fuzzy scoring, raw-vector rescoring, complete ties and all supported lifecycle states/targets. | -| Resource/read interfaces | Typed request/result/error contracts and charging units; minimal shared-budget refusal and snapshot/fingerprint round-trip proofs; every unaccounted native allocation/dispatch assigned a route decision | Phases 1/4/5: actual preallocation/dispatch gates, bounded cancellation/cleanup, every renderer and policy/retention path, whole-query and fallback accounting. | -| Retrieval evaluation | Frozen mixed tasks/corpus, judgment provenance, budgets, comparisons and acceptance criteria before tuning | Phase 5: measured correctness/quality/cost, source defaults and each enabled ANN family's effort mapping; retain failures and unsupported cases. | - -An unresolved design choice or a missing required prototype keeps Phase 0 -open. Production work scheduled above does not. An explicit upstream dependency -can close a route decision, but cannot qualify or enable that route; the release -still needs a working bounded implementation of every required capability. -No production gate is waived by this classification. The four recorded GQT -regressions remain release blockers even though they need not be repaired to -accept the design. +| Language and composition | Checked syntax/types/plans for the initial stages; the kernel decision; invalid-rewrite counterexamples; one compiler/engine/GQT slice with refusal | Phase D: every enabled stage, walker, result type and optimized population boundary through GQT and mechanism owners, plus the in-context instrument. C1/C3/C4 execution is Phase G. | +| Schema and identity | Default lifetime, space/binding separation, concrete `.pg` syntax and canonical serialization prototype; export/reapply/rename/drop-readd fixtures; opt-in feature disposition | Phase C: accepted SchemaIR feature, provider qualification, no rebuild for non-adopters, predecessor tests. No unresolved provider identity silently accepted. | +| Lexical/vector numerics | Complete numeric policies and independent oracles; discriminating native probes; exact scan/fallback decisions | Phase C (lexical) and D (vector): NFC/index parity, live statistics, fuzzy scoring, raw-vector rescoring, complete ties, all lifecycle states. Phase F: native parity. | +| Resource/read interfaces | Read options as settings; typed result/error contracts; snapshot/fingerprint round-trip proofs; every unaccounted native allocation/dispatch assigned a route decision or handed to engine version 2 | Phase A: descriptors, diagnostics, coarse budgets. Phase D: coherent follow-up. Engine version 2: whole-query admission. | +| Retrieval evaluation | Frozen mixed tasks/corpus, judgment provenance, budgets, comparisons and acceptance criteria before tuning | Phase B: the instrument and its baseline. Phase F: measured correctness/quality/cost, source defaults, each ANN family's effort mapping. | ### Implementation phases -Phase 0 resolves the contracts; Phases 1–6 implement and qualify one coordinated -release. Fixtures and transport/migration tooling can proceed once their -required interfaces are fixed. Native experiments can begin against exact -references, but enabling a route requires its semantic and resource gates. +Phase 0 resolves the contracts; Phases A–G implement and qualify them one +reversible step at a time, ordered by risk rather than by the sections of +this RFC. Fixtures and tooling can proceed once their interfaces are fixed; +native experiments can begin against exact references, but enabling a route +requires its semantic and resource gates. These are required outcomes, not a claim that the proposed mechanisms work. -For each package, inspect the actual checkout/lockfile and complete relevant +For each phase, inspect the actual checkout/lockfile and the relevant [Lance documentation](../dev/lance.md), extend the [existing owner](../dev/testing.md), and record the revision, toolchain/configuration, command, result and limits. Use a test that would fail if the claim were false. Revalidate changed @@ -1826,195 +1828,186 @@ Keep evidence in the [native matrix](#contract-to-code-qualification) and status in frontmatter. A contradictory probe requires revising the proposed mechanism, not weakening the promised result. -#### Phase 0: resolve contracts and build the oracles +| Phase | Ships | Risk and reversibility | Owner | Gate | +|---|---|---|---|---| +| A — Truth first | Refusals (`T26`, `T27`, `FullTextIndexRequired`); retrieval as typed IR; projectable metrics; deterministic ties; `warnings`/`metrics`/`retrievals`/usage descriptors; the diagnostics contract; `explain` v0 (logical plan); coarse budgets through the existing `ResourceLimitExceeded` owner; session-scoped caching of version-pinned state | Low: no syntax or format change; refusals and additive fields | RFC 0047 | Characterization goldens; the traversal-target and unindexed regression cases green; per-request overhead measured | +| B — The agent door | Stored queries as typed tools; `@description` on `.pg` declarations and in `schema show`; the one-page card; the in-context competence instrument with graph-computed ground truth; baseline measurements on today's language | None: the PG annotation is metadata | This RFC (agent-facing surface); composition RFC (instrument) | Instrument runs end to end; baseline recorded | +| C — Representations, opt-in per field | `@analyzed` and analyzer profiles as an additive SchemaIR feature (unchanged graphs need no rebuild); resolved embedding recipes and defaults; `terms`/`match_terms` as a predicate; the exact scan baseline across index states; `bm25_v1` against the Decimal oracle; zero-edit membership through the existing FTS index, fuzzy through a budgeted scan | Medium: the format commitment is opt-in | Lexical RFC; this RFC (representation identity) | Membership laws across absent/partial/full/rebuilt indexes; oracle parity; the fuzzy and index-state regression cases green | +| D — Kernel stages, additive, behind a setting | `filter`, `let`, `rank` (`lexical`, `knn`, `ann`, `rrf`, `ties:`), `group`, `order`/`limit` at any position with `of`/`per`, pure `return` with the aggregate sugar, `metric()`; typed stage IR with metric origin and fingerprints; coherent follow-up reads; exact physical paths only; the old grammar still parses and a session setting selects the kernel | Medium: additive syntax, no format; reversible | This RFC | Instrument A/B (first-try validity, turns) kernel versus current grammar; C2's terminal subset; barrier cost measured | +| E — Cutover | Release N: legacy spellings compile to the kernel with deprecation diagnostics and a rewrite tool for stored queries; GQ minor bump. Release N+1: removal and major bump under the compatibility-surfaces RFC | User-facing, bounded by the window; reversible until N+1 | This RFC and the lexical RFC (mappings) | Kernel not worse than the old grammar on the instrument; no un-migrated stored query | +| F — Acceleration and qualification | Native lexical routes with parity oracles; ANN families qualified for recall against `knn`; statistics-driven prefilter/postfilter and route choice with differential oracles; late hydration; plan cache; engine version 2 planner integration through the rewrite catalogue | Performance only; every route behind a parity gate; piecemeal | This RFC with the engine version 2 planner RFC | Each route: oracle identical, cost measured; defaults chosen inside the evaluation envelope | +| G — Programmability and composition | `fuse(expr)`, `define`, the typed plan surface, multi-statement requests, `sub()` reductions, type-union targets, path selectors, each its own increment | Additive, one at a time | Composition RFC | Each gated by the instrument and its own oracle | + +#### Phase A: truth first + +RFC 0047 is this phase and its ordered stages are the internal order. It +ships first and alone because every item is a refusal of a wrong answer or +an additive field. The session-scoped caching of immutable version-pinned +state rides with it because the fixed per-request overhead, not plan choice, +bounds the agent workload, and the cached state is invariant-12-clean. + +**Exit:** characterization goldens before and after the retrieval-IR +refactor; every ranked result totally ordered; descriptors in every +transport; the two regression cases RFC 0047 owns green. + +**Open:** fuzzy analysis and index-state divergence remain until Phase C; +whole-query admission remains engine version 2's. + +#### Phase B: the agent door + +**Ships:** the [agent-facing surface](#agent-facing-surface): stored queries +as typed tools with descriptions that state the population, the result and +the one trade-off; `@description` on schema declarations, printed by +`schema show`; the one-page card; the composition RFC's +[in-context competence](2026-09-18-gq-composition-and-language-evolution.md#in-context-competence) instrument with a task +set whose ground truth exact queries compute at a pinned snapshot. + +**Exit:** the instrument runs against today's language and records +first-try validity, turns to a correct query, task success and tokens per +model; those numbers are the baseline every later spelling decision is +measured against. + +**Open:** nothing that blocks another phase; this phase exists so that +Phase D is decided on evidence. + +#### Phase C: representations, opt-in per field + +**Input:** Phase 0's numeric oracles and schema-default decisions. + +Build resolved per-field representations, `@analyzed` with its analyzer and +scorer defaults and embedding recipes with the schema-owned default, as an +additive SchemaIR feature that a graph adopts field by field. A graph that +declares neither is unchanged and needs no rebuild; a field that adopts +`@analyzed` takes the NFC certificate and an index rebuild for that field. +Build bounded NFC/analysis, complete `Terms` matching, unified exact/fuzzy +scoring against the Decimal oracle, and the exact scan baseline that holds +across every index state, with zero-edit membership served by the existing +FTS index and fuzzy membership by a budgeted scan. Extend search/substrate +owners and the lexical RFC's qualification matrix; observable rows, shapes +and errors belong in GQT. + +**Exit:** the [lexical RFC's phase](2026-09-18-analyzed-lexical-search.md#rollout) exit; export/reapplication, +rename versus drop/re-add, default/override and no-drift fixtures for both +representation kinds; the fuzzy and index-state regression cases green. + +**Open:** native fuzzy analysis, float32 BM25, stale statistics and native +cuts are not substitutes for the accepted exact contract (Phase F). A +provider label is not an immutable encoder; unverifiable aliases are refused. + +#### Phase D: kernel stages, additive, behind a setting + +**Input:** Phase C representations, Phase B's baseline, Phase 0's kernel +decision. + +Add the kernel stages to the production grammar without removing anything: +`filter`, `let`, `rank` with `lexical`/`knn`/`ann`/`rrf` and `ties:`, +`group`, `order`/`limit` at any position with `of`/`per`, pure `return` with +the aggregate sugar, `metric()`. Lower them through typed stage IR carrying +metric origin and fingerprints; preserve distinct target identity, +missing-arm metrics and every winning binding row; exact physical paths only +(scan-based lexical, exhaustive `knn`, the existing `ann`). Complete the +coherent follow-up read path: `@id` lookup at a pinned snapshot and replay +identity through every renderer. A session setting selects the kernel while +both grammars parse, so the instrument compares them on the same tasks. -**Input:** fresh production baseline, capability matrix, open gates and -archived experiments inspected at their recorded bases. +**Exit:** executable counterexamples distinguish filter-before/after-cut, +source windows from final limits, target counts from path counts, and +missing membership from a new feature; C2's terminal subset returns three +binding rows from p1/p2 and excludes p3; null and multiple group membership, +metric reductions and inherited order tested; the instrument shows the kernel +at least as good as the old grammar on first-try validity and turns; barrier +cost measured. -**2026-09-13 decision checkpoint:** C3 retains partial term scores, distinguishes -zero from null and preserves membership. Shared expressions have explicit -precedence, typing, null and checked-arithmetic rules. Global execution is -deferred with compatibility obligations retained. On base `dd6959c8` plus this -change, Rust 1.97.1 passes all 367 compiler tests, the 13-case lexical oracle -and both `staged_composition` native tests. This settles the stated policies; -it does not complete any production evaluator or the phase's remaining gates. +**Open:** native relational probes do not prove GQ lowering, masks, metric +carriage or order after payload reads; compare optimized plans with the +independent oracles and retain the nullable dynamic-filter fence. -```bash -# on the evidence branch at dd6959c8 -cargo +1.97.1 test --locked -p omnigraph-compiler -cargo +1.97.1 test --locked -p omnigraph-engine --test search lexical_scoring_v1_reference_oracle -- --exact -cargo +1.97.1 test --locked -p omnigraph-engine --test rrf_prefilter_gate staged_composition -``` +#### Phase E: cutover -| Decision package | Required disposition and proof | -|---|---| -| [Language](2026-09-18-gq-composition-and-language-evolution.md#language-evolution-and-compatibility) | Shared expressions, explicit output, namespaces, precedence, parameters and scope; parser/type/plan fixtures | -| [Composition](2026-09-18-gq-composition-and-language-evolution.md#required-composition-examples) | C1–C4 syntax, type derivation, golden plans, physical feasibility and invalid-rewrite counterexamples | -| [Selection/scoring](#target-identity-fan-out-grouping-and-metrics) | Target/binding/group multiplicity, total ties, live statistics, lexical/vector numeric policy and checked fusion arithmetic | -| [Representations](#representation-identity-and-source-attribution) | Schema defaults and overrides, resolved encoder/Unicode/analyzer identity, export/reapplication and format coordination with RFCs 0040/0043/0044 | -| [Global search](#graph-wide-discovery-across-entity-types) | Execution deferred by explicit decision; prove typed identity/source/table separation and future union/narrowing/projection compatibility before stabilization | -| [Execution/read contract](#result-metadata-coherent-continuation-and-budgets) | Resource units, admission/interfaces, error/result types, fingerprints, replay and retention | -| [Workload](#mixed-workload-qualification) | Fixed tasks/corpus, exact oracles, relevance judgments, recipes, budgets and acceptance criteria before tuning | - -**Exit:** concrete design dispositions for every frontmatter gate and -native-matrix row: a prototype-supported route, specified bounded fallback -with its feasibility proof, or explicit upstream dependency. Carry a minimal -query through the actual compiler/engine/GQT with success and shared-resource -refusal; include later-stage descriptors, column demand, GQT detection and -pre-scan parameter admission. Existing compiler, search, schema and read owners -supply the evidence. - -**Open:** logical goldens and native primitives do not establish integrated -C1–C4 lowering, numeric/null evaluation or whole-query bounds. The archived -lexical integration proves only its recorded slice. Full production C1/C3/C4 -and general C2 grouping are deferred, but their design proofs are due here. -Global execution is explicitly deferred; its compatibility proof is still due. -Do not start a dependent package before its interfaces are resolved. - -#### Phase 1: build representation, plan, and resource foundations - -**Input:** Phase 0's accepted interfaces and ordinary-query compatibility fixtures. - -Implement resolved per-field representations/defaults, shared expressions, -typed stage IR, metric/population identity and fingerprints. Analytical rows -need no score. All operators share one snapshot, admission, cancellation and -resource context. Extend compiler AST/IR/SchemaIR/descriptors, engine query -execution, sealed TableStore and schema/rebuild owners; every stage walker -must retain later reads and hidden columns. - -**Exit:** serialization, rename versus drop/re-add, parameter/refusal, -default/override and export/reapplication fixtures; no drift with runtime -defaults; one budget across stages/fallbacks. Close native allocation, -shared-buffer and dispatched-I/O gates before claiming whole-query bounds. -Exercise active reads/decode, retained plans/caches, output and encoder calls. - -**Open:** a shared DataFusion pool does not govern all native allocations or -tasks. Inspect ownership and prove preallocation refusal, cancellation and -cleanup. Typed extension points do not make deferred public operators available. - -#### Phase 2: implement complete lexical and vector retrieval - -**Input:** Phase 1 representations/resources and Phase 0 numeric oracles. - -The lexical half of this phase (bounded NFC analysis, complete `Terms` -matching, unified exact/fuzzy scoring and qualified index acceleration) is -owned by [Analyzed lexical search](2026-09-18-analyzed-lexical-search.md#rollout). This RFC's half builds -exhaustive `knn` and declared `ann` with qualified fallback, charges -statistics, coverage, scoring and selection to the shared context, and -extends the search and substrate owners. Observable rows, shapes and errors -belong in GQT. - -**Exit:** predicate/retriever membership agreement before cuts, independent -score and vector fixtures, total ties, all index/lifecycle states, and typed -budget/cancellation failures. Assert that intended index/compaction states -were reached; distinguish current from pinned answers and reopen snapshots. - -**Open:** native cuts are not substitutes for the accepted exact contract. -Numerical parity does not establish relevance quality. Candidate-scoring -operators and richer representations remain deferred; Phase 5 chooses -measured defaults. - -#### Phase 3: compose graph scope, fusion, and selection - -**Input:** Phase 2 reference retrievers and Phase 1 stage/resource interfaces. - -Connect graph-defined populations, named weighted RRF, inter-stage traversal, -group quotas, final projection/order and existing terminal aggregates. -Preserve distinct target identity, missing-arm metrics and every winning -binding row. Extend IR/query execution, traversal/projection/aggregation, -GQT and existing search/selection owners. +**Input:** Phase D green on the instrument; the deprecation mappings fixed in +the lexical RFC (`fuzzy`, `search`, `match_text`) and here (`nearest`, +retrieval in `order`, positional `rrf`). -**Exit:** executable counterexamples distinguish filter-before/after-cut, -source windows from final limits, target counts from path counts, and -missing membership from a new feature. C2's initial terminal subset returns -three binding rows from p1/p2 and excludes p3. Test null/multiple group -membership, metric reductions, inherited order and fan-out/sort refusal despite -a small final result. No successful truncated aggregate. +Release N: legacy spellings compile to the kernel and emit the deprecation +diagnostic with the replacement; a rewrite tool migrates stored queries and +`cluster.yaml` inline queries; GQ language minor bump. Release N+1: the +spellings are removed with the GQ language major bump under the +compatibility-surfaces RFC; the kernel becomes the only grammar and the +session setting retires. -**Open:** native relational probes do not prove GQ lowering, masks, metric -carriage, descriptor walkers or order after payload reads. Compare optimized -plans with independent oracles and retain the nullable dynamic-filter fence. -Reusable intermediate groups, general distinct aggregates and full per-parent -collection remain deferred. - -#### Phase 4: complete the agent-facing read path - -**Input:** Phase 0 read/error/fingerprint decisions and Phase 3 result semantics; -transport work can begin earlier against fixed interfaces. - -Expose inline/stored queries, inspectable plans, metrics, completion/coverage/ -selection metadata and coherent identity-based source reads. Reuse existing -snapshot carriers, descriptions/instructions and authorization. Extend API -types, server data/stored-query/policy/OpenAPI owners and CLI parity. - -**Exit:** discovery → pinned source read → expansion → verification, and -graph-scoped retrieval → terminal aggregate through embedded, HTTP, stored-query -and CLI paths. Include an entity without an application key. Change head so -pinned/current answers differ; test expired/unavailable snapshots, revoked -access, exact-coverage refusal, JSON/JSONL metadata and result types. - -**Open:** existing request fields do not prove renderer/client compatibility, -general system-ID lookup or retention. A graph snapshot does not freeze an -external encoder. Qualify identity and refusal boundaries; stable ranked -cursors and durable result storage remain deferred. - -#### Phase 5: qualify physical execution and measure retrieval - -**Input:** exact references from Phase 2; full-pipeline trials additionally need -Phases 3–4 and the fixed mixed-workload protocol. - -Compare each native route with the exact evaluator and composition oracle: -graph masks, analyzer certificates, expansions, live statistics, tails, raw -vectors, ties, partitions, fallback and cancellation. ANN is judged for recall -against exact `knn`, not exact candidate equality. Use existing search/ -substrate owners and benchmark instruments. - -**Exit:** every enabled path passes semantic/resource gates. Freeze -corpus/schema/snapshot, representations, model/recipes, windows, budgets, -expected facts, judgments and pass/refusal criteria before trials. Report -task/population errors, query repairs, round trips, context, quality and cost; -retain failed trials and judgment provenance. Choose source-window defaults -and each index family's `ann_default_v1` mapping only within that envelope. - -**Open:** the document pilot lacks independent human adjudication and broader -analytical coverage. No universal join strategy, candidate window or effort -mapping is established. Keep an unqualified native route disabled while a -qualified fallback serves the contract; future operators need fresh trials. - -#### Phase 6: ship the coordinated language and data migration - -**Input:** qualified Phases 1–5, final schema/format identity and wire decisions. - -Follow the [migration sequence](#migration-sequence): finalize identities; -prepare schema/query diagnostics and examples; export/init/load real predecessor -fixtures; regenerate incompatible representations and reconcile indexes; -verify ordinary and migrated queries, result types and follow-up; publish -coordinated docs/release notes and remove legacy syntax/scaffolding. - -**Exit:** existing schema/rebuild, cross-version, API/OpenAPI, CLI and query -owners prove mutual format refusal, compatible-value preservation, unresolved -encoding refusal, rewrite idempotence and successful migrated journeys. -Run the current [canonical checks](../dev/testing.md), including separate -GQT and required environments. Users encounter one supported cutover. - -**Open:** recheck format ownership as related RFCs land, actual client parsers, -and loss of old histories/snapshot references across rebuild. A skipped -predecessor test is not upgrade evidence. Deferred composition and any explicitly -deferred global-search capability must remain absent from release claims. +**Exit:** the existing schema/rebuild, cross-version, API/OpenAPI, CLI and +query owners prove rewrite idempotence and migrated journeys; no stored query +in the fleet still uses a removed spelling; coordinated docs and release +notes. + +**Open:** actual client parsers; the recheck of format ownership as related +RFCs land. + +#### Phase F: acceleration and qualification + +**Input:** exact references from Phases C and D; the fixed mixed-workload +protocol. + +Compare each native route with the exact evaluator and the composition +oracle: graph masks, analyzer certificates, expansions, live statistics, +tails, raw vectors, ties, partitions, fallback and cancellation. ANN is +judged for recall against exact `knn`, not exact candidate equality. Enable +statistics-driven prefilter/postfilter and route choice through the +[rewrite catalogue](#rewrite-catalogue-and-explain-contract) with a +differential oracle per route pair; add late payload hydration and the plan +cache; integrate with the engine version 2 planner as it lands. + +**Exit:** every enabled route passes its semantic and resource gates; source +window defaults and each index family's `ann_default_v1` mapping chosen +within the frozen evaluation envelope; task/population errors, repairs, round +trips, context, quality and cost reported with failed trials retained. + +**Open:** an unqualified route stays disabled while the exact fallback serves +the contract; no universal join strategy or effort mapping is established. + +#### Phase G: programmability and composition + +Each increment is its own PR against the composition RFC's kernel: +`fuse(expr)` with `rrf_v1` as its named policy, transparent `define`, the +typed plan surface, multi-statement requests at one snapshot, `sub()` +reductions, type-union targets for graph-wide search, path selectors. Each is +additive, each is gated by the in-context instrument and its own oracle, and +none is required for this RFC's `complete`. ### Release and completion criteria -Phases 1–4 establish the full initial feature path; Phase 5 qualifies its -execution and defaults; Phase 6 makes it a supported release. Fuzzy ranked -retrieval, coherent follow-up, and whole-query resource bounds are required -for that release. They cannot be moved to the extension list to mark the -implementation complete. An acceleration path may remain disabled only when -the qualified fallback preserves the promised semantics and resource behavior. +Phases A–D establish the feature path; Phase E makes the kernel the one +grammar; Phase F qualifies execution and defaults. This RFC's +`implementation` is `complete` when Phases D, E and F have exited. Fuzzy +ranked retrieval, coherent follow-up, and typed budget failures are required +for that; they cannot be moved to the extension list. An acceleration path +may remain disabled only when the qualified fallback preserves the promised +semantics and resource behavior. Whole-query admission is engine version 2's +completion criterion, not this RFC's. Frontmatter remains the implementation-status authority. Advance it when implementation lands, and mark it complete only after the release criteria -and their owned evidence pass. Historical prototypes and current-code baseline -tests do not count as completion of a new phase. +and their owned evidence pass. Historical prototypes and current-code +baseline tests do not count as completion of a new phase. + +### What is off the critical path + +The coordinated cutover with a mandatory export/init/load rebuild (replaced +by opt-in representations and the deprecation window); native fuzzy +acceleration (Phase F, after the scan baseline is the oracle); the +whole-query resource ledger (engine version 2 owns it; Phase A ships coarse +budgets as settings); cross-type search, `sub()` reductions and every +`Foundation` row (Phase G). + +### Risk register + +| Assumption | Tested in | If wrong | +|---|---|---| +| An in-context agent uses the kernel at least as well as today's grammar | B and D, before any removal | Keep the old grammar as the surface and revisit spellings with the instrument; nothing shipped is lost | +| Exact-scan fuzzy matching is fast enough on real graphs (tables of 200k rows exist) | C | Zero-edit stays index-served; fuzzy stays budgeted and refuses loudly; acceleration moves earlier in F | +| `@analyzed` can be an opt-in SchemaIR feature with no stamp bump for non-adopters | C's feature design | A versioned feature flag with the same no-rebuild property for non-adopters | +| Session-scoped caching removes the fixed per-request overhead | A | Multi-statement requests carry the load; planner work in F is not the answer either way | +| Stage barriers cost little | D | Widen the rewrite catalogue with proven equivalences before E | ### Extensions after the initial release @@ -2026,6 +2019,7 @@ Every later extension retains the stated semantic and qualification boundary. `Foundation` rows require their grammar/type/composition proofs before syntax stabilization while their broader operators remain deferred. These design proofs do not enlarge the advertised initial-release feature set. + ### Next delivery milestone: composed analytical answers Owned by [GQ composition and language evolution](2026-09-18-gq-composition-and-language-evolution.md#next-delivery-milestone-composed-analytical-answers): @@ -2033,7 +2027,7 @@ C1, C4 and C2's general grouping extensions after this RFC's initial release. ## Unresolved questions -The [acceptance/release matrix](#phase-0-acceptance-versus-release-qualification) +The [acceptance/release matrix](#acceptance-versus-release-qualification) assigns the completion point for each proof. In particular, decided interfaces still require prototypes; full production qualification belongs to its phase. @@ -2053,17 +2047,22 @@ still require prototypes; full production qualification belongs to its phase. handed to the engine version 2 memory/admission component; resolve native allocation and dispatch gaps for graph masks, scoring, coverage, sort/spill and output there. Actual default limits, complete enforcement - and cancellation qualification belong to Phases 1/4/5. + and cancellation qualification belong to Phases D and F and to engine + version 2. 4. Prototype the decided read options, envelope, retention/refusal boundaries - and canonical definition/semantic/execution identities. Phase 4 qualifies + and canonical definition/semantic/execution identities. Phase D qualifies all transports and authorization paths. Stable ranked cursors remain deferred. 5. Freeze the mixed-workload corpus, judgments and acceptance protocol in - Phase 0. Phase 5 chooses ANN effort mappings and agent recipe defaults from + Phase 0. Phase F chooses ANN effort mappings and agent recipe defaults from the resulting evaluation. The document pilot alone is insufficient. Further multilingual profiles require matched-set evidence and versioned identities. ## Decision log +- 2026-09-18 — replaced the single coordinated cutover with Phases A–G ordered + by risk: truth first, the agent door, opt-in representations, additive + kernel stages behind a setting, a deprecation cutover, acceleration, + programmability; added the risk register and what is off the critical path. - 2026-09-18 — decided that combination is an expression and retrieval is named: `fuse(expr)` as the general fusion source with `rrf_v1` as its named policy, explicit-mixing and missing-arm rules, structural identity, diff --git a/docs/rfcs/2026-09-18-analyzed-lexical-search.md b/docs/rfcs/2026-09-18-analyzed-lexical-search.md index 0ecf297d9..013d1f47c 100644 --- a/docs/rfcs/2026-09-18-analyzed-lexical-search.md +++ b/docs/rfcs/2026-09-18-analyzed-lexical-search.md @@ -769,36 +769,41 @@ The exact lexical qualification matrix retains all preceding requirements: ## Rollout -Phases are sequenced by [RFC 0048's implementation phases](0048-search-contracts.md#implementation-phases); -this RFC's `implementation` status advances with Phase 2 and completes when -the deprecation window has closed. The phase, restated here as the owner of -the lexical work: - -**Input:** Phase 1 representations/resources and Phase 0 numeric oracles. - -Build bounded NFC/analysis, complete `Terms` matching and unified -exact/fuzzy scoring; exhaustive `knn` and declared `ann` are the vector half -that RFC 0048 keeps. Charge statistics, coverage, scoring and selection to -the shared context. -Extend search/substrate owners and the -[lexical qualification matrix](#qualification-matrix); observable rows, -shapes and errors belong in GQT. - -**Exit:** predicate/retriever membership agreement before cuts, independent -score and vector fixtures, total ties, all index/lifecycle states, and typed -budget/cancellation failures. Assert that intended index/compaction states -were reached; distinguish current from pinned answers and reopen snapshots. - -**Open:** native fuzzy analysis, float32 BM25, stale statistics and native cuts -are not substitutes for the accepted exact contract. Reproduce their -counterexamples before reuse. Numerical parity does not establish relevance -quality. Candidate-scoring/model operators and richer representations remain -deferred; Phase 5 chooses measured defaults. +This RFC is Phase C of [RFC 0048's rollout](0048-search-contracts.md#implementation-phases): +representations, opt-in per field. Its `implementation` status advances with +that phase and completes when Phase E's deprecation window has closed on the +lexical spellings. + +**Input:** Phase 0's numeric oracles; RFC 0048's Phase A refusals in place. + +`@analyzed` and its analyzer profiles ship as an additive SchemaIR feature +that a graph adopts field by field: a graph that declares no `@analyzed` +field is unchanged and needs no rebuild; a field that adopts it takes the NFC +certificate and an index rebuild for that field. Build bounded NFC/analysis, +complete `Terms` matching, `match_terms` as a predicate, and unified +exact/fuzzy scoring against the Decimal oracle. The exact scan is the +baseline at every index state; zero-edit membership is served through the +existing FTS index and fuzzy membership through a budgeted scan until Phase F +qualifies a native route. Charge statistics, coverage and scoring to the +shared context. Extend search/substrate owners and the +[qualification matrix](#qualification-matrix); observable rows, shapes and +errors belong in GQT. + +**Exit:** predicate and retriever membership agree before any cut; the +membership laws hold across absent, partial, full and rebuilt indexes; oracle +parity for scores; total ties; typed budget failures; export and +reapplication preserve the resolved profile; the fuzzy and index-state +regression cases green under `issue_N` names. + +**Open:** native fuzzy analysis, float32 BM25, stale statistics and native +cuts are not substitutes for the accepted exact contract; reproduce their +counterexamples before reuse in Phase F. Numerical parity does not establish +relevance quality; Phase F chooses measured defaults. ## Unresolved questions 1. Close the Phase 0 numeric policies/oracles and native route dispositions. - Phases 2/5 must qualify BM25 across supported targets, exact live-row + Phases C and F must qualify BM25 across supported targets, exact live-row statistics and native/fallback score/winner parity. Polymorphic field-corpus execution is deferred. Validate edit weights and maximum reduction on the owned task corpus before release; numerical fixtures alone do not establish @@ -811,6 +816,9 @@ deferred; Phase 5 chooses measured defaults. ## Decision log +- 2026-09-18 — became Phase C of RFC 0048's rollout: `@analyzed` opt-in per + field with no rebuild for non-adopters, zero-edit membership through the + existing index, fuzzy through a budgeted scan. - 2026-09-18 — split out of RFC 0048 at its 2026-09-13 revision with the lexical decisions unchanged. Replaced the hard removal of `fuzzy`, `search` and `match_text` with a one-release deprecation window and fixed the diff --git a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md index 27fc870d5..6daee19ad 100644 --- a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md +++ b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md @@ -1013,8 +1013,9 @@ retain failed trajectories; a model revision reopens the measurement. ## Rollout No production work starts from this document. Its rules gate RFC 0048's -syntax stabilization (the `Foundation` rows of the matrix) and the milestone -below follows RFC 0048's initial release. +Phase D syntax and its instrument gates Phases B, D and E; the +programmability constructs and the milestone below are RFC 0048's +[Phase G](0048-search-contracts.md#implementation-phases), after the cutover. ### Next delivery milestone: composed analytical answers @@ -1073,6 +1074,8 @@ before optimizing batching, and measure per-group rescan cost. collapses `select`, `take`, `score`, `collect` and `optional` and moves scalar predicates out of `match`. Recorded as a decision RFC 0048 must take before its syntax stabilizes; the moved-in text above is unchanged. +- 2026-09-18 — programmability and composition became Phase G of RFC 0048's + rollout; the in-context instrument gates Phases B, D and E. - 2026-09-18 — added programmability: transparent `define`, a typed plan input surface, multi-statement requests at one snapshot; procedural control flow, opaque functions and free inference calls excluded. From 7284dfc55b0790a849b5c54ad16453412d8b9af3 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 18 Sep 2026 12:42:51 +0200 Subject: [PATCH 08/16] rfc: adopt the kernel in RFC 0048, the composition examples and the lexical RFC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RFC 0048's per-group selection is `limit n of $x per { … }` on the active order; `let` carries scoring features; `filter` carries predicates; the kernel note records the adoption. The composition RFC's C1–C4 are rewritten to `filter`, `order` + `limit`, `let` scoring features and `sub()` under `one`/`collect`; its capability rows and operator table follow; the duplicated C4 in the kernel section is dropped in favour of the example. The lexical RFC moves `match_terms` from the `match` block to a `filter` stage and its migration mapping says so. No semantic decision changed. --- docs/rfcs/0048-search-contracts.md | 64 ++++---- .../2026-09-18-analyzed-lexical-search.md | 29 ++-- ...8-gq-composition-and-language-evolution.md | 142 +++++++----------- 3 files changed, 107 insertions(+), 128 deletions(-) diff --git a/docs/rfcs/0048-search-contracts.md b/docs/rfcs/0048-search-contracts.md index 4a08ae40b..03eb37179 100644 --- a/docs/rfcs/0048-search-contracts.md +++ b/docs/rfcs/0048-search-contracts.md @@ -177,7 +177,7 @@ queries to adopt ranking. Existing search queries still need the rewrites above. score or distance available for projection. Missing arm membership remains distinguishable from a computed score. - Ranking within a graph-defined population, further traversal of selected - targets, and `take` selection with per-group quotas. Local and final ordering + targets, and `limit … of … per` selection with per-group quotas. Local and final ordering can explicitly place missing values with `nulls first` or `nulls last`; omitting the modifier retains current ascending/descending defaults. - Inspectable query definitions and plans, retrieval/coverage metadata, and @@ -299,21 +299,21 @@ cut is the shared `Terms` relation; ranking uses the field's resolved scoring policy (`bm25_v1` by default) with the [field-corpus statistics](2026-09-18-analyzed-lexical-search.md#lexical-scoring-and-shared-matching-semantics) that RFC owns. A source cannot change analysis, edit budget or statistics -scope; those are field and query facts. `match_terms` in `match` is the same +scope; those are field and query facts. `match_terms` in `filter` is the same description used as a Boolean predicate and introduces no rank or window. ### Clause composition and named stages -**Kernel alignment (proposed 2026-09-18).** The +**Kernel (adopted 2026-09-18).** This RFC's syntax is the [kernel](2026-09-18-gq-composition-and-language-evolution.md#kernel-one-stage-per-job) -in the composition RFC keeps this section's `rank` block, `yield`, named -arms and `metric()` unchanged and collapses the rest of this RFC's sketch: -`select` becomes `order` + `limit`, `take` becomes `limit n of $x per { … }`, -`score` becomes `let`, `collect`/`optional` blocks become `sub(…)` -expressions under a reduction, scalar predicates leave `match` for -`filter`, and stage tie keys are a `ties:` option on a source. The -semantics below are unchanged by that proposal; the spellings in the -examples are not yet rewritten to it. +the composition RFC defines: `match` (patterns only), `filter`, `let`, +`rank`, `group`, `order`, `limit n [of $x] [per { … }]`, `return`. The +`rank` block, `yield`, named arms and `metric()` below are kernel forms; +the earlier sketch's `select`, `take`, `score`, `collect` and `optional` +spellings are replaced by `order` + `limit`, `limit … of … per`, `let`, +and `sub(…)` under a reduction, scalar predicates live in `filter`, and +stage tie keys are a `ties:` option on a source. The semantics of every +section below are unchanged by the adoption; only spellings moved. The current grammar has one `match`, followed by `return`, optional `order`, and optional `limit`. The proposed extension admits explicit rank boundaries @@ -414,7 +414,7 @@ as a batch of per-row queries. Correlated per-target retrieval requires a separate bounded operator contract; it is not implicit in repeated `match`. Analyzed filtering alone remains an ordinary match predicate; the -[`match_terms` example](2026-09-18-analyzed-lexical-search.md#analyzed-filtering-in-match) and the rule that +[`match_terms` example](2026-09-18-analyzed-lexical-search.md#analyzed-filtering-with-filter) and the rule that no adjacent source inherits its edit budget are in the lexical RFC. ### Language evolution @@ -693,8 +693,8 @@ C1–C4, the grammar-evolution acceptance cases, are in [GQ composition and language evolution](2026-09-18-gq-composition-and-language-evolution.md#required-composition-examples). Of them only C2's terminal subset is in this release: retrieve, traverse, then terminally aggregate the selected population, with the candidate barrier -kept before traversal and aggregation. Intermediate grouping, `let`, -`select`, `score`, `optional` and `collect` are deferred there. +kept before traversal and aggregation. Intermediate `group`, scoring +features in `let`, and `sub()` reductions are deferred there. ### Target identity, fan-out, grouping, and metrics @@ -760,8 +760,9 @@ A later rank stage establishes a new active order while earlier metrics keep their origin. Aggregation does not implicitly inherit a constituent target's rank. These ordering rules require golden plans and fan-out fixtures. -Selection per group uses a `take` stage, with a named node/edge target, a -nonempty `per` key tuple, an optional local `order`, and a required `limit`: +Selection per group is the `limit` stage with a target and a partition: +`limit n of $x per { key, … }`, where `$x` is a named node or edge binding +and the key tuple is nonempty: ```gq query incidents_per_organization($q: String) { @@ -770,7 +771,7 @@ query incidents_per_organization($q: String) { lexical($i.title, terms($q), candidates: 100) as incidents yield incidents } - take $i { per { $o.slug } limit 2 } + limit 2 of $i per { $o.slug } return { $o.slug, $i.slug, metric(incidents, rank) as rank } order { $o.slug asc, rank asc, $i.@id } limit 20 @@ -782,16 +783,18 @@ population, the quota retains at most two of those incidents per organization, and the final limit returns at most 20 binding rows. Neither later limit refills the source. `per { $o.slug, $o.category }` illustrates a composite group key. -An omitted local `order` uses the latest ranking stage's rank. An explicit -local order, such as `order { metric(incidents, rank) asc nulls last }`, chooses -the pair winners without establishing a new global output order. With no -ranking stage in scope, a local comparator is required; this also permits -ordinary graph selection such as latest events per entity. Such input remains -subject to whole-query budgets rather than requiring a synthetic search stage. -Quota `limit` takes a non-null integer literal or parameter, must be nonnegative, -and permits zero, consistent with ordinary limit semantics. It does not impose -a source candidate-window cap; resource admission and checked arithmetic still -apply. Without a final order or an earlier ranking, output remains unordered. +The cut uses the active order: the latest ranking stage's rank, or an +explicit `order { … }` stage immediately before it, such as +`order { metric(incidents, rank) asc nulls last }`, which chooses the pair +winners and orders the surviving rows until a later `order` reorders them. +With no ranking stage in scope, a preceding `order` is required; this also +permits ordinary graph selection such as latest events per entity. Such +input remains subject to whole-query budgets rather than requiring a +synthetic search stage. The count takes a non-null integer literal or +parameter, must be nonnegative, and permits zero, consistent with ordinary +limit semantics. It does not impose a source candidate-window cap; resource +admission and checked arithmetic still apply. A `limit` without `of` counts +rows; without a final order or an earlier ranking, output remains unordered. Ordering expressions in either location may append `nulls first` or `nulls last`. Omission preserves current GQ defaults: ascending puts nulls first, @@ -857,7 +860,7 @@ ordinary graph operations. ### Lexical scoring The `bm25_v1` definition, the unified exact/fuzzy formula, the field-corpus -statistics decision and the feature table for the deferred `score` operator +statistics decision and the feature table for scoring features in `let` are owned by [Analyzed lexical search](2026-09-18-analyzed-lexical-search.md#lexical-scoring-and-shared-matching-semantics). Metrics produced by a `lexical` source carry the domain `Score`. @@ -1744,7 +1747,7 @@ goldens and API/CLI contract tests still cover boundaries GQT does not invoke. The test-only staged compiler prototype that exercised the GQ examples in this RFC, multi-stage node and edge targets, non-leaking negation scopes, alias namespaces, metric domains and origins, aggregate output identity, -final order/window separation, `take` key and comparator rules and the +final order/window separation, per-group key and comparator rules and the rejection cases is retained on the evidence branch ([`staged_probe.rs`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-compiler/src/query/staged_probe.rs)). It is partial compiler evidence, not a staged AST/IR or execution, and it is @@ -2059,6 +2062,9 @@ still require prototypes; full production qualification belongs to its phase. ## Decision log +- 2026-09-18 — adopted the kernel: this RFC's examples and the per-group + selection section now use `limit … of … per`, `let`, `filter` and `sub(…)`; + no semantic decision changed. - 2026-09-18 — replaced the single coordinated cutover with Phases A–G ordered by risk: truth first, the agent door, opt-in representations, additive kernel stages behind a setting, a deprecation cutover, acceleration, diff --git a/docs/rfcs/2026-09-18-analyzed-lexical-search.md b/docs/rfcs/2026-09-18-analyzed-lexical-search.md index 013d1f47c..af21abdbd 100644 --- a/docs/rfcs/2026-09-18-analyzed-lexical-search.md +++ b/docs/rfcs/2026-09-18-analyzed-lexical-search.md @@ -25,7 +25,7 @@ A String property becomes searchable by declaring `@analyzed`, which binds an immutable analyzer profile and a default scoring policy into the accepted schema. One typed lexical description, `terms(text, mode, max_edits)`, is consumed by two operators: `match_terms(field, terms(...))`, a Boolean -predicate in `match`, and the `lexical` ranking source that +predicate in `filter`, and the `lexical` ranking source that [RFC 0048](0048-search-contracts.md#lexical-sources) places inside a `rank` stage. Both apply the field's analyzer to document and query text at every edit budget, so the same query has one matched set whether the field is indexed, partially @@ -203,16 +203,16 @@ Adding an analyzed query term cannot widen `mode: all`. Appending unrelated documents cannot remove existing predicate matches. A resource failure is a typed query failure, not an empty or truncated successful matched set. -### Analyzed filtering in `match` +### Analyzed filtering with `filter` -Analyzed filtering alone remains an ordinary match predicate: +Analyzed filtering alone is an ordinary predicate in a `filter` stage; it +never sits inside the pattern block (the composition RFC's +[pattern/predicate rule](2026-09-18-gq-composition-and-language-evolution.md#kernel-one-stage-per-job)): ```gq query organization_names($q: String) { - match { - $o: Organization - match_terms($o.name, terms($q, mode: all, max_edits: 1)) - } + match { $o: Organization } + filter { match_terms($o.name, terms($q, mode: all, max_edits: 1)) } return { $o.slug, $o.name } order { $o.slug asc } } @@ -238,7 +238,8 @@ rewrites stay there. The deprecation mapping is fixed here: `search(f, q)` and `match_text(f, q)` become `match_terms(f, terms(q))` (all terms, zero edits) and `fuzzy(f, q, n)` becomes `match_terms(f, terms(q, mode: all, max_edits: n))` -with `n` clamped to the admitted range or refused. The mapping changes +with `n` clamped to the admitted range or refused; the predicate moves from +the `match` block to a `filter` stage after it. The mapping changes membership where today's answer depended on index state; the diagnostic says so. The legacy spellings never gain new semantics during the window. @@ -280,7 +281,7 @@ or fuzzy scorer selector. Query-term repetition and order carry no extra weight; intentional source weighting belongs to explicit fusion or a separately specified future scoring operator. -The deferred membership-preserving `score` operator uses this same kernel +The deferred membership-preserving scoring feature in `let` uses this same kernel over its incoming targets, including targets that fail `Terms` membership: | Field/query outcome | Retrieval membership before its cut | Lexical feature | @@ -690,10 +691,10 @@ later `main` revision. | Existing GQT owner | First observed failure | Required implementation and migration proof | |---|---|---| -| [`fuzzy_query_bypasses_index_analyzer`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/fuzzy_query_bypasses_index_analyzer.gqt) | Step 2: capitalized `Introductio` returns no rows; `intro` is expected. | Phase 2 must apply the accepted field analyzer at every edit budget. Retain the lowercase and zero-edit controls and reach the later assertions. | -| [`index_state_changes_text_matches`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/index_state_changes_text_matches.gqt) | Step 5, including the two mutation steps: `running` finds only the appended row and loses the indexed row. | Phase 2 must use one matching definition for indexed and uncovered rows. Complete the later `beto` assertion as well; passing the first repaired step is insufficient. | -| [`search_on_traversal_target_is_dropped`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/search_on_traversal_target_is_dropped.gqt) | Step 2: traversal returns B, C and D where only B and D match. | Phase 3 must retain the target predicate and rank the traversal target. The later ranking assertion must return D, rather than fail for a missing score column. | -| [`unindexed_search_is_case_sensitive`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/unindexed_search_is_case_sensitive.gqt) | Step 3: unindexed `deep` finds only the lowercase row; both rows are expected. | Phase 2 must preserve matching with and without an index. Migrate both field declarations to the intended accepted analyzer, and run both query-case controls. | +| [`fuzzy_query_bypasses_index_analyzer`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/fuzzy_query_bypasses_index_analyzer.gqt) | Step 2: capitalized `Introductio` returns no rows; `intro` is expected. | Phase C must apply the accepted field analyzer at every edit budget. Retain the lowercase and zero-edit controls and reach the later assertions. | +| [`index_state_changes_text_matches`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/index_state_changes_text_matches.gqt) | Step 5, including the two mutation steps: `running` finds only the appended row and loses the indexed row. | Phase C must use one matching definition for indexed and uncovered rows. Complete the later `beto` assertion as well; passing the first repaired step is insufficient. | +| [`search_on_traversal_target_is_dropped`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/search_on_traversal_target_is_dropped.gqt) | Step 2: traversal returns B, C and D where only B and D match. | Phase D must retain the target predicate and rank the traversal target. The later ranking assertion must return D, rather than fail for a missing score column. | +| [`unindexed_search_is_case_sensitive`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/unindexed_search_is_case_sensitive.gqt) | Step 3: unindexed `deep` finds only the lowercase row; both rows are expected. | Phase C must preserve matching with and without an index. Migrate both field declarations to the intended accepted analyzer, and run both query-case controls. | These cases were added in `b1df2041` and remain active regressions. During the coordinated query migration, rewrite their legacy `search`, `fuzzy` and `bm25` @@ -816,6 +817,8 @@ relevance quality; Phase F chooses measured defaults. ## Decision log +- 2026-09-18 — `match_terms` moved from the `match` block to a `filter` + stage under the adopted kernel. - 2026-09-18 — became Phase C of RFC 0048's rollout: `@analyzed` opt-in per field with no rebuild for non-adopters, zero-edit membership through the existing index, fuzzy through a budgeted scan. diff --git a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md index 6daee19ad..3eafd1386 100644 --- a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md +++ b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md @@ -46,7 +46,7 @@ and retrieve then aggregate. The stage boundaries RFC 0048 introduces expose information loss that a single `match`/`return` cannot express. Before that syntax freezes, the shared expression model, contextual keywords, scope transitions and null rules must be fixed once, or every later operator -(intermediate grouping, `let`, `select`, `optional`, `collect`, typed unions) +(intermediate grouping, `let`, subquery reductions, typed unions) will be a special case. RFC 0048's [agent workload](0048-search-contracts.md#agent-workload-and-design-objective) table states the requirements these rules serve. @@ -96,7 +96,7 @@ Every operator declares its scope and population effect: | Predicate / ordinary projection | Preserve rows, duplicates, bindings and metric origins | | Traversal | Extend bindings with existing endpoint/edge-instance semantics; bounded reachability does not become path enumeration | | `rank` | Select distinct targets and retain their incoming binding rows | -| `take` | Select target/group pairs; it does not create reusable group features | +| `limit … of … per` | Select target/group pairs; it does not create reusable group features | | Group/reduction | Export keys and reduced values, discard unreduced member bindings/metrics | | Future optional/branch/collection | Explicit imports/exports, nullability, multiplicity and local scope | @@ -238,9 +238,9 @@ change the tested RFC query examples. | Reusable intermediate grouping/reduction | No | Foundation | Examples C1–C2 export group keys/entities and reduced values; drop unreduced member bindings and active ranks. Group-stage implementation remains deferred. | | General distinct rows/entities and distinct aggregates | No dedicated syntax | Defer | Specify the identity/value tuple and duplicate equivalence; retrieval deduplication is not general `DISTINCT`. | | Distinct-target ranking with retained binding rows | Limited; legacy search exists without the new stage contract | Deliver | Each target gets one source rank; selecting it retains its associated binding rows. | -| Quotas per explicit group, including ordinary non-search selection | No | Deliver through `take` | Select target/group pairs; use explicit reductions when order varies per pair; require local order without an active rank. | -| Parent selection using reduced child metrics | Limited; terminal aggregates cannot feed a quota | Deliver through `take` reductions | Selection does not create reusable group features or implicit parent-source ranks. | -| Top-N retrieval within every full group; row-dependent query inputs | No | Foundation | Examples C1/C4 require explicit correlation/partition scope, empty-group behavior and one total budget; implementation remains deferred. Global top-K plus `take` is different. | +| Quotas per explicit group, including ordinary non-search selection | No | Deliver through `limit … of … per` | Select target/group pairs; use explicit reductions when order varies per pair; require local order without an active rank. | +| Parent selection using reduced child metrics | Limited; terminal aggregates cannot feed a quota | Deliver through `limit … of … per` with explicit reductions | Selection does not create reusable group features or implicit parent-source ranks. | +| Top-N retrieval within every full group; row-dependent query inputs | No | Foundation | Examples C1/C4 require explicit correlation/partition scope, empty-group behavior and one total budget; implementation remains deferred. Global top-K plus `limit … of … per` is different. | | Final ordering by properties/aliases and identity tie-breaks | Yes, with documented legacy search boundary-tie gaps | Keep; repair retrieval ties | Final ordering cannot change a previous source cutoff; each selection owns its complete comparator. | | Explicit `nulls first` / `nulls last` | No syntax; fixed ascending/descending defaults | Deliver | Apply consistently in local and final ordering; omission preserves existing defaults. | | Final row limit, including zero | Yes; integer literal | Keep | Counts output rows; it does not resize retrieval or bound all intermediate work. | @@ -336,11 +336,8 @@ and enrichment. ```gq query composition_c1($q: String) { - match { - $i: Incident - $s hasIncident $i - $i.period = "prior" or $i.period = "current" - } + match { $i: Incident $s hasIncident $i } + filter { $i.period = "prior" or $i.period = "current" } group { per { $s } reduce { @@ -349,7 +346,8 @@ query composition_c1($q: String) { } } let { current_count - prior_count as increase } - select { order { increase desc, $s.@id asc } limit 1 } + order { increase desc, $s.@id asc } + limit 1 match { $s hasReport $p } rank $p { lexical($p.text, terms($q), candidates: 2) as reports @@ -364,10 +362,11 @@ query composition_c1($q: String) { `group` exports entity keys under their binding names and named reductions, discarding incident bindings. `let` adds row values whose sibling expressions -read the incoming scope; numeric results do not become identities. `select` -cuts rows, while `take` selects target/group pairs. Retain computed facts -through the service cut and later report reads. Multiple selected services -require C4's explicit correlation, not repeated global top-K plus `take`. +read the incoming scope; numeric results do not become identities. `order` +then `limit` cuts rows, while `limit … of … per` selects target/group pairs. +Retain computed facts through the service cut and later report reads. +Multiple selected services require C4's explicit correlation, not repeated +global top-K plus a per-group cut. **C2 — Retrieve, traverse, then aggregate the selected population.** Selected p1/p2 have three bindings to project P (two from p1); eligible p3 falls outside @@ -416,19 +415,17 @@ query composition_c3($q: String, $vector: Vector(3)) { knn($p.embedding, $vector, candidates: 2) as dense yield dense } - score $p { - lexical($p.text, terms($q), scoring: bm25_v1) as words_feature - } + let { lexical_score($p.text, terms($q), scoring: bm25_v1) as words_feature } return { $p.@id as passage_id, metric(words, rank) as lexical_rank, - metric(words, score) as lexical_score, feature(words_feature) as lexical_feature + metric(words, score) as lexical_score, words_feature as lexical_feature } } ``` -`score` preserves targets, bindings and comparator; it has no candidate window, -output selector, rank or fusion vote. `feature` names a scorer; `metric` -names retrieval membership. Reordering/cutting requires an explicit operation. +A scoring feature in `let` preserves targets, bindings and comparator; it +has no candidate window, output selector, rank or fusion vote. Its alias +names a feature; `metric` names retrieval membership. Reordering/cutting requires an explicit operation. A hidden lexical top-K followed by a join can drop d2 and is invalid. Scoring uses accepted corpus statistics and charges their work even for a small target set. @@ -448,11 +445,8 @@ null owner and empty typed list. ```gq query composition_c4($q: String) { - match { - $i: Incident - $s hasIncident $i - $i.period = "prior" or $i.period = "current" - } + match { $i: Incident $s hasIncident $i } + filter { $i.period = "prior" or $i.period = "current" } group { per { $s } reduce { @@ -461,20 +455,20 @@ query composition_c4($q: String) { } } let { current_count - prior_count as increase } - select { order { increase desc, $s.@id asc } limit 2 } - optional ($s) as owner { - match { $s ownedBy $person } - return { $person as person } - } - collect ($s) as reports { - match { $s hasReport $p } - rank $p { - lexical($p.text, terms($q), candidates: 2) as relevant - yield relevant - } - return { $p.@id as id, $p.text as text } - order { metric(relevant, rank) asc, $p.@id asc } - limit 2 + order { increase desc, $s.@id asc } + limit 2 + let { + one(sub($s) { match { $s ownedBy $person } return { $person as person } }) as owner, + collect(sub($s, $q) { + match { $s hasReport $p } + rank $p { + lexical($p.text, terms($q), candidates: 2) as relevant + yield relevant + } + return { $p.@id as id, $p.text as text } + order { metric(relevant, rank) asc, $p.@id asc } + limit 2 + }) as reports } return { $s as service, prior_count as prior_count, current_count as current_count, @@ -484,13 +478,13 @@ query composition_c4($q: String) { ``` Imports explicitly distinguish `$entity` from computed row values; query -parameters stay available. Child bindings/sources do not escape. `optional` -returns a nullable object: zero rows produce null, one produces an object, -and more than one requires explicit selection/collection or a cardinality -error. `OwnedBy` is zero-or-one in this example. +parameters stay available. Child bindings/sources do not escape. +`one(sub …)` returns a nullable object: zero rows produce null, one produces +an object, and more than one is a cardinality error unless the subquery +selects or collects explicitly. `OwnedBy` is zero-or-one in this example. -`collect` consumes ordered binding rows and requires a local comparator and -output limit. Unique reports need an explicit identity reduction; source +`collect(sub …)` consumes ordered binding rows and requires a local +comparator and output limit. Unique reports need an explicit identity reduction; source windows do not bound later fan-out or cumulative parent work. Both children preserve the parent row, accepted snapshot and one whole-query budget. @@ -723,9 +717,11 @@ and `optional (…) as …` blocks and maps Cypher's `CALL { }`, `EXISTS { }`, row-producing outer join (Cypher `OPTIONAL MATCH`), which fans out; the two are different operations, not two spellings. -**Spellings this removes from RFC 0048's sketch.** +**Spellings the kernel replaced in RFC 0048's earlier sketch** (adopted +2026-09-18; RFC 0048 and the examples in this document now use the kernel +forms). -| RFC 0048 sketch | Kernel form | +| Earlier sketch | Kernel form | |---|---| | `select { order { … } limit n }` | `order { … }` then `limit n` | | `take $i { per { $o.slug } order { … } limit 2 }` | `order { … }` (or the active rank) then `limit 2 of $i per { $o.slug }` | @@ -887,45 +883,17 @@ query incidents_per_organization($q: String) { order { $o.slug asc, rank asc, $i.@id asc } limit 20 } - -query composition_c4($q: String) { - match { $i: Incident $s hasIncident $i } - filter { $i.period = "prior" or $i.period = "current" } - group { - per { $s } - reduce { - count_if($i.period = "prior") as prior_count, - count_if($i.period = "current") as current_count - } - } - let { current_count - prior_count as increase } - order { increase desc, $s.@id asc } - limit 2 - let { - one(sub($s) { match { $s ownedBy $person } return { $person as person } }) as owner, - collect(sub($s, $q) { - match { $s hasReport $p } - rank $p { lexical($p.text, terms($q), candidates: 2) as relevant yield relevant } - return { $p.@id as id, $p.text as text } - order { metric(relevant, rank) asc, $p.@id asc } - limit 2 - }) as reports - } - return { - $s as service, prior_count, current_count, increase, owner, reports - } -} ``` -Each example has one spelling per operation; the second no longer needs a -`take` block, and the third no longer needs `select`, `optional` or -`collect` stages. The semantics RFC 0048 specifies for pair selection, -missing arms and nested budgets are unchanged; only the spellings moved. +Each example has one spelling per operation; the second needs no separate +per-group stage. [C1–C4](#required-composition-examples) are written in the +same forms. The semantics RFC 0048 specifies for pair selection, missing arms +and nested budgets are unchanged; only the spellings moved. -**Decision needed.** Adopt the kernel as the grammar RFC 0048's Phase 0 -stabilizes against, and rewrite its sketch and the `take`/`score`/`select` -sections accordingly, or keep the separate stage kinds and record why each -extra spelling earns its place. +**Decision (2026-09-18):** the kernel is adopted as the grammar RFC 0048's +Phase 0 stabilizes against; RFC 0048's per-group selection section and this +document's examples use its forms. A future stage kind earns its place by +the rule in the principle above: it cannot be desugared. ## Invariants @@ -936,7 +904,7 @@ extra spelling earns its place. - **Integrity failures are loud (8):** integer overflow, non-finite results, chained comparisons, unknown constructors and unresolved aliases are typed failures. -- **Bounded resource use (11):** deferred operators (`collect`, `optional`, +- **Bounded resource use (11):** deferred operators (subquery reductions, intermediate grouping) enter the language only with one whole-query budget and explicit empty-group, duplicate and cardinality rules. @@ -1074,6 +1042,8 @@ before optimizing batching, and measure per-group rescan cost. collapses `select`, `take`, `score`, `collect` and `optional` and moves scalar predicates out of `match`. Recorded as a decision RFC 0048 must take before its syntax stabilizes; the moved-in text above is unchanged. +- 2026-09-18 — kernel adopted; C1–C4 rewritten to `filter`, `order` + + `limit`, `let` scoring features and `sub()` reductions. - 2026-09-18 — programmability and composition became Phase G of RFC 0048's rollout; the in-context instrument gates Phases B, D and E. - 2026-09-18 — added programmability: transparent `define`, a typed plan From 42427ee37c53cc63bdb9a2a14c28f7fc40599121 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 18 Sep 2026 13:19:53 +0200 Subject: [PATCH 09/16] =?UTF-8?q?rfc:=20link=20the=20four=20regression=20i?= =?UTF-8?q?ssues=20(#747=E2=80=93#750)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/rfcs/0047-search-plan-truth.md | 3 ++- docs/rfcs/0048-search-contracts.md | 5 +++-- docs/rfcs/2026-09-18-analyzed-lexical-search.md | 10 +++++----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/rfcs/0047-search-plan-truth.md b/docs/rfcs/0047-search-plan-truth.md index 4fec3cd0a..b374dbba4 100644 --- a/docs/rfcs/0047-search-plan-truth.md +++ b/docs/rfcs/0047-search-plan-truth.md @@ -374,7 +374,8 @@ qualification required for the replacement. Regression cases for the refusals: a `.gqt` case with a String property without `@index` expecting `T27`, and a `search.rs` owner that drops the physical index of a declared property and expects `FullTextIndexRequired` -(the PR #606 case `unindexed_search_is_case_sensitive` seeds both). +(#747's case `unindexed_search_is_case_sensitive` seeds both; #750 owns +the traversal-target shape). Revalidation against the pinned Lance source also leaves a separate determinism gate: its plain FTS collector can drop equal-score boundary rows diff --git a/docs/rfcs/0048-search-contracts.md b/docs/rfcs/0048-search-contracts.md index 03eb37179..db4773351 100644 --- a/docs/rfcs/0048-search-contracts.md +++ b/docs/rfcs/0048-search-contracts.md @@ -1609,8 +1609,9 @@ remaining assumptions and falsifiers. The four GQT regressions recorded on PR #606 and their required implementation proof are in -[Analyzed lexical search](2026-09-18-analyzed-lexical-search.md#regression-cases); they are held out of the -corpus until each fix lands under an `issue_N` name. +[Analyzed lexical search](2026-09-18-analyzed-lexical-search.md#regression-cases); they are filed as #747, +#748, #749 and #750 and held out of the corpus until each fix lands under +its `issue_N` name. ### Implementation handoff and validation checkpoint diff --git a/docs/rfcs/2026-09-18-analyzed-lexical-search.md b/docs/rfcs/2026-09-18-analyzed-lexical-search.md index af21abdbd..d63d3c52c 100644 --- a/docs/rfcs/2026-09-18-analyzed-lexical-search.md +++ b/docs/rfcs/2026-09-18-analyzed-lexical-search.md @@ -691,10 +691,10 @@ later `main` revision. | Existing GQT owner | First observed failure | Required implementation and migration proof | |---|---|---| -| [`fuzzy_query_bypasses_index_analyzer`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/fuzzy_query_bypasses_index_analyzer.gqt) | Step 2: capitalized `Introductio` returns no rows; `intro` is expected. | Phase C must apply the accepted field analyzer at every edit budget. Retain the lowercase and zero-edit controls and reach the later assertions. | -| [`index_state_changes_text_matches`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/index_state_changes_text_matches.gqt) | Step 5, including the two mutation steps: `running` finds only the appended row and loses the indexed row. | Phase C must use one matching definition for indexed and uncovered rows. Complete the later `beto` assertion as well; passing the first repaired step is insufficient. | -| [`search_on_traversal_target_is_dropped`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/search_on_traversal_target_is_dropped.gqt) | Step 2: traversal returns B, C and D where only B and D match. | Phase D must retain the target predicate and rank the traversal target. The later ranking assertion must return D, rather than fail for a missing score column. | -| [`unindexed_search_is_case_sensitive`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/unindexed_search_is_case_sensitive.gqt) | Step 3: unindexed `deep` finds only the lowercase row; both rows are expected. | Phase C must preserve matching with and without an index. Migrate both field declarations to the intended accepted analyzer, and run both query-case controls. | +| #748 [`fuzzy_query_bypasses_index_analyzer`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/fuzzy_query_bypasses_index_analyzer.gqt) | Step 2: capitalized `Introductio` returns no rows; `intro` is expected. | Phase C must apply the accepted field analyzer at every edit budget. Retain the lowercase and zero-edit controls and reach the later assertions. | +| #749 [`index_state_changes_text_matches`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/index_state_changes_text_matches.gqt) | Step 5, including the two mutation steps: `running` finds only the appended row and loses the indexed row. | Phase C must use one matching definition for indexed and uncovered rows. Complete the later `beto` assertion as well; passing the first repaired step is insufficient. | +| #750 [`search_on_traversal_target_is_dropped`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/search_on_traversal_target_is_dropped.gqt) | Step 2: traversal returns B, C and D where only B and D match. | Phase D must retain the target predicate and rank the traversal target. The later ranking assertion must return D, rather than fail for a missing score column. | +| #747 [`unindexed_search_is_case_sensitive`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph-gqt/cases/unindexed_search_is_case_sensitive.gqt) | Step 3: unindexed `deep` finds only the lowercase row; both rows are expected. | Phase C must preserve matching with and without an index. Migrate both field declarations to the intended accepted analyzer, and run both query-case controls. | These cases were added in `b1df2041` and remain active regressions. During the coordinated query migration, rewrite their legacy `search`, `fuzzy` and `bm25` @@ -717,7 +717,7 @@ implementation above; later native probes do not close them. The four cases are held out of the corpus until each fix lands: a red case in `main` blocks unrelated work, and the fix regression gate keys on an -`issue_N` name. Each will be filed as an issue and land as +`issue_N` name. Each is filed (#747, #748, #749, #750) and lands as `issue_N_.gqt` with the fix that turns it green. ### Qualification matrix From 0eb39c1a9cc2db9dfe19f6d8e0385553427de0fe Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 18 Sep 2026 13:29:50 +0200 Subject: [PATCH 10/16] rfc: replace Phase A's caching item with attribution of the measured read floor (#752) --- docs/rfcs/0048-search-contracts.md | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/docs/rfcs/0048-search-contracts.md b/docs/rfcs/0048-search-contracts.md index db4773351..7130856ad 100644 --- a/docs/rfcs/0048-search-contracts.md +++ b/docs/rfcs/0048-search-contracts.md @@ -1424,12 +1424,12 @@ engine's meaning-preserving rewrites and the caller's meaning-changing ones distinguishable. One measurement outranks the catalogue. For the agent workload of many -small calls, the fixed per-request cost of reloading version-pinned control -state (on a fragmented graph, measured in hundreds of object-store reads -per request for the manifest alone) dwarfs any plan choice. Session-scoped -caching of immutable version-pinned state and multi-statement requests at -one snapshot are worth more than every rewrite above, and both are -invariant-12-clean because the cached state is immutable. +small calls, the fixed per-request floor dwarfs any plan choice: a trivial +served read measures about 0.35 s on the 0.11.0 personal graph with a +0.10–0.15 s round trip, and the engine's warm-read contract already +excludes manifest reloads, so the rest is unattributed server-side work +(#752). Attributing and cutting that floor, and multi-statement requests at +one snapshot, are worth more than every rewrite above. ## Invariants @@ -1834,7 +1834,7 @@ mechanism, not weakening the promised result. | Phase | Ships | Risk and reversibility | Owner | Gate | |---|---|---|---|---| -| A — Truth first | Refusals (`T26`, `T27`, `FullTextIndexRequired`); retrieval as typed IR; projectable metrics; deterministic ties; `warnings`/`metrics`/`retrievals`/usage descriptors; the diagnostics contract; `explain` v0 (logical plan); coarse budgets through the existing `ResourceLimitExceeded` owner; session-scoped caching of version-pinned state | Low: no syntax or format change; refusals and additive fields | RFC 0047 | Characterization goldens; the traversal-target and unindexed regression cases green; per-request overhead measured | +| A — Truth first | Refusals (`T26`, `T27`, `FullTextIndexRequired`); retrieval as typed IR; projectable metrics; deterministic ties; `warnings`/`metrics`/`retrievals`/usage descriptors; the diagnostics contract; `explain` v0 (logical plan); coarse budgets through the existing `ResourceLimitExceeded` owner; attribution of the served read floor through the usage descriptor | Low: no syntax or format change; refusals and additive fields | RFC 0047 | Characterization goldens; the traversal-target and unindexed regression cases green; the served read floor attributed (#752) | | B — The agent door | Stored queries as typed tools; `@description` on `.pg` declarations and in `schema show`; the one-page card; the in-context competence instrument with graph-computed ground truth; baseline measurements on today's language | None: the PG annotation is metadata | This RFC (agent-facing surface); composition RFC (instrument) | Instrument runs end to end; baseline recorded | | C — Representations, opt-in per field | `@analyzed` and analyzer profiles as an additive SchemaIR feature (unchanged graphs need no rebuild); resolved embedding recipes and defaults; `terms`/`match_terms` as a predicate; the exact scan baseline across index states; `bm25_v1` against the Decimal oracle; zero-edit membership through the existing FTS index, fuzzy through a budgeted scan | Medium: the format commitment is opt-in | Lexical RFC; this RFC (representation identity) | Membership laws across absent/partial/full/rebuilt indexes; oracle parity; the fuzzy and index-state regression cases green | | D — Kernel stages, additive, behind a setting | `filter`, `let`, `rank` (`lexical`, `knn`, `ann`, `rrf`, `ties:`), `group`, `order`/`limit` at any position with `of`/`per`, pure `return` with the aggregate sugar, `metric()`; typed stage IR with metric origin and fingerprints; coherent follow-up reads; exact physical paths only; the old grammar still parses and a session setting selects the kernel | Medium: additive syntax, no format; reversible | This RFC | Instrument A/B (first-try validity, turns) kernel versus current grammar; C2's terminal subset; barrier cost measured | @@ -1846,9 +1846,14 @@ mechanism, not weakening the promised result. RFC 0047 is this phase and its ordered stages are the internal order. It ships first and alone because every item is a refusal of a wrong answer or -an additive field. The session-scoped caching of immutable version-pinned -state rides with it because the fixed per-request overhead, not plan choice, -bounds the agent workload, and the cached state is invariant-12-clean. +an additive field. The served read floor rides with it: the engine's +warm-read contract (`warm_read_cost.rs`, since v0.7.1) already makes a warm +same-branch read zero manifest reads and one version probe, and a trivial +served read on the 0.11.0 personal graph measures about 0.35 s end to end +with a 0.10–0.15 s round trip, so about 0.2 s of server-side work is +unattributed. Phase A attributes it through the usage descriptor (#752) and +cuts what the attribution shows; caching of immutable version-pinned state +is invariant-12-clean if that is where the time is. **Exit:** characterization goldens before and after the retrieval-IR refactor; every ranked result totally ordered; descriptors in every @@ -2010,7 +2015,7 @@ budgets as settings); cross-type search, `sub()` reductions and every | An in-context agent uses the kernel at least as well as today's grammar | B and D, before any removal | Keep the old grammar as the surface and revisit spellings with the instrument; nothing shipped is lost | | Exact-scan fuzzy matching is fast enough on real graphs (tables of 200k rows exist) | C | Zero-edit stays index-served; fuzzy stays budgeted and refuses loudly; acceleration moves earlier in F | | `@analyzed` can be an opt-in SchemaIR feature with no stamp bump for non-adopters | C's feature design | A versioned feature flag with the same no-rebuild property for non-adopters | -| Session-scoped caching removes the fixed per-request overhead | A | Multi-statement requests carry the load; planner work in F is not the answer either way | +| The served read floor (about 0.2 s server-side on a trivial read) is attributable and reducible | A (#752) | Multi-statement requests carry the load; planner work in F is not the answer either way | | Stage barriers cost little | D | Widen the rewrite catalogue with proven equivalences before E | ### Extensions after the initial release @@ -2063,6 +2068,9 @@ still require prototypes; full production qualification belongs to its phase. ## Decision log +- 2026-09-18 — corrected Phase A's performance item after measuring the + 0.11.0 personal server: manifest reloads are already excluded by the + warm-read contract; the ~0.2 s server-side floor is attributed first (#752). - 2026-09-18 — adopted the kernel: this RFC's examples and the per-group selection section now use `limit … of … per`, `let`, `filter` and `sub(…)`; no semantic decision changed. From a2529a3ac92514b540ff16aca7139f190e932a94 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 18 Sep 2026 15:11:21 +0200 Subject: [PATCH 11/16] rfc: record the first in-context measurement and its diagnostic --- docs/rfcs/0047-search-plan-truth.md | 12 +++++++++++- ...-18-gq-composition-and-language-evolution.md | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/rfcs/0047-search-plan-truth.md b/docs/rfcs/0047-search-plan-truth.md index b374dbba4..a3d9f9074 100644 --- a/docs/rfcs/0047-search-plan-truth.md +++ b/docs/rfcs/0047-search-plan-truth.md @@ -160,6 +160,15 @@ depend on recognizing other languages' idioms and no catalogue of them is maintained; a reader that generalizes from the schema and the card is what the design leans on. +The measured motivating case: in the composition RFC's first in-context run, +every first-try failure was `query name {` without `()`, refused with +`parse error --> 1:1 … expected query_file` wrapped in terminal colour codes +and a Rust backtrace footer. Under this contract the same input reports a +`Q…` code at the name's end, "expected `(`: a query declares its parameters +even when it has none", the fix `query name()`, and nothing else; the CLI's +machine formats (`--json`, `jsonl`) carry the four fields without colour or +backtrace text. + **Metric projection.** ```gq @@ -469,7 +478,8 @@ language release or duplicate execution path. - 2026-09-18 — became Phase A of RFC 0048's rollout. - 2026-09-18 — added the diagnostics contract (code, position or stage, expectation, one fix; unknown names enumerate their set) for every - compile diagnostic and typed read failure. + compile diagnostic and typed read failure; recorded the measured + `query name {` case as its motivating example. - 2026-09-18 — replaced the `full_text_search_unindexed` warning with a compile-time `T27` refusal of undeclared targets and a plan-time `FullTextIndexRequired` refusal of unbuilt indexes (RFC 0043 precedent); a diff --git a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md index 3eafd1386..7e5c9ef24 100644 --- a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md +++ b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md @@ -978,6 +978,21 @@ RFC 0048's [mixed workload qualification](0048-search-contracts.md#mixed-workloa Hold the model, card, schema and snapshot fixed within a comparison and retain failed trajectories; a model revision reopens the measurement. +**First measurement (2026-09-18, today's grammar).** Claude Opus 5 at effort +`high`, a one-page card, the personal graph's schema (about 90k tokens, +served from the prompt cache), fifteen private tasks (counts, filters, one- +and two-hop traversals, negation, a date range, an ordered aggregate, +full-text `search`) at one pinned commit, three repeats each: 45/45 tasks +correct, 37/45 first queries valid, 1.18 queries per task, about 11 s per +task. Every first-try failure was the same construct: a parameterless +query written `query name {` without `()`, refused with +`parse error --> 1:1 … expected query_file`, a grammar-rule name at column +one with no fix; every model repaired it from the card on the next turn. +The baseline says two things: today's grammar is already learnable in +context by a current model, so the kernel must not regress that; and the +first diagnostic a caller meets is the worst one in the language. The +instrument's private task set and results live outside the repository. + ## Rollout No production work starts from this document. Its rules gate RFC 0048's @@ -1049,6 +1064,8 @@ before optimizing batching, and measure per-group rescan cost. - 2026-09-18 — added programmability: transparent `define`, a typed plan input surface, multi-statement requests at one snapshot; procedural control flow, opaque functions and free inference calls excluded. +- 2026-09-18 — recorded the first in-context measurement on today's grammar + (45/45 correct, 37/45 first-try valid, one error class: the missing `()`). - 2026-09-18 — made the in-context consumer the design target: card-sized kernel, case-insensitive atomic keywords, spellings decided by measured first-try validity rather than analogy to any prior language. From 4a223ca836d429567463aeb49c10c147ed5b6ec1 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 18 Sep 2026 15:19:44 +0200 Subject: [PATCH 12/16] rfc: Phase A as nine PRs against the current tree, with the stored-query deployment gate --- docs/rfcs/0047-search-plan-truth.md | 72 ++++++++++++++++++----------- docs/rfcs/0048-search-contracts.md | 5 +- 2 files changed, 49 insertions(+), 28 deletions(-) diff --git a/docs/rfcs/0047-search-plan-truth.md b/docs/rfcs/0047-search-plan-truth.md index a3d9f9074..32248b047 100644 --- a/docs/rfcs/0047-search-plan-truth.md +++ b/docs/rfcs/0047-search-plan-truth.md @@ -406,33 +406,48 @@ prototype's retirement label must not be reused as a current-code fact. This RFC is Phase A of [RFC 0048's rollout](0048-search-contracts.md#implementation-phases): it ships first and alone, with no syntax or format change, because every -item is a refusal of a wrong answer or an additive field. The ordered stages -below are its internal order. - -The retained prototype supplies starting points; each stage must be checked -against the revised scope: - -1. Substrate fences for the Lance 11 update→optimize stale-vector window - (test-only; can land before acceptance as an ordinary change). -2. `T27`, the `FullTextIndexRequired` refusal and the warning carrier - (engine → API → CLI → OpenAPI). -3. Characterization goldens, then the retrieval-IR refactor (behavior- - equivalent by construction; goldens prove it). -4. `T26` scan-rooted targets (compiler pass + engine backstops). -5. Projectable metrics and deterministic ties (single-search, fusion, - aggregated). -6. `metrics`/`retrievals` metadata with embedding coverage. - -For the standalone slice, `implementation` advances to `in-progress` at the -first landed stage and `complete` when stage 6 ships. For the combined release, -record completion against the corresponding RFC 0048 phase evidence instead -of requiring this interim sequence. Stages 2+ reference this RFC once accepted. -The lexical replacement and staged language are sequenced by RFC 0048, not -by a retirement commit in this rollout. Before porting prototype code, capture -the current compiler/engine baseline and identify which guarantees already -exist. If both designs are implemented together, build named stage IR and -metadata directly, preserving this RFC's correctness gates without an interim -language release or duplicate execution path. +item is a refusal of a wrong answer or an additive field. It lands as the +pull requests below, in order; each is green on its own and carries its own +regression test. + +The retained prototype (closed PR #595, branch `search-contracts-p0-p1` at +`3e459aad`, 11 commits on a 2026-09-01 base) is a design reference, not a +cherry-pick source: `main` has since changed the same files by thousands of +lines (`exec/query.rs`, `typecheck.rs`, `table_store.rs`, the API types), and +a trial merge conflicts in eleven files. Its commit order and its tests are +the specification; the code is re-derived against the current tree. + +| PR | Lands | Where it touches today's tree | Regression owner | +|---|---|---|---| +| 1 | The diagnostics contract: a structured compiler diagnostic (code, span, expectation, fix); pest's parse error mapped to the position of the deepest failure with the expected tokens and one fix (the measured `query name {` case needs a committal grammar rule or a recognizer for that shape); the CLI's `--json`/`jsonl` error output as `ErrorOutput` JSON with no colour or backtrace text; `ErrorOutput` gains additive `position`, `expected` and `fix` fields; OpenAPI regenerated | `CompilerError::Type("T33: …")` string prefixes in `typecheck.rs`; `parse_query` in `query/parser.rs`; `color_eyre::install()` in the CLI's `main.rs`; `ErrorOutput` in `omnigraph-api-types` | compiler parser/typecheck tests; CLI `cli_queries`; server `data_routes` and `openapi` | +| 2 | `T27` and the `FullTextIndexRequired` refusal, replacing the flat bare-tokenizer fallback | the search and rank arms of `typecheck.rs`; the FTS scan path in `exec/query.rs` that chooses index or flat, keyed on `TableStore::has_fts_index_on`; a new `OmniError` variant shaped like `FullTextIndexRebuildRequired` | `issue_747_unindexed_search_is_case_sensitive.gqt` (its unindexed steps become `T27` refusals); a `search.rs` test that drops a declared property's physical index and expects the refusal. The refusal keys on index absence only: an uncovered tail of a built index keeps scanning with the index analyzer | +| 3 | Characterization goldens for every retrieval shape, captured before the refactor | `search.rs` | the goldens themselves | +| 4 | Retrieval as typed IR: `QueryIR::retrieval` (`Nearest { k }`, `Bm25 { scan_cap }`, `FuseRrf`) fixed by lowering; the executor's `SearchMode` resolved from it instead of from `order_by[0]` | `extract_search_mode` and `extract_sub_search_mode` in `exec/query.rs`; `ir/lower.rs`, `ir/mod.rs` | PR 3's goldens (behaviour-equivalent); the #587 gate composes unchanged | +| 5 | `T26`: a `scan_root_variables` computation shared by lowering and typecheck, negation scopes checked with their own roots, an engine backstop; the `rrf_prefilter_gate` expand-dst fixture ported to assert the diagnostic | `ir/lower.rs`, `typecheck.rs`, `exec/query.rs` | `issue_750_search_on_traversal_target_is_dropped.gqt`; compiler typecheck/lower tests | +| 6 | Projectable `rrf` (retire `T37`) and the total order: score, then trailing keys inside score ties, then stable id; boundary-tie plateau retained within an explicit budget or a typed failure | `exec/projection.rs`, fusion in `exec/query.rs` | `search.rs`, `ordering.rs`, `aggregation.rs`; the tie fixture with more tied rows than the native window and reversed id order | +| 7 | Read descriptors: `warnings` (`search_order_ignored_by_aggregation`, `embedding_coverage_pending`), `metrics`, `retrievals` with known/unknown coverage by default and exact counts under a request-scoped `coverage` setting, `usage`; carried in JSON, the JSONL metadata record and Arrow metadata | `ReadOutput` in the API types; `read_format.rs`; a read-only `count_rows_matching` on `TableStore` registered in `forbidden_apis`; the settings table | server `data_routes`/`openapi`; CLI parity; `search.rs` coverage tests | +| 8 | `explain` v0: the logical plan and the retrieval descriptor for a query, as a CLI flag and a route; no surface exists today | CLI `cli.rs`, server routes, API types | CLI and server tests; OpenAPI | +| 9 | #752: attribute the served read floor per phase and report it through `usage`; independent of 1–8 | server instrumentation | a cost test pinning the object-store request count of a trivial served read | + +PR 6 is the one with open design work: retaining a boundary-tie plateau +within a budget is this RFC's `blocked_on`. PRs 1–5 and 7–9 have no open +questions. + +**Deployment gate.** `T26` and `T27` refuse queries the compiler accepts +today. Stored queries persist as text and recompile when a server starts, +and a registry that fails to compile quarantines its graph +(`omnigraph-cluster`'s serve path), so a deployment carrying one such +stored query would go dark on restart. PRs 2 and 5 therefore ship with the +new diagnostics exposed through `queries validate` and `cluster plan`, and +the release note tells operators to run them before upgrading; a refused +stored query is a pre-upgrade finding, never a boot failure. The +compatibility-surfaces RFC's cross-version job is the place this is +checked mechanically once it exists. + +`implementation` advances to `in-progress` when PR 1 lands and to +`complete` when PR 8 does; PR 9 is tracked by its issue. The lexical +replacement and the staged language are sequenced by RFC 0048, not by this +rollout. ## Unresolved questions @@ -475,6 +490,9 @@ language release or duplicate execution path. blanket claim that all RRF arm windows follow the output limit: BM25 arms are uncapped; vector arms inherit that limit. Distinguished historical prototype results from current evidence and total ordering from ANN replay. +- 2026-09-18 — replaced the six-stage rollout with the nine-PR order against + the current tree, recorded that the #595 prototype is a reference rather than + a cherry-pick source, and added the stored-query deployment gate. - 2026-09-18 — became Phase A of RFC 0048's rollout. - 2026-09-18 — added the diagnostics contract (code, position or stage, expectation, one fix; unknown names enumerate their set) for every diff --git a/docs/rfcs/0048-search-contracts.md b/docs/rfcs/0048-search-contracts.md index 7130856ad..c5b11d63e 100644 --- a/docs/rfcs/0048-search-contracts.md +++ b/docs/rfcs/0048-search-contracts.md @@ -1860,7 +1860,9 @@ refactor; every ranked result totally ordered; descriptors in every transport; the two regression cases RFC 0047 owns green. **Open:** fuzzy analysis and index-state divergence remain until Phase C; -whole-query admission remains engine version 2's. +whole-query admission remains engine version 2's. The PR order, the +prototype's status as a reference, and the stored-query deployment gate are +in [RFC 0047's rollout](0047-search-plan-truth.md#rollout). #### Phase B: the agent door @@ -2016,6 +2018,7 @@ budgets as settings); cross-type search, `sub()` reductions and every | Exact-scan fuzzy matching is fast enough on real graphs (tables of 200k rows exist) | C | Zero-edit stays index-served; fuzzy stays budgeted and refuses loudly; acceleration moves earlier in F | | `@analyzed` can be an opt-in SchemaIR feature with no stamp bump for non-adopters | C's feature design | A versioned feature flag with the same no-rebuild property for non-adopters | | The served read floor (about 0.2 s server-side on a trivial read) is attributable and reducible | A (#752) | Multi-statement requests carry the load; planner work in F is not the answer either way | +| A deployed graph has no stored query that `T26`/`T27` refuse; if one exists, operators find it before upgrading | A, through `queries validate` and `cluster plan` before the release | A refused stored query quarantines its graph at boot; the diagnostics ship in the validators first, and the release note makes the check a precondition | | Stage barriers cost little | D | Widen the rewrite catalogue with proven equivalences before E | ### Extensions after the initial release From d6398da5b26be5debb7e875698e998efc5284c67 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 18 Sep 2026 15:20:22 +0200 Subject: [PATCH 13/16] rfc: keep only the source-audit receipt under assets; probes and fixtures cite the evidence branch --- docs/rfcs/0048-search-contracts.md | 4 +- .../2026-09-18-analyzed-lexical-search.md | 10 +- docs/rfcs/assets/0048-arrow-pool-probe.py | 75 ---- docs/rfcs/assets/0048-arrow-pool-probe.rs | 55 --- docs/rfcs/assets/0048-lexical-scoring-v1.json | 412 ------------------ docs/rfcs/assets/0048-lexical-scoring-v1.py | 81 ---- 6 files changed, 7 insertions(+), 630 deletions(-) delete mode 100644 docs/rfcs/assets/0048-arrow-pool-probe.py delete mode 100644 docs/rfcs/assets/0048-arrow-pool-probe.rs delete mode 100644 docs/rfcs/assets/0048-lexical-scoring-v1.json delete mode 100644 docs/rfcs/assets/0048-lexical-scoring-v1.py diff --git a/docs/rfcs/0048-search-contracts.md b/docs/rfcs/0048-search-contracts.md index c5b11d63e..644fee282 100644 --- a/docs/rfcs/0048-search-contracts.md +++ b/docs/rfcs/0048-search-contracts.md @@ -1577,7 +1577,7 @@ Reproduce these native checks from the repository root: ```sh cargo test -p omnigraph-engine --test lance_surface_guards --locked -python3 docs/rfcs/assets/0048-arrow-pool-probe.py +python3 docs/rfcs/assets/0048-arrow-pool-probe.py # on the evidence branch ``` The six guard probes cited above are proposed in a separate test-only pull @@ -1586,7 +1586,7 @@ request; the DataFusion selection probes run on the evidence branch The Arrow runner creates an isolated temporary crate, checks all resolved registry versions/checksums against the workspace lockfile, and enables only -there the optional Arrow pool features. Its [Rust probe](assets/0048-arrow-pool-probe.rs) +there the optional Arrow pool features. Its [Rust probe](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/docs/rfcs/assets/0048-arrow-pool-probe.rs) and result receipt are reviewable; production dependency features are unchanged. These deterministic mechanism checks do not add latency, peak-memory or retrieval-quality claims. They supplement the frozen integrated prototype and diff --git a/docs/rfcs/2026-09-18-analyzed-lexical-search.md b/docs/rfcs/2026-09-18-analyzed-lexical-search.md index d63d3c52c..ebd34ebd2 100644 --- a/docs/rfcs/2026-09-18-analyzed-lexical-search.md +++ b/docs/rfcs/2026-09-18-analyzed-lexical-search.md @@ -417,7 +417,7 @@ positive IDF to zero at `N = 2^24`; the allocating that corpus. Native scores or top-k cuts require new qualification; rescoring an incomplete native candidate set cannot establish exact winners. -The [numerical fixtures](assets/0048-lexical-scoring-v1.json) and the +The [numerical fixtures](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph/tests/fixtures/lexical_scoring_v1.json) and the [`lexical_scoring_v1_reference_oracle`](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph/tests/search.rs) probe on the evidence branch cover the chosen formula, zero-edit behavior, repeated/reordered terms, alternative aggregation, overlapping groups, `all`/`any`, null/token-empty @@ -661,7 +661,7 @@ The pinned tokenizer/edit-distance probe passed 73,008 comparisons against an independent Unicode-scalar evaluator at budgets zero through two. This covers the primitive, not the revised NFC pipeline, indexed completeness or budgets. -The [Decimal oracle](assets/0048-lexical-scoring-v1.py) +The [Decimal oracle](https://github.com/ModernRelay/omnigraph/blob/ce5a3012d655f5a47c4475ada6ac5b8d4e488fbd/crates/omnigraph/tests/fixtures/lexical_scoring_v1.py) generates thirteen 80-digit reference cases. The float64 evaluator uses pinned `libm 0.2.16`, tolerance `2e-14 * max(1, expected)`, exact fixture order, repeated-term invariance, eligibility-independent scores/features, all/any @@ -825,9 +825,9 @@ relevance quality; Phase F chooses measured defaults. - 2026-09-18 — split out of RFC 0048 at its 2026-09-13 revision with the lexical decisions unchanged. Replaced the hard removal of `fuzzy`, `search` and `match_text` with a one-release deprecation window and fixed the - mapping, following the compatibility-surfaces RFC. Moved the numerical - oracle fixtures to `assets/`; the oracle test and native probes stay on the - evidence branch until an evaluator exists. + mapping, following the compatibility-surfaces RFC. The numerical oracle + fixtures, the oracle test and the native probes stay on the evidence branch + until an evaluator exists. - 2026-09-13 — decided membership-independent lexical features, null and checked arithmetic policies, and deferred global execution. Extended the existing Decimal, compiler and DataFusion probes; default native arithmetic is not diff --git a/docs/rfcs/assets/0048-arrow-pool-probe.py b/docs/rfcs/assets/0048-arrow-pool-probe.py deleted file mode 100644 index 2e9ac4fb7..000000000 --- a/docs/rfcs/assets/0048-arrow-pool-probe.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python3 -"""Run the optional Arrow-pool qualification outside the production workspace. - -Requires Python 3.11+, Cargo, and the registry dependencies cached by the -workspace build. Offline resolution must retain versions/checksums from the -workspace lockfile. The temporary crate and output remain available for audit. -""" - -import hashlib -import json -from pathlib import Path -import shutil -import subprocess -import tempfile -import tomllib - - -def main(): - assets = Path(__file__).resolve().parent - root = assets.parents[2] - lock_bytes = (root / "Cargo.lock").read_bytes() - baseline = tomllib.loads(lock_bytes.decode()) - allowed = { - (p["name"], p["version"], p.get("source"), p.get("checksum")) - for p in baseline["package"] - } - directory = Path(tempfile.mkdtemp(prefix="omnigraph-0048-arrow-pool-")) - print(f"Probe directory: {directory}", flush=True) - (directory / "Cargo.toml").write_text('''[package] -name = "omnigraph-0048-arrow-pool-probe" -version = "0.0.0" -edition = "2024" - -[workspace] - -[dependencies] -arrow-array = { version = "=58.3.0", features = ["pool"] } -arrow-buffer = { version = "=58.3.0", features = ["pool"] } -datafusion-execution = { version = "=54.0.0", features = ["arrow_buffer_pool"] } - -[profile.dev] -debug = 0 -''') - (directory / "Cargo.lock").write_bytes(lock_bytes) - (directory / "src").mkdir() - source = assets / "0048-arrow-pool-probe.rs" - shutil.copyfile(source, directory / "src/main.rs") - subprocess.run( - ["cargo", "metadata", "--offline", "--format-version", "1"], - cwd=directory, check=True, stdout=subprocess.DEVNULL, - ) - resolved = tomllib.loads((directory / "Cargo.lock").read_text()) - for package in resolved["package"]: - if package["name"] == "omnigraph-0048-arrow-pool-probe": - continue - identity = (package["name"], package["version"], package.get("source"), package.get("checksum")) - if identity not in allowed: - raise RuntimeError(f"Dependency differs from workspace lock: {identity}") - output = subprocess.check_output( - ["cargo", "run", "--offline", "--locked", "--target-dir", str(directory / "target")], - cwd=directory, text=True, - ) - result = { - "workspace_lock_sha256": hashlib.sha256(lock_bytes).hexdigest(), - "probe_source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), - "resolved_lock_sha256": hashlib.sha256((directory / "Cargo.lock").read_bytes()).hexdigest(), - "registry_dependencies_match_workspace": True, - "observations": json.loads(output), - } - (directory / "result.json").write_text(json.dumps(result, indent=2) + "\n") - print(json.dumps(result, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/docs/rfcs/assets/0048-arrow-pool-probe.rs b/docs/rfcs/assets/0048-arrow-pool-probe.rs deleted file mode 100644 index 2dadd14ac..000000000 --- a/docs/rfcs/assets/0048-arrow-pool-probe.rs +++ /dev/null @@ -1,55 +0,0 @@ -// RFC 0048 qualification only. Run through 0048-arrow-pool-probe.py so the -// optional upstream features do not change OmniGraph's production feature set. -use std::sync::Arc; - -use arrow_array::{Array, Int32Array}; -use arrow_buffer::MemoryPool as ArrowPool; -use datafusion_execution::memory_pool::arrow::ArrowMemoryPool; -use datafusion_execution::memory_pool::{GreedyMemoryPool, MemoryConsumer, MemoryPool}; - -fn main() { - let first: Arc = Arc::new(GreedyMemoryPool::new(1)); - let second: Arc = Arc::new(GreedyMemoryPool::new(1)); - let first_arrow = ArrowMemoryPool::new(first.clone(), MemoryConsumer::new("query-a")); - let second_arrow = ArrowMemoryPool::new(second.clone(), MemoryConsumer::new("query-b")); - let original = Int32Array::from(vec![1, 2, 3, 4, 5]); - let bytes = original.get_buffer_memory_size(); - assert!(bytes > 1); - - // Positive refusal control: DataFusion's fallible operator reservation. - let admission = MemoryConsumer::new("fallible-control").register(&first); - assert!(admission.try_grow(bytes).is_err()); - assert_eq!(first.reserved(), 0); - - // Arrow's claim API tracks an already allocated buffer, infallibly. - original.claim(&first_arrow); - assert_eq!(first.reserved(), bytes); - assert!(first_arrow.available() < 0); - let over_limit_available = first_arrow.available(); - let shared = original.slice(0, 2); - shared.claim(&first_arrow); - assert_eq!( - first.reserved(), - bytes, - "same-owner slices are not double charged" - ); - - // The buffer has one reservation. Claiming it for another query transfers - // that reservation even though the first query still retains the array. - shared.claim(&second_arrow); - assert_eq!(first.reserved(), 0); - assert_eq!(second.reserved(), bytes); - drop(shared); - assert_eq!( - second.reserved(), - bytes, - "the first query still owns an alias" - ); - assert_eq!(original.value(4), 5); - drop(original); - assert_eq!(first.reserved(), 0); - assert_eq!(second.reserved(), 0); - println!( - "{{\"buffer_bytes\":{bytes},\"pool_capacity\":1,\"available_after_claim\":{over_limit_available},\"fallible_admission_refused\":true,\"same_owner_shared_once\":true,\"cross_owner_reservation_transferred\":true,\"released_after_last_alias\":true}}" - ); -} diff --git a/docs/rfcs/assets/0048-lexical-scoring-v1.json b/docs/rfcs/assets/0048-lexical-scoring-v1.json deleted file mode 100644 index e22fb35f9..000000000 --- a/docs/rfcs/assets/0048-lexical-scoring-v1.json +++ /dev/null @@ -1,412 +0,0 @@ -[ - { - "name": "rare_variant", - "docs": [ - "beta", - "beta", - "beta", - "beto", - "gamma" - ], - "query": [ - "beta" - ], - "edits": 1, - "all_terms": false, - "expected_scores": [ - "0.28768207245178092743921900599382743150350971089776105650666568534929295072078044", - "0.28768207245178092743921900599382743150350971089776105650666568534929295072078044", - "0.28768207245178092743921900599382743150350971089776105650666568534929295072078044", - "0.14384103622589046371960950299691371575175485544888052825333284267464647536039022", - null - ], - "expected_features": [ - "0.28768207245178092743921900599382743150350971089776105650666568534929295072078044", - "0.28768207245178092743921900599382743150350971089776105650666568534929295072078044", - "0.28768207245178092743921900599382743150350971089776105650666568534929295072078044", - "0.14384103622589046371960950299691371575175485544888052825333284267464647536039022", - "0" - ], - "expected_order": [ - 0, - 1, - 2, - 3 - ] - }, - { - "name": "zero_edits", - "docs": [ - "beta", - "beta", - "beta", - "beto", - "gamma" - ], - "query": [ - "beta" - ], - "edits": 0, - "all_terms": false, - "expected_scores": [ - "0.53899650073268700512435673639569911116140609696139877151666420268670167440592912", - "0.53899650073268700512435673639569911116140609696139877151666420268670167440592912", - "0.53899650073268700512435673639569911116140609696139877151666420268670167440592912", - null, - null - ], - "expected_features": [ - "0.53899650073268700512435673639569911116140609696139877151666420268670167440592912", - "0.53899650073268700512435673639569911116140609696139877151666420268670167440592912", - "0.53899650073268700512435673639569911116140609696139877151666420268670167440592912", - "0", - "0" - ], - "expected_order": [ - 0, - 1, - 2 - ] - }, - { - "name": "zero_edits_multi_term", - "docs": [ - "beta gamma gamma", - "beta", - "gamma", - "zeta" - ], - "query": [ - "gamma", - "beta" - ], - "edits": 0, - "all_terms": false, - "expected_scores": [ - "1.2357761692752447451923272912857813558293338351379531123819519288056883929058444", - "0.80259147222730509511468982484630971040321068189082187319236632678182419385964649", - "0.80259147222730509511468982484630971040321068189082187319236632678182419385964649", - null - ], - "expected_features": [ - "1.2357761692752447451923272912857813558293338351379531123819519288056883929058444", - "0.80259147222730509511468982484630971040321068189082187319236632678182419385964649", - "0.80259147222730509511468982484630971040321068189082187319236632678182419385964649", - "0" - ], - "expected_order": [ - 0, - 1, - 2 - ] - }, - { - "name": "alternatives_do_not_add", - "docs": [ - "beta", - "beta beto", - "beta gamma", - "beto", - "beta beta", - "gamma" - ], - "query": [ - "beta" - ], - "edits": 1, - "all_terms": false, - "expected_scores": [ - "0.27924027631429145000705833224194705018943769579491494115542388982151562830461054", - "0.21222260999886150200536433250387975814397264880413535527812215626435187751150402", - "0.21222260999886150200536433250387975814397264880413535527812215626435187751150402", - "0.13962013815714572500352916612097352509471884789745747057771194491075781415230527", - "0.30317515714123071715052047500554251163424664114876479325446022323478839644500575", - null - ], - "expected_features": [ - "0.27924027631429145000705833224194705018943769579491494115542388982151562830461054", - "0.21222260999886150200536433250387975814397264880413535527812215626435187751150402", - "0.21222260999886150200536433250387975814397264880413535527812215626435187751150402", - "0.13962013815714572500352916612097352509471884789745747057771194491075781415230527", - "0.30317515714123071715052047500554251163424664114876479325446022323478839644500575", - "0" - ], - "expected_order": [ - 4, - 0, - 1, - 2, - 3 - ] - }, - { - "name": "one_term_satisfies_two_groups", - "docs": [ - "beto", - "beta", - "beta beto", - "gamma" - ], - "query": [ - "beta", - "beto" - ], - "edits": 1, - "all_terms": true, - "expected_scores": [ - "0.58268678960287962891668700351282612736695807171030143514554187049454260636308639", - "0.58268678960287962891668700351282612736695807171030143514554187049454260636308639", - "0.57276268369723447708598917133620865074513640138920141070267376077809054250775402", - null - ], - "expected_features": [ - "0.58268678960287962891668700351282612736695807171030143514554187049454260636308639", - "0.58268678960287962891668700351282612736695807171030143514554187049454260636308639", - "0.57276268369723447708598917133620865074513640138920141070267376077809054250775402", - "0" - ], - "expected_order": [ - 0, - 1, - 2 - ] - }, - { - "name": "all_counts_each_family_before_query_filter", - "docs": [ - "beta gamma", - "beto gamma", - "beta", - "gamma", - "gamma beta beta", - "other" - ], - "query": [ - "beta", - "gamma" - ], - "edits": 1, - "all_terms": true, - "expected_scores": [ - "0.81683365967553471412656105909568970967960356827946099667520948129677880355126166", - "0.61262524475665103559492079432176728225970267620959574750640711097258410266344625", - null, - null, - "0.82882241201910909838477868716460883457705371849882686257553055978939297385778704", - null - ], - "expected_features": [ - "0.81683365967553471412656105909568970967960356827946099667520948129677880355126166", - "0.61262524475665103559492079432176728225970267620959574750640711097258410266344625", - "0.52827829076841647272315633713253845354278709035465140545842352322998194360108769", - "0.52827829076841647272315633713253845354278709035465140545842352322998194360108769", - "0.82882241201910909838477868716460883457705371849882686257553055978939297385778704", - "0" - ], - "expected_order": [ - 4, - 0, - 1 - ] - }, - { - "name": "any_uses_the_same_statistics", - "docs": [ - "beta gamma", - "beto gamma", - "beta", - "gamma", - "gamma beta beta", - "other" - ], - "query": [ - "beta", - "gamma" - ], - "edits": 1, - "all_terms": false, - "expected_scores": [ - "0.81683365967553471412656105909568970967960356827946099667520948129677880355126166", - "0.61262524475665103559492079432176728225970267620959574750640711097258410266344625", - "0.52827829076841647272315633713253845354278709035465140545842352322998194360108769", - "0.52827829076841647272315633713253845354278709035465140545842352322998194360108769", - "0.82882241201910909838477868716460883457705371849882686257553055978939297385778704", - null - ], - "expected_features": [ - "0.81683365967553471412656105909568970967960356827946099667520948129677880355126166", - "0.61262524475665103559492079432176728225970267620959574750640711097258410266344625", - "0.52827829076841647272315633713253845354278709035465140545842352322998194360108769", - "0.52827829076841647272315633713253845354278709035465140545842352322998194360108769", - "0.82882241201910909838477868716460883457705371849882686257553055978939297385778704", - "0" - ], - "expected_order": [ - 4, - 0, - 1, - 2, - 3 - ] - }, - { - "name": "null_and_token_empty_are_outside_corpus", - "docs": [ - null, - "", - "beta", - "beto", - "beta gamma" - ], - "query": [ - "beta" - ], - "edits": 1, - "all_terms": false, - "expected_scores": [ - null, - null, - "0.14874382975896190932757264103745313625150100339118484181560999128796061111161147", - "0.074371914879480954663786320518726568125750501695592420907804995643980305555805734", - "0.11085625048073576261205885511281884682894886101795851418333197463914045545110666" - ], - "expected_features": [ - null, - "0", - "0.14874382975896190932757264103745313625150100339118484181560999128796061111161147", - "0.074371914879480954663786320518726568125750501695592420907804995643980305555805734", - "0.11085625048073576261205885511281884682894886101795851418333197463914045545110666" - ], - "expected_order": [ - 2, - 4, - 3 - ] - }, - { - "name": "two_edits_and_transposition", - "docs": [ - "abcd", - "abce", - "abef", - "ab", - "acbd", - "zxyz" - ], - "query": [ - "abcd" - ], - "edits": 2, - "all_terms": false, - "expected_scores": [ - "0.24116205681688807046064128693622699789087801000469926736150245030039986080852729", - "0.12058102840844403523032064346811349894543900500234963368075122515019993040426364", - "0.060290514204222017615160321734056749472719502501174816840375612575099965202131822", - "0.060290514204222017615160321734056749472719502501174816840375612575099965202131822", - "0.060290514204222017615160321734056749472719502501174816840375612575099965202131822", - null - ], - "expected_features": [ - "0.24116205681688807046064128693622699789087801000469926736150245030039986080852729", - "0.12058102840844403523032064346811349894543900500234963368075122515019993040426364", - "0.060290514204222017615160321734056749472719502501174816840375612575099965202131822", - "0.060290514204222017615160321734056749472719502501174816840375612575099965202131822", - "0.060290514204222017615160321734056749472719502501174816840375612575099965202131822", - "0" - ], - "expected_order": [ - 0, - 1, - 2, - 3, - 4 - ] - }, - { - "name": "unicode_scalar_edit", - "docs": [ - "café", - "cafe", - "other" - ], - "query": [ - "cafe" - ], - "edits": 1, - "all_terms": false, - "expected_scores": [ - "0.23500181462286777682546851557417103235044952440612402022469606850300093910071319", - "0.47000362924573555365093703114834206470089904881224804044939213700600187820142638", - null - ], - "expected_features": [ - "0.23500181462286777682546851557417103235044952440612402022469606850300093910071319", - "0.47000362924573555365093703114834206470089904881224804044939213700600187820142638", - "0" - ], - "expected_order": [ - 1, - 0 - ] - }, - { - "name": "empty_corpus", - "docs": [], - "query": [ - "beta" - ], - "edits": 1, - "all_terms": false, - "expected_scores": [], - "expected_features": [], - "expected_order": [] - }, - { - "name": "only_missing_and_token_empty", - "docs": [ - null, - "", - " " - ], - "query": [ - "beta" - ], - "edits": 1, - "all_terms": false, - "expected_scores": [ - null, - null, - null - ], - "expected_features": [ - null, - "0", - "0" - ], - "expected_order": [] - }, - { - "name": "no_matching_family", - "docs": [ - null, - "", - "gamma" - ], - "query": [ - "beta" - ], - "edits": 1, - "all_terms": false, - "expected_scores": [ - null, - null, - null - ], - "expected_features": [ - null, - "0", - "0" - ], - "expected_order": [] - } -] diff --git a/docs/rfcs/assets/0048-lexical-scoring-v1.py b/docs/rfcs/assets/0048-lexical-scoring-v1.py deleted file mode 100644 index f05a05cef..000000000 --- a/docs/rfcs/assets/0048-lexical-scoring-v1.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Regenerate lexical_scoring_v1.json after an intentional RFC 0048 change. - -Uses 80-digit Decimal arithmetic over already-analyzed fixture terms. This -is a numerical design oracle, not a tokenizer or retrieval benchmark. Cargo -does not run this generator: the checked-in expectations require review. -""" -from collections import Counter -from decimal import Decimal, localcontext -import json -from pathlib import Path - -def distance(a, b): - grid = [[0] * (len(b) + 1) for _ in range(len(a) + 1)] - for i in range(len(a) + 1): - grid[i][0] = i - for j in range(len(b) + 1): - grid[0][j] = j - for i, ca in enumerate(a, 1): - for j, cb in enumerate(b, 1): - grid[i][j] = min(grid[i-1][j] + 1, grid[i][j-1] + 1, - grid[i-1][j-1] + (ca != cb)) - return grid[-1][-1] - -cases = [ - ("rare_variant", ["beta", "beta", "beta", "beto", "gamma"], ["beta"], 1, False), - ("zero_edits", ["beta", "beta", "beta", "beto", "gamma"], ["beta"], 0, False), - ("zero_edits_multi_term", ["beta gamma gamma", "beta", "gamma", "zeta"], ["gamma", "beta"], 0, False), - ("alternatives_do_not_add", ["beta", "beta beto", "beta gamma", "beto", "beta beta", "gamma"], ["beta"], 1, False), - ("one_term_satisfies_two_groups", ["beto", "beta", "beta beto", "gamma"], ["beta", "beto"], 1, True), - ("all_counts_each_family_before_query_filter", ["beta gamma", "beto gamma", "beta", "gamma", "gamma beta beta", "other"], ["beta", "gamma"], 1, True), - ("any_uses_the_same_statistics", ["beta gamma", "beto gamma", "beta", "gamma", "gamma beta beta", "other"], ["beta", "gamma"], 1, False), - ("null_and_token_empty_are_outside_corpus", [None, "", "beta", "beto", "beta gamma"], ["beta"], 1, False), - ("two_edits_and_transposition", ["abcd", "abce", "abef", "ab", "acbd", "zxyz"], ["abcd"], 2, False), - ("unicode_scalar_edit", ["café", "cafe", "other"], ["cafe"], 1, False), - ("empty_corpus", [], ["beta"], 1, False), - ("only_missing_and_token_empty", [None, "", " "], ["beta"], 1, False), - ("no_matching_family", [None, "", "gamma"], ["beta"], 1, False), -] - -out = [] -with localcontext() as ctx: - ctx.prec = 80 - D = Decimal - for name, docs, query, edits, all_terms in cases: - terms = sorted(set(query)) - counts = [Counter((doc or "").split()) for doc in docs] - n = sum(bool(c) for c in counts) - total_length = sum(sum(c.values()) for c in counts) - scores = [None] * len(docs) - features = [None if doc is None else D(0) for doc in docs] - if n: - avg = D(total_length) / n - dfs = {q: sum(any(distance(q, t) <= edits for t in c) for c in counts) for q in terms} - for i, c in enumerate(counts): - contributions = [] - for q in terms: - weights = [] - for t, freq in c.items(): - edit = distance(q, t) - if edit <= edits: - norm = D("1.2") * (D("0.25") + D("0.75") * sum(c.values()) / avg) - weights.append((D(2) ** -edit) * D("2.2") * freq / (D(freq) + norm)) - if not weights: - contributions.append(D(0)) - else: - idf = (1 + (D(n - dfs[q]) + D("0.5")) / (D(dfs[q]) + D("0.5"))).ln() - contributions.append(idf * max(weights)) - matched = all(v > 0 for v in contributions) if all_terms else any(v > 0 for v in contributions) - if docs[i] is not None: - features[i] = sum(contributions) - if matched: - scores[i] = features[i] - rank = sorted((i for i, s in enumerate(scores) if s is not None), key=lambda i: (-scores[i], i)) - out.append(dict(name=name, docs=docs, query=query, edits=edits, all_terms=all_terms, - expected_scores=[str(s) if s is not None else None for s in scores], - expected_features=[str(s) if s is not None else None for s in features], - expected_order=rank)) - print(name, rank, [round(float(s), 9) if s is not None else None for s in scores]) - -path = Path(__file__).with_suffix(".json") -path.write_text(json.dumps(out, indent=2, ensure_ascii=False) + "\n") From 2318e74edba7befe35472509e36200892431b3b9 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 18 Sep 2026 15:21:54 +0200 Subject: [PATCH 14/16] rfc: state the search contracts in the terms of the first-principles data-systems guide --- docs/rfcs/0048-search-contracts.md | 14 ++++++++++++++ docs/rfcs/2026-09-18-analyzed-lexical-search.md | 7 ++++++- ...-09-18-gq-composition-and-language-evolution.md | 9 ++++++++- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/rfcs/0048-search-contracts.md b/docs/rfcs/0048-search-contracts.md index 644fee282..f062b657d 100644 --- a/docs/rfcs/0048-search-contracts.md +++ b/docs/rfcs/0048-search-contracts.md @@ -95,6 +95,18 @@ Lance owns versioned datasets and indexes; OmniGraph owns graph semantics, snapshot coordination and admission. DataFusion supplies relational operators where qualified. Missing acceleration may change cost, never logical meaning. +In the terms of [evaluating data systems from first principles](../dev/systems.md), +the first two rows are the observable contract: the accepted schema and the +typed plan fix state, operations, guarantees and interface; the third row is +the physical realization evaluated against that contract; the +[agent workload](#agent-workload-and-design-objective) is the distribution of +work it is optimized for. Every law under Design is a contract statement, +every route in the native matrix is a mechanism judged against it under the +stated work, `ann` is the one operation whose contract admits a +nondeterministic result and says so, and the result descriptors report the +guarantee record the guide's checklist asks for: completion, coverage, +approximation and selection as separate facts, never one "consistency" flag. + ### Agent workload and design objective Optimize reliable task completion within latency, execution and working-context @@ -2071,6 +2083,8 @@ still require prototypes; full production qualification belongs to its phase. ## Decision log +- 2026-09-18 — stated the contract / physical realization / distribution of + work split in the terms of `docs/dev/systems.md` (#745). - 2026-09-18 — corrected Phase A's performance item after measuring the 0.11.0 personal server: manifest reloads are already excluded by the warm-read contract; the ~0.2 s server-side floor is attributed first (#752). diff --git a/docs/rfcs/2026-09-18-analyzed-lexical-search.md b/docs/rfcs/2026-09-18-analyzed-lexical-search.md index ebd34ebd2..19d41d879 100644 --- a/docs/rfcs/2026-09-18-analyzed-lexical-search.md +++ b/docs/rfcs/2026-09-18-analyzed-lexical-search.md @@ -192,7 +192,10 @@ retrieval. Its contract is: | Empty text | A query yielding no searchable terms is a typed error. A null or token-empty document does not match. | | Completeness | Every successful result satisfies the predicate exactly; an edit-tolerant predicate does not advertise approximate recall. | -For a fixed schema, query, and document population, these are normative laws: +For a fixed schema, query, and document population, these are normative laws +(the operation's contract in the sense of +[evaluating data systems from first principles](../dev/systems.md); an index +is part of the physical realization and cannot appear in them): ```text matches(edits=0) ⊆ matches(edits=1) ⊆ matches(edits=2) @@ -817,6 +820,8 @@ relevance quality; Phase F chooses measured defaults. ## Decision log +- 2026-09-18 — named the membership laws as the operation's contract and the + index as physical realization, in the terms of `docs/dev/systems.md`. - 2026-09-18 — `match_terms` moved from the `match` block to a `filter` stage under the adopted kernel. - 2026-09-18 — became Phase C of RFC 0048's rollout: `@analyzed` opt-in per diff --git a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md index 7e5c9ef24..d339c4c68 100644 --- a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md +++ b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md @@ -662,7 +662,12 @@ design direction and does not claim implemented support. ### Kernel: one stage per job **Principle.** The language is a small kernel of stage kinds over binding -tables plus one expression grammar. Each kernel stage has one job and one +tables plus one expression grammar. In the terms of +[evaluating data systems from first principles](../dev/systems.md), the +kernel fixes operations before interface: each stage's semantics (population, +multiplicity, order, metric origin) is stated as a contract, and a spelling +is only ever an interface for one such operation, which is why a construct +that cannot be desugared is a new operation and needs its own RFC. Each kernel stage has one job and one spelling. A convenience spelling is admitted only when it desugars to exactly one kernel form, `explain` shows that form, and the RFC that adds the spelling names the desugaring. A construct that cannot be desugared is a new kernel @@ -1057,6 +1062,8 @@ before optimizing batching, and measure per-group rescan cost. collapses `select`, `take`, `score`, `collect` and `optional` and moves scalar predicates out of `match`. Recorded as a decision RFC 0048 must take before its syntax stabilizes; the moved-in text above is unchanged. +- 2026-09-18 — stated the kernel as operations-before-interface in the terms + of `docs/dev/systems.md` (#745). - 2026-09-18 — kernel adopted; C1–C4 rewritten to `filter`, `order` + `limit`, `let` scoring features and `sub()` reductions. - 2026-09-18 — programmability and composition became Phase G of RFC 0048's From e92932f7bfd4b61bf93d6402a18e27f5f6467466 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sat, 19 Sep 2026 13:37:31 +0200 Subject: [PATCH 15/16] rfc: apply the five review findings on #606 (pushdown boundary, negation-scoped migration, no whole-graph rebuild, filter in Phase C, one nullable-metric rule) --- docs/rfcs/0048-search-contracts.md | 84 ++++++++++++------- .../2026-09-18-analyzed-lexical-search.md | 46 +++++++--- ...8-gq-composition-and-language-evolution.md | 9 +- 3 files changed, 94 insertions(+), 45 deletions(-) diff --git a/docs/rfcs/0048-search-contracts.md b/docs/rfcs/0048-search-contracts.md index f062b657d..621dfa5a9 100644 --- a/docs/rfcs/0048-search-contracts.md +++ b/docs/rfcs/0048-search-contracts.md @@ -67,8 +67,9 @@ deferred; their typed union/projection and source contracts remain required language-evolution work, not an implied capability of same-type fusion. The pre-stable cutover deliberately breaks search queries and representation -declarations, with one coordinated format rebuild after a one-release -deprecation window for the legacy spellings. The +declarations, after a one-release deprecation window for the legacy +spellings and without a whole-graph rebuild: representation changes are +adopted per field. The [migration table](#user-facing-changes-and-migration) owns those consequences. **Phase 0 is incomplete.** There are checked logical plans, native probes and @@ -173,7 +174,7 @@ graphs still cross the storage-format upgrade described below. | Vector candidate depth inherited from final `limit`, while BM25 fusion arms scan uncapped | Each source and fusion stage has its own candidate window. Final `limit` counts output rows; graph fan-out can produce several rows per selected target. Stage comparators determine selection before final ordering. | Choose source/fusion windows explicitly and review tie keys and expected row counts. A migration cannot infer the intended recall/cost tradeoff from the old limit. | | Implicit vector geometry or an unresolved `@embed` model | A field's encoding recipe and geometry must resolve at schema acceptance. The model may inherit a schema-owned default recipe; distance may inherit that recipe's declared default. Raw vectors require explicit distance. | Declare the source and dimensions, then resolve a compatible recipe and geometry. A new default cannot identify how old vectors were produced; unresolved legacy vectors still need operator resolution or regeneration. | | Queries relying on silently ignored search constructs or permissive parameter handling | Invalid shapes, incompatible representations, token-empty queries, and exhausted budgets produce typed failures. A successful partial candidate set cannot stand in for an exact result. | Handle the declared errors and size queries explicitly; do not interpret a failure as an empty successful search. | -| Existing graph files open directly after the upgrade | The accepted-schema change requires an export/init/load rebuild. Compatible values and logical graph content are carried over; commit history, branches, and physical indexes are not preserved by that rebuild. | Plan the data upgrade even if no query uses search. Retain the predecessor graph if its history is needed, rebuild indexes explicitly, and obtain fresh snapshot references from the rebuilt graph. | +| Existing graph files open directly after the upgrade | Yes. `@analyzed` and resolved embedding recipes are additive schema features applied in place, in the class of an `@index` addition; a graph that adopts neither is unchanged, and commit history, branches and existing indexes are preserved. A field that adopts `@analyzed` needs its full-text index rebuilt for the NFC certificate; a field whose embedding recipe changes needs its vectors regenerated. | Plan per field, not per graph: apply the schema, rebuild the affected fields' indexes explicitly, regenerate vectors only where the recipe changed. No export/init/load is required by this change. | #### Additive capabilities @@ -209,8 +210,9 @@ queries to adopt ranking. Existing search queries still need the rewrites above. mechanisms. Stored-query bodies that use removed search constructs must be updated, even though the invocation mechanism is retained. - Graph commits, branches, coherent reads, and existing authorization retain - their contracts. Index maintenance remains explicit. These guarantees do not - imply preservation of old branches or snapshots across the format rebuild. + their contracts. Index maintenance remains explicit. Branches and snapshots + survive the upgrade; only the explicitly rebuilt indexes and regenerated + vectors of adopting fields change. #### HTTP, CLI, and SDK compatibility still to confirm @@ -226,13 +228,13 @@ claim that existing HTTP/SDK clients work unchanged. #### Migration sequence 1. Review the schema and resolve analyzer, scoring, vector geometry, and - encoding choices before rebuilding data. + encoding choices before applying it. 2. Rewrite application and stored queries against that schema, choosing matching modes, exact/approximate retrieval, and candidate windows. -3. Use the existing export/init/load upgrade path, regenerate incompatible - representations, and reconcile indexes explicitly. +3. Apply the schema in place, regenerate the representations whose recipe + changed, and rebuild the affected fields' indexes explicitly. 4. Validate rewritten queries, expected results, client response/error handling, - and snapshot follow-up before switching applications to the rebuilt graph. + and snapshot follow-up before switching applications to the upgraded graph. The release must include the schema/query migration diagnostics, updated examples, user guides, and release notes. Detailed semantics and remaining @@ -571,13 +573,13 @@ Rewritten application/stored queries and examples ship with the implementation. A rewrite must ask the caller to choose exact versus approximate retrieval and semantic windows when the old query never specified them. -The accepted-schema boundary uses the existing export/init/load rebuild. -An offline rewrite makes implicit analyzer and L2 choices explicit and -preserves exact key/index annotations. Legacy embeddings without recoverable +The accepted-schema boundary is an in-place apply of additive features; no +export/init/load is required. A schema rewrite tool makes implicit analyzer +and L2 choices explicit and preserves exact key/index annotations. Legacy embeddings without recoverable revision/recipe identity require operator resolution; a newly declared default cannot establish historical coordinate spaces. For new or regenerated values, `@embed("source")` may omit its model when the schema default resolves it. -The generated schema and every expanded default are reviewed before init. +The generated schema and every expanded default are reviewed before apply. Rebuilt vectors are needed when the chosen encoding recipe differs from that which produced existing values. The NFC/profile change also requires index rebuild or proven parity; an old certificate does not establish new analysis. @@ -922,11 +924,15 @@ Rules that make an open expression safe in that position: by the author's act of writing it. The typed domains exist to refuse *implicit* mixing: `order { metric(a, score) desc, metric(b, distance) asc }` is two orderings, not one combined score. -- *Missing arms are explicit.* A target absent from an arm has a null metric - there; an expression that can be null is refused as a `fuse` argument or - an `order` key unless wrapped in `coalesce` or the ordering states `nulls - first`/`nulls last`. A missing arm can never silently drop or demote a - target. +- *Missing arms are explicit in `fuse`.* A target absent from an arm has a + null metric there. A `fuse` expression that can be null is refused unless + the null is handled (`coalesce`), because a null is not a score and would + silently demote the target one arm missed. An `order` key may be nullable: + it takes the documented placement (ascending nulls first, descending nulls + last) or an explicit `nulls first`/`nulls last`, and a missing-arm metric + sorts as a null does. That is the one rule for both positions; the + [selection section](#target-identity-fan-out-grouping-and-metrics) states + the `order` half. - *Identity is structural.* The fingerprint of an inline expression is the hash of its normalized typed AST, so `explain`, replay descriptors and the semantic digest treat an on-the-fly formula exactly as a named policy; a @@ -1419,7 +1425,7 @@ differential oracle. Nothing here decides a planner. | Class | Rewrites | Precondition | |---|---|---| -| Always legal | Predicate pushdown through pattern joins; predicate pushdown into a retriever as a prefilter; projection pruning; join order and traversal direction inside one `match`; fusing adjacent `filter`/`let`; top-k pushdown of `order` + `limit` over one scan | The predicate precedes the `rank` it is pushed into; pruning retains hidden metric and identity columns; the scan's order is known and the cut unit is rows | +| Always legal | Predicate pushdown through inner pattern joins; predicate pushdown into a retriever as a prefilter; projection pruning; join order and traversal direction inside one `match`; fusing adjacent `filter`/`let`; top-k pushdown of `order` + `limit` over one scan | The join is an inner pattern join: a predicate never crosses an `optional { }` or `not { }` boundary in either direction, since inside such a block it is part of the join condition and outside it filters the joined rows (SQL's `ON` versus `WHERE`; pushing `owner.name IS NULL` into an optional match manufactures the unmatched row it should reject); the predicate precedes the `rank` it is pushed into; pruning retains hidden metric and identity columns; the scan's order is known and the cut unit is rows | | Barriers | None across `rank`, `group`, `limit` | A rewrite that crosses one changes the question; it is refused, and `explain` names the law that refused it | | Cost-based | Prefilter versus postfilter for a graph-scoped `rank` (the #587 gate; `rrf_plan` overrides); index versus exact scan; qualified fallback; late payload hydration; dense versus distinct-pair group selection | Statistics are visible, derived, explicitly maintained state (row counts, index coverage, selectivity estimates), never hidden; every route pair has a checked-in differential oracle proving identical results, `rrf_prefilter_gate.rs` being the pattern | | Approximation | Recall may be traded inside `ann` only, through the `oversample` mapping | That is the declared contract; `knn`, `lexical` and every exact operator admit no recall trade | @@ -1701,7 +1707,7 @@ Extend existing test owners rather than creating a parallel search harness: | Boundary | Required evidence and owner | |---|---| | Grammar / types / lowering | Compiler parser/typecheck/IR fixtures for schema default declarations and resolution, rank blocks, typed lexical queries, named metrics, invalid references, parameter bounds, removed syntax, and aggregate scope | -| Query semantics | `.gqt` cases for filter-before/after-rank, traversal-introduced targets, target/pair/binding-row counts, per-group multi-membership and null buckets, final limit independence, and exact verification | +| Query semantics | `.gqt` cases for filter-before/after-rank, traversal-introduced targets, target/pair/binding-row counts, per-group multi-membership and null buckets, final limit independence, and exact verification; a predicate inside versus after `optional { }` and `not { }`; ordering by a missing-arm metric under default and explicit placement, and `fuse` over a nullable expression refused without `coalesce` | | Search mechanisms | `search.rs`, `rrf_prefilter_gate.rs`, `ordering.rs`, `aggregation.rs`, and traversal owners for arm ranks through fan-out, missing arm versus rescore, common-identity deduplication, and graph populations | | Substrate qualification | `lance_surface_guards.rs` and search owners for NFC/analyzer/index parity, native row-mask mapping, score statistics, vector metric/precision, tail coverage, complete boundaries and different partition layouts | | Snapshot / policy / transport | `point_in_time.rs`, policy owners, server `data_routes`/`stored_queries`/`openapi`, and CLI parity for coherent follow-up, expiry/refusal, metadata, policy-safe counts and resolved query identity | @@ -1848,8 +1854,8 @@ mechanism, not weakening the promised result. |---|---|---|---|---| | A — Truth first | Refusals (`T26`, `T27`, `FullTextIndexRequired`); retrieval as typed IR; projectable metrics; deterministic ties; `warnings`/`metrics`/`retrievals`/usage descriptors; the diagnostics contract; `explain` v0 (logical plan); coarse budgets through the existing `ResourceLimitExceeded` owner; attribution of the served read floor through the usage descriptor | Low: no syntax or format change; refusals and additive fields | RFC 0047 | Characterization goldens; the traversal-target and unindexed regression cases green; the served read floor attributed (#752) | | B — The agent door | Stored queries as typed tools; `@description` on `.pg` declarations and in `schema show`; the one-page card; the in-context competence instrument with graph-computed ground truth; baseline measurements on today's language | None: the PG annotation is metadata | This RFC (agent-facing surface); composition RFC (instrument) | Instrument runs end to end; baseline recorded | -| C — Representations, opt-in per field | `@analyzed` and analyzer profiles as an additive SchemaIR feature (unchanged graphs need no rebuild); resolved embedding recipes and defaults; `terms`/`match_terms` as a predicate; the exact scan baseline across index states; `bm25_v1` against the Decimal oracle; zero-edit membership through the existing FTS index, fuzzy through a budgeted scan | Medium: the format commitment is opt-in | Lexical RFC; this RFC (representation identity) | Membership laws across absent/partial/full/rebuilt indexes; oracle parity; the fuzzy and index-state regression cases green | -| D — Kernel stages, additive, behind a setting | `filter`, `let`, `rank` (`lexical`, `knn`, `ann`, `rrf`, `ties:`), `group`, `order`/`limit` at any position with `of`/`per`, pure `return` with the aggregate sugar, `metric()`; typed stage IR with metric origin and fingerprints; coherent follow-up reads; exact physical paths only; the old grammar still parses and a session setting selects the kernel | Medium: additive syntax, no format; reversible | This RFC | Instrument A/B (first-try validity, turns) kernel versus current grammar; C2's terminal subset; barrier cost measured | +| C — Representations, opt-in per field | `@analyzed` and analyzer profiles as an additive SchemaIR feature (unchanged graphs need no rebuild); resolved embedding recipes and defaults; `filter` as the first kernel stage, with the pattern/predicate rule that admits predicates inside `not { }`; `terms`/`match_terms` as a predicate in it; the exact scan baseline across index states; `bm25_v1` against the Decimal oracle; zero-edit membership through the existing FTS index, fuzzy through a budgeted scan | Medium: the format commitment is opt-in | Lexical RFC; this RFC (representation identity) | Membership laws across absent/partial/full/rebuilt indexes; oracle parity; the fuzzy and index-state regression cases green | +| D — Kernel stages, additive, behind a setting | `let`, `rank` (`lexical`, `knn`, `ann`, `rrf`, `ties:`), `group`, `order`/`limit` at any position with `of`/`per`, pure `return` with the aggregate sugar, `metric()`; typed stage IR with metric origin and fingerprints; coherent follow-up reads; exact physical paths only; the old grammar still parses and a session setting selects the kernel | Medium: additive syntax, no format; reversible | This RFC | Instrument A/B (first-try validity, turns) kernel versus current grammar; C2's terminal subset; barrier cost measured | | E — Cutover | Release N: legacy spellings compile to the kernel with deprecation diagnostics and a rewrite tool for stored queries; GQ minor bump. Release N+1: removal and major bump under the compatibility-surfaces RFC | User-facing, bounded by the window; reversible until N+1 | This RFC and the lexical RFC (mappings) | Kernel not worse than the old grammar on the instrument; no un-migrated stored query | | F — Acceleration and qualification | Native lexical routes with parity oracles; ANN families qualified for recall against `knn`; statistics-driven prefilter/postfilter and route choice with differential oracles; late hydration; plan cache; engine version 2 planner integration through the rewrite catalogue | Performance only; every route behind a parity gate; piecemeal | This RFC with the engine version 2 planner RFC | Each route: oracle identical, cost measured; defaults chosen inside the evaluation envelope | | G — Programmability and composition | `fuse(expr)`, `define`, the typed plan surface, multi-statement requests, `sub()` reductions, type-union targets, path selectors, each its own increment | Additive, one at a time | Composition RFC | Each gated by the instrument and its own oracle | @@ -1905,9 +1911,15 @@ declares neither is unchanged and needs no rebuild; a field that adopts Build bounded NFC/analysis, complete `Terms` matching, unified exact/fuzzy scoring against the Decimal oracle, and the exact scan baseline that holds across every index state, with zero-edit membership served by the existing -FTS index and fuzzy membership by a budgeted scan. Extend search/substrate -owners and the lexical RFC's qualification matrix; observable rows, shapes -and errors belong in GQT. +FTS index and fuzzy membership by a budgeted scan. Ship `filter` here as +the first kernel stage, an additive stage keyword plus the pattern/predicate +rule that admits a predicate inside `not { }`, so that `match_terms` has +its kernel position and its membership gates run through the production +parser instead of through the spelling it replaces; `rank` and `lexical` +wait for D, and so does the gate that needs them (predicate and retriever +membership agreeing before a cut). Extend search/substrate owners and the +lexical RFC's qualification matrix; observable rows, shapes and errors +belong in GQT. **Exit:** the [lexical RFC's phase](2026-09-18-analyzed-lexical-search.md#rollout) exit; export/reapplication, rename versus drop/re-add, default/override and no-drift fixtures for both @@ -1922,8 +1934,9 @@ provider label is not an immutable encoder; unverifiable aliases are refused. **Input:** Phase C representations, Phase B's baseline, Phase 0's kernel decision. -Add the kernel stages to the production grammar without removing anything: -`filter`, `let`, `rank` with `lexical`/`knn`/`ann`/`rrf` and `ties:`, +Add the remaining kernel stages to the production grammar without removing +anything (`filter` landed in C): `let`, `rank` with `lexical`/`knn`/`ann`/`rrf` +and `ties:`, `group`, `order`/`limit` at any position with `of`/`per`, pure `return` with the aggregate sugar, `metric()`. Lower them through typed stage IR carrying metric origin and fingerprints; preserve distinct target identity, @@ -1933,9 +1946,14 @@ coherent follow-up read path: `@id` lookup at a pinned snapshot and replay identity through every renderer. A session setting selects the kernel while both grammars parse, so the instrument compares them on the same tasks. -**Exit:** executable counterexamples distinguish filter-before/after-cut, -source windows from final limits, target counts from path counts, and -missing membership from a new feature; C2's terminal subset returns three +**Exit:** predicate and retriever membership agree before any cut (the +lexical gate that needs `lexical`); executable counterexamples distinguish +filter-before/after-cut, a predicate inside versus after an `optional { }` +(the `IS NULL` shape) and inside versus after a `not { }`, source windows +from final limits, target counts from path counts, and missing membership +from a new feature; ordering by a missing-arm metric with default and +explicit placement, and a `fuse` over a nullable expression refused without +`coalesce`; C2's terminal subset returns three binding rows from p1/p2 and excludes p3; null and multiple group membership, metric reductions and inherited order tested; the instrument shows the kernel at least as good as the old grammar on first-try validity and turns; barrier @@ -2083,6 +2101,12 @@ still require prototypes; full production qualification belongs to its phase. ## Decision log +- 2026-09-19 — review fixes: no whole-graph rebuild anywhere (in-place + per-field adoption, matching Phase C); pushdown never crosses an + `optional { }`/`not { }` boundary; `filter` ships in Phase C so + `match_terms` has a callable kernel position and C's ranking gate moves to + D; one nullable-metric rule (`order` takes the documented placement, `fuse` + requires `coalesce`); fixtures for each added to Phase D's exit. - 2026-09-18 — stated the contract / physical realization / distribution of work split in the terms of `docs/dev/systems.md` (#745). - 2026-09-18 — corrected Phase A's performance item after measuring the diff --git a/docs/rfcs/2026-09-18-analyzed-lexical-search.md b/docs/rfcs/2026-09-18-analyzed-lexical-search.md index 19d41d879..9f2bc2b46 100644 --- a/docs/rfcs/2026-09-18-analyzed-lexical-search.md +++ b/docs/rfcs/2026-09-18-analyzed-lexical-search.md @@ -208,8 +208,9 @@ typed query failure, not an empty or truncated successful matched set. ### Analyzed filtering with `filter` -Analyzed filtering alone is an ordinary predicate in a `filter` stage; it -never sits inside the pattern block (the composition RFC's +Analyzed filtering alone is an ordinary predicate in a `filter` stage; inside +the pattern block it appears only within `not { }` or `optional { }`, where +its placement changes the answer (the composition RFC's [pattern/predicate rule](2026-09-18-gq-composition-and-language-evolution.md#kernel-one-stage-per-job)): ```gq @@ -229,8 +230,9 @@ an explicitly tolerant query. Both resolve the same default scoring policy. These rows are the lexical part of RFC 0048's [migration table](0048-search-contracts.md#user-facing-changes-and-migration); the coordinated -release, the format rebuild and the `nearest`/`order`/positional-`rrf` -rewrites stay there. +release and the `nearest`/`order`/positional-`rrf` rewrites stay there. No +graph is rebuilt: `@analyzed` applies in place and only an adopting field's +full-text index is rebuilt. | Existing usage or expectation | Proposed change | What users must do | |---|---|---| @@ -241,8 +243,12 @@ rewrites stay there. The deprecation mapping is fixed here: `search(f, q)` and `match_text(f, q)` become `match_terms(f, terms(q))` (all terms, zero edits) and `fuzzy(f, q, n)` becomes `match_terms(f, terms(q, mode: all, max_edits: n))` -with `n` clamped to the admitted range or refused; the predicate moves from -the `match` block to a `filter` stage after it. The mapping changes +with `n` clamped to the admitted range or refused. The rewrite is +scope-preserving: a predicate at the top level of `match` moves to a +`filter` stage after it, and a predicate inside `not { … }` stays inside +that block as `match_terms`, because the kernel admits predicates there and +hoisting it would either change the excluded population or leave its inner +binding unbound. The mapping changes membership where today's answer depended on index state; the diagnostic says so. The legacy spellings never gain new semantics during the window. @@ -594,8 +600,9 @@ results. with a diagnostic; the removal release bumps the GQ language major per the compatibility-surfaces RFC. Stored queries and `cluster.yaml` inline queries must be rewritten inside the window. -- **Schema and format:** `@analyzed` is a SchemaIR feature; incompatible - binaries refuse rather than reinterpret. The NFC profiles invalidate +- **Schema and format:** `@analyzed` is an additive SchemaIR feature applied + in place; graphs that do not adopt it are unchanged and no graph is + rebuilt; incompatible binaries refuse rather than reinterpret. The NFC profiles invalidate existing full-text certificates; RFC 0043's rebuild procedure applies. - **Results:** analyzed matching can change rows where the old answer depended on index state. That is the correction this RFC exists for; it is @@ -742,6 +749,9 @@ Default resolution requires its own compiler/schema and query fixtures: The exact lexical qualification matrix retains all preceding requirements: +- A search inside `not { }` migrates to `match_terms` in place and keeps the + excluded population; the same search hoisted to a `filter` is the + counterexample. - Positive and negative matches; `all`/`any`; repeated/reordered terms; null/token-empty values; empty and stop-word-only queries; validation even on empty populations. Edit budgets reject negative, non-integer, oversized, @@ -783,9 +793,14 @@ lexical spellings. `@analyzed` and its analyzer profiles ship as an additive SchemaIR feature that a graph adopts field by field: a graph that declares no `@analyzed` field is unchanged and needs no rebuild; a field that adopts it takes the NFC -certificate and an index rebuild for that field. Build bounded NFC/analysis, -complete `Terms` matching, `match_terms` as a predicate, and unified -exact/fuzzy scoring against the Decimal oracle. The exact scan is the +certificate and an index rebuild for that field. `filter` ships in this +phase as the first kernel stage, with the pattern/predicate rule that admits +a predicate inside `not { }`, so `match_terms` has its kernel position and +its gates run through the production parser. Build bounded NFC/analysis, +complete `Terms` matching, `match_terms` as a predicate in `filter` and +inside `not { }`, and unified exact/fuzzy scoring against the Decimal +oracle; the `lexical` ranking source and the gate that needs it (predicate +and retriever membership agreeing before a cut) belong to Phase D. The exact scan is the baseline at every index state; zero-edit membership is served through the existing FTS index and fuzzy membership through a budgeted scan until Phase F qualifies a native route. Charge statistics, coverage and scoring to the @@ -793,8 +808,9 @@ shared context. Extend search/substrate owners and the [qualification matrix](#qualification-matrix); observable rows, shapes and errors belong in GQT. -**Exit:** predicate and retriever membership agree before any cut; the -membership laws hold across absent, partial, full and rebuilt indexes; oracle +**Exit:** the membership laws hold across absent, partial, full and rebuilt +indexes, through `filter` and inside `not { }`; a nested-negation search +migrates in place; oracle parity for scores; total ties; typed budget failures; export and reapplication preserve the resolved profile; the fuzzy and index-state regression cases green under `issue_N` names. @@ -820,6 +836,10 @@ relevance quality; Phase F chooses measured defaults. ## Decision log +- 2026-09-19 — review fixes: the migration mapping is scope-preserving (a + search inside `not { }` stays there); `filter` ships in Phase C so + `match_terms` is callable; the ranking gate moves to Phase D; no graph is + rebuilt for adoption. - 2026-09-18 — named the membership laws as the operation's contract and the index as physical realization, in the terms of `docs/dev/systems.md`. - 2026-09-18 — `match_terms` moved from the `match` block to a `filter` diff --git a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md index d339c4c68..4f4fda369 100644 --- a/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md +++ b/docs/rfcs/2026-09-18-gq-composition-and-language-evolution.md @@ -754,8 +754,11 @@ pattern `WHERE` versus its `FILTER` statement, SQL/PGQ's `GRAPH_TABLE` rule here is the SQL `ON`/`WHERE` rule: a predicate may sit inside a pattern block only in the sub-patterns where moving it outside would change the answer, `optional { }` and `not { }`; inline constraints remain pattern -properties; every other predicate is a `filter` stage. Gremlin, which makes -no distinction and lets step order carry the meaning, is the counterexample: +properties; every other predicate is a `filter` stage. The same boundary +binds the optimizer: a predicate never crosses an `optional { }` or +`not { }` boundary in either direction (RFC 0048's rewrite catalogue), which +is the `IS NULL` counterexample stated as a rule. Gremlin, which makes no +distinction and lets step order carry the meaning, is the counterexample: the planner cannot tell a shape from a selection. **Open sets.** @@ -1062,6 +1065,8 @@ before optimizing batching, and measure per-group rescan cost. collapses `select`, `take`, `score`, `collect` and `optional` and moves scalar predicates out of `match`. Recorded as a decision RFC 0048 must take before its syntax stabilizes; the moved-in text above is unchanged. +- 2026-09-19 — review fix: the pattern/predicate boundary is also the + optimizer's pushdown boundary. - 2026-09-18 — stated the kernel as operations-before-interface in the terms of `docs/dev/systems.md` (#745). - 2026-09-18 — kernel adopted; C1–C4 rewritten to `filter`, `order` + From ad37e5c42ecd49c9cff38ba9d49d5f3962415652 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sat, 19 Sep 2026 15:00:20 +0200 Subject: [PATCH 16/16] rfc: accept RFC 0047 (search plan truth) --- docs/rfcs/0047-search-plan-truth.md | 23 ++++++++++++++++------- docs/rfcs/README.md | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/docs/rfcs/0047-search-plan-truth.md b/docs/rfcs/0047-search-plan-truth.md index 32248b047..54f762ffe 100644 --- a/docs/rfcs/0047-search-plan-truth.md +++ b/docs/rfcs/0047-search-plan-truth.md @@ -2,17 +2,16 @@ rfc: "0047" title: "Search plan truth: projectable ranking, deterministic order, and loud search failures" track: public -status: draft +status: accepted implementation: not-started authors: - Ragnor Comerford (@ragnorc) created: 2026-09-01 -updated: 2026-09-18 +updated: 2026-09-19 discussion: "https://github.com/ModernRelay/omnigraph/pull/606" supersedes: [] superseded_by: [] -blocked_on: - - "Complete boundary-tie handling at every native candidate cut, with bounded retention or explicit resource failure" +blocked_on: [] --- # RFC 0047: Search plan truth: projectable ranking, deterministic order, and loud search failures @@ -429,9 +428,11 @@ the specification; the code is re-derived against the current tree. | 8 | `explain` v0: the logical plan and the retrieval descriptor for a query, as a CLI flag and a route; no surface exists today | CLI `cli.rs`, server routes, API types | CLI and server tests; OpenAPI | | 9 | #752: attribute the served read floor per phase and report it through `usage`; independent of 1–8 | server instrumentation | a cost test pinning the object-store request count of a trivial served read | -PR 6 is the one with open design work: retaining a boundary-tie plateau -within a budget is this RFC's `blocked_on`. PRs 1–5 and 7–9 have no open -questions. +PR 6 is the one with open design work: the mechanism that retains a +boundary-tie plateau within a budget. Its contract is fixed here (retention +within an explicit budget, or a typed failure); the mechanism is PR 6's to +prove, and it is the second unresolved question below. PRs 1–5 and 7–9 have +no open questions. **Deployment gate.** `T26` and `T27` refuse queries the compiler accepts today. Stored queries persist as text and recompile when a server starts, @@ -454,6 +455,10 @@ rollout. 1. Is `warnings`' human-format contract (stderr for every non-full-JSON CLI format) acceptable, or should the JSONL metadata record grow a warnings field in the same change? +2. How PR 6 retains a boundary-tie plateau at every native candidate cut + within its budget: the retention mechanism and the budget's unit. The + contract is settled (bounded retention or a typed failure); the + mechanism is settled by PR 6's evidence. ## Decision log @@ -507,6 +512,10 @@ rollout. explicit unknown by default, exact counts when explicitly requested. This keeps query completion separate from representation knowledge and avoids mandatory exhaustive counting solely for default discovery metadata. +- 2026-09-19 — accepted by the maintainer on PR #606. The former `blocked_on` + entry (boundary-tie retention within a budget) is not a gate on the + decision: PR 6's contract is stated in the rollout, and the retention + mechanism is recorded as unresolved question 2 for PR 6 to settle. ## Appendix: agent context (non-normative) diff --git a/docs/rfcs/README.md b/docs/rfcs/README.md index 6ef253802..f69e0acbf 100644 --- a/docs/rfcs/README.md +++ b/docs/rfcs/README.md @@ -198,7 +198,7 @@ then dated RFCs by date. | [0044](0044-edge-keys.md) | Edge keys: derived edge identity | maintainer | draft | in-progress | | [0045](0045-gq-logic-tests.md) | GQ logic tests | maintainer | draft | partial | | [0046](0046-index-status.md) | Read-only index status | maintainer | draft | not-started | -| [0047](0047-search-plan-truth.md) | Search plan truth: projectable ranking, deterministic order, and loud search failures | public | draft | not-started | +| [0047](0047-search-plan-truth.md) | Search plan truth: projectable ranking, deterministic order, and loud search failures | public | accepted | not-started | | [0048](0048-search-contracts.md) | Search contracts and retrieval algebra | public | draft | not-started | | [0049](0049-control-plane-seams.md) | Control-plane seams: observe, readiness witness, bounded shutdown | maintainer | accepted | partial | | [0051](0051-json-output-via-arrow.md) | JSON output via Arrow | maintainer | draft | partial |