diff --git a/docs/reference/database-indexes.md b/docs/reference/database-indexes.md index 751dd36c1..ae7a10fea 100644 --- a/docs/reference/database-indexes.md +++ b/docs/reference/database-indexes.md @@ -36,7 +36,7 @@ Create indexes on encrypted columns when: - The table has a significant number of rows (typically > 1000) - You frequently query by equality on that column - Query performance is important -- The column contains searchable index terms (hmac_256, blake3, or ore) +- The column contains searchable index terms (hmac_256, blake3, ore, or ope) --- @@ -49,7 +49,7 @@ For PostgreSQL to use an index on encrypted columns, **all** of these conditions The encrypted data must contain the index term types that support the operation: - **Equality queries** - Require `unique` index config (adds `hm` hmac_256 or `b3` blake3 terms) -- **Range queries** - Require `ore` index config (adds `ob` ore_block_u64_8_256 terms) +- **Range queries** - Require `ore` index config (adds `ob` ore_block_u64_8_256 terms) **or** `ope` index config (adds `opf` ope_cllw_u64_65 / `opv` ope_cllw_var_8 terms) - **Pattern matching** - Typically scans (bloom filters don't use B-tree indexes) **Example:** @@ -149,7 +149,7 @@ Bitmap Heap Scan on users ### Range Queries -When encrypted column has `ob` (ore_block_u64_8_256) index terms: +When encrypted column has `ob` (ore_block_u64_8_256), `opf` (ope_cllw_u64_65), or `opv` (ope_cllw_var_8) index terms: ```sql SELECT * FROM events @@ -157,6 +157,8 @@ WHERE encrypted_date < $1::eql_v2_encrypted ORDER BY encrypted_date DESC; ``` +The encrypted operator class transparently dispatches to whichever ordered term is present on the column, so range queries against an `ore`-configured column and an `ope`-configured column have identical SQL. + ### GROUP BY Encrypted columns can be used in GROUP BY with indexes: @@ -215,6 +217,8 @@ B-tree indexes **only work** with: - `hm` (hmac_256) - for equality - `b3` (blake3) - for equality - `ob` (ore_block_u64_8_256) - for range queries +- `opf` (ope_cllw_u64_65) - for range queries (fixed-width OPE) +- `opv` (ope_cllw_var_8) - for range queries (variable-width OPE) They **do not work** with: - `bf` (bloom_filter) - pattern matching @@ -404,7 +408,7 @@ If you see `Seq Scan`, ensure: | Feature | B-tree Index | GIN Index | |---------|-------------|-----------| | **Use case** | Equality, range queries | JSONB containment | -| **Index terms** | `hm`, `b3`, `ob` | `sv` (via jsonb_array) | +| **Index terms** | `hm`, `b3`, `ob`, `opf`, `opv` | `sv` (via jsonb_array) | | **Operators** | `=`, `<`, `>`, `<=`, `>=` | `@>`, `<@` | | **Function** | Direct column reference | `eql_v2.jsonb_array()` | @@ -417,10 +421,13 @@ If you see `Seq Scan`, ensure: **Check 1: Verify data has index terms** ```sql --- Check if data contains hm (hmac_256) or b3 (blake3) for equality +-- Check if data contains hm (hmac_256) or b3 (blake3) for equality, +-- ob (ore) for range, or opf/opv (ope) for range SELECT encrypted_email::jsonb ? 'hm' AS has_hmac, encrypted_email::jsonb ? 'b3' AS has_blake3, - encrypted_email::jsonb ? 'ob' AS has_ore + encrypted_email::jsonb ? 'ob' AS has_ore, + encrypted_email::jsonb ? 'opf' AS has_ope_fixed, + encrypted_email::jsonb ? 'opv' AS has_ope_var FROM users LIMIT 1; ``` @@ -457,7 +464,7 @@ WHERE tablename = 'users' 1. **Ensure index exists and is being used** - Use `EXPLAIN ANALYZE` 2. **Check table has been ANALYZEd** - Run `ANALYZE table_name` 3. **Consider index selectivity** - Very small tables might not use indexes -4. **Check for appropriate search config** - Equality needs `unique`, ranges need `ore` +4. **Check for appropriate search config** - Equality needs `unique`, ranges need `ore` or `ope` --- diff --git a/docs/reference/index-config.md b/docs/reference/index-config.md index 88587ae8a..c5e01746c 100644 --- a/docs/reference/index-config.md +++ b/docs/reference/index-config.md @@ -18,7 +18,7 @@ Add an index to an encrypted column. Returns the updated configuration as JSONB. SELECT eql_v2.add_search_config( 'table_name', -- Name of the table 'column_name', -- Name of the column - 'index_name', -- Index kind ('unique', 'match', 'ore', 'ste_vec') + 'index_name', -- Index kind ('unique', 'match', 'ore', 'ope', 'ste_vec') 'cast_as', -- PostgreSQL type to cast decrypted data ('text', 'int', etc.) 'opts' -- Index options as JSONB (optional) ); @@ -109,6 +109,15 @@ If you're using n-gram as a token filter, then a token that is already shorter t However, if that same short string only appears as a part of a larger token, then it will not match that record. Try to ensure that the string you search for is at least as long as the `tokenLength` of the index, except in the specific case where you know that there are shorter tokens to match, _and_ you are explicitly OK with not returning records that have that short string as part of a larger token. +#### `ore` vs `ope` + +Both `ore` and `ope` enable the same ordered-comparison surface (`<`, `<=`, `=`, `>`, `>=`, `BETWEEN`, `ORDER BY`, `MIN`/`MAX`). + +- **`ore`** uses Order-Revealing Encryption (`ore_block_u64_8_256`, payload field `ob`). Ciphertexts compare via a custom per-byte protocol implemented in `eql_v2.compare_ore_block_u64_8_256`. This is the default ordered-search index. +- **`ope`** uses CLWW Order-Preserving Encryption — `ope_cllw_u64_65` (fixed-width, payload field `opf`) for numeric types and `ope_cllw_var_8` (variable-width, payload field `opv`) for text-shaped values. OPE ciphertexts compare with **standard lexicographic byte ordering**, which makes them usable in environments that can only sort `bytea` natively (e.g. some pluggable storage layers without custom comparators). + +`eql_v2.compare()` and the `<` / `<=` / `>` / `>=` operators dispatch automatically to whichever ordered terms are present on the encrypted value, so application queries do not change when switching between `ore` and `ope`. + #### Options for ste_vec indexes (`opts`) An ste_vec index on an encrypted JSONB column enables the use of PostgreSQL's `@>` and `<@` [containment operators](https://www.postgresql.org/docs/16/functions-json.html#FUNCTIONS-JSONB-OP-TABLE). diff --git a/docs/reference/sql-support.md b/docs/reference/sql-support.md index e2babce2d..5a11e430a 100644 --- a/docs/reference/sql-support.md +++ b/docs/reference/sql-support.md @@ -2,15 +2,18 @@ This page summarises which SQL operators and language features work against `eql_v2_encrypted` columns/values, and which EQL searchable-encryption index (configured via [`eql_v2.add_search_config`](./index-config.md)) each one requires. -EQL ships four search index kinds that encrypt data in ways that preserve specific query capabilities: +EQL ships five search index kinds that encrypt data in ways that preserve specific query capabilities: | Search index (config `index_name`) | Underlying encrypted term(s) | Enables | | ---------------------------------- | ---------------------------- | ------------------------------------------------------ | | `unique` | `hmac_256` (`hm`) or `blake3` (`b3`) | Exact equality | | `ore` | `ore_block_u64_8_256` (`ob`) | Ordered comparison (`<`, `<=`, `=`, `>`, `>=`), range (`BETWEEN`), `ORDER BY`, aggregates (`MIN`/`MAX`) | +| `ope` | `ope_cllw_u64_65` (`opf`) or `ope_cllw_var_8` (`opv`) | Ordered comparison (`<`, `<=`, `=`, `>`, `>=`), range (`BETWEEN`), `ORDER BY`, aggregates (`MIN`/`MAX`) — see note below | | `match` | `bloom_filter` (`bf`) | Substring / token matching via `LIKE` / `ILIKE` | | `ste_vec` | Structured encryption (`sv`) | JSONB containment and JSONB path / field access | +> **`ore` vs `ope`** — both index kinds support the same ordered-comparison surface. `ore` (Order-Revealing Encryption) is the default. `ope` (CLWW Order-Preserving Encryption) is an alternative for environments that need plain lexicographic byte comparison (e.g. pluggable storage that cannot run a custom comparator). On a column configured for `ope`, `eql_v2.compare()` and the `<` / `<=` / `>` / `>=` operators dispatch to OPE terms automatically. + Every column must also be registered with `eql_v2.add_column(...)` — that alone gives the column storage and decryption, but none of the operators below will produce results until at least one search index is added for the operation you need. @@ -20,30 +23,30 @@ Every column must also be registered with `eql_v2.add_column(...)` — that alon Each row lists an operator that EQL either implements natively on `eql_v2_encrypted` or that CipherStash Proxy rewrites into an EQL equivalent. A ✅ means the operator is supported on a column when that index is configured. A ❌ means the index does not support the operator (the database will either error, return no rows, or fall back to a scan that decrypts nothing useful). -| SQL operator | Meaning | `unique` | `ore` | `match` | `ste_vec` | -| --------------------------------- | ------------------------------- | :------: | :---: | :-----: | :-------: | -| `=` | Equality | ✅ | ✅ | ❌ | ❌ | -| `<>` / `!=` | Inequality | ✅ | ✅ | ❌ | ❌ | -| `<` | Less than | ❌ | ✅ | ❌ | ❌ | -| `<=` | Less than or equal | ❌ | ✅ | ❌ | ❌ | -| `>` | Greater than | ❌ | ✅ | ❌ | ❌ | -| `>=` | Greater than or equal | ❌ | ✅ | ❌ | ❌ | -| `LIKE` (`~~`) | Case-sensitive pattern match | ❌ | ❌ | ✅ | ❌ | -| `NOT LIKE` (`!~~`) | Negated case-sensitive match | ❌ | ❌ | ✅ | ❌ | -| `ILIKE` (`~~*`) | Case-insensitive pattern match | ❌ | ❌ | ✅\* | ❌ | -| `NOT ILIKE` (`!~~*`) | Negated case-insensitive match | ❌ | ❌ | ✅\* | ❌ | -| `@>` | JSONB contains | ❌ | ❌ | ❌ | ✅ | -| `<@` | JSONB is contained by | ❌ | ❌ | ❌ | ✅ | -| `->` (text, int, encrypted) | JSONB field / element access | ❌ | ❌ | ❌ | ✅ | -| `->>` | JSONB field as text (ciphertext) | ❌ | ❌ | ❌ | ✅ | -| `IS NULL` / `IS NOT NULL` | Null check | ✅ | ✅ | ✅ | ✅ | +| SQL operator | Meaning | `unique` | `ore` | `ope` | `match` | `ste_vec` | +| --------------------------------- | ------------------------------- | :------: | :---: | :---: | :-----: | :-------: | +| `=` | Equality | ✅ | ✅ | ✅ | ❌ | ❌ | +| `<>` / `!=` | Inequality | ✅ | ✅ | ✅ | ❌ | ❌ | +| `<` | Less than | ❌ | ✅ | ✅ | ❌ | ❌ | +| `<=` | Less than or equal | ❌ | ✅ | ✅ | ❌ | ❌ | +| `>` | Greater than | ❌ | ✅ | ✅ | ❌ | ❌ | +| `>=` | Greater than or equal | ❌ | ✅ | ✅ | ❌ | ❌ | +| `LIKE` (`~~`) | Case-sensitive pattern match | ❌ | ❌ | ❌ | ✅ | ❌ | +| `NOT LIKE` (`!~~`) | Negated case-sensitive match | ❌ | ❌ | ❌ | ✅ | ❌ | +| `ILIKE` (`~~*`) | Case-insensitive pattern match | ❌ | ❌ | ❌ | ✅\* | ❌ | +| `NOT ILIKE` (`!~~*`) | Negated case-insensitive match | ❌ | ❌ | ❌ | ✅\* | ❌ | +| `@>` | JSONB contains | ❌ | ❌ | ❌ | ❌ | ✅ | +| `<@` | JSONB is contained by | ❌ | ❌ | ❌ | ❌ | ✅ | +| `->` (text, int, encrypted) | JSONB field / element access | ❌ | ❌ | ❌ | ❌ | ✅ | +| `->>` | JSONB field as text (ciphertext) | ❌ | ❌ | ❌ | ❌ | ✅ | +| `IS NULL` / `IS NOT NULL` | Null check | ✅ | ✅ | ✅ | ✅ | ✅ | \* Case-insensitivity for `ILIKE` / `NOT ILIKE` is only effective when the `match` index is configured with a case-normalising token filter (e.g. `{"token_filters": [{"kind": "downcase"}]}`). Without it, `ILIKE` behaves identically to `LIKE` on the encrypted terms. Notes: - Binary operators have overloads that accept `jsonb` literals on either side; CipherStash Proxy typically rewrites those to `::eql_v2_encrypted` casts so the encrypted operator is selected. -- `=` and `<>` on a column that has **only** a `ste_vec` index will not match anything useful — the underlying comparison requires `hm`, `b3`, or `ob` terms. Configure `unique` (or `ore`) alongside `ste_vec` if you need equality on the outer value. +- `=` and `<>` on a column that has **only** a `ste_vec` index will not match anything useful — the underlying comparison requires `hm`, `b3`, `ob`, `opf`, or `opv` terms. Configure `unique` (or `ore` / `ope`) alongside `ste_vec` if you need equality on the outer value. - JSONB path operators (`->`, `->>`) return an `eql_v2_encrypted` value (or ciphertext for `->>`). The value they return is itself searchable only if the parent `ste_vec` index covers that path. ### Unsupported JSONB operators @@ -60,32 +63,32 @@ Use the equivalent [`jsonb_path_query`](#jsonb-functions-and-selectors-enabled-b This matrix covers higher-level SQL constructs rather than individual operators. As above, ✅ requires the listed index to be configured on the column; ❌ means the construct cannot be used against that column (without first decrypting via CipherStash Proxy or Protect.js). -| SQL feature | Notes | `unique` | `ore` | `match` | `ste_vec` | -| ---------------------------------- | ------------------------------------- | :------: | :---: | :-----: | :-------: | -| `WHERE col = …` / `<>` | | ✅ | ✅ | ❌ | ❌ | -| `WHERE col <` / `<=` / `>` / `>=` | | ❌ | ✅ | ❌ | ❌ | -| `WHERE col BETWEEN … AND …` | desugars to `>=` and `<=` | ❌ | ✅ | ❌ | ❌ | -| `WHERE col LIKE …` / `NOT LIKE` | | ❌ | ❌ | ✅ | ❌ | -| `WHERE col ILIKE …` / `NOT ILIKE` | requires `downcase` filter | ❌ | ❌ | ✅ | ❌ | -| `WHERE col IN (…)` | | ✅ | ✅ | ❌ | ❌ | -| `WHERE col @> …` / `<@ …` | | ❌ | ❌ | ❌ | ✅ | -| `ORDER BY col` | | ❌ | ✅ | ❌ | ❌ | -| `GROUP BY col` | requires `unique` on the whole column; `ore` not yet supported (see note below). Extracted JSON paths have separate caveats — see [ste_vec section](#index-terms-by-json-node-type). | ✅ | ❌ | ❌ | ❌ | -| `DISTINCT` / `DISTINCT ON (col)` | `unique` or `ore` | ✅ | ✅ | ❌ | ❌ | -| `HAVING` | same index requirements as the predicates used in `HAVING` (see operator matrix) | varies | varies | varies | varies | -| `MIN(col)` / `MAX(col)` | | ❌ | ✅ | ❌ | ❌ | -| `COUNT(col)` / `COUNT(DISTINCT col)` | `ore` or `unique` for `DISTINCT`; none for plain `COUNT(col)` | ✅ | ✅ | ✅ | ✅ | -| `JOIN … ON lhs.col = rhs.col` | same index and keyset on both sides | ✅ | ✅ | ❌ | ❌ | -| `JOIN … ON lhs.col < rhs.col` etc. | same index and keyset on both sides | ❌ | ✅ | ❌ | ❌ | -| `UNION` / `EXCEPT` / `INTERSECT` (set operations) | | ✅ | ✅ | ❌ | ❌ | -| `IS NULL` / `IS NOT NULL` | works because `NULL` values are not encrypted | ✅ | ✅ | ✅ | ✅ | -| Window functions over encrypted columns | works like the equivalent clauses in normal SQL (e.g. window `ORDER BY` needs `ore`) | varies | varies | varies | varies | +| SQL feature | Notes | `unique` | `ore` | `ope` | `match` | `ste_vec` | +| ---------------------------------- | ------------------------------------- | :------: | :---: | :---: | :-----: | :-------: | +| `WHERE col = …` / `<>` | | ✅ | ✅ | ✅ | ❌ | ❌ | +| `WHERE col <` / `<=` / `>` / `>=` | | ❌ | ✅ | ✅ | ❌ | ❌ | +| `WHERE col BETWEEN … AND …` | desugars to `>=` and `<=` | ❌ | ✅ | ✅ | ❌ | ❌ | +| `WHERE col LIKE …` / `NOT LIKE` | | ❌ | ❌ | ❌ | ✅ | ❌ | +| `WHERE col ILIKE …` / `NOT ILIKE` | requires `downcase` filter | ❌ | ❌ | ❌ | ✅ | ❌ | +| `WHERE col IN (…)` | | ✅ | ✅ | ✅ | ❌ | ❌ | +| `WHERE col @> …` / `<@ …` | | ❌ | ❌ | ❌ | ❌ | ✅ | +| `ORDER BY col` | | ❌ | ✅ | ✅ | ❌ | ❌ | +| `GROUP BY col` | requires `unique` on the whole column; `ore` / `ope` not yet supported (see note below). Extracted JSON paths have separate caveats — see [ste_vec section](#index-terms-by-json-node-type). | ✅ | ❌ | ❌ | ❌ | ❌ | +| `DISTINCT` / `DISTINCT ON (col)` | `unique`, `ore`, or `ope` | ✅ | ✅ | ✅ | ❌ | ❌ | +| `HAVING` | same index requirements as the predicates used in `HAVING` (see operator matrix) | varies | varies | varies | varies | varies | +| `MIN(col)` / `MAX(col)` | | ❌ | ✅ | ✅ | ❌ | ❌ | +| `COUNT(col)` / `COUNT(DISTINCT col)` | `ore` / `ope` or `unique` for `DISTINCT`; none for plain `COUNT(col)` | ✅ | ✅ | ✅ | ✅ | ✅ | +| `JOIN … ON lhs.col = rhs.col` | same index and keyset on both sides | ✅ | ✅ | ✅ | ❌ | ❌ | +| `JOIN … ON lhs.col < rhs.col` etc. | same index and keyset on both sides | ❌ | ✅ | ✅ | ❌ | ❌ | +| `UNION` / `EXCEPT` / `INTERSECT` (set operations) | | ✅ | ✅ | ✅ | ❌ | ❌ | +| `IS NULL` / `IS NOT NULL` | works because `NULL` values are not encrypted | ✅ | ✅ | ✅ | ✅ | ✅ | +| Window functions over encrypted columns | works like the equivalent clauses in normal SQL (e.g. window `ORDER BY` needs `ore` or `ope`) | varies | varies | varies | varies | varies | Notes: - **Cross-column / cross-table comparisons** (joins, `IN (subquery)`, `UNION` dedup, etc.) require both sides to have been encrypted with the *same* keyset and the matching search index. Encrypted values from different `ste_vec` prefixes are deliberately incomparable. -- **`GROUP BY`** on encrypted columns relies on an operator class which currently only supports encrypted values with a `unique` index term. This is a surprising limitation because it would be natural to expect `ore` index terms to also work. This limitation will be lifted in the future. See [Database Indexes](./database-indexes.md#group-by) for performance considerations. -- **`ORDER BY`** without an `ore` index will still *run* (the EQL `compare` function has a deterministic literal fallback to avoid btree errors), but the resulting order is not meaningful. Configure `ore` whenever ordering matters. +- **`GROUP BY`** on encrypted columns relies on an operator class which currently only supports encrypted values with a `unique` index term. This is a surprising limitation because it would be natural to expect `ore` / `ope` index terms to also work. This limitation will be lifted in the future. See [Database Indexes](./database-indexes.md#group-by) for performance considerations. +- **`ORDER BY`** without an `ore` or `ope` index will still *run* (the EQL `compare` function has a deterministic literal fallback to avoid btree errors), but the resulting order is not meaningful. Configure `ore` (or `ope`) whenever ordering matters. - **Aggregates beyond `MIN`/`MAX`** (e.g. `SUM`, `AVG`) are not supported on encrypted values — decrypt and perform those aggregate operations on the client-side instead. - **Parameter binding**: CipherStash Proxy rewrites bound parameters in `WHERE`, `JOIN`, and `RETURNING` clauses with `::JSONB::eql_v2_encrypted` casts so that the encrypted operator and any B-tree / GIN indexes are selected. Writing those casts yourself is only required when bypassing the proxy. @@ -136,8 +139,8 @@ When the `ste_vec` index is configured, CipherStash Proxy rewrites these standar | `jsonb_array_elements(arr)` | `eql_v2.jsonb_array_elements(arr)` | Path must resolve to a JSON array node | Set-returning; yields `eql_v2_encrypted`. | | `jsonb_array_elements_text(arr)` | `eql_v2.jsonb_array_elements_text(arr)` | Path must resolve to a JSON array node | Set-returning; yields ciphertext as `text`. | | `COUNT(col)` | plain `count(*)` | — | No encrypted term required. | -| `COUNT(DISTINCT col)` | deterministic dedup | `unique` **or** `ore` on the extracted node | For a JSON leaf, that means Object / Array / Bool / Null (dedup via `b3`) or String / Number (dedup via `ocv`/`ocf`). | -| `MIN(col)` / `MAX(col)` | `eql_v2` ORE aggregates | `ore` **or** ste_vec-extracted String / Number node | Requires a node that emits `ocv` / `ocf` (or a sibling `ore` index). | +| `COUNT(DISTINCT col)` | deterministic dedup | `unique`, `ore`, **or** `ope` on the extracted node | For a JSON leaf, that means Object / Array / Bool / Null (dedup via `b3`) or String / Number (dedup via `ocv`/`ocf`). | +| `MIN(col)` / `MAX(col)` | `eql_v2` ORE/OPE aggregates | `ore`, `ope`, **or** ste_vec-extracted String / Number node | Requires a node that emits `ocv` / `ocf` (or a sibling `ore` / `ope` index). | Additionally, `eql_v2.jsonb_array`, `eql_v2.jsonb_contains`, and `eql_v2.jsonb_contained_by` are EQL helpers (not automatic rewrites) used when building **GIN-indexed** containment queries. See [GIN Indexes for JSONB Containment](./database-indexes.md#gin-indexes-for-jsonb-containment) for the full setup. diff --git a/src/config/constraints.sql b/src/config/constraints.sql index 183790fd1..ceef85b8c 100644 --- a/src/config/constraints.sql +++ b/src/config/constraints.sql @@ -33,7 +33,7 @@ END; --! @internal --! --! Checks that all index types specified in the configuration are valid. ---! Valid index types are: match, ore, unique, ste_vec. +--! Valid index types are: match, ore, ope, unique, ste_vec. --! --! @param jsonb Configuration data to validate --! @return boolean True if all index types are valid @@ -49,10 +49,10 @@ AS $$ BEGIN IF (SELECT EXISTS (SELECT eql_v2.config_get_indexes(val))) THEN - IF (SELECT bool_and(index = ANY('{match, ore, unique, ste_vec}')) FROM eql_v2.config_get_indexes(val) AS index) THEN + IF (SELECT bool_and(index = ANY('{match, ore, ope, unique, ste_vec}')) FROM eql_v2.config_get_indexes(val) AS index) THEN RETURN true; END IF; - RAISE 'Configuration has an invalid index (%). Index should be one of {match, ore, unique, ste_vec}', val; + RAISE 'Configuration has an invalid index (%). Index should be one of {match, ore, ope, unique, ste_vec}', val; END IF; RETURN true; END; diff --git a/src/config/functions.sql b/src/config/functions.sql index 8628cee65..3513f1871 100644 --- a/src/config/functions.sql +++ b/src/config/functions.sql @@ -4,13 +4,13 @@ --! @brief Add a search index configuration for an encrypted column --! ---! Configures a searchable encryption index (unique, match, ore, or ste_vec) on an ---! encrypted column. Creates or updates the pending configuration, then migrates ---! and activates it unless migrating flag is set. +--! Configures a searchable encryption index (unique, match, ore, ope, or ste_vec) +--! on an encrypted column. Creates or updates the pending configuration, then +--! migrates and activates it unless migrating flag is set. --! --! @param table_name Text Name of the table containing the column --! @param column_name Text Name of the column to configure ---! @param index_name Text Type of index ('unique', 'match', 'ore', 'ste_vec') +--! @param index_name Text Type of index ('unique', 'match', 'ore', 'ope', 'ste_vec') --! @param cast_as Text PostgreSQL type for decrypted values (default: 'text') --! @param opts JSONB Index-specific options (default: '{}') --! @param migrating Boolean Skip auto-migration if true (default: false) diff --git a/src/ope_cllw_u64_65/compare.sql b/src/ope_cllw_u64_65/compare.sql new file mode 100644 index 000000000..b799f9dfa --- /dev/null +++ b/src/ope_cllw_u64_65/compare.sql @@ -0,0 +1,71 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/ope_cllw_u64_65/types.sql +-- REQUIRE: src/ope_cllw_u64_65/functions.sql + + +--! @brief Compare two encrypted values using CLWW OPE index terms +--! +--! Performs a three-way comparison (returns -1/0/1) of encrypted values using +--! their fixed-width CLWW OPE ciphertext index terms. Used internally by range +--! operators (<, <=, >, >=) for order-preserving comparisons without decryption. +--! +--! @param a eql_v2_encrypted First encrypted value to compare +--! @param b eql_v2_encrypted Second encrypted value to compare +--! @return Integer -1 if a < b, 0 if a = b, 1 if a > b +--! +--! @note NULL values are sorted before non-NULL values +--! @note OPE ciphertexts compare via standard lexicographic bytea ordering — +--! no custom per-byte protocol required (unlike the ORE CLWW variants) +--! +--! @see eql_v2.ope_cllw_u64_65 +--! @see eql_v2.has_ope_cllw_u64_65 +CREATE FUNCTION eql_v2.compare_ope_cllw_u64_65(a eql_v2_encrypted, b eql_v2_encrypted) + RETURNS integer + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + DECLARE + a_term eql_v2.ope_cllw_u64_65; + b_term eql_v2.ope_cllw_u64_65; + BEGIN + IF a IS NULL AND b IS NULL THEN + RETURN 0; + END IF; + + IF a IS NULL THEN + RETURN -1; + END IF; + + IF b IS NULL THEN + RETURN 1; + END IF; + + IF eql_v2.has_ope_cllw_u64_65(a) THEN + a_term := eql_v2.ope_cllw_u64_65(a); + END IF; + + IF eql_v2.has_ope_cllw_u64_65(b) THEN + b_term := eql_v2.ope_cllw_u64_65(b); + END IF; + + IF a_term IS NULL AND b_term IS NULL THEN + RETURN 0; + END IF; + + IF a_term IS NULL THEN + RETURN -1; + END IF; + + IF b_term IS NULL THEN + RETURN 1; + END IF; + + -- OPE: standard lex byte compare is exact + IF a_term.bytes < b_term.bytes THEN + RETURN -1; + ELSIF a_term.bytes > b_term.bytes THEN + RETURN 1; + ELSE + RETURN 0; + END IF; + END; +$$ LANGUAGE plpgsql; diff --git a/src/ope_cllw_u64_65/functions.sql b/src/ope_cllw_u64_65/functions.sql new file mode 100644 index 000000000..e06ddd9be --- /dev/null +++ b/src/ope_cllw_u64_65/functions.sql @@ -0,0 +1,89 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/common.sql +-- REQUIRE: src/ope_cllw_u64_65/types.sql + + +--! @brief Extract CLWW OPE index term from JSONB payload +--! +--! Extracts the fixed-width CLWW OPE ciphertext from the 'opf' field of an +--! encrypted data payload. Used internally for range query comparisons. +--! +--! @param jsonb containing encrypted EQL payload +--! @return eql_v2.ope_cllw_u64_65 CLWW OPE ciphertext +--! @throws Exception if 'opf' field is missing when ope index is expected +--! +--! @see eql_v2.has_ope_cllw_u64_65 +--! @see eql_v2.compare_ope_cllw_u64_65 +CREATE FUNCTION eql_v2.ope_cllw_u64_65(val jsonb) + RETURNS eql_v2.ope_cllw_u64_65 + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + + IF NOT (eql_v2.has_ope_cllw_u64_65(val)) THEN + RAISE 'Expected a ope_cllw_u64_65 index (opf) value in json: %', val; + END IF; + + RETURN ROW(decode(val->>'opf', 'hex')); + END; +$$ LANGUAGE plpgsql; + + +--! @brief Extract CLWW OPE index term from encrypted column value +--! +--! Extracts the fixed-width CLWW OPE ciphertext from an encrypted column value +--! by accessing its underlying JSONB data field. +--! +--! @param eql_v2_encrypted Encrypted column value +--! @return eql_v2.ope_cllw_u64_65 CLWW OPE ciphertext +--! +--! @see eql_v2.ope_cllw_u64_65(jsonb) +CREATE FUNCTION eql_v2.ope_cllw_u64_65(val eql_v2_encrypted) + RETURNS eql_v2.ope_cllw_u64_65 + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN (SELECT eql_v2.ope_cllw_u64_65(val.data)); + END; +$$ LANGUAGE plpgsql; + + +--! @brief Check if JSONB payload contains CLWW OPE index term +--! +--! Tests whether the encrypted data payload includes an 'opf' field, +--! indicating a fixed-width CLWW OPE ciphertext is available for range queries. +--! +--! @param jsonb containing encrypted EQL payload +--! @return Boolean True if 'opf' field is present and non-null +--! +--! @see eql_v2.ope_cllw_u64_65 +CREATE FUNCTION eql_v2.has_ope_cllw_u64_65(val jsonb) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN val ->> 'opf' IS NOT NULL; + END; +$$ LANGUAGE plpgsql; + + +--! @brief Check if encrypted column value contains CLWW OPE index term +--! +--! Tests whether an encrypted column value includes a fixed-width CLWW OPE +--! ciphertext by checking its underlying JSONB data field. +--! +--! @param eql_v2_encrypted Encrypted column value +--! @return Boolean True if CLWW OPE ciphertext is present +--! +--! @see eql_v2.has_ope_cllw_u64_65(jsonb) +CREATE FUNCTION eql_v2.has_ope_cllw_u64_65(val eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN eql_v2.has_ope_cllw_u64_65(val.data); + END; +$$ LANGUAGE plpgsql; diff --git a/src/ope_cllw_u64_65/types.sql b/src/ope_cllw_u64_65/types.sql new file mode 100644 index 000000000..8d9226f3b --- /dev/null +++ b/src/ope_cllw_u64_65/types.sql @@ -0,0 +1,18 @@ +-- REQUIRE: src/schema.sql + +--! @brief CLWW OPE index term type for fixed-width numeric range queries +--! +--! Composite type for CLWW (Chenette, Lewi, Weis, Wu) Order-Preserving Encryption +--! over 64-bit integers. Ciphertexts are 65 bytes (8 bytes per plaintext bit plus +--! one reserved carry byte). +--! +--! Ciphertexts compare with **standard lexicographic byte ordering** — unlike +--! the ORE variants there is no custom per-byte compare protocol. The ciphertext +--! is stored in the 'opf' field of encrypted data payloads. +--! +--! @see eql_v2.add_search_config +--! @see eql_v2.compare_ope_cllw_u64_65 +--! @note This is a transient type used only during query execution +CREATE TYPE eql_v2.ope_cllw_u64_65 AS ( + bytes bytea +); diff --git a/src/ope_cllw_var_8/compare.sql b/src/ope_cllw_var_8/compare.sql new file mode 100644 index 000000000..7d2be1a4c --- /dev/null +++ b/src/ope_cllw_var_8/compare.sql @@ -0,0 +1,72 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/ope_cllw_var_8/types.sql +-- REQUIRE: src/ope_cllw_var_8/functions.sql + + +--! @brief Compare two encrypted values using variable-width CLWW OPE index terms +--! +--! Performs a three-way comparison (returns -1/0/1) of encrypted values using +--! their variable-width CLWW OPE ciphertext index terms. Used internally by +--! range operators (<, <=, >, >=) for order-preserving comparisons without +--! decryption. +--! +--! @param a eql_v2_encrypted First encrypted value to compare +--! @param b eql_v2_encrypted Second encrypted value to compare +--! @return Integer -1 if a < b, 0 if a = b, 1 if a > b +--! +--! @note NULL values are sorted before non-NULL values +--! @note OPE ciphertexts compare via standard lexicographic bytea ordering — +--! bytea compare handles variable-length inputs (shorter prefix is less) +--! +--! @see eql_v2.ope_cllw_var_8 +--! @see eql_v2.has_ope_cllw_var_8 +CREATE FUNCTION eql_v2.compare_ope_cllw_var_8(a eql_v2_encrypted, b eql_v2_encrypted) + RETURNS integer + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + DECLARE + a_term eql_v2.ope_cllw_var_8; + b_term eql_v2.ope_cllw_var_8; + BEGIN + IF a IS NULL AND b IS NULL THEN + RETURN 0; + END IF; + + IF a IS NULL THEN + RETURN -1; + END IF; + + IF b IS NULL THEN + RETURN 1; + END IF; + + IF eql_v2.has_ope_cllw_var_8(a) THEN + a_term := eql_v2.ope_cllw_var_8(a); + END IF; + + IF eql_v2.has_ope_cllw_var_8(b) THEN + b_term := eql_v2.ope_cllw_var_8(b); + END IF; + + IF a_term IS NULL AND b_term IS NULL THEN + RETURN 0; + END IF; + + IF a_term IS NULL THEN + RETURN -1; + END IF; + + IF b_term IS NULL THEN + RETURN 1; + END IF; + + -- OPE: standard lex byte compare is exact (shorter prefix sorts less) + IF a_term.bytes < b_term.bytes THEN + RETURN -1; + ELSIF a_term.bytes > b_term.bytes THEN + RETURN 1; + ELSE + RETURN 0; + END IF; + END; +$$ LANGUAGE plpgsql; diff --git a/src/ope_cllw_var_8/functions.sql b/src/ope_cllw_var_8/functions.sql new file mode 100644 index 000000000..f3b7c4072 --- /dev/null +++ b/src/ope_cllw_var_8/functions.sql @@ -0,0 +1,89 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/common.sql +-- REQUIRE: src/ope_cllw_var_8/types.sql + + +--! @brief Extract variable-width CLWW OPE index term from JSONB payload +--! +--! Extracts the variable-width CLWW OPE ciphertext from the 'opv' field of an +--! encrypted data payload. Used internally for range query comparisons. +--! +--! @param jsonb containing encrypted EQL payload +--! @return eql_v2.ope_cllw_var_8 Variable-width CLWW OPE ciphertext +--! @throws Exception if 'opv' field is missing when ope index is expected +--! +--! @see eql_v2.has_ope_cllw_var_8 +--! @see eql_v2.compare_ope_cllw_var_8 +CREATE FUNCTION eql_v2.ope_cllw_var_8(val jsonb) + RETURNS eql_v2.ope_cllw_var_8 + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + + IF NOT (eql_v2.has_ope_cllw_var_8(val)) THEN + RAISE 'Expected a ope_cllw_var_8 index (opv) value in json: %', val; + END IF; + + RETURN ROW(decode(val->>'opv', 'hex')); + END; +$$ LANGUAGE plpgsql; + + +--! @brief Extract variable-width CLWW OPE index term from encrypted column value +--! +--! Extracts the variable-width CLWW OPE ciphertext from an encrypted column value +--! by accessing its underlying JSONB data field. +--! +--! @param eql_v2_encrypted Encrypted column value +--! @return eql_v2.ope_cllw_var_8 Variable-width CLWW OPE ciphertext +--! +--! @see eql_v2.ope_cllw_var_8(jsonb) +CREATE FUNCTION eql_v2.ope_cllw_var_8(val eql_v2_encrypted) + RETURNS eql_v2.ope_cllw_var_8 + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN (SELECT eql_v2.ope_cllw_var_8(val.data)); + END; +$$ LANGUAGE plpgsql; + + +--! @brief Check if JSONB payload contains variable-width CLWW OPE index term +--! +--! Tests whether the encrypted data payload includes an 'opv' field, +--! indicating a variable-width CLWW OPE ciphertext is available for range queries. +--! +--! @param jsonb containing encrypted EQL payload +--! @return Boolean True if 'opv' field is present and non-null +--! +--! @see eql_v2.ope_cllw_var_8 +CREATE FUNCTION eql_v2.has_ope_cllw_var_8(val jsonb) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN val ->> 'opv' IS NOT NULL; + END; +$$ LANGUAGE plpgsql; + + +--! @brief Check if encrypted column value contains variable-width CLWW OPE index term +--! +--! Tests whether an encrypted column value includes a variable-width CLWW OPE +--! ciphertext by checking its underlying JSONB data field. +--! +--! @param eql_v2_encrypted Encrypted column value +--! @return Boolean True if variable-width CLWW OPE ciphertext is present +--! +--! @see eql_v2.has_ope_cllw_var_8(jsonb) +CREATE FUNCTION eql_v2.has_ope_cllw_var_8(val eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN eql_v2.has_ope_cllw_var_8(val.data); + END; +$$ LANGUAGE plpgsql; diff --git a/src/ope_cllw_var_8/types.sql b/src/ope_cllw_var_8/types.sql new file mode 100644 index 000000000..b730e7301 --- /dev/null +++ b/src/ope_cllw_var_8/types.sql @@ -0,0 +1,19 @@ +-- REQUIRE: src/schema.sql + +--! @brief CLWW OPE index term type for variable-width range queries +--! +--! Composite type for variable-width CLWW (Chenette, Lewi, Weis, Wu) +--! Order-Preserving Encryption. Unlike ope_cllw_u64_65, supports +--! variable-length ciphertexts (strings / byte slices). Ciphertext length is +--! `8 * plaintext_bytes + 1` (one carry byte + 8 bytes per plaintext byte). +--! +--! Ciphertexts compare with **standard lexicographic byte ordering** — unlike +--! the ORE variants there is no custom per-byte compare protocol. The ciphertext +--! is stored in the 'opv' field of encrypted data payloads. +--! +--! @see eql_v2.add_search_config +--! @see eql_v2.compare_ope_cllw_var_8 +--! @note This is a transient type used only during query execution +CREATE TYPE eql_v2.ope_cllw_var_8 AS ( + bytes bytea +); diff --git a/src/operators/compare.sql b/src/operators/compare.sql index b434539e6..2268d3b2e 100644 --- a/src/operators/compare.sql +++ b/src/operators/compare.sql @@ -17,6 +17,14 @@ -- REQUIRE: src/ore_cllw_var_8/types.sql -- REQUIRE: src/ore_cllw_var_8/functions.sql +-- REQUIRE: src/ope_cllw_u64_65/types.sql +-- REQUIRE: src/ope_cllw_u64_65/functions.sql +-- REQUIRE: src/ope_cllw_u64_65/compare.sql + +-- REQUIRE: src/ope_cllw_var_8/types.sql +-- REQUIRE: src/ope_cllw_var_8/functions.sql +-- REQUIRE: src/ope_cllw_var_8/compare.sql + --! @brief Core comparison function for encrypted values --! --! Compares two encrypted values using their index terms without decryption. @@ -27,8 +35,10 @@ --! 1. ore_block_u64_8_256 (Order-Revealing Encryption) --! 2. ore_cllw_u64_8 (Order-Revealing Encryption) --! 3. ore_cllw_var_8 (Order-Revealing Encryption) ---! 4. hmac_256 (Hash-based equality) ---! 5. blake3 (Hash-based equality) +--! 4. ope_cllw_u64_65 (Order-Preserving Encryption) +--! 5. ope_cllw_var_8 (Order-Preserving Encryption) +--! 6. hmac_256 (Hash-based equality) +--! 7. blake3 (Hash-based equality) --! --! The first index term type present in both values is used for comparison. --! If no matching index terms are found, falls back to JSONB literal comparison @@ -76,6 +86,14 @@ AS $$ RETURN eql_v2.compare_ore_cllw_var_8(a, b); END IF; + IF eql_v2.has_ope_cllw_u64_65(a) AND eql_v2.has_ope_cllw_u64_65(b) THEN + RETURN eql_v2.compare_ope_cllw_u64_65(a, b); + END IF; + + IF eql_v2.has_ope_cllw_var_8(a) AND eql_v2.has_ope_cllw_var_8(b) THEN + RETURN eql_v2.compare_ope_cllw_var_8(a, b); + END IF; + IF eql_v2.has_hmac_256(a) AND eql_v2.has_hmac_256(b) THEN RETURN eql_v2.compare_hmac_256(a, b); END IF; diff --git a/src/operators/order_by.sql b/src/operators/order_by.sql index 8269baae2..5d9959f6b 100644 --- a/src/operators/order_by.sql +++ b/src/operators/order_by.sql @@ -3,6 +3,10 @@ -- REQUIRE: src/ore_block_u64_8_256/functions.sql -- REQUIRE: src/ore_cllw_u64_8/types.sql -- REQUIRE: src/ore_cllw_u64_8/functions.sql +-- REQUIRE: src/ope_cllw_u64_65/types.sql +-- REQUIRE: src/ope_cllw_u64_65/functions.sql +-- REQUIRE: src/ope_cllw_var_8/types.sql +-- REQUIRE: src/ope_cllw_var_8/functions.sql --! @brief Extract ORE index term for ordering encrypted values --! @@ -30,4 +34,41 @@ AS $$ $$ LANGUAGE plpgsql; +--! @brief Extract OPE ciphertext bytes for ordering encrypted values +--! +--! Returns the raw CLWW Order-Preserving Encryption ciphertext as `bytea` so +--! it can be used as an order key. OPE ciphertexts compare with standard +--! lexicographic byte ordering, so the returned bytea can be ordered directly +--! with `<`, `=`, `>` (no custom protocol required). +--! +--! Prefers the fixed-width variant (`opf`, ope_cllw_u64_65) when present and +--! falls back to the variable-width variant (`opv`, ope_cllw_var_8). Returns +--! NULL when neither is present. +--! +--! @param a eql_v2_encrypted Encrypted value to extract order key from +--! @return bytea OPE ciphertext bytes, or NULL if no OPE term is available +--! +--! @note Requires 'ope' index configuration on the column +--! @see eql_v2.ope_cllw_u64_65 +--! @see eql_v2.ope_cllw_var_8 +--! @see eql_v2.add_search_config +CREATE FUNCTION eql_v2.order_by_ope(a eql_v2_encrypted) + RETURNS bytea + IMMUTABLE STRICT PARALLEL SAFE + SET search_path = pg_catalog, extensions, public +AS $$ + BEGIN + IF eql_v2.has_ope_cllw_u64_65(a) THEN + RETURN (eql_v2.ope_cllw_u64_65(a)).bytes; + END IF; + + IF eql_v2.has_ope_cllw_var_8(a) THEN + RETURN (eql_v2.ope_cllw_var_8(a)).bytes; + END IF; + + RETURN NULL; + END; +$$ LANGUAGE plpgsql; + + diff --git a/src/operators/sort.sql b/src/operators/sort.sql index ae0c7ff6a..b097179f7 100644 --- a/src/operators/sort.sql +++ b/src/operators/sort.sql @@ -2,6 +2,10 @@ -- REQUIRE: src/encrypted/types.sql -- REQUIRE: src/ore_block_u64_8_256/types.sql -- REQUIRE: src/ore_block_u64_8_256/functions.sql +-- REQUIRE: src/ope_cllw_u64_65/types.sql +-- REQUIRE: src/ope_cllw_u64_65/functions.sql +-- REQUIRE: src/ope_cllw_var_8/types.sql +-- REQUIRE: src/ope_cllw_var_8/functions.sql -- REQUIRE: src/operators/compare.sql -- REQUIRE: src/operators/order_by.sql @@ -11,6 +15,14 @@ --! Provides O(n log n) quicksort-based sorting using eql_v2.compare() for environments --! where btree operator classes are unavailable (e.g., Supabase). This is significantly --! faster than the O(n^2) correlated subquery workaround. +--! +--! When all input rows share an ORE term (`ob`) the sort path pre-extracts the +--! ORE order key once per row and compares those keys directly. When all rows +--! share the *same* OPE subtype (every non-NULL row carries `opf`, or every +--! non-NULL row carries `opv`) the matching OPE ciphertext is pre-extracted as +--! `bytea` and compared lexicographically (OPE ciphertexts are designed to be +--! ordered that way within a subtype). Mixed-subtype OPE batches and rows +--! lacking ORE/OPE entirely fall back to `eql_v2.compare()` per pair. --! @internal @@ -49,21 +61,78 @@ $$ LANGUAGE plpgsql; --! @internal ---! @brief Compare two elements from aligned arrays using generic or ORE-key ordering +--! @brief Compare pre-extracted OPE ciphertext bytes with encrypted NULL semantics +--! +--! OPE ciphertexts within a single subtype are ordered lexicographically by +--! construction, so once the bytea has been extracted (per-subtype, by the +--! caller) we can dispatch directly to the native bytea comparison operators. +--! Callers must ensure every non-NULL key originates from the same OPE subtype +--! (`opf` or `opv`); mixed-subtype batches must use `strategy = 'compare'`. +--! +--! @param a bytea First OPE ciphertext (or NULL) +--! @param b bytea Second OPE ciphertext (or NULL) +--! @return integer -1 if a < b, 0 if a = b, 1 if a > b +CREATE FUNCTION eql_v2._compare_ope_key( + a bytea, + b bytea +) +RETURNS integer +IMMUTABLE PARALLEL SAFE + SET search_path = pg_catalog, extensions, public +AS $$ +BEGIN + IF a IS NULL AND b IS NULL THEN + RETURN 0; + END IF; + + IF a IS NULL THEN + RETURN -1; + END IF; + + IF b IS NULL THEN + RETURN 1; + END IF; + + IF a < b THEN + RETURN -1; + ELSIF a > b THEN + RETURN 1; + ELSE + RETURN 0; + END IF; +END; +$$ LANGUAGE plpgsql; + + +--! @internal +--! @brief Compare two elements from aligned arrays using the selected sort strategy +--! +--! @param vals eql_v2_encrypted[] Encrypted values (used when strategy = 'compare') +--! @param ore_keys eql_v2.ore_block_u64_8_256[] Pre-extracted ORE keys (strategy = 'ore') +--! @param ope_keys bytea[] Pre-extracted OPE ciphertext bytes (strategy = 'ope') +--! @param left_idx integer Index of the left element +--! @param right_idx integer Index of the right element +--! @param strategy text One of 'ore', 'ope', or 'compare' +--! @return integer -1 if left < right, 0 if equal, 1 if left > right CREATE FUNCTION eql_v2._compare_sort_elements( vals eql_v2_encrypted[], - keys eql_v2.ore_block_u64_8_256[], + ore_keys eql_v2.ore_block_u64_8_256[], + ope_keys bytea[], left_idx integer, right_idx integer, - use_ore boolean + strategy text ) RETURNS integer IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - IF use_ore THEN - RETURN eql_v2._compare_order_key(keys[left_idx], keys[right_idx]); + IF strategy = 'ore' THEN + RETURN eql_v2._compare_order_key(ore_keys[left_idx], ore_keys[right_idx]); + END IF; + + IF strategy = 'ope' THEN + RETURN eql_v2._compare_ope_key(ope_keys[left_idx], ope_keys[right_idx]); END IF; RETURN eql_v2.compare(vals[left_idx], vals[right_idx]); @@ -72,30 +141,38 @@ $$ LANGUAGE plpgsql; --! @internal ---! @brief Compare an array element against a captured pivot value or ORE key +--! @brief Compare an array element against a captured pivot using the selected strategy --! --! @param vals eql_v2_encrypted[] Array of encrypted values ---! @param keys eql_v2.ore_block_u64_8_256[] Array of pre-extracted ORE order keys +--! @param ore_keys eql_v2.ore_block_u64_8_256[] Array of pre-extracted ORE keys +--! @param ope_keys bytea[] Array of pre-extracted OPE ciphertext bytes --! @param idx integer Index of the element to compare ---! @param pivot_val eql_v2_encrypted Pivot encrypted value (used when use_ore is false) ---! @param pivot_key eql_v2.ore_block_u64_8_256 Pivot ORE key (used when use_ore is true) ---! @param use_ore boolean When true compare ORE keys, otherwise compare encrypted values +--! @param pivot_val eql_v2_encrypted Pivot encrypted value (strategy = 'compare') +--! @param pivot_ore_key eql_v2.ore_block_u64_8_256 Pivot ORE key (strategy = 'ore') +--! @param pivot_ope_key bytea Pivot OPE ciphertext bytes (strategy = 'ope') +--! @param strategy text One of 'ore', 'ope', or 'compare' --! @return integer -1 if element < pivot, 0 if equal, 1 if element > pivot CREATE FUNCTION eql_v2._compare_sort_pivot( vals eql_v2_encrypted[], - keys eql_v2.ore_block_u64_8_256[], + ore_keys eql_v2.ore_block_u64_8_256[], + ope_keys bytea[], idx integer, pivot_val eql_v2_encrypted, - pivot_key eql_v2.ore_block_u64_8_256, - use_ore boolean + pivot_ore_key eql_v2.ore_block_u64_8_256, + pivot_ope_key bytea, + strategy text ) RETURNS integer IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - IF use_ore THEN - RETURN eql_v2._compare_order_key(keys[idx], pivot_key); + IF strategy = 'ore' THEN + RETURN eql_v2._compare_order_key(ore_keys[idx], pivot_ore_key); + END IF; + + IF strategy = 'ope' THEN + RETURN eql_v2._compare_ope_key(ope_keys[idx], pivot_ope_key); END IF; RETURN eql_v2.compare(vals[idx], pivot_val); @@ -108,20 +185,23 @@ $$ LANGUAGE plpgsql; --! --! @param ids bigint[] Array of row identifiers (reordered in place) --! @param vals eql_v2_encrypted[] Array of encrypted values (reordered in place) ---! @param keys eql_v2.ore_block_u64_8_256[] Array of pre-extracted ORE order keys (reordered in place) +--! @param ore_keys eql_v2.ore_block_u64_8_256[] Array of pre-extracted ORE keys (reordered in place) +--! @param ope_keys bytea[] Array of pre-extracted OPE bytes (reordered in place) --! @param lo integer Lower bound index (1-based, inclusive) --! @param hi integer Upper bound index (1-based, inclusive) ---! @param use_ore boolean When true compare ORE keys, otherwise compare encrypted values +--! @param strategy text One of 'ore', 'ope', or 'compare' --! @return ids bigint[] Sorted array of row identifiers --! @return vals eql_v2_encrypted[] Sorted array of encrypted values ---! @return keys eql_v2.ore_block_u64_8_256[] Sorted array of pre-extracted order keys +--! @return ore_keys eql_v2.ore_block_u64_8_256[] Sorted array of pre-extracted ORE keys +--! @return ope_keys bytea[] Sorted array of pre-extracted OPE bytes CREATE FUNCTION eql_v2._insertion_sort( INOUT ids bigint[], INOUT vals eql_v2_encrypted[], - INOUT keys eql_v2.ore_block_u64_8_256[], + INOUT ore_keys eql_v2.ore_block_u64_8_256[], + INOUT ope_keys bytea[], lo integer, hi integer, - use_ore boolean + strategy text ) SET search_path = pg_catalog, extensions, public AS $$ @@ -130,7 +210,8 @@ DECLARE j integer; key_id bigint; key_val eql_v2_encrypted; - sort_key eql_v2.ore_block_u64_8_256; + sort_ore_key eql_v2.ore_block_u64_8_256; + sort_ope_key bytea; BEGIN IF lo >= hi THEN RETURN; @@ -139,22 +220,29 @@ BEGIN FOR i IN lo + 1..hi LOOP key_id := ids[i]; key_val := vals[i]; - sort_key := keys[i]; + sort_ore_key := ore_keys[i]; + sort_ope_key := ope_keys[i]; j := i - 1; WHILE j >= lo LOOP - EXIT WHEN use_ore = FALSE AND eql_v2.compare(vals[j], key_val) <= 0; - EXIT WHEN use_ore = TRUE AND eql_v2._compare_order_key(keys[j], sort_key) <= 0; + EXIT WHEN strategy = 'compare' + AND eql_v2.compare(vals[j], key_val) <= 0; + EXIT WHEN strategy = 'ore' + AND eql_v2._compare_order_key(ore_keys[j], sort_ore_key) <= 0; + EXIT WHEN strategy = 'ope' + AND eql_v2._compare_ope_key(ope_keys[j], sort_ope_key) <= 0; ids[j + 1] := ids[j]; vals[j + 1] := vals[j]; - keys[j + 1] := keys[j]; + ore_keys[j + 1] := ore_keys[j]; + ope_keys[j + 1] := ope_keys[j]; j := j - 1; END LOOP; ids[j + 1] := key_id; vals[j + 1] := key_val; - keys[j + 1] := sort_key; + ore_keys[j + 1] := sort_ore_key; + ope_keys[j + 1] := sort_ope_key; END LOOP; END; $$ LANGUAGE plpgsql; @@ -169,28 +257,32 @@ $$ LANGUAGE plpgsql; --! --! @param ids bigint[] Array of row identifiers (reordered in place) --! @param vals eql_v2_encrypted[] Array of encrypted values to compare (reordered in place) ---! @param keys eql_v2.ore_block_u64_8_256[] Optional pre-extracted order keys (reordered in place) +--! @param ore_keys eql_v2.ore_block_u64_8_256[] Pre-extracted ORE keys (reordered in place) +--! @param ope_keys bytea[] Pre-extracted OPE ciphertext bytes (reordered in place) --! @param lo integer Lower bound index (1-based, inclusive) --! @param hi integer Upper bound index (1-based, inclusive) ---! @param use_ore boolean When true compare keys, otherwise compare vals +--! @param strategy text One of 'ore', 'ope', or 'compare' --! --! @return ids bigint[] Sorted array of row identifiers --! @return vals eql_v2_encrypted[] Sorted array of encrypted values ---! @return keys eql_v2.ore_block_u64_8_256[] Sorted array of pre-extracted order keys +--! @return ore_keys eql_v2.ore_block_u64_8_256[] Sorted array of pre-extracted ORE keys +--! @return ope_keys bytea[] Sorted array of pre-extracted OPE bytes CREATE FUNCTION eql_v2._quicksort_sorter( INOUT ids bigint[], INOUT vals eql_v2_encrypted[], - INOUT keys eql_v2.ore_block_u64_8_256[], + INOUT ore_keys eql_v2.ore_block_u64_8_256[], + INOUT ope_keys bytea[], lo integer, hi integer, - use_ore boolean + strategy text ) SET search_path = pg_catalog, extensions, public AS $$ DECLARE insertion_threshold CONSTANT integer := 16; pivot_val eql_v2_encrypted; - pivot_key eql_v2.ore_block_u64_8_256; + pivot_ore_key eql_v2.ore_block_u64_8_256; + pivot_ope_key bytea; mid integer; i integer; j integer; @@ -198,44 +290,56 @@ DECLARE right_lo integer; tmp_id bigint; tmp_val eql_v2_encrypted; - tmp_key eql_v2.ore_block_u64_8_256; + tmp_ore_key eql_v2.ore_block_u64_8_256; + tmp_ope_key bytea; BEGIN WHILE lo < hi LOOP IF hi - lo <= insertion_threshold THEN - SELECT q.ids, q.vals, q.keys INTO ids, vals, keys - FROM eql_v2._insertion_sort(ids, vals, keys, lo, hi, use_ore) q; + SELECT q.ids, q.vals, q.ore_keys, q.ope_keys + INTO ids, vals, ore_keys, ope_keys + FROM eql_v2._insertion_sort(ids, vals, ore_keys, ope_keys, lo, hi, strategy) q; RETURN; END IF; -- Median-of-three pivot selection: sort lo, mid, hi then use mid as pivot mid := lo + (hi - lo) / 2; - IF eql_v2._compare_sort_elements(vals, keys, lo, mid, use_ore) > 0 THEN + IF eql_v2._compare_sort_elements(vals, ore_keys, ope_keys, lo, mid, strategy) > 0 THEN tmp_id := ids[lo]; ids[lo] := ids[mid]; ids[mid] := tmp_id; tmp_val := vals[lo]; vals[lo] := vals[mid]; vals[mid] := tmp_val; - tmp_key := keys[lo]; keys[lo] := keys[mid]; keys[mid] := tmp_key; + tmp_ore_key := ore_keys[lo]; ore_keys[lo] := ore_keys[mid]; ore_keys[mid] := tmp_ore_key; + tmp_ope_key := ope_keys[lo]; ope_keys[lo] := ope_keys[mid]; ope_keys[mid] := tmp_ope_key; END IF; - IF eql_v2._compare_sort_elements(vals, keys, lo, hi, use_ore) > 0 THEN + IF eql_v2._compare_sort_elements(vals, ore_keys, ope_keys, lo, hi, strategy) > 0 THEN tmp_id := ids[lo]; ids[lo] := ids[hi]; ids[hi] := tmp_id; tmp_val := vals[lo]; vals[lo] := vals[hi]; vals[hi] := tmp_val; - tmp_key := keys[lo]; keys[lo] := keys[hi]; keys[hi] := tmp_key; + tmp_ore_key := ore_keys[lo]; ore_keys[lo] := ore_keys[hi]; ore_keys[hi] := tmp_ore_key; + tmp_ope_key := ope_keys[lo]; ope_keys[lo] := ope_keys[hi]; ope_keys[hi] := tmp_ope_key; END IF; - IF eql_v2._compare_sort_elements(vals, keys, mid, hi, use_ore) > 0 THEN + IF eql_v2._compare_sort_elements(vals, ore_keys, ope_keys, mid, hi, strategy) > 0 THEN tmp_id := ids[mid]; ids[mid] := ids[hi]; ids[hi] := tmp_id; tmp_val := vals[mid]; vals[mid] := vals[hi]; vals[hi] := tmp_val; - tmp_key := keys[mid]; keys[mid] := keys[hi]; keys[hi] := tmp_key; + tmp_ore_key := ore_keys[mid]; ore_keys[mid] := ore_keys[hi]; ore_keys[hi] := tmp_ore_key; + tmp_ope_key := ope_keys[mid]; ope_keys[mid] := ope_keys[hi]; ope_keys[hi] := tmp_ope_key; END IF; pivot_val := vals[mid]; - pivot_key := keys[mid]; + pivot_ore_key := ore_keys[mid]; + pivot_ope_key := ope_keys[mid]; i := lo; j := hi; LOOP - WHILE eql_v2._compare_sort_pivot(vals, keys, i, pivot_val, pivot_key, use_ore) < 0 LOOP + WHILE eql_v2._compare_sort_pivot( + vals, ore_keys, ope_keys, i, + pivot_val, pivot_ore_key, pivot_ope_key, strategy + ) < 0 LOOP i := i + 1; END LOOP; - WHILE eql_v2._compare_sort_pivot(vals, keys, j, pivot_val, pivot_key, use_ore) > 0 LOOP + WHILE eql_v2._compare_sort_pivot( + vals, ore_keys, ope_keys, j, + pivot_val, pivot_ore_key, pivot_ope_key, strategy + ) > 0 LOOP j := j - 1; END LOOP; @@ -243,7 +347,8 @@ BEGIN tmp_id := ids[i]; ids[i] := ids[j]; ids[j] := tmp_id; tmp_val := vals[i]; vals[i] := vals[j]; vals[j] := tmp_val; - tmp_key := keys[i]; keys[i] := keys[j]; keys[j] := tmp_key; + tmp_ore_key := ore_keys[i]; ore_keys[i] := ore_keys[j]; ore_keys[j] := tmp_ore_key; + tmp_ope_key := ope_keys[i]; ope_keys[i] := ope_keys[j]; ope_keys[j] := tmp_ope_key; i := i + 1; j := j - 1; @@ -254,14 +359,16 @@ BEGIN IF left_hi - lo < hi - right_lo THEN IF lo < left_hi THEN - SELECT q.ids, q.vals, q.keys INTO ids, vals, keys - FROM eql_v2._quicksort_sorter(ids, vals, keys, lo, left_hi, use_ore) q; + SELECT q.ids, q.vals, q.ore_keys, q.ope_keys + INTO ids, vals, ore_keys, ope_keys + FROM eql_v2._quicksort_sorter(ids, vals, ore_keys, ope_keys, lo, left_hi, strategy) q; END IF; lo := right_lo; ELSE IF right_lo < hi THEN - SELECT q.ids, q.vals, q.keys INTO ids, vals, keys - FROM eql_v2._quicksort_sorter(ids, vals, keys, right_lo, hi, use_ore) q; + SELECT q.ids, q.vals, q.ore_keys, q.ope_keys + INTO ids, vals, ore_keys, ope_keys + FROM eql_v2._quicksort_sorter(ids, vals, ore_keys, ope_keys, right_lo, hi, strategy) q; END IF; hi := left_hi; END IF; @@ -310,17 +417,27 @@ $$ LANGUAGE plpgsql; --! @internal ---! @brief Sort encrypted values using precomputed ORE keys when available +--! @brief Sort encrypted values using precomputed ORE or OPE keys when available --! ---! Shared implementation for public sorting entrypoints. When `use_ore` is true ---! the caller must provide an aligned `keys` array; otherwise `eql_v2.compare()` ---! is used on the encrypted values directly. +--! Shared implementation for public sorting entrypoints. The `strategy` +--! parameter selects the comparison path: `'ore'` uses the aligned `ore_keys` +--! array; `'ope'` uses the aligned `ope_keys` array (lex bytea comparison); +--! `'compare'` falls back to `eql_v2.compare()` on the encrypted values directly. +--! +--! @param ids bigint[] Row identifiers aligned with `vals` +--! @param vals eql_v2_encrypted[] Encrypted values to sort +--! @param ore_keys eql_v2.ore_block_u64_8_256[] Pre-extracted ORE keys (used when strategy = 'ore') +--! @param ope_keys bytea[] Pre-extracted OPE ciphertext bytes (used when strategy = 'ope'); must all originate from the same OPE subtype +--! @param direction text Sort direction: 'ASC' (default) or 'DESC' +--! @param strategy text One of 'ore', 'ope', or 'compare' +--! @return TABLE(id bigint, val eql_v2_encrypted) Sorted rows CREATE FUNCTION eql_v2._sort_compare_precomputed( ids bigint[], vals eql_v2_encrypted[], - keys eql_v2.ore_block_u64_8_256[], + ore_keys eql_v2.ore_block_u64_8_256[], + ope_keys bytea[], direction text DEFAULT 'ASC', - use_ore boolean DEFAULT true + strategy text DEFAULT 'ore' ) RETURNS TABLE(id bigint, val eql_v2_encrypted) IMMUTABLE PARALLEL SAFE @@ -332,7 +449,8 @@ DECLARE k integer; sorted_ids bigint[]; sorted_vals eql_v2_encrypted[]; - sorted_keys eql_v2.ore_block_u64_8_256[]; + sorted_ore_keys eql_v2.ore_block_u64_8_256[]; + sorted_ope_keys bytea[]; BEGIN n := coalesce(array_length(ids, 1), 0); m := coalesce(array_length(vals, 1), 0); @@ -341,10 +459,15 @@ BEGIN RAISE EXCEPTION 'ids and vals must have the same length'; END IF; - IF use_ore THEN - k := coalesce(array_length(keys, 1), 0); + IF strategy = 'ore' THEN + k := coalesce(array_length(ore_keys, 1), 0); + IF n <> k THEN + RAISE EXCEPTION 'ids and ore_keys must have the same length when strategy = ''ore'''; + END IF; + ELSIF strategy = 'ope' THEN + k := coalesce(array_length(ope_keys, 1), 0); IF n <> k THEN - RAISE EXCEPTION 'ids and keys must have the same length when use_ore is true'; + RAISE EXCEPTION 'ids and ope_keys must have the same length when strategy = ''ope'''; END IF; END IF; @@ -359,8 +482,9 @@ BEGIN RETURN; END IF; - SELECT q.ids, q.vals, q.keys INTO sorted_ids, sorted_vals, sorted_keys - FROM eql_v2._quicksort_sorter(ids, vals, keys, 1, n, use_ore) q; + SELECT q.ids, q.vals, q.ore_keys, q.ope_keys + INTO sorted_ids, sorted_vals, sorted_ore_keys, sorted_ope_keys + FROM eql_v2._quicksort_sorter(ids, vals, ore_keys, ope_keys, 1, n, strategy) q; RETURN QUERY SELECT emitted.id, emitted.val @@ -375,6 +499,12 @@ $$ LANGUAGE plpgsql; --! quicksort with eql_v2.compare(). Returns sorted rows as a table, avoiding --! the need for unnest() or other array manipulation by callers. --! +--! When all input rows share an `ore` term the sort uses pre-extracted ORE +--! keys; when every non-NULL row shares the *same* OPE subtype (all `opf` or +--! all `opv`) the matching OPE ciphertext is pre-extracted as `bytea` and +--! compared lexicographically. Mixed-subtype OPE batches and other mixed +--! inputs fall back to `eql_v2.compare()` per pair. +--! --! This function is designed for environments without operator classes (e.g., Supabase) --! where direct ORDER BY on encrypted columns is not available. --! @@ -417,26 +547,75 @@ IMMUTABLE STRICT PARALLEL SAFE AS $$ DECLARE n integer; - sorted_keys eql_v2.ore_block_u64_8_256[]; + sorted_ore_keys eql_v2.ore_block_u64_8_256[]; + sorted_ope_u64_keys bytea[]; + sorted_ope_var_keys bytea[]; + selected_ope_keys bytea[]; i integer; use_ore boolean := true; + use_ope_u64 boolean := true; + use_ope_var boolean := true; + strategy text; BEGIN n := coalesce(array_length(ids, 1), 0); + -- Pre-extract per-subtype keys. Mixed OPE subtypes (some opf, some opv) + -- cannot share the bytea fast path because the ciphertexts are not + -- comparable across subtypes — fall back to eql_v2.compare() in that case + -- (which itself requires both sides to share a subtype). FOR i IN 1..n LOOP IF vals[i] IS NULL THEN - sorted_keys[i] := NULL; - ELSIF eql_v2.has_ore_block_u64_8_256(vals[i]) THEN - sorted_keys[i] := eql_v2.order_by(vals[i]); + sorted_ore_keys[i] := NULL; + sorted_ope_u64_keys[i] := NULL; + sorted_ope_var_keys[i] := NULL; ELSE - use_ore := false; - EXIT; + IF use_ore THEN + IF eql_v2.has_ore_block_u64_8_256(vals[i]) THEN + sorted_ore_keys[i] := eql_v2.order_by(vals[i]); + ELSE + use_ore := false; + END IF; + END IF; + + IF use_ope_u64 THEN + IF eql_v2.has_ope_cllw_u64_65(vals[i]) THEN + sorted_ope_u64_keys[i] := (eql_v2.ope_cllw_u64_65(vals[i])).bytes; + ELSE + use_ope_u64 := false; + END IF; + END IF; + + IF use_ope_var THEN + IF eql_v2.has_ope_cllw_var_8(vals[i]) THEN + sorted_ope_var_keys[i] := (eql_v2.ope_cllw_var_8(vals[i])).bytes; + ELSE + use_ope_var := false; + END IF; + END IF; + + EXIT WHEN NOT use_ore AND NOT use_ope_u64 AND NOT use_ope_var; END IF; END LOOP; + IF use_ore THEN + strategy := 'ore'; + selected_ope_keys := NULL; + ELSIF use_ope_u64 THEN + strategy := 'ope'; + selected_ope_keys := sorted_ope_u64_keys; + ELSIF use_ope_var THEN + strategy := 'ope'; + selected_ope_keys := sorted_ope_var_keys; + ELSE + strategy := 'compare'; + selected_ope_keys := NULL; + END IF; + RETURN QUERY SELECT sc.id, sc.val - FROM eql_v2._sort_compare_precomputed(ids, vals, sorted_keys, direction, use_ore) sc; + FROM eql_v2._sort_compare_precomputed( + ids, vals, sorted_ore_keys, selected_ope_keys, direction, strategy + ) sc; END; $$ LANGUAGE plpgsql; @@ -509,9 +688,12 @@ $$ LANGUAGE plpgsql; --! --! Convenience wrapper that accepts a SQL query string, executes it, collects the --! results, and returns them sorted. For ORE-backed values this pre-extracts the ---! order key once per row and sorts on that key; other values fall back to ---! eql_v2.compare(). The query must return ---! exactly two columns: a bigint identifier and an eql_v2_encrypted value. +--! order key once per row and sorts on that key; for OPE-backed values where +--! every non-NULL row shares the *same* OPE subtype (all `opf` or all `opv`) +--! the matching OPE ciphertext is pre-extracted as `bytea` and compared +--! lexicographically. Mixed-subtype OPE batches and other mixed inputs fall +--! back to eql_v2.compare(). The query must return exactly two columns: a +--! bigint identifier and an eql_v2_encrypted value. --! --! @param query text SQL query returning (bigint, eql_v2_encrypted) columns --! @param direction text Sort direction: 'ASC' (default) or 'DESC' @@ -542,9 +724,19 @@ AS $$ DECLARE all_ids bigint[]; all_vals eql_v2_encrypted[]; - all_keys eql_v2.ore_block_u64_8_256[]; - all_have_order_keys boolean; + all_ore_keys eql_v2.ore_block_u64_8_256[]; + all_ope_u64_keys bytea[]; + all_ope_var_keys bytea[]; + selected_ope_keys bytea[]; + all_have_ore_keys boolean; + all_have_ope_u64_keys boolean; + all_have_ope_var_keys boolean; + strategy text; BEGIN + -- Pre-extract per-subtype keys. The OPE bytea fast path is only valid when + -- every non-NULL row carries the same OPE subtype; mixed opf/opv batches + -- must fall back to eql_v2.compare() per pair (which itself requires + -- matching subtypes on both sides). EXECUTE format( 'WITH input_rows AS ( SELECT row_number() OVER () AS ord, @@ -554,33 +746,73 @@ BEGIN WHEN sub.val IS NULL THEN NULL WHEN eql_v2.has_ore_block_u64_8_256(sub.val) THEN eql_v2.order_by(sub.val) ELSE NULL - END AS sort_key, + END AS ore_key, + CASE + WHEN sub.val IS NULL THEN NULL + WHEN eql_v2.has_ope_cllw_u64_65(sub.val) + THEN (eql_v2.ope_cllw_u64_65(sub.val)).bytes + ELSE NULL + END AS ope_u64_key, + CASE + WHEN sub.val IS NULL THEN NULL + WHEN eql_v2.has_ope_cllw_var_8(sub.val) + THEN (eql_v2.ope_cllw_var_8(sub.val)).bytes + ELSE NULL + END AS ope_var_key, CASE WHEN sub.val IS NULL THEN TRUE ELSE eql_v2.has_ore_block_u64_8_256(sub.val) - END AS has_order_key + END AS has_ore_key, + CASE + WHEN sub.val IS NULL THEN TRUE + ELSE eql_v2.has_ope_cllw_u64_65(sub.val) + END AS has_ope_u64_key, + CASE + WHEN sub.val IS NULL THEN TRUE + ELSE eql_v2.has_ope_cllw_var_8(sub.val) + END AS has_ope_var_key FROM (%s) sub(id, val) ) SELECT array_agg(id ORDER BY ord), array_agg(val ORDER BY ord), - array_agg(sort_key ORDER BY ord), - coalesce(bool_and(has_order_key), TRUE) + array_agg(ore_key ORDER BY ord), + array_agg(ope_u64_key ORDER BY ord), + array_agg(ope_var_key ORDER BY ord), + coalesce(bool_and(has_ore_key), TRUE), + coalesce(bool_and(has_ope_u64_key), TRUE), + coalesce(bool_and(has_ope_var_key), TRUE) FROM input_rows', query - ) INTO all_ids, all_vals, all_keys, all_have_order_keys; + ) INTO all_ids, all_vals, all_ore_keys, all_ope_u64_keys, all_ope_var_keys, + all_have_ore_keys, all_have_ope_u64_keys, all_have_ope_var_keys; IF all_ids IS NULL THEN RETURN; END IF; + IF all_have_ore_keys THEN + strategy := 'ore'; + selected_ope_keys := NULL; + ELSIF all_have_ope_u64_keys THEN + strategy := 'ope'; + selected_ope_keys := all_ope_u64_keys; + ELSIF all_have_ope_var_keys THEN + strategy := 'ope'; + selected_ope_keys := all_ope_var_keys; + ELSE + strategy := 'compare'; + selected_ope_keys := NULL; + END IF; + RETURN QUERY SELECT sc.id, sc.val FROM eql_v2._sort_compare_precomputed( all_ids, all_vals, - all_keys, + all_ore_keys, + selected_ope_keys, direction, - all_have_order_keys + strategy ) sc; END; $$ LANGUAGE plpgsql; diff --git a/src/ste_vec/functions.sql b/src/ste_vec/functions.sql index 16d8bd239..0da3b799d 100644 --- a/src/ste_vec/functions.sql +++ b/src/ste_vec/functions.sql @@ -291,9 +291,9 @@ $$; --! @brief Extract deterministic fields as array for GIN indexing --! ---! Extracts only deterministic search term fields (s, b3, hm, ocv, ocf) from each ---! STE vector element. Excludes non-deterministic ciphertext for correct containment ---! comparison using PostgreSQL's native @> operator. +--! Extracts only deterministic search term fields (s, b3, hm, ocv, ocf, opf, opv) +--! from each STE vector element. Excludes non-deterministic ciphertext for correct +--! containment comparison using PostgreSQL's native @> operator. --! --! @param val jsonb containing encrypted EQL payload --! @return jsonb[] Array of JSONB elements with only deterministic fields @@ -311,7 +311,7 @@ AS $$ CASE WHEN val ? 'sv' THEN val->'sv' ELSE jsonb_build_array(val) END ) AS elem, LATERAL jsonb_each(elem) AS kv(key, value) - WHERE kv.key IN ('s', 'b3', 'hm', 'ocv', 'ocf') + WHERE kv.key IN ('s', 'b3', 'hm', 'ocv', 'ocf', 'opf', 'opv') GROUP BY elem ); $$; diff --git a/tests/sqlx/Cargo.lock b/tests/sqlx/Cargo.lock index fabc05728..8adf15f58 100644 --- a/tests/sqlx/Cargo.lock +++ b/tests/sqlx/Cargo.lock @@ -195,6 +195,7 @@ name = "eql_tests" version = "0.1.0" dependencies = [ "anyhow", + "hex", "serde", "serde_json", "sqlx", diff --git a/tests/sqlx/Cargo.toml b/tests/sqlx/Cargo.toml index 057e40c78..f2681e9c3 100644 --- a/tests/sqlx/Cargo.toml +++ b/tests/sqlx/Cargo.toml @@ -9,6 +9,7 @@ tokio = { version = "1", features = ["full"] } serde = { version = "1", features = ["derive"] } serde_json = "1" anyhow = "1" +hex = "0.4" [dev-dependencies] # None needed - tests live in this crate diff --git a/tests/sqlx/tests/ope_tests.rs b/tests/sqlx/tests/ope_tests.rs new file mode 100644 index 000000000..819e37593 --- /dev/null +++ b/tests/sqlx/tests/ope_tests.rs @@ -0,0 +1,471 @@ +//! OPE (CLWW Order-Preserving Encryption) tests +//! +//! Exercises the `ope_cllw_u64_65` and `ope_cllw_var_8` support wired into +//! `eql_v2_encrypted`. Unlike the ORE CLLW variants, OPE ciphertexts compare +//! under standard lexicographic `bytea` ordering — these tests verify that +//! property end-to-end through the SEM extraction, `compare_*` helpers, and +//! the generic `eql_v2.compare` dispatch. +//! +//! Fixture data for OPE is constructed inline as hand-crafted JSONB payloads +//! (there is no Rust-side fixture generator yet; the ORE fixture tables are +//! built from `ore_rs`, not `cllw_ore::encrypt_ope`). + +use anyhow::Result; +use eql_tests::QueryAssertion; +use sqlx::{PgPool, Row}; + +/// Build a 65-byte OPE fixed ciphertext from a single "signal" byte at index 8 +/// (the first plaintext body byte). All other bytes are zero. Larger signal → +/// larger ciphertext under lex compare. +fn opf_payload(signal: u8) -> String { + let mut bytes = vec![0u8; 65]; + bytes[8] = signal; + format!( + r#"{{"v":2,"i":{{"t":"t","c":"c"}},"opf":"{}"}}"#, + hex::encode(&bytes) + ) +} + +fn opv_payload(bytes: &[u8]) -> String { + format!( + r#"{{"v":2,"i":{{"t":"t","c":"c"}},"opv":"{}"}}"#, + hex::encode(bytes) + ) +} + +#[sqlx::test] +async fn opf_extracts_to_65_bytes(pool: PgPool) -> Result<()> { + let sql = format!( + "SELECT length((eql_v2.ope_cllw_u64_65('{}'::jsonb)).bytes)", + opf_payload(1) + ); + QueryAssertion::new(&pool, &sql).returns_int_value(65).await; + Ok(()) +} + +#[sqlx::test] +async fn has_opf_true_when_field_present(pool: PgPool) -> Result<()> { + let sql = format!( + "SELECT eql_v2.has_ope_cllw_u64_65('{}'::jsonb)", + opf_payload(1) + ); + QueryAssertion::new(&pool, &sql) + .returns_bool_value(true) + .await; + Ok(()) +} + +#[sqlx::test] +async fn has_opf_false_when_field_absent(pool: PgPool) -> Result<()> { + // Same shape but 'opf' replaced with 'ob' — should not trigger ope detection. + let sql = + r#"SELECT eql_v2.has_ope_cllw_u64_65('{"v":2,"i":{"t":"t","c":"c"},"ob":["00"]}'::jsonb)"#; + QueryAssertion::new(&pool, sql) + .returns_bool_value(false) + .await; + Ok(()) +} + +#[sqlx::test] +async fn compare_opf_three_way(pool: PgPool) -> Result<()> { + let a = opf_payload(1); + let b = opf_payload(2); + + let cmp = |l: &str, r: &str| { + format!( + "SELECT eql_v2.compare_ope_cllw_u64_65(eql_v2.to_encrypted('{}'::jsonb), eql_v2.to_encrypted('{}'::jsonb))", + l, r + ) + }; + + QueryAssertion::new(&pool, cmp(&a, &b)) + .returns_int_value(-1) + .await; + QueryAssertion::new(&pool, cmp(&b, &a)) + .returns_int_value(1) + .await; + QueryAssertion::new(&pool, cmp(&a, &a)) + .returns_int_value(0) + .await; + Ok(()) +} + +#[sqlx::test] +async fn generic_compare_dispatches_to_opf(pool: PgPool) -> Result<()> { + let a = opf_payload(1); + let b = opf_payload(2); + + let sql = format!( + "SELECT eql_v2.compare(eql_v2.to_encrypted('{}'::jsonb), eql_v2.to_encrypted('{}'::jsonb))", + a, b + ); + QueryAssertion::new(&pool, &sql).returns_int_value(-1).await; + Ok(()) +} + +#[sqlx::test] +async fn encrypted_lt_operator_uses_opf(pool: PgPool) -> Result<()> { + let a = opf_payload(1); + let b = opf_payload(2); + + let sql = format!( + "SELECT eql_v2.to_encrypted('{}'::jsonb) < eql_v2.to_encrypted('{}'::jsonb)", + a, b + ); + QueryAssertion::new(&pool, &sql) + .returns_bool_value(true) + .await; + Ok(()) +} + +#[sqlx::test] +async fn encrypted_gt_operator_uses_opf(pool: PgPool) -> Result<()> { + let a = opf_payload(1); + let b = opf_payload(2); + + let sql = format!( + "SELECT eql_v2.to_encrypted('{}'::jsonb) > eql_v2.to_encrypted('{}'::jsonb)", + b, a + ); + QueryAssertion::new(&pool, &sql) + .returns_bool_value(true) + .await; + Ok(()) +} + +#[sqlx::test] +async fn compare_opv_short_prefix_sorts_less(pool: PgPool) -> Result<()> { + // Shorter ciphertext that is a lex prefix of the longer one. + let short = opv_payload(&[0xaa, 0x11, 0x11, 0x11, 0x11]); + let long = opv_payload(&[0xaa, 0x11, 0x11, 0x11, 0x11, 0x00]); + + let sql = format!( + "SELECT eql_v2.compare_ope_cllw_var_8(eql_v2.to_encrypted('{}'::jsonb), eql_v2.to_encrypted('{}'::jsonb))", + short, long + ); + QueryAssertion::new(&pool, &sql).returns_int_value(-1).await; + Ok(()) +} + +#[sqlx::test] +async fn compare_opv_three_way_same_length(pool: PgPool) -> Result<()> { + let a = opv_payload(&[0xaa, 0x11, 0x11]); + let b = opv_payload(&[0xbb, 0x11, 0x11]); + + let cmp = |l: &str, r: &str| { + format!( + "SELECT eql_v2.compare_ope_cllw_var_8(eql_v2.to_encrypted('{}'::jsonb), eql_v2.to_encrypted('{}'::jsonb))", + l, r + ) + }; + + QueryAssertion::new(&pool, cmp(&a, &b)) + .returns_int_value(-1) + .await; + QueryAssertion::new(&pool, cmp(&b, &a)) + .returns_int_value(1) + .await; + QueryAssertion::new(&pool, cmp(&a, &a)) + .returns_int_value(0) + .await; + Ok(()) +} + +#[sqlx::test] +async fn generic_compare_dispatches_to_opv(pool: PgPool) -> Result<()> { + let a = opv_payload(&[0xaa, 0x11]); + let b = opv_payload(&[0xbb, 0x11]); + + let sql = format!( + "SELECT eql_v2.compare(eql_v2.to_encrypted('{}'::jsonb), eql_v2.to_encrypted('{}'::jsonb))", + a, b + ); + QueryAssertion::new(&pool, &sql).returns_int_value(-1).await; + Ok(()) +} + +#[sqlx::test] +async fn config_check_accepts_ope_index(pool: PgPool) -> Result<()> { + let sql = r#"SELECT eql_v2.config_check_indexes('{"v":1,"tables":{"t":{"c":{"cast_as":"int","indexes":{"ope":{}}}}}}'::jsonb)"#; + QueryAssertion::new(&pool, sql) + .returns_bool_value(true) + .await; + Ok(()) +} + +#[sqlx::test] +async fn order_by_ope_extracts_opf_bytes(pool: PgPool) -> Result<()> { + let payload = opf_payload(7); + let sql = format!( + "SELECT length(eql_v2.order_by_ope(eql_v2.to_encrypted('{}'::jsonb)))", + payload + ); + QueryAssertion::new(&pool, &sql).returns_int_value(65).await; + Ok(()) +} + +#[sqlx::test] +async fn order_by_ope_extracts_opv_bytes(pool: PgPool) -> Result<()> { + let payload = opv_payload(&[0xaa, 0x11, 0x22, 0x33]); + let sql = format!( + "SELECT length(eql_v2.order_by_ope(eql_v2.to_encrypted('{}'::jsonb)))", + payload + ); + QueryAssertion::new(&pool, &sql).returns_int_value(4).await; + Ok(()) +} + +#[sqlx::test] +async fn sort_compare_orders_opf_lexicographically(pool: PgPool) -> Result<()> { + let payloads = [opf_payload(3), opf_payload(1), opf_payload(2)]; + let sql = format!( + "SELECT id FROM eql_v2.sort_compare( + ARRAY[1::bigint, 2::bigint, 3::bigint], + ARRAY[ + eql_v2.to_encrypted('{}'::jsonb), + eql_v2.to_encrypted('{}'::jsonb), + eql_v2.to_encrypted('{}'::jsonb) + ]::eql_v2_encrypted[], + 'ASC' + )", + payloads[0], payloads[1], payloads[2] + ); + + let rows = sqlx::query(&sql).fetch_all(&pool).await?; + let ids: Vec = rows.iter().map(|r| r.try_get(0).unwrap()).collect(); + assert_eq!( + ids, + vec![2, 3, 1], + "opf ASC should be id=2 (1) < 3 (2) < 1 (3)" + ); + Ok(()) +} + +#[sqlx::test] +async fn sort_compare_uses_ope_fast_path(pool: PgPool) -> Result<()> { + let mut tx = pool.begin().await?; + + sqlx::query( + "CREATE TABLE encrypted_ope( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + e eql_v2_encrypted + )", + ) + .execute(&mut *tx) + .await?; + + for signal in [3u8, 1, 2] { + let sql = format!( + "INSERT INTO encrypted_ope(e) VALUES (eql_v2.to_encrypted('{}'::jsonb))", + opf_payload(signal) + ); + sqlx::query(&sql).execute(&mut *tx).await?; + } + + sqlx::query( + "SELECT pg_stat_reset_single_function_counters(p.oid) + FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'eql_v2' + AND p.proname IN ('ope_cllw_u64_65', 'ope_cllw_var_8', 'order_by')", + ) + .execute(&mut *tx) + .await?; + + let rows = sqlx::query( + "SELECT id FROM eql_v2.sort_compare( + (SELECT array_agg(id ORDER BY id) FROM encrypted_ope), + (SELECT array_agg(e ORDER BY id) FROM encrypted_ope), + 'ASC' + )", + ) + .fetch_all(&mut *tx) + .await?; + let ids: Vec = rows.iter().map(|r| r.try_get(0).unwrap()).collect(); + assert_eq!( + ids, + vec![2, 3, 1], + "OPE ASC should be id=2 (1) < 3 (2) < 1 (3)" + ); + + // sort_compare extracts the OPE key per subtype directly. Verify the opf + // extractor is invoked (the precise count varies because the encrypted + // overload delegates to the jsonb overload, but it must be called at all) + // and the var_8 extractor is never invoked because the homogeneity check + // gives up after the first row. + let opf_calls: i64 = sqlx::query_scalar( + "SELECT coalesce(sum(calls), 0)::bigint + FROM pg_stat_xact_user_functions + WHERE schemaname = 'eql_v2' AND funcname = 'ope_cllw_u64_65'", + ) + .fetch_one(&mut *tx) + .await?; + assert!( + opf_calls >= 3, + "sort_compare should extract opf key for every row (got {opf_calls} calls)" + ); + + let opv_calls: i64 = sqlx::query_scalar( + "SELECT coalesce(sum(calls), 0)::bigint + FROM pg_stat_xact_user_functions + WHERE schemaname = 'eql_v2' AND funcname = 'ope_cllw_var_8'", + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!( + opv_calls, 0, + "sort_compare on opf-only data must not invoke the opv extractor" + ); + + let ore_calls: i64 = sqlx::query_scalar( + "SELECT coalesce(sum(calls), 0)::bigint + FROM pg_stat_xact_user_functions + WHERE schemaname = 'eql_v2' AND funcname = 'order_by'", + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!( + ore_calls, 0, + "sort_compare on OPE-only data must not call the ORE order_by extractor" + ); + + tx.rollback().await?; + Ok(()) +} + +#[sqlx::test] +async fn sort_compare_mixed_ope_subtypes_falls_back_to_compare(pool: PgPool) -> Result<()> { + // Mixing `opf` and `opv` payloads in the same batch must not take the OPE + // bytea fast path: those ciphertexts are not comparable across subtypes. + // sort_compare should fall back to eql_v2.compare() (which itself rejects + // mixed subtypes and uses literal JSONB ordering) and still return all + // rows without error. + let mut tx = pool.begin().await?; + + sqlx::query( + "SELECT pg_stat_reset_single_function_counters(p.oid) + FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'eql_v2' + AND p.proname = '_compare_ope_key'", + ) + .execute(&mut *tx) + .await?; + + let opf = opf_payload(5); + let opv = opv_payload(&[0xaa, 0x11]); + + let sql = format!( + "SELECT id FROM eql_v2.sort_compare( + ARRAY[1::bigint, 2::bigint, 3::bigint], + ARRAY[ + eql_v2.to_encrypted('{}'::jsonb), + eql_v2.to_encrypted('{}'::jsonb), + eql_v2.to_encrypted('{}'::jsonb) + ]::eql_v2_encrypted[], + 'ASC' + )", + opf, opv, opf + ); + let rows = sqlx::query(&sql).fetch_all(&mut *tx).await?; + assert_eq!( + rows.len(), + 3, + "mixed OPE batch should still sort and return all rows" + ); + + // _compare_ope_key is only invoked by the sort path when strategy='ope'. + // A mixed-subtype batch must select the compare-fallback strategy, so this + // helper must never run. + let ope_compare_calls: i64 = sqlx::query_scalar( + "SELECT coalesce(sum(calls), 0)::bigint + FROM pg_stat_xact_user_functions + WHERE schemaname = 'eql_v2' AND funcname = '_compare_ope_key'", + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!( + ope_compare_calls, 0, + "mixed-subtype batches must not take the OPE bytea fast path" + ); + + tx.rollback().await?; + Ok(()) +} + +#[sqlx::test] +async fn order_by_compare_mixed_ope_subtypes_falls_back(pool: PgPool) -> Result<()> { + // Same homogeneity contract for the dynamic-SQL entrypoint: a query that + // returns mixed `opf` and `opv` rows must not lex-compare the bytea + // ciphertexts (different subtypes are not order-comparable). + let mut tx = pool.begin().await?; + + sqlx::query( + "CREATE TABLE encrypted_ope_mixed( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + e eql_v2_encrypted + )", + ) + .execute(&mut *tx) + .await?; + + let payloads = [opf_payload(5), opv_payload(&[0xaa, 0x11]), opf_payload(2)]; + for payload in &payloads { + let sql = format!( + "INSERT INTO encrypted_ope_mixed(e) VALUES (eql_v2.to_encrypted('{}'::jsonb))", + payload + ); + sqlx::query(&sql).execute(&mut *tx).await?; + } + + sqlx::query( + "SELECT pg_stat_reset_single_function_counters(p.oid) + FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'eql_v2' + AND p.proname = '_compare_ope_key'", + ) + .execute(&mut *tx) + .await?; + + let rows = sqlx::query( + "SELECT id FROM eql_v2.order_by_compare( + 'SELECT id, e FROM encrypted_ope_mixed', 'ASC' + )", + ) + .fetch_all(&mut *tx) + .await?; + assert_eq!( + rows.len(), + 3, + "order_by_compare should return every row even on mixed-subtype input" + ); + + let ope_compare_calls: i64 = sqlx::query_scalar( + "SELECT coalesce(sum(calls), 0)::bigint + FROM pg_stat_xact_user_functions + WHERE schemaname = 'eql_v2' AND funcname = '_compare_ope_key'", + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!( + ope_compare_calls, 0, + "order_by_compare must not select the OPE bytea fast path on mixed-subtype input" + ); + + tx.rollback().await?; + Ok(()) +} + +#[sqlx::test] +async fn config_check_rejects_unknown_index(pool: PgPool) -> Result<()> { + let sql = r#"SELECT eql_v2.config_check_indexes('{"v":1,"tables":{"t":{"c":{"cast_as":"int","indexes":{"bogus":{}}}}}}'::jsonb)"#; + // Should raise; use sqlx directly to assert the error message mentions `ope`. + let err = sqlx::query(sql) + .fetch_one(&pool) + .await + .expect_err("expected check_indexes to reject unknown index"); + let msg = err.to_string(); + assert!( + msg.contains("match, ore, ope, unique, ste_vec"), + "expected error to list valid indexes including 'ope'; got: {msg}" + ); + Ok(()) +}