diff --git a/CHANGELOG.md b/CHANGELOG.md index 398d0b41..134101a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +### Changed + +- **EQL v3 (searchable encryption)**: Proxy now targets EQL v3. Encrypted columns are declared with self-configuring, typed `jsonb` domains (for example `eql_v3_text_search`, `eql_v3_integer_ord`, `eql_v3_json_search`) that encode both the scalar type and the column's searchable capabilities in the column type itself, replacing EQL v2's opaque `eql_v2_encrypted` composite type and its separate `eql_v2_configuration` table. The bundled `cipherstash-client` is upgraded to 0.42.0 and EQL to 3.0.2. Existing v2-encrypted data and schemas must be migrated to v3. + +### Added + +- **Encrypted full-text match with `@@`**: The `@@` operator is now supported on encrypted text columns whose domain carries a match (bloom-filter) term, rewritten to the EQL v3 `eql_v3.match_term` form. + +### Fixed + +- **`LIKE`/`ILIKE` capability checking**: `LIKE` and `ILIKE` on an encrypted column are now gated by the column's token-match capability. Previously these predicates bypassed capability checking and were silently accepted on columns that do not support fuzzy match; they are now rejected with a capability error. + ## [2.2.4] - 2026-06-18 ### Fixed diff --git a/packages/cipherstash-proxy/src/postgresql/data/from_sql.rs b/packages/cipherstash-proxy/src/postgresql/data/from_sql.rs index f70e4962..1b3f757a 100644 --- a/packages/cipherstash-proxy/src/postgresql/data/from_sql.rs +++ b/packages/cipherstash-proxy/src/postgresql/data/from_sql.rs @@ -81,7 +81,20 @@ pub fn literal_from_sql( // #[cfg(not(feature = "bigdecimal"))] // Value::Number(s, _) => todo!("Number parsed type not implemented"), // #[cfg(feature = "bigdecimal")] - Value::Number(d, _) => Some(decimal_from_sql(d, col_type)?), + Value::Number(d, _) => { + // A JSON ordering operand (`col -> sel > 4`) is a scalar SteVec + // ordering term, encoded as a float like the stored JSON number leaf + // — NOT a JSON document (`decimal_from_sql` would make a `Json` + // plaintext, which `SteVecTerm` rejects). + if eql_term == EqlTermVariant::JsonOrd { + use bigdecimal::ToPrimitive; + Some(Plaintext::new( + d.to_f64().ok_or(MappingError::CouldNotParseParameter)?, + )) + } else { + Some(decimal_from_sql(d, col_type)?) + } + } Value::Placeholder(_) => { return Err(MappingError::Internal(String::from( @@ -93,6 +106,21 @@ pub fn literal_from_sql( Ok(pt) } +/// A JSON ordering operand (`EqlTerm::JsonOrd`) arriving as a jsonb param +/// carries a single scalar. Encode a number as a float (matching the stored +/// JSON number leaf's SteVec `op` encoding) and a string as text — the only +/// scalar shapes `SteVecTerm` accepts (a full JSON value is rejected). +fn json_ord_scalar_plaintext(value: serde_json::Value) -> Result { + match value { + serde_json::Value::Number(n) => n + .as_f64() + .ok_or(MappingError::CouldNotParseParameter) + .map(Plaintext::new), + serde_json::Value::String(s) => Ok(Plaintext::new(s)), + _ => Err(MappingError::CouldNotParseParameter), + } +} + /// Converts a string value to a Plaintext value based on input postgres type and target column type. /// Usually, the input type is a string and the target type is parsed appropriately (for example, a string to a number). /// However, other input postgres types are possible. @@ -181,6 +209,21 @@ fn text_from_sql( .map_err(|_| MappingError::CouldNotParseParameter) .map(Plaintext::new) } + // A JSON ordering operand reaches here in two textual shapes: a bare SQL + // literal (`col -> sel > '4'` / `> 'C'` → `4` / `C`) and a text-format + // jsonb param (`4` / `"C"`, the value's jsonb rendering). Parse as JSON to + // recover the scalar type and its content: a number encodes as a float + // (matching the stored leaf's `for_json_value` SteVec `op`), a JSON string + // as its unquoted text. A bare word (`C`) is not valid JSON, so fall back + // to raw text. Mirrors `json_ord_scalar_plaintext` on the binary param path. + (EqlTermVariant::JsonOrd, ColumnType::Json) => { + match serde_json::from_str::(val) { + Ok(value @ (serde_json::Value::Number(_) | serde_json::Value::String(_))) => { + json_ord_scalar_plaintext(value) + } + _ => Ok(Plaintext::new(val)), + } + } (EqlTermVariant::Tokenized, ColumnType::Text) => Ok(Plaintext::new(val)), (eql_term, col_type) => Err(MappingError::UnsupportedParameterType { @@ -273,6 +316,13 @@ fn binary_from_sql( Plaintext::new(val) }) } + // A JSON ordering operand (`col -> sel > $2`) arrives as a jsonb scalar; + // encode it as the scalar shape SteVecTerm accepts (number → float, + // string → text). + (EqlTermVariant::JsonOrd, ColumnType::Json, _) => { + parse_bytes_from_sql::(bytes, pg_type) + .and_then(json_ord_scalar_plaintext) + } // Python psycopg sends JSON/B as BYTEA ( EqlTermVariant::Full | EqlTermVariant::Partial, diff --git a/packages/cipherstash-proxy/src/postgresql/frontend.rs b/packages/cipherstash-proxy/src/postgresql/frontend.rs index 3fb24af1..b3e7822d 100644 --- a/packages/cipherstash-proxy/src/postgresql/frontend.rs +++ b/packages/cipherstash-proxy/src/postgresql/frontend.rs @@ -27,7 +27,7 @@ use crate::prometheus::{ STATEMENTS_PASSTHROUGH_TOTAL, STATEMENTS_UNMAPPABLE_TOTAL, }; use crate::proxy::EncryptionService; -use crate::EqlOutput; +use crate::{EqlOutput, EqlQueryPayload}; use bytes::BytesMut; use cipherstash_client::encryption::Plaintext; use eql_mapper::{self, EqlMapperError, EqlTerm, TypeCheckedStatement}; @@ -649,6 +649,14 @@ where let mut encrypted_expressions = vec![]; for encrypted in encrypted_literals { let e = match encrypted { + // A JSON selector (RHS of `->`/`->>`, or the `jsonb_path_query` + // path) is a bare tokenized-selector hash used directly as `text` + // by the eql_v3 functions (`eql_v3."->"(json, text)`). Bind the raw + // token: JSON-serializing it (below) would re-quote the bare string + // (`""`), so it would never match the stored per-entry `s`. + Some(EqlOutput::Query(EqlQueryPayload::Selector(s))) => { + Some(Value::SingleQuotedString(s.clone())) + } Some(en) => Some(to_json_literal_value(&en)?), None => None, }; diff --git a/packages/cipherstash-proxy/src/postgresql/messages/bind.rs b/packages/cipherstash-proxy/src/postgresql/messages/bind.rs index a4701974..54b53821 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/bind.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/bind.rs @@ -5,7 +5,7 @@ use crate::postgresql::context::column::Column; use crate::postgresql::data::bind_param_from_sql; use crate::postgresql::format_code::FormatCode; use crate::postgresql::protocol::BytesMutReadString; -use crate::EqlOutput; +use crate::{EqlOutput, EqlQueryPayload}; use crate::{SIZE_I16, SIZE_I32}; use bytes::{Buf, BufMut, BytesMut}; use cipherstash_client::encryption::Plaintext; @@ -84,12 +84,24 @@ impl Bind { pub fn rewrite(&mut self, encrypted: Vec>) -> Result<(), Error> { for (idx, ct) in encrypted.iter().enumerate() { if let Some(ct) = ct { - let json = serde_json::to_value(ct)?; - - // convert json to bytes - let bytes = json.to_string().into_bytes(); - - self.param_values[idx].rewrite(&bytes); + match ct { + // A JSON selector (`->`/`->>`/`jsonb_path_query`) is a bare + // tokenized-selector hash bound directly as `text`, NOT jsonb. + // Use the raw token: JSON-serializing it re-quotes the bare + // string (`""`), which never matches the stored per-entry + // `s`. It must also skip the jsonb version header a binary + // rewrite would prepend — the binary wire form of `text` is + // just its raw bytes, and a leading `0x01` corrupts the + // selector so `->` matches nothing. + EqlOutput::Query(EqlQueryPayload::Selector(s)) => { + self.param_values[idx].rewrite_text(s.clone().into_bytes()); + } + // convert json to bytes + _ => { + let bytes = serde_json::to_value(ct)?.to_string().into_bytes(); + self.param_values[idx].rewrite(&bytes); + } + } } } Ok(()) @@ -156,6 +168,20 @@ impl BindParam { self.dirty = true; } + /// Rewrite this param as a bare `text` value, without the jsonb version + /// header [`rewrite`] prepends for binary jsonb payloads. + /// + /// Used for the tokenized selector of a JSON field access (`->`/`->>`/ + /// `jsonb_path_query`), which is bound as `text`, not jsonb. The binary + /// wire form of `text` is simply its raw UTF-8 bytes, so no header is + /// added in either format — a stray `0x01` would corrupt the selector and + /// stop `->` from matching any stored entry. + pub fn rewrite_text(&mut self, bytes: Vec) { + self.bytes.clear(); + self.bytes.extend_from_slice(&bytes); + self.dirty = true; + } + pub fn requires_rewrite(&self) -> bool { self.dirty } diff --git a/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs b/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs index 3fc3f725..d4d3bd51 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs @@ -35,7 +35,7 @@ impl DataRow { .filter(|_| data_column.is_not_null()) .and_then(|config| { data_column - .try_into() + .to_eql_ciphertext() .inspect_err(|err| match err { Error::Encrypt(EncryptError::ColumnIsNull) => { debug!(target: DECRYPT, msg ="ColumnIsNull", ?config); @@ -179,24 +179,36 @@ impl TryFrom for BytesMut { } } -impl TryFrom<&mut DataColumn> for EqlCiphertext { - type Error = Error; - - fn try_from(col: &mut DataColumn) -> Result { - // EQL v3 column types (`eql_v3_text_eq`, `eql_v3_integer_ord`, …) are - // DOMAINS over `jsonb`, so a value arrives with jsonb's representation. - // - // EQL v2's `eql_v2_encrypted` was a composite type, which is why this - // used to strip a `("…")` wrapper in text and a 12-byte rowtype header - // in binary. Neither exists any more — a domain is wire-identical to its - // base type. - // - // text — the JSON object itself, no wrapper and no doubled quotes - // binary — a 1-byte jsonb version header followed by the JSON text - // - // The two are told apart by the leading byte: the version header is - // `0x01`, and JSON text for an EQL payload always starts with `{`. - let Some(bytes) = &col.bytes else { +impl DataColumn { + /// Parse this column's bytes into an [`EqlCiphertext`]. + /// + /// EQL v3 column types (`eql_v3_text_eq`, `eql_v3_integer_ord`, …) are + /// DOMAINS over `jsonb`, so a value arrives with jsonb's representation. + /// + /// EQL v2's `eql_v2_encrypted` was a composite type, which is why this + /// used to strip a `("…")` wrapper in text and a 12-byte rowtype header + /// in binary. Neither exists any more — a domain is wire-identical to its + /// base type. + /// + /// text — the JSON object itself, no wrapper and no doubled quotes + /// binary — a 1-byte jsonb version header followed by the JSON text + /// + /// The two are told apart by the leading byte: the version header is + /// `0x01`, and JSON text for an EQL payload always starts with `{`. + /// + /// The JSON is usually a self-describing payload — a scalar `{v,i,c,…}` or + /// a SteVec document `{v,k:"sv",i,h,sv}` — and deserialises directly. The + /// exception is a JSON field access (`eql_v3."->"(…)` / + /// `eql_v3.jsonb_path_query(…)`), whose result is a single + /// `eql_v3_json_entry` (`{v,i,h,s,c,op}`) — one SteVec entry merged with + /// its document envelope. That has a `c`, so it would masquerade as a + /// scalar `Encrypted` payload, but its `c` is an *entry* ciphertext that + /// only decrypts with the entry's selector-derived nonce. So when the + /// payload is a bare entry (see [`is_json_entry`]) it is reshaped into a + /// one-entry SteVec document (see [`json_entry_into_ste_vec_document`]) and + /// the ordinary SteVec decrypt path recovers the field value. + fn to_eql_ciphertext(&self) -> Result { + let Some(bytes) = &self.bytes else { return Err(EncryptError::ColumnCouldNotBeParsed.into()); }; @@ -206,11 +218,58 @@ impl TryFrom<&mut DataColumn> for EqlCiphertext { None => return Err(EncryptError::ColumnCouldNotBeParsed.into()), }; - serde_json::from_slice(json).map_err(|err| { - debug!(target: DECRYPT, error = err.to_string()); - err.into() - }) + let mut value: serde_json::Value = + serde_json::from_slice(json).map_err(log_deserialise_error)?; + + if is_json_entry(&value) { + json_entry_into_ste_vec_document(&mut value)?; + } + + serde_json::from_value(value).map_err(log_deserialise_error) + } +} + +/// Whether a decoded EQL payload is a bare `eql_v3_json_entry` — the result of +/// a JSON field access (`eql_v3."->"(…)` / `eql_v3.jsonb_path_query(…)`). +/// +/// A root-level selector `s` is the tell: a scalar `Encrypted` payload has no +/// selector at all, and a SteVec document carries selectors only inside its +/// `sv[]` entries, never at the root. +fn is_json_entry(value: &serde_json::Value) -> bool { + value.get("s").is_some() +} + +/// Reshape a single `eql_v3_json_entry` into a one-entry SteVec document. +/// +/// The entry is `{v,i,h,s,c,op}`: document-envelope fields (`v`, `i`, `h`) +/// alongside one SteVec entry's fields (`s`, `c`, the optional array marker +/// `a`, and the optional ordering term `op`). Move the entry fields under +/// `sv:[{…}]` and tag the object as a SteVec (`k:"sv"`), yielding +/// `{v,k:"sv",i,h,sv:[{s,c,a?,op?}]}` — the shape an [`EqlCiphertext`] SteVec +/// document deserialises from and the decrypt path knows how to open. +fn json_entry_into_ste_vec_document(value: &mut serde_json::Value) -> Result<(), Error> { + use serde_json::Value; + + let object = value + .as_object_mut() + .ok_or(EncryptError::ColumnCouldNotBeParsed)?; + + let mut entry = serde_json::Map::new(); + for key in ["s", "c", "a", "op"] { + if let Some(field) = object.remove(key) { + entry.insert(key.to_owned(), field); + } } + + object.insert("k".to_owned(), Value::String("sv".to_owned())); + object.insert("sv".to_owned(), Value::Array(vec![Value::Object(entry)])); + + Ok(()) +} + +fn log_deserialise_error(err: serde_json::Error) -> Error { + debug!(target: DECRYPT, error = err.to_string()); + err.into() } #[cfg(test)] diff --git a/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs b/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs index ce4cd105..61f83766 100644 --- a/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs +++ b/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs @@ -272,6 +272,17 @@ impl EncryptionService for ZeroKms { EqlOperation::Query(&index.index_type, QueryOp::SteVecSelector) }) .unwrap_or(EqlOperation::Store), + + // JsonOrd is the scalar value operand of a JSON field ordering + // comparison (`col -> sel < value`): a SteVec ordering term + // (`{v,i,op}`) compared via `eql_v3.ord_term`. + EqlTermVariant::JsonOrd => col + .config + .indexes + .iter() + .find(|i| matches!(i.index_type, IndexType::SteVec { .. })) + .map(|index| EqlOperation::Query(&index.index_type, QueryOp::SteVecTerm)) + .unwrap_or(EqlOperation::Store), }; let prepared = PreparedPlaintext::new( diff --git a/packages/eql-mapper-macros/src/parse_type_decl.rs b/packages/eql-mapper-macros/src/parse_type_decl.rs index 29786bb0..37b300ee 100644 --- a/packages/eql-mapper-macros/src/parse_type_decl.rs +++ b/packages/eql-mapper-macros/src/parse_type_decl.rs @@ -169,7 +169,7 @@ impl Parse for EqlTrait { Err(syn::Error::new( input.span(), format!( - "Expected Eq, Ord, TokenMatch or JsonLike while parsing EqlTrait; got: {}", + "Expected Eq, Ord, TokenMatch, JsonLike or Contain while parsing EqlTrait; got: {}", input.cursor().token_stream() ), )) @@ -477,6 +477,11 @@ impl Parse for SqltkBinOp { if input.peek(token::At) { let _: token::At = input.parse()?; + // `@@` (fuzzy match) or `@>` (containment). + if input.peek(token::At) { + let _: token::At = input.parse()?; + return Ok(Self(quote!(::sqltk::parser::ast::BinaryOperator::AtAt))); + } let _: token::Gt = input.parse()?; return Ok(Self(quote!(::sqltk::parser::ast::BinaryOperator::AtArrow))); } @@ -552,7 +557,7 @@ impl Parse for SqltkBinOp { Err(syn::Error::new( input.span(), - "Expected an operator corresponding to one of the EQL traits Eq, Ord, TokenMatch or JsonLike".to_string(), + "Expected an operator corresponding to one of the EQL traits Eq, Ord, TokenMatch, JsonLike or Contain".to_string(), )) } } diff --git a/packages/eql-mapper/CONTEXT.md b/packages/eql-mapper/CONTEXT.md index 8e2eb0ad..820695d5 100644 --- a/packages/eql-mapper/CONTEXT.md +++ b/packages/eql-mapper/CONTEXT.md @@ -4,11 +4,13 @@ Parses SQL, infers a type for every node, and rewrites statements that touch enc columns into their EQL v3 equivalents. It knows nothing about the PostgreSQL wire protocol, ZeroKMS, or ciphertext — it reasons about types and rewrites syntax. -> **Migration in flight.** The code still emits the EQL v2 surface -> (`eql_v2_encrypted` casts, `eql_v2.*` calls); the vocabulary below is the **v3 -> target** the type-checker extension is being designed against. See -> [`docs/adr/`](./docs/adr/) for the load-bearing decisions and -> `docs/plans/2026-07-20-eql-v3-type-checker-handoff.md` for the impact maps. +> **v3 migration.** The mapper now emits the EQL **v3** surface — v3 domain casts +> (`::public.eql_v3_*` for stored values, `::eql_v3.query_*` for operands) and the +> `eql_v3.*` functional-index form (term-extraction functions, `eql_v3.jsonb_*`, +> `eql_v3."->"`, `match_term`). No `eql_v2.*` names remain in its output. +> End-to-end validation against a live database with EQL v3 installed is still +> pending. See [`docs/adr/`](./docs/adr/) for the load-bearing decisions and +> `docs/plans/2026-07-20-eql-v3-type-checker-handoff.md` for the original impact maps. ## Language diff --git a/packages/eql-mapper/docs/adr/0003-v3-rewrite-pipeline.md b/packages/eql-mapper/docs/adr/0003-v3-rewrite-pipeline.md new file mode 100644 index 00000000..8377bdbf --- /dev/null +++ b/packages/eql-mapper/docs/adr/0003-v3-rewrite-pipeline.md @@ -0,0 +1,118 @@ +--- +status: accepted +--- + +# The EQL v3 rewrite pipeline: term functions, cast targets, and operand context + +ADR-0001 fixes *what* the mapper emits (the `eql_v3.*_term()` functional-index form); +this ADR fixes *how* the transformation pipeline produces it, because the v2→v3 change +is structural, not a find-and-replace of names. + +## Two contexts, two cast targets + +An encrypted value appears in a statement in one of two roles, and they cast differently: + +- **Stored value** — an INSERT `VALUES` item or an UPDATE `SET` right-hand side. It casts + to the column domain, `''::jsonb::public.eql_v3__`, and is **not** + wrapped in a term function. +- **Query operand** — the right-hand side of a predicate (`col = $1`, `col > 'x'`). It casts + to the query twin, `''::jsonb::eql_v3.query__`, and the whole + predicate is rewritten through term functions (below). + +The v2 pipeline did not need this distinction: every encrypted value cast to the single +opaque `eql_v2_encrypted`, and the opaque type carried its own operators. Under v3 the two +roles produce different SQL, so the pipeline must know a value's role. + +**Decision:** the *type checker* records each encrypted literal/param's role while it walks +the AST (it already visits the INSERT/UPDATE targets and the predicate operands during +inference), and the transformation rules read that role. Re-deriving role from AST context +inside the transform is the fallback if threading it through inference proves awkward, but +the inference pass is where the context is already known. + +## Operator rewriting + +A comparison with an encrypted operand is rewritten by wrapping **both** operands in the +term function the operator's capability selects: + +``` +col operand → eql_v3.(col) eql_v3.(operand) +``` + +The term function is chosen by `(operator, the terms the column's domain stores)` — verified +against the `eql_v3.eq`/`lt`/… bodies in the installed `cipherstash-encrypt.sql`: + +| Operator | Term function | +|---|---| +| `=` `<>` | `eq_term` **if the domain stores `hm`**, else `ord_term` (`op`), else `ord_term_ore` (`ob`) | +| `<` `<=` `>` `>=` | `ord_term` (domain stores `op`) / `ord_term_ore` (domain stores `ob`) | +| `@@` | `match_term` (domain stores `bf`) | + +Two verified subtleties: + +- **`=` is not always `eq_term`.** A term-extraction function exists exactly where its term + does: `eq_term` only on domains storing `hm` (`_eq`, `_search*`, and — the text exception — + `text_ord*`). On an ord-only scalar such as `integer_ord` there is no `hm`, so `eql_v3.eq` + itself is `ord_term(a) = ord_term(b)`. The mapper mirrors this: `=`/`<>` fall back to the + ordering term when the domain has no `hm`. +- **`ord_term` vs `ord_term_ore`** is not derivable from the coarse `Ord` trait — it comes + from the domain identity (ADR-0002): a `*_ord` / `*_ord_ope` domain stores `op` ⇒ + `ord_term`; a `*_ord_ore` / `*_search_ore` domain stores `ob` ⇒ `ord_term_ore`. + +Which terms a domain stores is recoverable from its typname (with the text `hm` exception); +the mapper derives them there rather than re-consulting `eql-bindings`. + +### Operand cast target — the query twin + +The right-hand operand casts to the **query twin** `eql_v3.query__` (schema +`eql_v3`), e.g. a `public.eql_v3_integer_ord` column's operand casts to +`eql_v3.query_integer_ord`. Verified: the twins exist for every scalar domain, carry the +**term-only** payload (`{v,i,}`, no stored ciphertext `c`) that a query value actually +is, and have their own `ord_term`/`eq_term` overloads. So: + +``` +salary > 'x' → eql_v3.ord_term(salary) > eql_v3.ord_term(''::jsonb::eql_v3.query_numeric_ord) +``` + +The column operand needs no cast (it is already the domain type); only the query operand is +cast, and to the twin — **not** the column domain, whose CHECK requires the ciphertext a +query value does not carry. (`eql_v3.eq(domain, jsonb)` casting to the column domain is a +separate convenience overload, not what the mapper emits.) + +**Selecting the term function *is* the capability check.** A column whose domain provides no +term function for the operator (e.g. `ORDER BY` / `>` on an `_eq` column, any operator on a +storage-only `boolean`/`json` column) has no valid rewrite target — that absence *is* the +capability error the type checker raises. This keeps one mechanism, not a separate bounds +check bolted on. + +## JSON is different (see ADR-0002 amendment) + +Encrypted JSON columns (`eql_v3_json_search`) keep `->`/`->>` (JsonLike) **and** `@>`/`<@` +(Contain) — verified against the installed `cipherstash-encrypt.sql`. `Contain` is therefore +retained as a JSON-only capability and its rewrite targets the SteVec containment surface +(`eql_v3.to_ste_vec_query` + `@>`), **not** deleted. `@>`/`<@` on *scalar* encrypted columns +still have no term/rewrite and so raise. + +## Rule inventory under v3 + +- `CastLiteralsAsEncrypted` / `CastParamsAsEncrypted` — retained; the cast target moves from + `eql_v2_encrypted` to the role-appropriate v3 domain (column domain vs query twin). These + gain access to each node's domain identity (via `node_types`) to name the target. +- **`RewriteEqlComparisonOps`** (new) — wraps scalar comparison operands in term functions + and performs the capability check. Models its node handling on `RewriteContainmentOps` + (`mem::replace` against a throwaway `Value::Null` to preserve operand `NodeKey` identity + for the cast rules that run after). +- `RewriteContainmentOps` — **retargeted, not retired**: `@>`/`<@` on JSON columns rewrite to + the v3 SteVec containment surface; on scalar columns they raise. +- `RewriteStandardSqlFnsOnEqlTypes` — retargeted from `eql_v2.{min,max,jsonb_*}` to the v3 + surface. Whether some of these become native overload resolution (and the rule shrinks) is + gated on v3 shipping operator/function overloads bound to the domains; verify per function. +- `PreserveEffectiveAliases`, `FailOnPlaceholderChange` — unchanged. + +## Consequences + +- The pipeline gains a stored-vs-operand notion it did not have; this is the load-bearing new + concept, and getting it wrong casts an operand to a column domain (or vice versa). +- Bound checking goes live here: the term-function selection raising on an absent capability + is the user-visible capability error, so the ADR-0001 "let the database do its job" stance + is refined — the mapper raises when there is no valid rewrite, rather than emitting SQL that + would fail at the database. diff --git a/packages/eql-mapper/src/inference/infer_type_impls/expr.rs b/packages/eql-mapper/src/inference/infer_type_impls/expr.rs index d2658b99..ea965625 100644 --- a/packages/eql-mapper/src/inference/infer_type_impls/expr.rs +++ b/packages/eql-mapper/src/inference/infer_type_impls/expr.rs @@ -1,10 +1,13 @@ use crate::{ get_sql_binop_rule, - inference::{unifier::Type, InferType, TypeError}, + inference::{ + unifier::{EqlTerm, EqlValue, TokenType, Type, Value}, + InferType, TypeError, + }, EqlTrait, IdentCase, TypeInferencer, }; use eql_mapper_macros::trace_infer; -use sqltk::parser::ast::{AccessExpr, Array, Expr, Ident, Subscript}; +use sqltk::parser::ast::{AccessExpr, Array, BinaryOperator, Expr, Ident, Subscript}; #[trace_infer] impl<'ast> InferType<'ast, Expr> for TypeInferencer<'ast> { @@ -109,26 +112,82 @@ impl<'ast> InferType<'ast, Expr> for TypeInferencer<'ast> { } Expr::BinaryOp { left, op, right } => { - get_sql_binop_rule(op).apply_constraints(self, left, right, expr_val)?; + // Encrypted JSON field ORDERING (`col -> sel < value`, `>`, `<=`, + // `>=`): the value operand is a scalar SteVec ordering term + // (`{v,i,op}`, `QueryOp::SteVecTerm`), not a JSON document, and the + // comparison runs through `eql_v3.ord_term` on both sides. Type the + // operand as `EqlTerm::JsonOrd` so it encrypts and casts as an + // ordering operand — the generic `T Ord T` rule would instead unify + // it to the whole JSON type (→ a full document, which cannot be an + // ordering operand). Equality (`=`) is intentionally NOT handled here + // (exact JSON equality is value-selector containment, not ordering). + let handled = if matches!( + op, + BinaryOperator::Lt + | BinaryOperator::LtEq + | BinaryOperator::Gt + | BinaryOperator::GtEq + ) { + match (self.eql_json_value(left), self.eql_json_value(right)) { + (Some(json), None) => { + self.unify_node_with_type( + &**right, + Type::Value(Value::Eql(EqlTerm::JsonOrd(json))), + )?; + self.unify_node_with_type(expr_val, Type::native())?; + true + } + (None, Some(json)) => { + self.unify_node_with_type( + &**left, + Type::Value(Value::Eql(EqlTerm::JsonOrd(json))), + )?; + self.unify_node_with_type(expr_val, Type::native())?; + true + } + _ => false, + } + } else { + false + }; + + if !handled { + get_sql_binop_rule(op).apply_constraints(self, left, right, expr_val)?; + } } - //customer_name LIKE 'A%'; + // `customer_name LIKE 'A%'`. Route LIKE/ILIKE through the `~~`/`~~*` + // operator rules so an encrypted LHS must implement `TokenMatch` (the + // pattern becomes its `Tokenized` type, the result is `Native`). + // Previously this only unified the result with `Native`, so LIKE on an + // encrypted column bypassed capability checking entirely. Expr::Like { - negated: _, + negated, expr, pattern, escape_char: _, any: false, + } => { + let op = if *negated { + BinaryOperator::PGNotLikeMatch + } else { + BinaryOperator::PGLikeMatch + }; + get_sql_binop_rule(&op).apply_constraints(self, expr, pattern, expr_val)?; } - | Expr::ILike { - negated: _, + Expr::ILike { + negated, expr, pattern, escape_char: _, any: false, } => { - self.unify_node_with_type(expr_val, Type::native())?; - self.unify_nodes(&**expr, &**pattern)?; + let op = if *negated { + BinaryOperator::PGNotILikeMatch + } else { + BinaryOperator::PGILikeMatch + }; + get_sql_binop_rule(&op).apply_constraints(self, expr, pattern, expr_val)?; } Expr::Like { any: true, .. } | Expr::ILike { any: true, .. } => { @@ -444,3 +503,20 @@ impl<'ast> InferType<'ast, Expr> for TypeInferencer<'ast> { Ok(()) } } + +impl<'ast> TypeInferencer<'ast> { + /// If `expr` resolves to an encrypted JSON (`JsonLike`) value — the field + /// access side of a JSON ordering comparison (`col -> sel`, `col ->> sel`, or + /// `jsonb_path_query_first(col, sel)`) — return its [`EqlValue`]. Returns + /// `None` for scalar EQL columns (which compare via the ordinary term rewrite) + /// and for non-EQL types. + fn eql_json_value(&self, expr: &'ast Expr) -> Option { + match &*self.get_node_type(expr) { + Type::Value(Value::Eql(eql_term)) => { + let eql_value = eql_term.eql_value(); + (eql_value.domain_identity().token == TokenType::Json).then(|| eql_value.clone()) + } + _ => None, + } + } +} diff --git a/packages/eql-mapper/src/inference/sql_types/sql_decls.rs b/packages/eql-mapper/src/inference/sql_types/sql_decls.rs index abedfec6..4f5b5896 100644 --- a/packages/eql-mapper/src/inference/sql_types/sql_decls.rs +++ b/packages/eql-mapper/src/inference/sql_types/sql_decls.rs @@ -28,6 +28,7 @@ static SQL_BINARY_OPERATORS: LazyLock> = (T !~~ ::Tokenized) -> Native where T: TokenMatch; // NOT LIKE (T ~~* ::Tokenized) -> Native where T: TokenMatch; // ILIKE (T !~~* ::Tokenized) -> Native where T: TokenMatch; // NOT ILIKE + (T @@ ::Tokenized) -> Native where T: TokenMatch; // @@ fuzzy match }; ops.into_iter() .map(|binary_op_spec| (binary_op_spec.op.clone(), binary_op_spec)) @@ -65,17 +66,20 @@ static SQL_FUNCTION_TYPES: LazyLock, FunctionDecl> pg_catalog.jsonb_array_length(T) -> Native where T: JsonLike; pg_catalog.jsonb_array_elements(T) -> SetOf where T: JsonLike; pg_catalog.jsonb_array_elements_text(T) -> SetOf where T: JsonLike; - eql_v2.min(T) -> T where T: Ord; - eql_v2.max(T) -> T where T: Ord; - eql_v2.jsonb_path_query(T, ::Path) -> T where T: JsonLike; - eql_v2.jsonb_path_query_first(T, ::Path) -> T where T: JsonLike; - eql_v2.jsonb_path_exists(T, ::Path) -> Native where T: JsonLike; - eql_v2.jsonb_array_length(T) -> Native where T: JsonLike; - eql_v2.jsonb_array_elements(T) -> SetOf where T: JsonLike; - eql_v2.jsonb_array_elements_text(T) -> SetOf where T: JsonLike; - eql_v2.jsonb_array(T) -> Native where T: Contain; - eql_v2.jsonb_contains(T, T) -> Native where T: Contain; - eql_v2.jsonb_contained_by(T, T) -> Native where T: Contain; + eql_v3.min(T) -> T where T: Ord; + eql_v3.max(T) -> T where T: Ord; + eql_v3.jsonb_path_query(T, ::Path) -> T where T: JsonLike; + eql_v3.jsonb_path_query_first(T, ::Path) -> T where T: JsonLike; + eql_v3.jsonb_path_exists(T, ::Path) -> Native where T: JsonLike; + eql_v3.jsonb_array_length(T) -> Native where T: JsonLike; + eql_v3.jsonb_array_elements(T) -> SetOf where T: JsonLike; + eql_v3.jsonb_array_elements_text(T) -> SetOf where T: JsonLike; + // JSON containment (`@>`/`<@`) — retained in v3, scoped to JSON + // columns (ADR-0002 amendment). `@>`/`<@` on scalar encrypted columns + // still raise. + eql_v3.jsonb_array(T) -> Native where T: Contain; + eql_v3.jsonb_contains(T, T) -> Native where T: Contain; + eql_v3.jsonb_contained_by(T, T) -> Native where T: Contain; }; HashMap::from_iter( @@ -102,6 +106,23 @@ pub(crate) fn get_sql_function(fn_name: &ObjectName) -> SqlFunction { .unwrap_or(SqlFunction::Fallback) } +/// The `eql_v3.` counterpart a `pg_catalog` function is rewritten to on EQL +/// types, or `None` if none is declared. `count`, for example, works on encrypted +/// values natively (Postgres counts the domain directly), so it has no counterpart +/// and is left untouched. +pub(crate) fn get_eql_v3_function_name(fn_name: &ObjectName) -> Option { + let bare = fn_name.0.last()?; + let eql_v3_name = ObjectName(vec![ + ObjectNamePart::Identifier(Ident::new("eql_v3")), + bare.clone(), + ]); + if SQL_FUNCTION_TYPES.contains_key(&IdentCase(eql_v3_name.clone())) { + Some(eql_v3_name) + } else { + None + } +} + #[cfg(test)] mod tests { use crate::inference::sql_types::sql_decls::{SQL_BINARY_OPERATORS, SQL_FUNCTION_TYPES}; diff --git a/packages/eql-mapper/src/inference/unifier/eql_traits.rs b/packages/eql-mapper/src/inference/unifier/eql_traits.rs index 10ad71a9..8ef5e4ce 100644 --- a/packages/eql-mapper/src/inference/unifier/eql_traits.rs +++ b/packages/eql-mapper/src/inference/unifier/eql_traits.rs @@ -326,6 +326,7 @@ impl EqlTerm { EqlTerm::JsonAccessor(_) => EqlTraits::none(), EqlTerm::JsonPath(_) => EqlTraits::none(), EqlTerm::Tokenized(_) => EqlTraits::none(), + EqlTerm::JsonOrd(_) => EqlTraits::none(), } } } diff --git a/packages/eql-mapper/src/inference/unifier/types.rs b/packages/eql-mapper/src/inference/unifier/types.rs index b55a784a..26eb1b0f 100644 --- a/packages/eql-mapper/src/inference/unifier/types.rs +++ b/packages/eql-mapper/src/inference/unifier/types.rs @@ -195,6 +195,13 @@ pub enum EqlTerm { /// [`EqlValue`] that implements the EQL trait `TokenMatch`. #[display("EQL:Tokenized({})", _0)] Tokenized(EqlValue), + + /// A scalar ordering operand for an encrypted JSON field comparison — the + /// non-JSON side of `col -> sel value` where `` is `<`/`<=`/`>`/`>=`. + /// Encrypted as a SteVec ordering term (`{v,i,op}`, `QueryOp::SteVecTerm`) and + /// compared via `eql_v3.ord_term`, so it is NOT a whole JSON document. + #[display("EQL:JsonOrd({})", _0)] + JsonOrd(EqlValue), } #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Display, Hash)] @@ -209,16 +216,25 @@ pub enum EqlTermVariant { JsonPath, #[display("EQL:Tokenized")] Tokenized, + #[display("EQL:JsonOrd")] + JsonOrd, } impl EqlTerm { pub fn table_column(&self) -> &TableColumn { + self.eql_value().table_column() + } + + /// The [`EqlValue`] every `EqlTerm` variant wraps — its `TableColumn`, inert + /// domain identity, and capabilities. + pub fn eql_value(&self) -> &EqlValue { match self { EqlTerm::Full(eql_value) | EqlTerm::Partial(eql_value, _) | EqlTerm::JsonAccessor(eql_value) | EqlTerm::JsonPath(eql_value) - | EqlTerm::Tokenized(eql_value) => eql_value.table_column(), + | EqlTerm::Tokenized(eql_value) + | EqlTerm::JsonOrd(eql_value) => eql_value, } } @@ -229,6 +245,7 @@ impl EqlTerm { EqlTerm::JsonAccessor(_) => EqlTermVariant::JsonAccessor, EqlTerm::JsonPath(_) => EqlTermVariant::JsonPath, EqlTerm::Tokenized(_) => EqlTermVariant::Tokenized, + EqlTerm::JsonOrd(_) => EqlTermVariant::JsonOrd, } } } @@ -355,6 +372,103 @@ impl DomainIdentity { }) } + /// The capability suffix of the domain typname (`eql_v3__`), + /// e.g. `ord_ore` for `eql_v3_text_ord_ore`, or `""` for a storage-only + /// domain like `eql_v3_integer`. + fn suffix(&self) -> &str { + let prefix_len = "eql_v3_".len() + self.token.as_domain_str().len(); + self.domain + .value + .get(prefix_len..) + .map(|rest| rest.strip_prefix('_').unwrap_or(rest)) + .unwrap_or("") + } + + // Which SEM terms the domain stores, derived from its typname. The catalog is + // the authority (ADR-0002) and these mirror the term → domain mapping the + // schema loader inverts. `text` is the exception: `text_ord*` stores `hm` + // alongside its ordering term, because lexicographic ORE/OPE over text is not + // equality-lossless. + + /// The domain stores the `hm` (HMAC equality) term ⇒ `eq_term` is available. + pub fn stores_hm(&self) -> bool { + matches!(self.suffix(), "eq" | "search" | "search_ore") + || (self.token == TokenType::Text + && matches!(self.suffix(), "ord" | "ord_ope" | "ord_ore")) + } + + /// The domain stores the `op` (CLLW-OPE) term ⇒ `ord_term` is available. + pub fn stores_op(&self) -> bool { + matches!(self.suffix(), "ord" | "ord_ope" | "search") + } + + /// The domain stores the `ob` (block-ORE) term ⇒ `ord_term_ore` is available. + pub fn stores_ob(&self) -> bool { + matches!(self.suffix(), "ord_ore" | "search_ore") + } + + /// The domain stores the `bf` (bloom-filter) term ⇒ `match_term` is available. + pub fn stores_bf(&self) -> bool { + matches!(self.suffix(), "match" | "search" | "search_ore") + } + + /// The `eql_v3` term-extraction function for equality (`=`, `<>`), or `None` + /// if the domain supports no equality. `eq_term` when the domain stores `hm`; + /// otherwise equality falls back to the ordering term (an ord-only scalar such + /// as `integer_ord` compares via `ord_term`, mirroring `eql_v3.eq`). + pub fn eq_term_fn(&self) -> Option<&'static str> { + if self.stores_hm() { + Some("eq_term") + } else { + self.ord_term_fn() + } + } + + /// The `eql_v3` term-extraction function for ordering (`<`, `<=`, `>`, `>=`, + /// `MIN`/`MAX`), or `None` if the domain is not orderable. `ord_term` for `op` + /// domains, `ord_term_ore` for `ob` (block-ORE) domains. + pub fn ord_term_fn(&self) -> Option<&'static str> { + if self.stores_op() { + Some("ord_term") + } else if self.stores_ob() { + Some("ord_term_ore") + } else { + None + } + } + + /// The `eql_v3` term-extraction function for fuzzy match (`@@`), or `None` if + /// the domain has no bloom filter. + pub fn match_term_fn(&self) -> Option<&'static str> { + if self.stores_bf() { + Some("match_term") + } else { + None + } + } + + /// The query-operand twin of this column domain — `(schema, typname)`, e.g. + /// `("eql_v3", "query_integer_ord")` for `public.eql_v3_integer_ord`. A query + /// operand casts to the twin (which carries the term-only payload), never to + /// the column domain (whose CHECK requires the stored ciphertext). + pub fn query_twin(&self) -> (&'static str, String) { + // Every JSON domain (`json`, `json_search`, `json_entry`) shares a single + // query-operand type in the catalog — `eql_v3.query_json` — because a + // jsonb query operand is a SteVec needle whose shape does not vary by the + // column's searchable capability. The generic `query_` rule below is + // correct only for the scalar families (e.g. `query_integer_ord`); applied + // to JSON it would emit a non-existent `eql_v3.query_json_search`. + if self.token == TokenType::Json { + return ("eql_v3", "query_json".to_string()); + } + let bare = self + .domain + .value + .strip_prefix("eql_v3_") + .unwrap_or(&self.domain.value); + ("eql_v3", format!("query_{bare}")) + } + /// A canonical identity for a `(token, capabilities)` pair. This is a /// **test/fixture convenience** for constructing identities where no live /// schema loader supplies the real domain name — production identities always @@ -804,3 +918,129 @@ impl From for Type { Type::Value(Value::Array(array)) } } + +#[cfg(test)] +mod domain_identity_tests { + use super::DomainIdentity; + + fn di(domain: &str) -> DomainIdentity { + DomainIdentity::from_domain_name(domain) + .unwrap_or_else(|| panic!("{domain} is not a v3 domain name")) + } + + #[test] + fn suffix_is_parsed_across_tokens_and_variants() { + assert_eq!(di("eql_v3_integer").suffix(), ""); + assert_eq!(di("eql_v3_integer_eq").suffix(), "eq"); + assert_eq!(di("eql_v3_integer_ord").suffix(), "ord"); + assert_eq!(di("eql_v3_integer_ord_ope").suffix(), "ord_ope"); + assert_eq!(di("eql_v3_integer_ord_ore").suffix(), "ord_ore"); + assert_eq!(di("eql_v3_text_search_ore").suffix(), "search_ore"); + assert_eq!(di("eql_v3_bigint_ord_ore").suffix(), "ord_ore"); + } + + #[test] + fn double_token_does_not_swallow_the_capability_suffix() { + // `double` is the one token whose plain-English name could be mistaken + // for a two-word `double precision`. The catalog spells the domain + // `eql_v3_double_ord` (see tests/sql/schema.sql), so `as_domain_str()` + // ("double", 6 chars) must line the prefix up exactly on the `_ord` + // boundary. Pins the invariant that `suffix()`/`stores_*` documents as a + // comment: a hypothetical `eql_v3_double_precision_ord` would parse the + // suffix as "precision_ord" and silently report the column as + // non-orderable. + assert_eq!(di("eql_v3_double").suffix(), ""); + assert_eq!(di("eql_v3_double_ord").suffix(), "ord"); + assert_eq!(di("eql_v3_double_ord_ore").suffix(), "ord_ore"); + assert_eq!(di("eql_v3_double_ord").ord_term_fn(), Some("ord_term")); + assert_eq!( + di("eql_v3_double_ord_ore").ord_term_fn(), + Some("ord_term_ore") + ); + } + + #[test] + fn eq_term_uses_eq_term_only_when_hm_is_stored() { + // _eq stores hm. + assert_eq!(di("eql_v3_integer_eq").eq_term_fn(), Some("eq_term")); + // ord-only scalar has no hm -> equality falls back to ord_term + // (mirrors eql_v3.eq(integer_ord, ...) = ord_term(a) = ord_term(b)). + assert_eq!(di("eql_v3_integer_ord").eq_term_fn(), Some("ord_term")); + assert_eq!( + di("eql_v3_integer_ord_ore").eq_term_fn(), + Some("ord_term_ore") + ); + // text is the exception: text_ord* stores hm, so eq_term is available. + assert_eq!(di("eql_v3_text_ord").eq_term_fn(), Some("eq_term")); + assert_eq!(di("eql_v3_text_ord_ore").eq_term_fn(), Some("eq_term")); + // storage-only and match-only have no equality. + assert_eq!(di("eql_v3_integer").eq_term_fn(), None); + assert_eq!(di("eql_v3_text_match").eq_term_fn(), None); + } + + #[test] + fn ord_term_picks_ope_vs_ore_from_the_domain() { + assert_eq!(di("eql_v3_integer_ord").ord_term_fn(), Some("ord_term")); + assert_eq!(di("eql_v3_integer_ord_ope").ord_term_fn(), Some("ord_term")); + assert_eq!( + di("eql_v3_integer_ord_ore").ord_term_fn(), + Some("ord_term_ore") + ); + assert_eq!(di("eql_v3_text_search").ord_term_fn(), Some("ord_term")); + assert_eq!( + di("eql_v3_text_search_ore").ord_term_fn(), + Some("ord_term_ore") + ); + // not orderable + assert_eq!(di("eql_v3_integer_eq").ord_term_fn(), None); + assert_eq!(di("eql_v3_text_match").ord_term_fn(), None); + assert_eq!(di("eql_v3_integer").ord_term_fn(), None); + } + + #[test] + fn match_term_needs_a_bloom_filter() { + assert_eq!(di("eql_v3_text_match").match_term_fn(), Some("match_term")); + assert_eq!(di("eql_v3_text_search").match_term_fn(), Some("match_term")); + assert_eq!( + di("eql_v3_text_search_ore").match_term_fn(), + Some("match_term") + ); + assert_eq!(di("eql_v3_text_ord").match_term_fn(), None); + assert_eq!(di("eql_v3_integer_eq").match_term_fn(), None); + } + + #[test] + fn storage_only_domain_supports_no_operations() { + let d = di("eql_v3_integer"); + assert_eq!(d.eq_term_fn(), None); + assert_eq!(d.ord_term_fn(), None); + assert_eq!(d.match_term_fn(), None); + } + + #[test] + fn query_twin_prefixes_the_bare_domain() { + assert_eq!( + di("eql_v3_integer_ord").query_twin(), + ("eql_v3", "query_integer_ord".to_string()) + ); + assert_eq!( + di("eql_v3_text_search_ore").query_twin(), + ("eql_v3", "query_text_search_ore".to_string()) + ); + } + + #[test] + fn json_domains_all_share_the_query_json_twin() { + // The catalog defines a single jsonb query operand type, eql_v3.query_json, + // for every JSON column domain — the generic query_ rule would emit a + // non-existent eql_v3.query_json_search / eql_v3.query_json_entry. + assert_eq!( + di("eql_v3_json").query_twin(), + ("eql_v3", "query_json".to_string()) + ); + assert_eq!( + di("eql_v3_json_search").query_twin(), + ("eql_v3", "query_json".to_string()) + ); + } +} diff --git a/packages/eql-mapper/src/inference/unifier/unify_types.rs b/packages/eql-mapper/src/inference/unifier/unify_types.rs index 44af5732..57b9a0ae 100644 --- a/packages/eql-mapper/src/inference/unifier/unify_types.rs +++ b/packages/eql-mapper/src/inference/unifier/unify_types.rs @@ -113,6 +113,10 @@ impl UnifyTypes for Unifier<'_> { Ok(EqlTerm::Tokenized(lhs.clone()).into()) } + (EqlTerm::JsonOrd(lhs), EqlTerm::JsonOrd(rhs)) if lhs == rhs => { + Ok(EqlTerm::JsonOrd(lhs.clone()).into()) + } + (_, _) => Err(TypeError::Conflict(format!( "cannot unify EQL terms {lhs} and {rhs}" ))), diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index 73df2541..e6030edf 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -120,6 +120,286 @@ mod test { } } + #[test] + fn like_on_token_match_column_type_checks() { + let schema = resolver(schema! { + tables: { + users: { + id, + email (EQL: TokenMatch), + } + } + }); + + let statement = parse("SELECT id FROM users WHERE email LIKE 'a%'"); + assert!( + type_check(schema, &statement).is_ok(), + "LIKE on a TokenMatch column should type check" + ); + } + + #[test] + fn like_rewrites_to_match_term() { + let schema = resolver(schema! { + tables: { + users: { + id, + email (EQL: TokenMatch), + } + } + }); + + let statement = parse("SELECT id FROM users WHERE email LIKE 'a%'"); + + let typed = match type_check(schema, &statement) { + Ok(typed) => typed, + Err(err) => panic!("type check failed: {err:#?}"), + }; + + match typed.transform(HashMap::from_iter([( + typed.literals[0].1.as_node_key(), + ast::Value::SingleQuotedString("ENCRYPTED".into()), + )])) { + Ok(transformed) => assert_eq!( + transformed.to_string(), + "SELECT id FROM users WHERE eql_v3.match_term(email) @> eql_v3.match_term('ENCRYPTED'::JSONB::eql_v3.query_text_match)" + ), + Err(err) => panic!("transformation failed: {err}"), + }; + } + + #[test] + fn ord_ore_column_rewrites_to_ord_term_ore() { + // The explicit EQL("") form pins a block-ORE ordering domain, so + // the rewrite must select ord_term_ore (not ord_term) and the query twin + // must be query_integer_ord_ore. + let schema = resolver(schema! { + tables: { + events: { + id, + seq (EQL("eql_v3_integer_ord_ore"): Ord), + } + } + }); + + let statement = parse("SELECT id FROM events WHERE seq > 5"); + + let typed = match type_check(schema, &statement) { + Ok(typed) => typed, + Err(err) => panic!("type check failed: {err:#?}"), + }; + + match typed.transform(HashMap::from_iter([( + typed.literals[0].1.as_node_key(), + ast::Value::SingleQuotedString("ENCRYPTED".into()), + )])) { + Ok(transformed) => assert_eq!( + transformed.to_string(), + "SELECT id FROM events WHERE eql_v3.ord_term_ore(seq) > eql_v3.ord_term_ore('ENCRYPTED'::JSONB::eql_v3.query_integer_ord_ore)" + ), + Err(err) => panic!("transformation failed: {err}"), + }; + } + + #[test] + fn at_at_rewrites_to_match_term() { + let schema = resolver(schema! { + tables: { + users: { + id, + email (EQL: TokenMatch), + } + } + }); + + let statement = parse("SELECT id FROM users WHERE email @@ 'a'"); + + let typed = match type_check(schema, &statement) { + Ok(typed) => typed, + Err(err) => panic!("type check failed: {err:#?}"), + }; + + match typed.transform(HashMap::from_iter([( + typed.literals[0].1.as_node_key(), + ast::Value::SingleQuotedString("ENCRYPTED".into()), + )])) { + Ok(transformed) => assert_eq!( + transformed.to_string(), + "SELECT id FROM users WHERE eql_v3.match_term(email) @> eql_v3.match_term('ENCRYPTED'::JSONB::eql_v3.query_text_match)" + ), + Err(err) => panic!("transformation failed: {err}"), + }; + } + + #[test] + fn like_on_non_match_encrypted_column_is_rejected() { + // Regression: LIKE used to unify to Native and bypass capability checking. + // An encrypted column that only implements Eq must not accept LIKE. + let schema = resolver(schema! { + tables: { + users: { + id, + email (EQL: Eq), + } + } + }); + + let statement = parse("SELECT id FROM users WHERE email LIKE 'a%'"); + assert!( + type_check(schema, &statement).is_err(), + "LIKE on a non-TokenMatch encrypted column should be a capability error" + ); + } + + #[test] + fn ilike_rewrites_to_match_term() { + // ILIKE takes the same match arm as LIKE (RewriteEqlMatchOps handles both), + // so it must rewrite to the identical match_term form. + let schema = resolver(schema! { + tables: { + users: { + id, + email (EQL: TokenMatch), + } + } + }); + + let statement = parse("SELECT id FROM users WHERE email ILIKE 'a%'"); + + let typed = match type_check(schema, &statement) { + Ok(typed) => typed, + Err(err) => panic!("type check failed: {err:#?}"), + }; + + match typed.transform(HashMap::from_iter([( + typed.literals[0].1.as_node_key(), + ast::Value::SingleQuotedString("ENCRYPTED".into()), + )])) { + Ok(transformed) => assert_eq!( + transformed.to_string(), + "SELECT id FROM users WHERE eql_v3.match_term(email) @> eql_v3.match_term('ENCRYPTED'::JSONB::eql_v3.query_text_match)" + ), + Err(err) => panic!("transformation failed: {err}"), + }; + } + + #[test] + fn not_like_rewrites_to_negated_match_term() { + // The `negated` arm wraps the containment in `NOT (...)` — otherwise + // untested (only the positive LIKE/ILIKE/@@ forms had rewrite coverage). + let schema = resolver(schema! { + tables: { + users: { + id, + email (EQL: TokenMatch), + } + } + }); + + let statement = parse("SELECT id FROM users WHERE email NOT LIKE 'a%'"); + + let typed = match type_check(schema, &statement) { + Ok(typed) => typed, + Err(err) => panic!("type check failed: {err:#?}"), + }; + + match typed.transform(HashMap::from_iter([( + typed.literals[0].1.as_node_key(), + ast::Value::SingleQuotedString("ENCRYPTED".into()), + )])) { + Ok(transformed) => assert_eq!( + transformed.to_string(), + "SELECT id FROM users WHERE NOT (eql_v3.match_term(email) @> eql_v3.match_term('ENCRYPTED'::JSONB::eql_v3.query_text_match))" + ), + Err(err) => panic!("transformation failed: {err}"), + }; + } + + #[test] + fn not_ilike_rewrites_to_negated_match_term() { + // NOT ILIKE takes the same negated match arm as NOT LIKE. + let schema = resolver(schema! { + tables: { + users: { + id, + email (EQL: TokenMatch), + } + } + }); + + let statement = parse("SELECT id FROM users WHERE email NOT ILIKE 'a%'"); + + let typed = match type_check(schema, &statement) { + Ok(typed) => typed, + Err(err) => panic!("type check failed: {err:#?}"), + }; + + match typed.transform(HashMap::from_iter([( + typed.literals[0].1.as_node_key(), + ast::Value::SingleQuotedString("ENCRYPTED".into()), + )])) { + Ok(transformed) => assert_eq!( + transformed.to_string(), + "SELECT id FROM users WHERE NOT (eql_v3.match_term(email) @> eql_v3.match_term('ENCRYPTED'::JSONB::eql_v3.query_text_match))" + ), + Err(err) => panic!("transformation failed: {err}"), + }; + } + + #[test] + fn native_like_still_type_checks() { + // Regression: routing LIKE/ILIKE through the TokenMatch-bounded rule must + // not regress plain LIKE on a native (non-encrypted) column — Native + // satisfies all bounds, so `WHERE native_col LIKE 'x'` still type checks. + let schema = resolver(schema! { + tables: { + users: { + id, + email, + } + } + }); + + let statement = parse("SELECT id FROM users WHERE email LIKE 'a%'"); + assert!( + type_check(schema, &statement).is_ok(), + "LIKE on a native column should still type check" + ); + } + + #[test] + fn update_set_casts_stored_value() { + // ADR-0003's second stored-value context: an UPDATE SET on an encrypted + // column casts the assigned literal to the column domain, exactly like + // INSERT. Only INSERT had cast-target rewrite coverage before this. + let schema = resolver(schema! { + tables: { + employees: { + id, + salary (EQL), + } + } + }); + + let statement = parse("UPDATE employees SET salary = 20000 WHERE id = 123"); + + let typed = match type_check(schema, &statement) { + Ok(typed) => typed, + Err(err) => panic!("type check failed: {err:#?}"), + }; + + match typed.transform(HashMap::from_iter([( + typed.literals[0].1.as_node_key(), + ast::Value::SingleQuotedString("ENCRYPTED".into()), + )])) { + Ok(transformed) => assert_eq!( + transformed.to_string(), + "UPDATE employees SET salary = 'ENCRYPTED'::JSONB::public.eql_v3_text WHERE id = 123" + ), + Err(err) => panic!("transformation failed: {err}"), + }; + } + #[test] fn insert_with_value() { // init_tracing(); @@ -1130,7 +1410,7 @@ mod test { )])) { Ok(transformed_statement) => assert_eq!( transformed_statement.to_string(), - "SELECT * FROM employees WHERE salary > 'ENCRYPTED'::JSONB::eql_v2_encrypted" + "SELECT * FROM employees WHERE eql_v3.ord_term(salary) > eql_v3.ord_term('ENCRYPTED'::JSONB::eql_v3.query_text_ord)" ), Err(err) => panic!("statement transformation failed: {err}"), }; @@ -1180,7 +1460,7 @@ mod test { )])) { Ok(transformed_statement) => assert_eq!( transformed_statement.to_string(), - "INSERT INTO employees (salary) VALUES ('ENCRYPTED'::JSONB::eql_v2_encrypted)" + "INSERT INTO employees (salary) VALUES ('ENCRYPTED'::JSONB::public.eql_v3_text)" ), Err(err) => panic!("statement transformation failed: {err}"), }; @@ -1460,7 +1740,7 @@ mod test { match typed.transform(HashMap::new()) { Ok(statement) => assert_eq!( statement.to_string(), - "SELECT eql_v2.min(salary), eql_v2.max(salary), department FROM employees GROUP BY department".to_string() + "SELECT eql_v3.min(salary), eql_v3.max(salary), department FROM employees GROUP BY department".to_string() ), Err(err) => panic!("transformation failed: {err}"), } @@ -1476,7 +1756,7 @@ mod test { tables: { employees: { id, - eql_col (EQL), + eql_col (EQL: Eq), native_col, } } @@ -1493,7 +1773,7 @@ mod test { Ok(statement) => { assert_eq!( statement.to_string(), - "SELECT * FROM employees WHERE eql_col = $1::JSONB::eql_v2_encrypted AND native_col = $2" + "SELECT * FROM employees WHERE eql_v3.eq_term(eql_col) = eql_v3.eq_term($1::JSONB::eql_v3.query_text_eq) AND native_col = $2" ); } Err(err) => panic!("transformation failed: {err}"), @@ -1538,8 +1818,8 @@ mod test { assert_eq!( statement.to_string(), "SELECT \ - eql_v2.jsonb_path_exists(eql_col, ''::JSONB::eql_v2_encrypted), \ - eql_v2.jsonb_path_query(eql_col, ''::JSONB::eql_v2_encrypted), \ + eql_v3.jsonb_path_exists(eql_col, ''), \ + eql_v3.jsonb_path_query(eql_col, ''), \ jsonb_path_query(native_col, '$.not-secret') \ FROM employees" ); @@ -1656,7 +1936,9 @@ mod test { value: ast::Value::SingleQuotedString(s), span: _, }) => { - format!("''::JSONB::eql_v2_encrypted") + // A jsonb_path_query selector is emitted as bare encrypted-selector + // text (eql_v3.jsonb_path_query(json, text)), not a jsonb cast. + format!("''") } _ => panic!("unsupported expr type in test util"), }) @@ -1677,7 +1959,7 @@ mod test { match type_check(schema, &statement) { Ok(typed) => match typed.transform(encrypted_literals) { Ok(statement) => { - let rewritten_fn_name = format!("eql_v2.{fn_name}"); + let rewritten_fn_name = format!("eql_v3.{fn_name}"); assert_eq!( statement.to_string(), format!( @@ -1714,10 +1996,13 @@ mod test { )) { Ok(statement) => { let expected = match op { - "@>" => "SELECT id, eql_v2.jsonb_contains(notes, ''::JSONB::eql_v2_encrypted) AS meds FROM patients".to_string(), - "<@" => "SELECT id, eql_v2.jsonb_contained_by(notes, ''::JSONB::eql_v2_encrypted) AS meds FROM patients".to_string(), - // Other operators are not transformed - _ => format!("SELECT id, notes {op} ''::JSONB::eql_v2_encrypted AS meds FROM patients"), + "@>" => "SELECT id, eql_v3.jsonb_contains(notes, ''::JSONB::public.eql_v3_text_search) AS meds FROM patients".to_string(), + "<@" => "SELECT id, eql_v3.jsonb_contained_by(notes, ''::JSONB::public.eql_v3_text_search) AS meds FROM patients".to_string(), + // -> / ->> field access: functionalised to eql_v3."->"/"->>", + // with the field selector passed as encrypted text. + "->" => "SELECT id, eql_v3.\"->\"(notes, '') AS meds FROM patients".to_string(), + "->>" => "SELECT id, eql_v3.\"->>\"(notes, '') AS meds FROM patients".to_string(), + _ => format!("SELECT id, notes {op} ''::JSONB::public.eql_v3_text_search AS meds FROM patients"), }; assert_eq!(statement.to_string(), expected) } @@ -1740,12 +2025,12 @@ mod test { }); let statement = parse( - "SELECT id FROM patients WHERE eql_v2.jsonb_array(notes) @> eql_v2.jsonb_array(notes)", + "SELECT id FROM patients WHERE eql_v3.jsonb_array(notes) @> eql_v3.jsonb_array(notes)", ); match type_check(schema, &statement) { Ok(_) => (), - Err(err) => panic!("type check failed for eql_v2.jsonb_array: {err}"), + Err(err) => panic!("type check failed for eql_v3.jsonb_array: {err}"), } } @@ -1760,11 +2045,11 @@ mod test { } }); - let statement = parse("SELECT id FROM patients WHERE eql_v2.jsonb_contains(notes, notes)"); + let statement = parse("SELECT id FROM patients WHERE eql_v3.jsonb_contains(notes, notes)"); match type_check(schema, &statement) { Ok(_) => (), - Err(err) => panic!("type check failed for eql_v2.jsonb_contains: {err}"), + Err(err) => panic!("type check failed for eql_v3.jsonb_contains: {err}"), } } @@ -1780,16 +2065,16 @@ mod test { }); let statement = - parse("SELECT id FROM patients WHERE eql_v2.jsonb_contained_by(notes, notes)"); + parse("SELECT id FROM patients WHERE eql_v3.jsonb_contained_by(notes, notes)"); match type_check(schema, &statement) { Ok(_) => (), - Err(err) => panic!("type check failed for eql_v2.jsonb_contained_by: {err}"), + Err(err) => panic!("type check failed for eql_v3.jsonb_contained_by: {err}"), } } #[test] - fn eql_v2_jsonb_contains_with_param() { + fn eql_v3_jsonb_contains_with_param() { let schema = resolver(schema! { tables: { patients: { @@ -1799,7 +2084,7 @@ mod test { } }); - let statement = parse("SELECT id FROM patients WHERE eql_v2.jsonb_contains(notes, $1)"); + let statement = parse("SELECT id FROM patients WHERE eql_v3.jsonb_contains(notes, $1)"); let typed = type_check(schema, &statement) .map_err(|err| err.to_string()) @@ -1812,7 +2097,7 @@ mod test { match typed.transform(HashMap::new()) { Ok(statement) => assert_eq!( statement.to_string(), - "SELECT id FROM patients WHERE eql_v2.jsonb_contains(notes, $1::JSONB::eql_v2_encrypted)" + "SELECT id FROM patients WHERE eql_v3.jsonb_contains(notes, $1::JSONB::public.eql_v3_text_search)" ), Err(err) => panic!("transformation failed: {err}"), } @@ -1840,15 +2125,15 @@ mod test { // Verify function call exists assert!( - sql.contains("eql_v2.jsonb_contains"), - "Expected @> to be transformed to eql_v2.jsonb_contains, got: {sql}" + sql.contains("eql_v3.jsonb_contains"), + "Expected @> to be transformed to eql_v3.jsonb_contains, got: {sql}" ); // CRITICAL: Verify the parameter is cast to enable GIN index usage - // The cast ::JSONB::eql_v2_encrypted is required for GIN indexes to work + // The cast ::JSONB::public.eql_v3_text_search is required for GIN indexes to work assert!( - sql.contains("::JSONB::eql_v2_encrypted") || sql.contains("::jsonb::eql_v2_encrypted"), - "Expected parameter to be cast as ::JSONB::eql_v2_encrypted for GIN index support, got: {sql}" + sql.contains("::JSONB::public.eql_v3_text_search") || sql.contains("::jsonb::public.eql_v3_text_search"), + "Expected parameter to be cast as ::JSONB::public.eql_v3_text_search for GIN index support, got: {sql}" ); } @@ -1874,14 +2159,14 @@ mod test { // Verify function call exists assert!( - sql.contains("eql_v2.jsonb_contained_by"), - "Expected <@ to be transformed to eql_v2.jsonb_contained_by, got: {sql}" + sql.contains("eql_v3.jsonb_contained_by"), + "Expected <@ to be transformed to eql_v3.jsonb_contained_by, got: {sql}" ); // CRITICAL: Verify the parameter is cast to enable GIN index usage assert!( - sql.contains("::JSONB::eql_v2_encrypted") || sql.contains("::jsonb::eql_v2_encrypted"), - "Expected parameter to be cast as ::JSONB::eql_v2_encrypted for GIN index support, got: {sql}" + sql.contains("::JSONB::public.eql_v3_text_search") || sql.contains("::jsonb::public.eql_v3_text_search"), + "Expected parameter to be cast as ::JSONB::public.eql_v3_text_search for GIN index support, got: {sql}" ); } @@ -1914,8 +2199,8 @@ mod test { // Verify function call exists inside the EXPLAIN assert!( - sql.contains("eql_v2.jsonb_contains"), - "Expected @> inside EXPLAIN to be transformed to eql_v2.jsonb_contains, got: {sql}" + sql.contains("eql_v3.jsonb_contains"), + "Expected @> inside EXPLAIN to be transformed to eql_v3.jsonb_contains, got: {sql}" ); } @@ -2076,7 +2361,7 @@ mod test { } }); - let statement = parse("SELECT eql_v2.jsonb_path_query(notes, $1) as notes FROM patients"); + let statement = parse("SELECT eql_v3.jsonb_path_query(notes, $1) as notes FROM patients"); let typed = type_check(schema, &statement) .map_err(|err| err.to_string()) diff --git a/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs b/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs index af2ae3d6..ea51b581 100644 --- a/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs +++ b/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs @@ -1,21 +1,30 @@ -use std::{any::type_name, collections::HashMap}; +use std::{any::type_name, collections::HashMap, sync::Arc}; use sqltk::parser::ast::{Expr, Value, ValueWithSpan}; +use sqltk::parser::tokenizer::Span; use sqltk::{NodeKey, NodePath, Visitable}; +use crate::unifier::{EqlTerm, Type, Value as UnifierValue}; use crate::EqlMapperError; -use super::helpers::cast_as_encrypted; +use super::helpers::{cast_to_v3_domain, json_ord_cast_target, v3_cast_target}; use super::TransformationRule; #[derive(Debug)] pub struct CastLiteralsAsEncrypted<'ast> { encrypted_literals: HashMap, Value>, + node_types: Arc, Type>>, } impl<'ast> CastLiteralsAsEncrypted<'ast> { - pub fn new(encrypted_literals: HashMap, Value>) -> Self { - Self { encrypted_literals } + pub fn new( + encrypted_literals: HashMap, Value>, + node_types: Arc, Type>>, + ) -> Self { + Self { + encrypted_literals, + node_types, + } } } @@ -26,11 +35,46 @@ impl<'ast> TransformationRule<'ast> for CastLiteralsAsEncrypted<'ast> { target_node: &mut N, ) -> Result { if self.would_edit(node_path, target_node) { - if let Some((Expr::Value(ValueWithSpan { value, .. }),)) = node_path.last_1_as::() + if let Some((original @ Expr::Value(ValueWithSpan { value, .. }),)) = + node_path.last_1_as::() { if let Some(replacement) = self.encrypted_literals.remove(&NodeKey::new(value)) { + // The literal's domain identity determines the cast target; its + // role (query operand vs stored value) determines which v3 + // domain (query twin vs column domain). + let Some(Type::Value(UnifierValue::Eql(eql_term))) = + self.node_types.get(&NodeKey::new(original)) + else { + return Err(EqlMapperError::Transform(format!( + "{}: encrypted literal has no EQL type", + type_name::() + ))); + }; + let target_node = target_node.downcast_mut::().unwrap(); - *target_node = cast_as_encrypted(replacement); + *target_node = match eql_term { + // A JSON selector — the RHS of `->`/`->>`, or the path + // argument of `jsonb_path_query` — is passed to the EQL v3 + // function as the encrypted-selector *text* + // (`eql_v3."->"(json, text)`, `jsonb_path_query(json, + // text)`), not a jsonb-domain payload. The encrypt pipeline + // produces that selector text (a SteVec `QueryOp::SteVecSelector` + // token), so it is emitted verbatim, uncast. + EqlTerm::JsonAccessor(_) | EqlTerm::JsonPath(_) => { + Expr::Value(ValueWithSpan { + value: replacement, + span: Span::empty(), + }) + } + _ => { + let (schema, domain) = + json_ord_cast_target(eql_term).unwrap_or_else(|| { + let identity = eql_term.eql_value().domain_identity().clone(); + v3_cast_target(node_path, &identity) + }); + cast_to_v3_domain(replacement, &schema, &domain) + } + }; return Ok(true); } } diff --git a/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs b/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs index b55b6e4d..f7259f58 100644 --- a/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs +++ b/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs @@ -1,6 +1,6 @@ -use super::helpers::cast_as_encrypted; +use super::helpers::{cast_to_v3_domain, json_ord_cast_target, v3_cast_target}; use super::TransformationRule; -use crate::unifier::Type; +use crate::unifier::{EqlTerm, Type, Value as UnifierValue}; use crate::EqlMapperError; use sqltk::parser::ast::{Expr, Value, ValueWithSpan}; use sqltk::parser::tokenizer::Span; @@ -26,6 +26,34 @@ impl<'ast> TransformationRule<'ast> for CastParamsAsEncrypted<'ast> { target_node: &mut N, ) -> Result { if self.would_edit(node_path, target_node) { + // Resolve the operand's v3 cast target from its domain identity and + // its role (query operand → query twin; stored value → column domain). + let Some((original,)) = node_path.last_1_as::() else { + return Ok(false); + }; + let Some(Type::Value(UnifierValue::Eql(eql_term))) = + self.node_types.get(&NodeKey::new(original)) + else { + return Ok(false); + }; + + // A JSON selector operand (RHS of `->`/`->>`, or the path argument of + // `jsonb_path_query`) is passed to the EQL v3 function as the encrypted + // selector *text* — `eql_v3."->"(json, text)`, `jsonb_path_query(json, + // text)` — not a jsonb query-domain payload. The proxy encrypts these + // params as SteVec selectors, so the placeholder is left bare (its type + // is inferred from the function signature) rather than cast to the + // `eql_v3.query_*` twin. Mirrors the JsonAccessor arm of + // `CastLiteralsAsEncrypted`. + if matches!(eql_term, EqlTerm::JsonAccessor(_) | EqlTerm::JsonPath(_)) { + return Ok(false); + } + + let (schema, domain) = json_ord_cast_target(eql_term).unwrap_or_else(|| { + let identity = eql_term.eql_value().domain_identity().clone(); + v3_cast_target(node_path, &identity) + }); + if let Some( expr @ Expr::Value(ValueWithSpan { value: Value::Placeholder(_), @@ -48,7 +76,7 @@ impl<'ast> TransformationRule<'ast> for CastParamsAsEncrypted<'ast> { unreachable!("the Expr is known to be Expr::Value(ValueWithSpan::{{ value: Value::Placeholder(_), .. }})") }; - *expr = cast_as_encrypted(value); + *expr = cast_to_v3_domain(value, &schema, &domain); return Ok(true); } } diff --git a/packages/eql-mapper/src/transformation_rules/helpers.rs b/packages/eql-mapper/src/transformation_rules/helpers.rs index dc55b3d2..5846f5cf 100644 --- a/packages/eql-mapper/src/transformation_rules/helpers.rs +++ b/packages/eql-mapper/src/transformation_rules/helpers.rs @@ -1,9 +1,88 @@ use sqltk::parser::{ - ast::{CastKind, DataType, Expr, Ident, ObjectName, ObjectNamePart}, + ast::{ + BinaryOperator, CastKind, DataType, Expr, Function, FunctionArg, FunctionArgExpr, + FunctionArgumentList, FunctionArguments, Ident, ObjectName, ObjectNamePart, + }, tokenizer::Span, }; +use sqltk::NodePath; -pub(crate) fn cast_as_encrypted(wrapped: sqltk::parser::ast::Value) -> Expr { +use crate::unifier::{DomainIdentity, EqlTerm}; + +/// The v3 cast target `(schema, domain)` for a JSON ordering operand +/// ([`EqlTerm::JsonOrd`]) — always the shape-only scalar ord twin +/// `eql_v3.query_integer_ord`, regardless of the JSON leaf's scalar type. +/// +/// `eql_v3.ord_term` is type-agnostic (it extracts the `op` bytes as +/// `ope_cllw` and compares them bytewise), and `query_integer_ord`'s domain +/// CHECK is shape-only (`{v,i,op}`, no `c`). So a single twin serves numbers, +/// text, dates, etc. — which is what makes JSON range work in the extended +/// protocol, where the operand's scalar type is unknown at rewrite time. +/// Returns `None` for any other term. +pub(crate) fn json_ord_cast_target(eql_term: &EqlTerm) -> Option<(String, String)> { + matches!(eql_term, EqlTerm::JsonOrd(_)) + .then(|| ("eql_v3".to_string(), "query_integer_ord".to_string())) +} + +/// The scalar comparison operators the v3 term-function rewrite handles. +pub(crate) fn is_comparison_op(op: &BinaryOperator) -> bool { + matches!( + op, + BinaryOperator::Eq + | BinaryOperator::NotEq + | BinaryOperator::Lt + | BinaryOperator::LtEq + | BinaryOperator::Gt + | BinaryOperator::GtEq + ) +} + +/// Whether an encrypted value at `node_path` is a **query operand** (the RHS of a +/// comparison or match predicate) rather than a **stored value** (an INSERT +/// `VALUES` item or UPDATE `SET` target). Walks the enclosing `Expr` ancestor +/// chain looking for a comparison `BinaryOp` or a `LIKE`/`ILIKE` predicate. The +/// traversal is post-order, so when a cast rule runs on the operand the enclosing +/// predicate is still intact in the path. +fn is_query_operand(node_path: &NodePath<'_>) -> bool { + let mut depth = 1; + while let Some(expr) = node_path.nth_last_as::(depth) { + match expr { + Expr::BinaryOp { op, .. } + if is_comparison_op(op) || matches!(op, BinaryOperator::AtAt) => + { + return true + } + Expr::Like { .. } | Expr::ILike { .. } => return true, + _ => {} + } + depth += 1; + } + false +} + +/// The v3 cast target `(schema, domain typname)` for an encrypted value carrying +/// `identity` at `node_path`. A query operand casts to the `eql_v3.query_*` twin +/// (term-only payload); a stored value casts to the `public` column domain. +pub(crate) fn v3_cast_target( + node_path: &NodePath<'_>, + identity: &DomainIdentity, +) -> (String, String) { + if is_query_operand(node_path) { + let (schema, twin) = identity.query_twin(); + (schema.to_string(), twin) + } else { + ("public".to_string(), identity.domain.value.clone()) + } +} + +/// Builds `::JSONB::.` — the cast that wraps an encrypted +/// value (a jsonb payload) as an EQL v3 domain. `schema` is `public` for a stored +/// column domain and `eql_v3` for a query-operand twin. +pub(crate) fn cast_to_v3_domain( + wrapped: sqltk::parser::ast::Value, + schema: &str, + domain: &str, +) -> Expr { let cast_jsonb = Expr::Cast { kind: CastKind::DoubleColon, expr: Box::new(Expr::Value(sqltk::parser::ast::ValueWithSpan { @@ -14,14 +93,37 @@ pub(crate) fn cast_as_encrypted(wrapped: sqltk::parser::ast::Value) -> Expr { format: None, }; - let encrypted_type = ObjectName(vec![ObjectNamePart::Identifier(Ident::new( - "eql_v2_encrypted", - ))]); + let domain_type = ObjectName(vec![ + ObjectNamePart::Identifier(Ident::new(schema)), + ObjectNamePart::Identifier(Ident::new(domain)), + ]); Expr::Cast { kind: CastKind::DoubleColon, expr: Box::new(cast_jsonb), - data_type: DataType::Custom(encrypted_type, vec![]), + data_type: DataType::Custom(domain_type, vec![]), format: None, } } + +/// Builds `eql_v3.()` — a call to an EQL v3 term-extraction function +/// (`eq_term`, `ord_term`, `ord_term_ore`, `match_term`). +pub(crate) fn eql_v3_term_call(fn_name: &str, arg: Expr) -> Expr { + Expr::Function(Function { + name: ObjectName(vec![ + ObjectNamePart::Identifier(Ident::new("eql_v3")), + ObjectNamePart::Identifier(Ident::new(fn_name)), + ]), + uses_odbc_syntax: false, + args: FunctionArguments::List(FunctionArgumentList { + args: vec![FunctionArg::Unnamed(FunctionArgExpr::Expr(arg))], + duplicate_treatment: None, + clauses: vec![], + }), + parameters: FunctionArguments::None, + filter: None, + null_treatment: None, + over: None, + within_group: vec![], + }) +} diff --git a/packages/eql-mapper/src/transformation_rules/mod.rs b/packages/eql-mapper/src/transformation_rules/mod.rs index 2d440042..d48cab5b 100644 --- a/packages/eql-mapper/src/transformation_rules/mod.rs +++ b/packages/eql-mapper/src/transformation_rules/mod.rs @@ -16,6 +16,8 @@ mod cast_params_as_encrypted; mod fail_on_placeholder_change; mod preserve_effective_aliases; mod rewrite_containment_ops; +mod rewrite_eql_comparison_ops; +mod rewrite_eql_match_ops; mod rewrite_standard_sql_fns_on_eql_types; use std::marker::PhantomData; @@ -25,6 +27,8 @@ pub(crate) use cast_params_as_encrypted::*; pub(crate) use fail_on_placeholder_change::*; pub(crate) use preserve_effective_aliases::*; pub(crate) use rewrite_containment_ops::*; +pub(crate) use rewrite_eql_comparison_ops::*; +pub(crate) use rewrite_eql_match_ops::*; pub(crate) use rewrite_standard_sql_fns_on_eql_types::*; use crate::EqlMapperError; diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs b/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs index 3142f8ca..11b03dee 100644 --- a/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs +++ b/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs @@ -15,13 +15,19 @@ use crate::EqlMapperError; use super::TransformationRule; -/// Rewrites `@>` and `<@` operators on EQL types to function calls. +/// Rewrites JSON binary operators on encrypted columns to `eql_v3` function +/// calls — containment (`@>`/`<@`, retained in v3 scoped to JSON, ADR-0002) and +/// field access (`->`/`->>`, functionalised because managed Postgres forbids the +/// operator DDL, ADR-0001): /// -/// - `col @> val` → `eql_v2.jsonb_contains(col, val)` -/// - `val <@ col` → `eql_v2.jsonb_contained_by(val, col)` +/// - `col @> val` → `eql_v3.jsonb_contains(col, val)` +/// - `val <@ col` → `eql_v3.jsonb_contained_by(val, col)` +/// - `col -> sel` → `eql_v3."->"(col, sel)` +/// - `col ->> sel` → `eql_v3."->>"(col, sel)` /// -/// This transformation enables GIN index usage when the index is created on -/// `eql_v2.jsonb_array(encrypted_col)`. +/// Containment enables GIN index usage via `eql_v3.jsonb_array(encrypted_col)`. +/// The `->`/`->>` field selector is passed as encrypted text (see +/// `CastLiteralsAsEncrypted`), matching the `eql_v3."->"(json, text)` signature. #[derive(Debug)] pub struct RewriteContainmentOps<'ast> { node_types: Arc, Type>>, @@ -50,10 +56,17 @@ impl<'ast> RewriteContainmentOps<'ast> { } fn make_function_call(fn_name: &str, left: Expr, right: Expr) -> Expr { + // Operator-symbol function names (`->`, `->>`) must be quoted; + // ordinary names (`jsonb_contains`) are not. + let fn_ident = if fn_name.chars().all(|c| c.is_alphanumeric() || c == '_') { + Ident::new(fn_name) + } else { + Ident::with_quote('"', fn_name) + }; Expr::Function(Function { name: ObjectName(vec![ - ObjectNamePart::Identifier(Ident::new("eql_v2")), - ObjectNamePart::Identifier(Ident::new(fn_name)), + ObjectNamePart::Identifier(Ident::new("eql_v3")), + ObjectNamePart::Identifier(fn_ident), ]), uses_odbc_syntax: false, args: FunctionArguments::List(FunctionArgumentList { @@ -85,6 +98,8 @@ impl<'ast> TransformationRule<'ast> for RewriteContainmentOps<'ast> { let fn_name = match op { BinaryOperator::AtArrow => "jsonb_contains", // @> BinaryOperator::ArrowAt => "jsonb_contained_by", // <@ + BinaryOperator::Arrow => "->", // -> field access + BinaryOperator::LongArrow => "->>", // ->> field access (as text) _ => return Ok(false), }; @@ -106,7 +121,13 @@ impl<'ast> TransformationRule<'ast> for RewriteContainmentOps<'ast> { fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { // Use node_path to get the original AST node (with correct NodeKey identity) if let Some((Expr::BinaryOp { left, op, right },)) = node_path.last_1_as::() { - if matches!(op, BinaryOperator::AtArrow | BinaryOperator::ArrowAt) { + if matches!( + op, + BinaryOperator::AtArrow + | BinaryOperator::ArrowAt + | BinaryOperator::Arrow + | BinaryOperator::LongArrow + ) { // Only rewrite if at least one operand is EQL-typed return self.uses_eql_type(left, right); } diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_eql_comparison_ops.rs b/packages/eql-mapper/src/transformation_rules/rewrite_eql_comparison_ops.rs new file mode 100644 index 00000000..042b04ec --- /dev/null +++ b/packages/eql-mapper/src/transformation_rules/rewrite_eql_comparison_ops.rs @@ -0,0 +1,123 @@ +use std::collections::HashMap; +use std::mem; +use std::sync::Arc; + +use sqltk::parser::ast::Value as SqltkValue; +use sqltk::parser::ast::{BinaryOperator, Expr, ValueWithSpan}; +use sqltk::parser::tokenizer::Span; +use sqltk::{NodeKey, NodePath, Visitable}; + +use crate::unifier::{DomainIdentity, Type, Value}; +use crate::EqlMapperError; + +use super::helpers::{eql_v3_term_call, is_comparison_op}; +use super::TransformationRule; + +/// Rewrites scalar comparison operators on encrypted columns into the EQL v3 +/// functional-index form (ADR-0001, ADR-0003): +/// +/// - `col = x` → `eql_v3.eq_term(col) = eql_v3.eq_term(x)` (or `ord_term` when +/// the domain stores no `hm`) +/// - `col > x` → `eql_v3.ord_term(col) > eql_v3.ord_term(x)` (`ord_term_ore` for +/// block-ORE domains) +/// +/// The term function is chosen from the column's domain identity; a column whose +/// domain provides no term for the operator is a capability error (this is the +/// same absence the type checker's bound check raises on — this rule is the +/// backstop at rewrite time). +/// +/// Operands are moved with `mem::replace` (not cloned) so their `NodeKey` +/// identity survives for the cast rules. Post-order traversal means the operand +/// literals/params have already been cast to their v3 domains by the time this +/// rule wraps them. +#[derive(Debug)] +pub struct RewriteEqlComparisonOps<'ast> { + node_types: Arc, Type>>, +} + +impl<'ast> RewriteEqlComparisonOps<'ast> { + pub fn new(node_types: Arc, Type>>) -> Self { + Self { node_types } + } + + fn eql_identity_of(&self, expr: &'ast Expr) -> Option { + match self.node_types.get(&NodeKey::new(expr)) { + Some(Type::Value(Value::Eql(eql_term))) => { + Some(eql_term.eql_value().domain_identity().clone()) + } + _ => None, + } + } + + /// The term function for `op` on a column with `identity`, or `None` if the + /// domain provides no term for that operator. + fn term_fn_for(op: &BinaryOperator, identity: &DomainIdentity) -> Option<&'static str> { + match op { + BinaryOperator::Eq | BinaryOperator::NotEq => identity.eq_term_fn(), + BinaryOperator::Lt + | BinaryOperator::LtEq + | BinaryOperator::Gt + | BinaryOperator::GtEq => identity.ord_term_fn(), + _ => None, + } + } +} + +impl<'ast> TransformationRule<'ast> for RewriteEqlComparisonOps<'ast> { + fn apply( + &mut self, + node_path: &NodePath<'ast>, + target_node: &mut N, + ) -> Result { + if !self.would_edit(node_path, target_node) { + return Ok(false); + } + + // Read the operator and the encrypted operand's domain identity from the + // ORIGINAL nodes (node_types is keyed by them); `target_node`'s children + // may already be rebuilt with different NodeKeys. + let Some((Expr::BinaryOp { left, op, right },)) = node_path.last_1_as::() else { + return Ok(false); + }; + if !is_comparison_op(op) { + return Ok(false); + } + let Some(identity) = self + .eql_identity_of(left) + .or_else(|| self.eql_identity_of(right)) + else { + return Ok(false); + }; + + let Some(term_fn) = Self::term_fn_for(op, &identity) else { + return Err(EqlMapperError::Transform(format!( + "encrypted column {} does not support operator {op} (domain {})", + identity.token, identity.domain.value + ))); + }; + + if let Expr::BinaryOp { left, right, .. } = target_node.downcast_mut::().unwrap() { + let dummy = Expr::Value(ValueWithSpan { + value: SqltkValue::Null, + span: Span::empty(), + }); + let left_expr = mem::replace(&mut **left, dummy.clone()); + let right_expr = mem::replace(&mut **right, dummy); + **left = eql_v3_term_call(term_fn, left_expr); + **right = eql_v3_term_call(term_fn, right_expr); + return Ok(true); + } + + Ok(false) + } + + fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { + if let Some((Expr::BinaryOp { left, op, right },)) = node_path.last_1_as::() { + if is_comparison_op(op) { + return self.eql_identity_of(left).is_some() + || self.eql_identity_of(right).is_some(); + } + } + false + } +} diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_eql_match_ops.rs b/packages/eql-mapper/src/transformation_rules/rewrite_eql_match_ops.rs new file mode 100644 index 00000000..671acbde --- /dev/null +++ b/packages/eql-mapper/src/transformation_rules/rewrite_eql_match_ops.rs @@ -0,0 +1,137 @@ +use std::collections::HashMap; +use std::mem; +use std::sync::Arc; + +use sqltk::parser::ast::Value as SqltkValue; +use sqltk::parser::ast::{BinaryOperator, Expr, UnaryOperator, ValueWithSpan}; +use sqltk::parser::tokenizer::Span; +use sqltk::{NodeKey, NodePath, Visitable}; + +use crate::unifier::{DomainIdentity, Type, Value}; +use crate::EqlMapperError; + +use super::helpers::eql_v3_term_call; +use super::TransformationRule; + +/// Rewrites fuzzy-match predicates on encrypted columns into the EQL v3 +/// match form (ADR-0001, ADR-0003): +/// +/// - `col LIKE pat` → `eql_v3.match_term(col) @> eql_v3.match_term(pat)` +/// - `col NOT LIKE pat` → `NOT (eql_v3.match_term(col) @> eql_v3.match_term(pat))` +/// - `col @@ pat` → `eql_v3.match_term(col) @> eql_v3.match_term(pat)` +/// +/// Fuzzy match compares bloom-filter terms with `@>` (containment), so unlike a +/// scalar comparison the operator *becomes* `@>` between the two `match_term` +/// calls — mirroring `eql_v3.matches`, whose body is `match_term(a) @> match_term(b)`. +/// A column whose domain stores no bloom filter (`bf`) has no `match_term` and is +/// a capability error. +#[derive(Debug)] +pub struct RewriteEqlMatchOps<'ast> { + node_types: Arc, Type>>, +} + +impl<'ast> RewriteEqlMatchOps<'ast> { + pub fn new(node_types: Arc, Type>>) -> Self { + Self { node_types } + } + + fn eql_identity_of(&self, expr: &'ast Expr) -> Option { + match self.node_types.get(&NodeKey::new(expr)) { + Some(Type::Value(Value::Eql(eql_term))) => { + Some(eql_term.eql_value().domain_identity().clone()) + } + _ => None, + } + } + + /// The encrypted column's `(domain identity, negated)` if `expr` is a fuzzy- + /// match predicate — `LIKE`/`ILIKE` or `@@` — with an encrypted operand. + fn match_predicate(&self, expr: &'ast Expr) -> Option<(DomainIdentity, bool)> { + match expr { + Expr::Like { + expr, negated, any, .. + } + | Expr::ILike { + expr, negated, any, .. + } if !*any => self.eql_identity_of(expr).map(|id| (id, *negated)), + Expr::BinaryOp { + left, + op: BinaryOperator::AtAt, + right, + } => self + .eql_identity_of(left) + .or_else(|| self.eql_identity_of(right)) + .map(|id| (id, false)), + _ => None, + } + } +} + +impl<'ast> TransformationRule<'ast> for RewriteEqlMatchOps<'ast> { + fn apply( + &mut self, + node_path: &NodePath<'ast>, + target_node: &mut N, + ) -> Result { + if !self.would_edit(node_path, target_node) { + return Ok(false); + } + + // Read the identity from the ORIGINAL node (node_types is keyed by it). + let Some((original,)) = node_path.last_1_as::() else { + return Ok(false); + }; + let Some((identity, negated)) = self.match_predicate(original) else { + return Ok(false); + }; + + let Some(term_fn) = identity.match_term_fn() else { + return Err(EqlMapperError::Transform(format!( + "encrypted column {} does not support fuzzy match (domain {})", + identity.token, identity.domain.value + ))); + }; + + let dummy = Expr::Value(ValueWithSpan { + value: SqltkValue::Null, + span: Span::empty(), + }); + let expr = target_node.downcast_mut::().unwrap(); + let (col_expr, pat_expr) = match expr { + Expr::Like { expr, pattern, .. } | Expr::ILike { expr, pattern, .. } => ( + mem::replace(&mut **expr, dummy.clone()), + mem::replace(&mut **pattern, dummy), + ), + Expr::BinaryOp { left, right, .. } => ( + mem::replace(&mut **left, dummy.clone()), + mem::replace(&mut **right, dummy), + ), + _ => return Ok(false), + }; + + // eql_v3.match_term(col) @> eql_v3.match_term(pat) + let matched = Expr::BinaryOp { + left: Box::new(eql_v3_term_call(term_fn, col_expr)), + op: BinaryOperator::AtArrow, + right: Box::new(eql_v3_term_call(term_fn, pat_expr)), + }; + + *expr = if negated { + Expr::UnaryOp { + op: UnaryOperator::Not, + expr: Box::new(Expr::Nested(Box::new(matched))), + } + } else { + matched + }; + + Ok(true) + } + + fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { + if let Some((expr,)) = node_path.last_1_as::() { + return self.match_predicate(expr).is_some(); + } + false + } +} diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_standard_sql_fns_on_eql_types.rs b/packages/eql-mapper/src/transformation_rules/rewrite_standard_sql_fns_on_eql_types.rs index b5440a82..25ad4a1f 100644 --- a/packages/eql-mapper/src/transformation_rules/rewrite_standard_sql_fns_on_eql_types.rs +++ b/packages/eql-mapper/src/transformation_rules/rewrite_standard_sql_fns_on_eql_types.rs @@ -1,13 +1,10 @@ -use std::mem; use std::{collections::HashMap, sync::Arc}; -use sqltk::parser::ast::{ - Expr, Function, FunctionArg, FunctionArguments, Ident, ObjectName, ObjectNamePart, -}; +use sqltk::parser::ast::{Expr, Function, FunctionArg, FunctionArguments}; use sqltk::{AsNodeKey, NodeKey, NodePath, Visitable}; use crate::unifier::{Type, Value}; -use crate::{get_sql_function, EqlMapperError}; +use crate::{get_eql_v3_function_name, get_sql_function, EqlMapperError}; use super::TransformationRule; @@ -56,10 +53,10 @@ impl<'ast> TransformationRule<'ast> for RewriteStandardSqlFnsOnEqlTypes<'ast> { ) -> Result { if self.would_edit(node_path, target_node) { let function = target_node.downcast_mut::().unwrap(); - let mut existing_name = mem::take(&mut function.name.0); - existing_name.insert(0, ObjectNamePart::Identifier(Ident::new("eql_v2"))); - function.name = ObjectName(existing_name); - return Ok(true); + if let Some(v3_name) = get_eql_v3_function_name(&function.name) { + function.name = v3_name; + return Ok(true); + } } Ok(false) @@ -67,8 +64,12 @@ impl<'ast> TransformationRule<'ast> for RewriteStandardSqlFnsOnEqlTypes<'ast> { fn would_edit(&mut self, node_path: &NodePath<'ast>, _target_node: &N) -> bool { if let Some((_expr, function)) = node_path.last_2_as::() { + // Rewrite a pg_catalog function on an EQL type to its eql_v3 + // counterpart — but only when one exists (e.g. `count` has none and is + // left to run natively on the encrypted value). return get_sql_function(&function.name).should_rewrite() - && self.uses_eql_type(function); + && self.uses_eql_type(function) + && get_eql_v3_function_name(&function.name).is_some(); } false diff --git a/packages/eql-mapper/src/type_checked_statement.rs b/packages/eql-mapper/src/type_checked_statement.rs index 68aa762f..e46cc10b 100644 --- a/packages/eql-mapper/src/type_checked_statement.rs +++ b/packages/eql-mapper/src/type_checked_statement.rs @@ -7,7 +7,8 @@ use crate::unifier::EqlTerm; use crate::{ CastLiteralsAsEncrypted, CastParamsAsEncrypted, DryRunnable, EqlMapperError, FailOnPlaceholderChange, Param, PreserveEffectiveAliases, RewriteContainmentOps, - RewriteStandardSqlFnsOnEqlTypes, TransformationRule, + RewriteEqlComparisonOps, RewriteEqlMatchOps, RewriteStandardSqlFnsOnEqlTypes, + TransformationRule, }; use crate::unifier::{Projection, Type, Value}; @@ -153,8 +154,10 @@ impl<'ast> TypeCheckedStatement<'ast> { DryRunnable::new(( RewriteStandardSqlFnsOnEqlTypes::new(Arc::clone(&self.node_types)), RewriteContainmentOps::new(Arc::clone(&self.node_types)), + RewriteEqlComparisonOps::new(Arc::clone(&self.node_types)), + RewriteEqlMatchOps::new(Arc::clone(&self.node_types)), PreserveEffectiveAliases, - CastLiteralsAsEncrypted::new(encrypted_literals), + CastLiteralsAsEncrypted::new(encrypted_literals, Arc::clone(&self.node_types)), FailOnPlaceholderChange::new(), CastParamsAsEncrypted::new(Arc::clone(&self.node_types)), ))