From e480d2458d1ef3c0f22c85b169221ad56a81732c Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Wed, 21 Oct 2026 11:06:43 +0200 Subject: [PATCH 01/17] Plan the dev-2101 Mode-B grammar OpenSpec change Proposal, design, tasks and delta specs: one pyparsing grammar for every Mode-B position, SQL forms, quoted identifiers and SQL string literals, CAST as the database's cast plus portable TRY_CAST, FILTER (WHERE) row semantics, literal-only {var} binding, NULLS FIRST/LAST, stored-text migration, and the importers' legacy parser retired. --- .../.openspec.yaml | 2 + .../design.md | 65 ++++++ .../proposal.md | 36 ++++ .../aggregations/functional-form/spec.md | 28 +++ .../specs/models/stored-upgrade/spec.md | 35 ++++ .../specs/queries/casts/spec.md | 67 ++++++ .../specs/queries/date-functions/spec.md | 58 ++++++ .../specs/queries/expression-grammar/spec.md | 190 ++++++++++++++++++ .../specs/queries/expression-keywords/spec.md | 44 ++++ .../specs/queries/positions/spec.md | 24 +++ .../specs/queries/variables/spec.md | 75 +++++++ .../tasks.md | 80 ++++++++ 12 files changed, 704 insertions(+) create mode 100644 openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/.openspec.yaml create mode 100644 openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/design.md create mode 100644 openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/proposal.md create mode 100644 openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/aggregations/functional-form/spec.md create mode 100644 openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/models/stored-upgrade/spec.md create mode 100644 openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/casts/spec.md create mode 100644 openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/date-functions/spec.md create mode 100644 openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/expression-grammar/spec.md create mode 100644 openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/expression-keywords/spec.md create mode 100644 openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/positions/spec.md create mode 100644 openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/variables/spec.md create mode 100644 openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/tasks.md diff --git a/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/.openspec.yaml b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/.openspec.yaml new file mode 100644 index 00000000..9076a660 --- /dev/null +++ b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-10-20 diff --git a/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/design.md b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/design.md new file mode 100644 index 00000000..f32fcc43 --- /dev/null +++ b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/design.md @@ -0,0 +1,65 @@ +## Context + +See proposal.md for motivation. Today `slayer/engine/syntax.py` runs about eight text pre-passes (colon, CASE, extract, count-distinct, star-args, LIKE, filter operators, `||`) and then `ast.parse`; filters alone go through `parse_filter_expr`. `slayer/core/formula.py` holds a second, older Python-AST parser (`parse_formula` / `FieldSpec`) used only by the cube, dbt and OSI importers (and, outside the repo, by Storyline). `{var}` substitution is textual (`escape="python"`), run before parsing. The IR already carries `OrderEntry.nulls`; nothing sets it. SQL generation already probes a cached per-engine `ServerProfile` (ClickHouse uses it to refuse correlated subqueries). + +This change is PR 1 of three: DEV-2119 (expressions in every column position) and DEV-2120 (portable function library, regex, JSON, coercion, collation, C9) stack on it. + +## Goals / Non-Goals + +**Goals:** one Mode-B grammar and one parse tree for every position and every consumer; SQL forms in every position; SQL string / identifier quoting; literal-only, structural `{var}` binding; casts; `NULLS FIRST/LAST`; stored text upgraded so no stored meaning changes. + +**Non-Goals:** new scalar functions, regex, JSON, TEXT↔number coercion, collation (DEV-2120); expressions in column-only positions (DEV-2119); removing query-time Mode A (DEV-2021); a `DECIMAL` member of `DataType`. + +## Decisions + +### D1. The parser lives in `core` and is pure syntax + +`slayer/engine/syntax.py` moves to `slayer/core/syntax.py` (declared child `core.syntax`) with `la:deterministic-refactor`. It imports only `slayer.core.*`: the transform tables (`ALL_TRANSFORMS`, `TIME_TRANSFORMS`, `TIMELESS_TRANSFORMS`) move from `core/formula.py` to `core/enums.py` next to `RANK_FAMILY_TRANSFORMS`. Engine P2 ("parsing is pure syntax") keeps governing it; `la-arch-check` model-truth enforces the boundary. Approved harness edits H1–H7: `model.c4` (`core.syntax` child; arrows `core.query → core.syntax`, `engine.binding → core.syntax`, `engine.bind_inputs → core.syntax`; legacy `core.query → engine.syntax` removed), `index.yaml` baseline 8 → 7, `system.arc42.md` P2 and P10, `core.arc42.md` §1 / §2, `engine.arc42.md` §2 / P9, regenerated diagrams. P10's "every Mode B construct means the same on every dialect" carries `[target: DEV-2120]`. + +### D2. pyparsing grammar, one entry point + +`Forward()` recursion; `infix_notation` ladder (loosest first) `OR`, `AND`, `NOT`, comparison / `IS` / `IN` / `BETWEEN` / `LIKE` / `ILIKE`, `||`, `+ -`, `* / %`, unary `-`, `**` (right-associative, right operand may be unary-negated: Python semantics, `-2**2 = -4`), `::`, postfix (call, `FILTER`, `.`); `ParserElement.enable_packrat()`; operator alternations as `Regex`; `lru_cache` on `parse_expr`. Keywords are case-insensitive, ASCII-exact (the DEV-1833 Unicode rules carry over) and reserved only in their slot, so `case`, `end`, `with`, `model`, `date`, … stay identifiers elsewhere; `CASE` is a conditional only when `WHEN` follows. CASE still lowers to `iif`, `EXTRACT` to `date_part`, `count(distinct x)` to `count_distinct`, `POSITION` / `SUBSTRING … FROM … FOR` to `instr` / `substr`, `BETWEEN` to its conjunction, `CURRENT_DATE` / `CURRENT_TIMESTAMP` to the clock calls, interval literals to `interval(n, unit)` — so every spelling collapses to one node and downstream stays spelling-insensitive (engine P2). `parse_filter_expr` is deleted; the kwarg-vs-comparison rule (an aggregation / transform / unknown call reads `name =` after `(` or `,` as a kwarg; a scalar call and everything else reads `=` as comparison) is the one rule for every position. Syntax errors carry the pyparsing location. + +Alternatives: keep `ast.parse` with more pre-passes (the status quo the issue retires); sqlglot as the parser (it would read Mode-B constructs — kwargs, colon, transforms, `{var}` — as SQL and need its own rewrite layer). + +### D3. New node and key kinds + +Casts are a new `Cast` parsed node and a new `CastKey(source, type_text, data_type, try_)` (system P13: a cast is not a scalar call). The key implements `children()` / `map_children()`, equality, phase (its source's), binding, checker typing (`data_type` per `queries/casts`; UNKNOWN for unmapped types) and render dispatch. `FILTER (WHERE c)` attaches to the `AggCall` node as an optional `filter` predicate and to its aggregate key; `IS [NOT] DISTINCT FROM` reuses the dialect's null-safe-equality builder; `x IS TRUE` lowers to `x IS NOT NULL AND x` (`IS FALSE`: `x IS NOT NULL AND NOT x`), and the `NOT` forms negate those, so NULL never propagates. + +### D4. Casts: the database's cast, plus a portable `TRY_CAST` + +`CAST` / `::` render `exp.Cast` with the author's `exp.DataType` (sqlglot-parsed, transpiled by the target dialect's generator). The plan-time "type the dialect cannot express" check is: sqlglot raises, or the round-trip of the type through the target dialect loses it (unsupported-type warning) → typed error. `TRY_CAST` lowers per dialect: native `TRY_CAST` (DuckDB, SQL Server, Snowflake, Trino), `SAFE_CAST` (BigQuery), `accurateCastOrNull` (ClickHouse); Postgres ≥ 16 `CASE WHEN pg_input_is_valid(CAST(x AS TEXT), '') THEN CAST(x AS ) END`; SQLite a registered `slayer_try_cast(value, target)` connection function; MySQL a numeric `REGEXP` guard (dates already yield NULL). The Postgres form needs the server version at render time: the Postgres dialect gains `server_profile_sql()` (`SELECT current_setting('server_version_num')`) and `parse_server_profile`; the engine fetches the cached profile before rendering a Postgres query that contains a `TRY_CAST` and passes it to the dialect; no profile (dry run without a connection, failed probe) → the plain `CAST`, which runs on every version. Guards cast the input to TEXT first so a non-text source never meets a regex. + +Rejected alternatives (user rulings): normalising `CAST` value semantics (rounding, boolean words, ISO-only, NULL-on-error) — `CAST` must be the author's database cast; a calendar-exact regex on old Postgres — performance on recent versions; refusing on old Postgres — a plain cast is strictly better. + +### D5. Strings, identifiers and `{var}` binding + +Strings are SQL: `'…'`, `''` escape, backslash literal. `"…"` is a quoted identifier everywhere (`""` escape), producing the same `Ref` / `DottedRef` as a bare name. A quoted name that fails to resolve gets the "strings use single quotes" hint in the binder's unresolved-reference error. + +Placeholders are masked before parsing with offset-preserving sentinels of the placeholder's exact length, so error positions and names stay true. The grammar admits a placeholder in three slots only: inside a string literal (a template literal of text and placeholder parts), a bare literal slot, and the whole `IN` list; anything else is a parse error naming "variables supply literal values". Values bind at the existing substitution points (`ir/variables.py`, `core/query.py`), now into the parse tree: a template literal renders the value's text; a bare slot takes a number or boolean (a numeric string reads as a number; a string value errors with the quote hint); the `IN` slot takes a list. Result names are computed from the text as written, before binding, for measures and computed dimensions alike (the `_template` / `template` plumbing stays, `ComputedDimension.substituted`'s text splice goes). `date_range` bounds are not Mode B: they keep their own literal-only substitution, then time-point validation. `escape="python"` for Mode B is deleted; Mode-A substitution is untouched. + +### D6. Stored-text migration + +New stored-only steps at SlayerModel 15 → 16, SlayerQuery 6 → 7 and Memory 4 → 5, registered in `slayer/storage/migrations.py` alongside `rank_direction_migration.py`, recursing like it does (`source_queries`, saved queries, inline `ModelExtension.measures`, memories' queries). The rewrite uses Python's `tokenize` over the legacy text: each STRING token is decoded with the old grammar's rules (`ast.literal_eval`) and re-emitted as a SQL single-quoted literal; an `OP` token `|` becomes `||`; everything else is copied byte-for-byte; untokenisable text is left unchanged. It runs before the existing load-path repairs (legacy time literals, aliases), which parse filter text with sqlglot and therefore already read `"…"` as an identifier. Mode-A fields are never visited. + +### D7. NULL ordering + +`OrderItem.nulls: Literal["first", "last"] | None` (validated; serialised; preserved by `_strip_order`, saved-query refinement, variable binding and every other path that rebuilds an order item) feeds `OrderEntry.nulls` at `engine/compile/stages.py`. `SqlDialect.build_ordered` returns a list of order terms; base returns one `exp.Ordered` with `nulls_first`; T-SQL emits a leading `CASE WHEN IS NULL THEN … END` key (NULLs-first or NULLs-last per the policy) followed by the expression, built over the resolved ordering expression rather than its bracketed alias, fixing the alias mis-resolution noted in `tsql.py`. + +### D8. One parser repo-wide (absorbs DEV-1831) + +`parse_formula`, the `FieldSpec` classes, `_rewrite_funcstyle_aggregations`, `_preprocess_agg_refs` and `_expand_named_measures` are deleted. The importers call `parse_expr`; the dbt importer's dangling-reference and cycle checks are re-implemented as a namespace-aware dependency walk over `ParsedExpr` (named measures and custom-aggregation names as the namespace, with cycle detection), shared by the three importers. The facade translator's and the importers' emitted Mode-B text is checked against the new grammar (single-quoted strings, no Python escapes, no `|`). + +## Risks / Trade-offs + +- [The parser swap touches every query] → SQL golden baselines must regenerate byte-identically for every pre-existing shape; the existing parser tests port to the new parser and stay green. +- [pyparsing performance on long formulas] → packrat plus `lru_cache`; a benchmark test over the longest formulas in the suite and docs, compared with the old parser. +- [Live payloads with `"…"` strings or backslash escapes break] → typed errors with the single-quote hint; release note; the migration covers everything stored. +- [Storyline imports `parse_formula`] → accepted break, recorded on DEV-2034; the port is `parse_expr` plus `walk_parsed_refs`. +- [Postgres before 16: `TRY_CAST` raises on invalid input] → documented; a plain cast is the only form that runs there. +- [Render-time server profile on Postgres] → read from the existing per-engine cache; probed only when the query contains a `TRY_CAST`. +- [Retargeting tests that assert on deleted helpers] → listed individually in tasks.md, approved by the user. + +## Migration Plan + +Stored data upgrades on load (D6) and re-saves in the new spelling. No rollback step beyond the usual: an older SLayer opening a store re-saved by this version reads `'…'` strings correctly but fails on `"…"` identifiers, as for any forward-written store. diff --git a/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/proposal.md b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/proposal.md new file mode 100644 index 00000000..85820d3f --- /dev/null +++ b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/proposal.md @@ -0,0 +1,36 @@ +## Why + +Mode B is parsed by Python's `ast` behind a stack of text pre-passes, so SQL spellings work only in some positions (`=`, `IS NULL`, `AND`, `LIKE` only in filters), `"abc"` is a string, and there is no `CAST`, `BETWEEN`, `ILIKE`, `FILTER (WHERE …)` or `NULLS FIRST/LAST`. The SLayer SQL programme (DEV-1768) needs Mode B to be a SQL-compatible grammar, and replacing query-time Mode-A columns (DEV-2021) needs Mode B to express what agents put there — `CAST` alone blocks a third of those expressions. + +## What Changes + +- Mode B gets its own grammar (pyparsing), replacing `ast.parse` and every text pre-pass; one entry point for every position. The parser moves to `slayer/core/syntax.py` (declared child `core.syntax`), killing the legacy arrow `core.query → engine.syntax`. +- SQL forms are valid in every position (measures, dimensions, order, filters), with every existing spelling kept: `=`, `<>`, `IS [NOT] NULL`, `[NOT] IN`, `AND`/`OR`/`NOT`, `[NOT] LIKE` over any operand, `[NOT] ILIKE`, `||`, `[NOT] BETWEEN`, `IS [NOT] DISTINCT FROM`, `IS [NOT] TRUE|FALSE`, `count(distinct x)`, `CAST(x AS t)` / `x::t`, `EXTRACT(f FROM x)`, `INTERVAL '3 day'` / `INTERVAL '3' DAY` / `INTERVAL 3 DAY`, `DATE '…'` / `TIMESTAMP '…'`, `CURRENT_DATE` / `CURRENT_TIMESTAMP`, `POSITION(a IN b)`, `SUBSTRING(x FROM a [FOR b])`, `agg(x) FILTER (WHERE c)`. +- `CAST` / `::` are the database's own cast of the author's type (spelling transpiled to the dialect, parameters kept); `TRY_CAST(x AS t)` is the portable cast that yields NULL for invalid input on every dialect (Postgres before 16: a plain cast). +- `ORDER BY … NULLS FIRST|LAST` through `OrderItem.nulls`, honoured on every dialect. +- **BREAKING** `"…"` is a quoted identifier everywhere; strings are `'…'` with `''` as the only escape (backslash is literal). Stored Mode-B text is migrated; a live double-quoted name that resolves to nothing errors with a "use single quotes for strings" hint. +- **BREAKING** `{var}` placeholders bind into the parse tree and supply literal values only: inside a string literal, in a bare number / boolean position, or as a whole `IN` list. Anything else is an error. +- **BREAKING** `|` as concatenation and `//` are rejected with hints (`||`; `div(a, b)` / `floor(a / b)`); stored `|` is migrated to `||`. `**` is lowered to `pow` (it crashed before). +- The importers' legacy parser (`parse_formula`, `FieldSpec`, the colon rewriters in `slayer/core/formula.py`) is deleted; cube / dbt / OSI importers use `parse_expr` with a namespace-aware dependency check (absorbs DEV-1831). **BREAKING** for Python callers of `parse_formula` (Storyline ports on its next bump). + +## Capabilities + +### New Capabilities +- `queries/expression-grammar`: the Mode-B grammar — SQL forms in every position, identifiers and quoting, string literals, precedence, legacy spellings and the forms rejected with hints. +- `queries/casts`: `CAST` / `::` as the database's cast of the author's type, the inferred result type, and the portable `TRY_CAST`. + +### Modified Capabilities +- `queries/expression-keywords`: CASE conditions accept SQL spellings natively; LIKE rewriting is replaced by the grammar's LIKE / ILIKE operators. +- `queries/variables`: placeholders bind as literals in three positions; the textual Mode-B escaping regime is gone. +- `aggregations/functional-form`: `agg(x) FILTER (WHERE c)`. +- `queries/date-functions`: SQL interval and typed-literal spellings; `CURRENT_DATE` / `CURRENT_TIMESTAMP`. +- `queries/positions`: order items carry an explicit NULL placement. +- `models/stored-upgrade`: stored Mode-B text is upgraded to the new string and identifier rules. + +## Impact + +- Code: `slayer/engine/syntax.py` → `slayer/core/syntax.py`; `slayer/core/query.py`, `slayer/core/keys.py` (cast key), `slayer/core/formula.py` (parser deleted; transform tables to `slayer/core/enums.py`), `slayer/engine/binding.py`, `bind_inputs.py`, `population.py`, `model_reads.py`, `schema_drift.py`, `unrelated.py`, `query_engine.py`, `slayer/memories/resolver.py`, `slayer/ir/variables.py`, `slayer/sql/render/*`, `slayer/sql/dialects/*` (casts, NULL ordering, server profile on Postgres), `slayer/sql/client.py`, `slayer/storage/migrations.py` + a new migration module, `slayer/dbt|cube|osi/converter.py`, `slayer/facade/translator.py` (emitted Mode-B text). +- Dependencies: `pyparsing` added. +- Architecture: `model.c4`, `index.yaml` (legacy baseline 8 → 7), `system.arc42.md` P2 / P10, `core.arc42.md`, `engine.arc42.md`. +- External: Storyline's `parse_formula` use (`storyline/api/adapters/models.py`); agents writing double-quoted strings or backslash escapes. +- Follow-ups stacked on this change: DEV-2119 (expressions in every column position), DEV-2120 (portable function library). diff --git a/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/aggregations/functional-form/spec.md b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/aggregations/functional-form/spec.md new file mode 100644 index 00000000..517afe4b --- /dev/null +++ b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/aggregations/functional-form/spec.md @@ -0,0 +1,28 @@ +## ADDED Requirements + +### Requirement: An aggregation accepts a FILTER clause +`agg(…) FILTER (WHERE c)` SHALL be accepted on every aggregation — built-in, aliased and model-defined custom — in every aggregation position, and SHALL mean the aggregation computed over only the rows where `c` is true. `c` SHALL be a row-level predicate over the aggregation's dataset; an aggregate, a transform or an attached value inside `c` SHALL be rejected with a typed query error naming the clause. Every row-level input of the aggregation SHALL be restricted by `c`: the source value, every column-valued parameter (`weight=`, `other=`), and the ranking key of `first` / `last`; literal parameters are unaffected. The clause SHALL compose with `partition_by=` and `window=`. An unnamed filtered aggregation SHALL take its result key from its full text by the existing expression-naming rule, so it never collides with the unfiltered aggregation. + +#### Scenario: Filtered count +- **WHEN** a query on `orders` by `region` has measure `count(*) FILTER (WHERE status = 'paid')` +- **THEN** each region's value is its number of paid orders, 0 where it has none + +#### Scenario: Filtered sum equals a masked sum +- **WHEN** one measure is `sum(amount) FILTER (WHERE status = 'paid')` and another `sum(iif(status = 'paid', amount, NULL))` +- **THEN** both return the same values + +#### Scenario: Filtered weighted average uses only passing rows +- **WHEN** a measure is `weighted_avg(price, weight=quantity) FILTER (WHERE status = 'paid')` +- **THEN** it equals the weighted average computed over the paid rows alone, numerator and denominator both + +#### Scenario: Filtered last picks the newest passing row +- **WHEN** a query by `customer_id` has measure `last(balance, updated_at) FILTER (WHERE status = 'active')` and a customer's newest row is inactive +- **THEN** the value is the balance of that customer's newest active row + +#### Scenario: Aggregate inside the condition rejected +- **WHEN** a measure is `sum(amount) FILTER (WHERE amount > avg(amount))` +- **THEN** it is rejected with the typed error naming the FILTER clause + +#### Scenario: With partition_by +- **WHEN** a query by `region, city` has measure `sum(amount, partition_by=region) FILTER (WHERE status = 'paid')` +- **THEN** each row carries its region's paid total diff --git a/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/models/stored-upgrade/spec.md b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/models/stored-upgrade/spec.md new file mode 100644 index 00000000..700c0a33 --- /dev/null +++ b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/models/stored-upgrade/spec.md @@ -0,0 +1,35 @@ +## ADDED Requirements + +### Requirement: Stored Mode-B text is upgraded to the SQL string and identifier rules + +On load, every Mode-B surface of a stored model, saved query or memory written before this change — `ModelMeasure.formula`, and a stored query's measures, dimensions, computed-dimension expressions, time-dimension columns, order columns and expressions, `main_time_dimension` and filters, recursively through stored `source_queries`, saved queries and inline `ModelExtension.measures` — SHALL be rewritten so that it means under the current grammar what it meant when stored: a double-quoted string literal becomes the equivalent single-quoted literal, a string literal using backslash escapes is re-encoded with its decoded value, and a `|` concatenation becomes `||`. The rewrite SHALL be token-exact: text inside string literals is never otherwise touched, and stored text that cannot be tokenised is left unchanged. Mode-A surfaces SHALL NOT be rewritten, and a client query without a version SHALL NOT be rewritten. + +#### Scenario: Double-quoted strings in a stored query + +- **WHEN** a stored query written before this change filters `status == "paid"` and has measure `sum(iif(region == "EU", amount, 0))` +- **THEN** it loads as `status == 'paid'` and `sum(iif(region == 'EU', amount, 0))` and returns the rows it returned before + +#### Scenario: Escapes and embedded quotes + +- **WHEN** a stored filter is `name == "O'Brien"`, another `name == 'O\'Brien'`, and a stored measure formula compares with `"tab\there"` +- **THEN** they load as `name == 'O''Brien'` (twice) and a single-quoted literal containing a real tab character, and match the same rows as before + +#### Scenario: Pipe concatenation + +- **WHEN** a stored computed dimension is `status | region` +- **THEN** it loads as `status || region` and returns the same values + +#### Scenario: Mode-A text untouched + +- **WHEN** a stored model has `Column.sql` `"Order Total" * 2` and a model filter `status = "x"` +- **THEN** both load byte-for-byte unchanged + +#### Scenario: Client queries are not rewritten + +- **WHEN** a client sends a query without a version that filters `status = "paid"` +- **THEN** it is not rewritten and fails with the double-quoted-name hint unless a column named `paid` exists + +#### Scenario: Released stores still open with their results + +- **WHEN** the committed 0.10.2 store is opened +- **THEN** every model and memory loads and every recorded query returns the rows recorded from 0.10.2 diff --git a/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/casts/spec.md b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/casts/spec.md new file mode 100644 index 00000000..3840f7a9 --- /dev/null +++ b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/casts/spec.md @@ -0,0 +1,67 @@ +## Purpose + +Defines casts in Mode-B expressions: `CAST` / `::` as the database's own cast of the author's type, and `TRY_CAST` as the one portable cast that turns an invalid input into NULL on every dialect. + +## ADDED Requirements + +### Requirement: CAST is the database's cast of the author's type + +`CAST(x AS )` and `x::` SHALL be accepted in every Mode-B position, chains included (`x::int::text`). The cast SHALL render the author's type, parameters included (`DECIMAL(10,2)`, `VARCHAR(3)`), changing only its spelling to the target dialect's; its value semantics — rounding, parsing, invalid input — SHALL be the database's own. A type the target dialect cannot express SHALL be rejected at plan time with a typed query error naming the type and the dialect. + +#### Scenario: Author's parameterised type is kept + +- **WHEN** a measure on Postgres is `sum(CAST(amount AS DECIMAL(10,2)))` +- **THEN** the generated SQL casts to `DECIMAL(10, 2)` and the value is Postgres's own + +#### Scenario: Spelling transpiled to the dialect + +- **WHEN** a dimension is `amount::text` on MySQL +- **THEN** the generated SQL is valid MySQL casting to its character type + +#### Scenario: Chained casts + +- **WHEN** a dimension is `amount::int::text` +- **THEN** it casts to INT, then to TEXT + +#### Scenario: Invalid input follows the database + +- **WHEN** a dimension is `CAST(status AS INT)` over a row holding `'abc'` +- **THEN** Postgres fails the query with its own error and SQLite returns 0, as each database's `CAST` does + +### Requirement: A cast's result type is inferred + +A cast SHALL type as SLayer's data type for its target: the integer family as INT; floating-point, `REAL`, `DECIMAL` and `NUMERIC` as DOUBLE; character types as TEXT; `BOOLEAN` as BOOLEAN; `DATE` as DATE; `TIMESTAMP` and `DATETIME` as TIMESTAMP; any other type as UNKNOWN. The inferred type SHALL drive checking and response types exactly as a column's declared type does. + +#### Scenario: Cast to DATE is a date operand + +- **WHEN** a dimension is `date_diff('day', CAST(created_txt AS DATE), DATE '2025-01-01')` over a TEXT column `created_txt` +- **THEN** it binds without a date-operand type error and the response reports INT + +#### Scenario: Unknown target type + +- **WHEN** a dimension is `CAST(payload AS JSON)` on Postgres +- **THEN** it renders and its response type is UNKNOWN + +### Requirement: TRY_CAST turns invalid input into NULL on every dialect + +`TRY_CAST(x AS )` SHALL render the author's type like `CAST` and SHALL yield NULL, never an error or a substitute value, for an input the cast cannot convert, on every Tier-1 dialect. Where the database cannot itself tell a valid input from an invalid one (SQLite, MySQL numeric targets), a number SHALL be valid when it is an optionally signed decimal with an optional fraction and exponent, surrounding spaces ignored; a boolean when it is one of `true`, `false`, `t`, `f`, `yes`, `no`, `1`, `0` in any case; a date or timestamp when it is an ISO time-point shape naming a real calendar value. On Postgres, a server older than 16 — or one whose version cannot be determined, including a dry run without a connection — SHALL get a plain `CAST`, so an invalid input fails the query there; this is documented. + +#### Scenario: Invalid text becomes NULL + +- **WHEN** a dimension is `TRY_CAST(status AS INT)` over rows holding `'12'` and `'abc'` +- **THEN** they yield 12 and NULL on SQLite, DuckDB, MySQL and Postgres 16 + +#### Scenario: SQLite does not truncate to a prefix + +- **WHEN** a dimension is `TRY_CAST(status AS INT)` over a row holding `'12abc'` on SQLite +- **THEN** it yields NULL, not 12 + +#### Scenario: Postgres before 16 + +- **WHEN** the same query runs against a Postgres server reporting version 15 +- **THEN** the generated SQL is a plain `CAST`, and a row holding `'abc'` fails the query with Postgres's error + +#### Scenario: Postgres 16 uses the native validity check + +- **WHEN** the query runs against a Postgres server reporting version 16 or later +- **THEN** the generated SQL guards the cast with Postgres's own input-validity check and `'abc'` yields NULL diff --git a/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/date-functions/spec.md b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/date-functions/spec.md new file mode 100644 index 00000000..ab5c339b --- /dev/null +++ b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/date-functions/spec.md @@ -0,0 +1,58 @@ +## MODIFIED Requirements + +### Requirement: The interval spelling is date_add +`x + interval(n, unit)`, `interval(n, unit) + x` and `x - interval(n, unit)` SHALL mean exactly +`date_add(x, n, unit)` (for subtraction, `date_add(x, -n, unit)`) and SHALL resolve to the same +value as that call, sharing its result key and SQL. The SQL interval literals `INTERVAL ' '`, +`INTERVAL '' ` and `INTERVAL ` (one unit; keyword and unit case-insensitive; a +unit's plural form accepted) SHALL mean `interval(n, unit)` in all of these positions. Chains SHALL +apply left to right. `interval` SHALL follow the same count and unit rules as `date_add`. An interval +in any other position — standalone, as the left side of `-`, negated, multiplied, compared, nested in +another interval, added to another interval, or as a function argument — SHALL be rejected before +binding with an error pointing to `date_add`, and an interval literal naming more than one unit +(`INTERVAL '1 day 2 hours'`) SHALL be rejected with an error suggesting a chain. + +#### Scenario: Operator spelling equals date_add +- **WHEN** one measure uses `max(shipped_at - interval(2, 'day'))` and another `max(date_add(shipped_at, -2, 'day'))` +- **THEN** both resolve to the same value and render identical SQL + +#### Scenario: Chained intervals +- **WHEN** a filter uses `created_at + interval(1, 'month') + interval(2, 'hour') < now()` +- **THEN** it binds as `date_add(date_add(created_at, 1, 'month'), 2, 'hour') < now()` + +#### Scenario: Commuted addition +- **WHEN** a filter uses `interval(7, 'day') + order_date > current_date()` +- **THEN** it binds as `date_add(order_date, 7, 'day') > current_date()` + +#### Scenario: Stray interval rejected +- **WHEN** a filter uses `interval(7, 'day') - order_date > 0`, `-interval(7, 'day') + order_date > order_date`, or `order_date + interval(1, 'day') * 2 > order_date` +- **THEN** each is rejected with an error pointing to `date_add` + +#### Scenario: SQL interval literals equal date_add +- **WHEN** filters use `order_date + INTERVAL '7 days' > CURRENT_DATE`, `order_date + INTERVAL '7' DAY > CURRENT_DATE` and `order_date + INTERVAL 7 DAY > CURRENT_DATE` +- **THEN** each binds as `date_add(order_date, 7, 'day') > current_date()` + +#### Scenario: Multi-unit interval literal rejected +- **WHEN** a filter uses `order_date + INTERVAL '1 day 2 hours' > now()` +- **THEN** it is rejected with an error suggesting `+ INTERVAL '1 day' + INTERVAL '2 hours'` + +## ADDED Requirements + +### Requirement: SQL date literals and clock keywords +`DATE ''` SHALL be a DATE value and `TIMESTAMP ''` a TIMESTAMP value, accepting +exactly the shapes and calendar checks of ISO literals in date positions, in every position; a +malformed or impossible value SHALL be rejected with the same typed query error. `CURRENT_DATE` SHALL +mean `current_date()` and `CURRENT_TIMESTAMP` SHALL mean `now()`, sharing their result keys, SQL and +result-cache bypass. + +#### Scenario: Typed literals +- **WHEN** a filter uses `created_at >= TIMESTAMP '2024-01-01 00:00'` and a dimension uses `date_diff('day', DATE '2024-01-01', created_at)` +- **THEN** both bind with typed temporal literals and execute + +#### Scenario: Impossible typed literal +- **WHEN** a filter uses `created_at >= DATE '2024-02-30'` +- **THEN** the query is rejected with the type error naming the literal + +#### Scenario: Clock keywords +- **WHEN** one filter uses `created_at >= CURRENT_DATE` and another `created_at >= current_date()` +- **THEN** both render identical SQL and neither query is served from or stored in the result cache diff --git a/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/expression-grammar/spec.md b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/expression-grammar/spec.md new file mode 100644 index 00000000..3a69a306 --- /dev/null +++ b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/expression-grammar/spec.md @@ -0,0 +1,190 @@ +## Purpose + +Defines the Mode-B expression grammar: one SQL-compatible syntax shared by every query and model formula position — its operators and predicates, identifiers and quoting, string literals, precedence, the legacy spellings it keeps, and the forms it rejects with a hint. + +## ADDED Requirements + +### Requirement: One grammar in every Mode-B position + +Query measures, dimensions (string and object form), filters, order targets, `ModelMeasure.formula` and `ModelExtension.measures` SHALL all be parsed by the same grammar: any construct accepted in one of these positions SHALL be accepted syntactically in all of them, and whether it is legal there SHALL be decided only by typing (field / measure / position rules), never by the position's parser. + +#### Scenario: SQL predicate spellings in a measure + +- **WHEN** a query on `orders` has measure `sum(iif(status = 'paid' AND amount IS NOT NULL, amount, 0))` +- **THEN** it parses, binds and returns the paid total — the same value as `sum(iif(status == 'paid' and amount is not None, amount, 0))` + +#### Scenario: Same text in filter, measure and order + +- **WHEN** `amount BETWEEN 10 AND 20` is used as a filter, inside `sum(iif(amount BETWEEN 10 AND 20, 1, 0))` as a measure, and `iif(amount BETWEEN 10 AND 20, 0, 1)` as an order target on a raw-rows query +- **THEN** each parses and evaluates the same predicate + +### Requirement: SQL operators and predicates + +The grammar SHALL accept, with keywords matched case-insensitively: + +- comparison `=`, `<>`, `!=`, `==`, `<`, `<=`, `>`, `>=`; +- `IS [NOT] NULL`, `IS [NOT] DISTINCT FROM`, `IS [NOT] TRUE`, `IS [NOT] FALSE`; +- `[NOT] IN (, …)` and `[NOT] IN [, …]`; +- `[NOT] BETWEEN a AND b`, meaning `x >= a AND x <= b` (negated: `NOT (x >= a AND x <= b)`); +- `AND`, `OR`, `NOT`; +- `||` string concatenation; +- arithmetic `+`, `-`, `*`, `/`, `%` (the result's sign is the dividend's on every dialect), `**` (power) and unary `-`. + +`IS [NOT] DISTINCT FROM` SHALL be a null-safe comparison on every dialect. + +#### Scenario: BETWEEN and NOT BETWEEN + +- **WHEN** a query on `orders` filters `amount BETWEEN 10 AND 20`, and another filters `amount NOT BETWEEN 10 AND 20` +- **THEN** the first keeps exactly the rows with `10 <= amount <= 20` and the second exactly the rows with a non-NULL amount outside that range + +#### Scenario: Null-safe comparison + +- **WHEN** a query filters `region IS NOT DISTINCT FROM NULL` +- **THEN** it keeps exactly the rows whose region is NULL, on every Tier-1 dialect + +#### Scenario: Boolean tests + +- **WHEN** a query filters `(amount > 100) IS TRUE` and another `(amount > 100) IS NOT TRUE` +- **THEN** the second keeps the rows the first drops, including rows whose amount is NULL + +#### Scenario: Concatenation operator + +- **WHEN** a dimension is `{"expression": "status || '-' || region", "name": "tag"}` +- **THEN** each row's `tag` is the two values joined by `-`, and NULL when either is NULL + +#### Scenario: Modulo sign + +- **WHEN** a measure is `min(-7 % 3)` +- **THEN** it returns `-1` on every Tier-1 dialect + +### Requirement: LIKE and ILIKE are operators over any operand + +`x [NOT] LIKE p` and `x [NOT] ILIKE p` SHALL accept any expression on either side. `ILIKE` SHALL match case-insensitively on every dialect. + +#### Scenario: LIKE over a nested call + +- **WHEN** a query filters `lower(trim(status)) LIKE 'pa%'` +- **THEN** it parses and keeps the rows whose trimmed, lower-cased status starts with `pa` + +#### Scenario: ILIKE + +- **WHEN** a query filters `status ILIKE 'PA%'` on data holding `paid` and `Pending` +- **THEN** it keeps `paid` and drops `Pending`, on every Tier-1 dialect + +#### Scenario: LIKE inside a string literal is text + +- **WHEN** a filter is `note = 'we like cats'` +- **THEN** the literal is compared byte-for-byte and no LIKE is formed + +### Requirement: SQL-standard call forms + +The grammar SHALL accept `POSITION(a IN b)` (meaning `instr(b, a)`) and `SUBSTRING(x FROM a [FOR b])` (meaning `substr(x, a[, b])`). + +#### Scenario: SQL substring and position + +- **WHEN** a dimension is `SUBSTRING(status FROM 1 FOR 3)` and another `POSITION('a' IN status)` +- **THEN** they equal `substr(status, 1, 3)` and `instr(status, 'a')`, with the same result keys and SQL + +### Requirement: Precedence and associativity + +Operators SHALL bind, loosest first: `OR`; `AND`; `NOT`; comparison, `IS`, `IN`, `BETWEEN`, `LIKE`, `ILIKE`; `||`; `+` `-`; `*` `/` `%`; unary `-`; `**`; `::`; postfix (call, `FILTER`, `.`). `**` SHALL be right-associative and its right operand MAY carry a unary minus. Parentheses override everything. + +#### Scenario: NOT and comparison + +- **WHEN** a filter is `NOT amount > 100 AND status = 'paid'` +- **THEN** it means `(NOT (amount > 100)) AND (status = 'paid')` + +#### Scenario: Power + +- **WHEN** measures are `min(2 ** 3 ** 2)`, `min(-2 ** 2)`, `min((-2) ** 2)` and `min(2 ** -2)` +- **THEN** they return 512, -4, 4 and 0.25 + +#### Scenario: Concatenation binds tighter than comparison + +- **WHEN** a filter is `status || 'x' = 'paidx'` +- **THEN** it means `(status || 'x') = 'paidx'` + +### Requirement: Identifiers and quoted identifiers + +A bare identifier or dotted path SHALL name a column, measure, alias or join path as before. A double-quoted name (`"order total"`, `customers."Region"`) SHALL be an identifier in every identifier position, with `""` standing for one `"` inside it. A name beginning `__slayer_` SHALL be rejected, quoted or not. A double-quoted name that resolves to nothing SHALL fail with the unresolved-reference error plus a hint that strings use single quotes. + +#### Scenario: Quoted column name + +- **WHEN** model `orders` has a column named `order total` and a query has measure `sum("order total")` +- **THEN** it sums that column + +#### Scenario: Quoted segment in a dotted path + +- **WHEN** a query on `orders` has dimension `customers."Region"` and `customers` has a column named `Region` +- **THEN** it groups by that column + +#### Scenario: Double-quoted string hint + +- **WHEN** a query on `orders` filters `status = "paid"` and no column or alias is named `paid` +- **THEN** it fails with the unresolved-reference error naming `paid` and stating that strings use single quotes + +### Requirement: String literals follow SQL rules + +A string literal SHALL be single-quoted, with `''` standing for one `'`; a backslash SHALL be an ordinary character. + +#### Scenario: Doubled quote + +- **WHEN** a query filters `customer_name = 'O''Brien'` +- **THEN** it compares against `O'Brien` + +#### Scenario: Backslash is literal + +- **WHEN** a query filters `path = 'C:\temp'` +- **THEN** it compares against the six characters `C:\temp` + +### Requirement: Legacy spellings keep their meaning + +`==`, `!=`, `is None`, `is not None`, `True`, `False`, `None`, `[…]` lists after `in`, the method-call aggregation form (`customers.sum(revenue)`) and the colon aggregation form (`revenue:sum`) SHALL keep their meaning, result keys and SQL. Every expression that parsed before this change and contains none of the forms this change removes SHALL bind to the same value with byte-identical generated SQL. + +#### Scenario: Python spellings + +- **WHEN** a filter is `status == 'paid' and region is not None` and another is `status = 'paid' AND region IS NOT NULL` +- **THEN** both render identical SQL + +#### Scenario: Pre-existing expressions are unchanged + +- **WHEN** the SQL golden baselines committed before this change are regenerated +- **THEN** they are byte-identical + +### Requirement: Keyword arguments versus comparison + +Inside an aggregation, a transform or an unknown call, `name = value` directly after `(` or `,` SHALL be a keyword argument. Inside a scalar function call and everywhere else, `=` SHALL be a comparison. + +#### Scenario: Keyword argument + +- **WHEN** a measure is `percentile(amount, p=0.9)` +- **THEN** `p` is a keyword argument + +#### Scenario: Comparison inside a scalar call + +- **WHEN** a measure is `sum(iif(status = 'paid', amount, 0))` +- **THEN** `status = 'paid'` is the condition of `iif` + +### Requirement: Forms rejected with a hint + +The following SHALL fail at parse time with an error naming the form and the remedy: `a | b` (use `||`); `a // b` (use `div(a, b)` or `floor(a / b)`); `TRIM(BOTH | LEADING | TRAILING …)`; a scalar `min(a, b)` / `max(a, b)` with two positional values (use `greatest` / `least`); `TIME '…'` literals; `SIMILAR TO`; `REGEXP` / `RLIKE` operators; raw `OVER (…)`. A syntax error SHALL report the character position where parsing failed. + +#### Scenario: Bitwise-or spelling + +- **WHEN** a measure is `min(status | region)` +- **THEN** it fails naming `|` and suggesting `||` + +#### Scenario: Floor-division spelling + +- **WHEN** a measure is `sum(amount) // 2` +- **THEN** it fails suggesting `div(a, b)` or `floor(a / b)` + +#### Scenario: Two-argument scalar max + +- **WHEN** a dimension is `max(amount, 10)` +- **THEN** it fails suggesting `greatest(amount, 10)` + +#### Scenario: Syntax error position + +- **WHEN** a filter is `amount > > 5` +- **THEN** the error names the position of the second `>` diff --git a/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/expression-keywords/spec.md b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/expression-keywords/spec.md new file mode 100644 index 00000000..950d8a1b --- /dev/null +++ b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/expression-keywords/spec.md @@ -0,0 +1,44 @@ +## MODIFIED Requirements + +### Requirement: CASE WHEN lowering + +A Mode-B expression SHALL accept SQL `CASE … END` conditionals — searched +(`CASE WHEN cond THEN val … [ELSE val] END`) and simple +(`CASE operand WHEN val THEN val … [ELSE val] END`) — lowering them to nested +`iif(cond, then, otherwise)` calls with `None` as the default otherwise. WHEN +conditions, THEN values and ELSE values SHALL accept the full Mode-B grammar in every +expression position, including measures. A `CASE` that is recognized as a conditional +(a `WHEN` follows it) but is malformed MUST raise a specific malformed-CASE error naming +the defect. + +#### Scenario: searched CASE lowers to iif + +- WHEN `CASE WHEN amount > 100 THEN 'big' ELSE 'small' END` is parsed as a Mode-B expression +- THEN it parses as `iif(amount > 100, 'big', 'small')` — a scalar call, usable wherever a scalar expression is legal + +#### Scenario: simple CASE compares the operand per branch + +- WHEN `CASE status WHEN 'a' THEN 1 WHEN 'b' THEN 2 END` is parsed +- THEN it parses as `iif(status == 'a', 1, iif(status == 'b', 2, None))` + +#### Scenario: nested CASE in THEN and ELSE values + +- WHEN a THEN or ELSE value itself contains a complete `CASE … END` +- THEN the nested conditional is lowered recursively and the enclosing branches are unaffected + +#### Scenario: SQL operator spellings inside WHEN conditions + +- WHEN `CASE WHEN region = 'EU' AND amount IS NOT NULL THEN 1 ELSE 0 END` appears in a measure formula +- THEN the WHEN condition parses with the same meaning as `region == 'EU' and amount is not None` + +#### Scenario: recognized-but-malformed CASE still errors specifically + +- WHEN `CASE WHEN a THEN 1` (missing END) or `CASE WHEN a 1 END` (missing THEN) is parsed +- THEN a malformed-CASE error is raised naming the missing keyword, not a generic syntax error + +## REMOVED Requirements + +### Requirement: LIKE operator rewriting + +**Reason**: LIKE is no longer a filter-only text rewrite to a `like()` call over a restricted left side; the grammar's `[NOT] LIKE` / `[NOT] ILIKE` operators accept any operand in every position, string literals follow SQL rules, and a double-quoted operand is an identifier. +**Migration**: See `queries/expression-grammar` › "LIKE and ILIKE are operators over any operand" and "String literals follow SQL rules". Stored formulas are upgraded on load (`models/stored-upgrade`). diff --git a/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/positions/spec.md b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/positions/spec.md new file mode 100644 index 00000000..13c12a92 --- /dev/null +++ b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/positions/spec.md @@ -0,0 +1,24 @@ +## ADDED Requirements + +### Requirement: Order items carry an explicit NULL placement +An order item SHALL accept `nulls` set to `first` or `last`. With it, the sorted rows SHALL place NULL values first or last on every Tier-1 dialect, SQL Server included, for every kind of order target — a projected field, a hidden filter- or order-only value, a measure, a transform or an expression. Without it, NULL placement SHALL be unchanged. `nulls` SHALL survive serialisation, saved-query refinement and the stage rewrites that rebuild order items. + +#### Scenario: NULLS FIRST on a descending order +- **WHEN** a raw-rows query on `orders` orders by `{"column": "discount", "direction": "desc", "nulls": "first"}` +- **THEN** the rows with a NULL discount come first, then the rest by discount descending, on every Tier-1 dialect + +#### Scenario: NULLS LAST on a hidden aggregate +- **WHEN** a query groups by `region` and orders by `{"column": "max(discount)", "direction": "asc", "nulls": "last"}` without projecting `max(discount)` +- **THEN** regions whose `max(discount)` is NULL come last, and the hidden value is not returned + +#### Scenario: SQL Server honours the policy over an aliased target +- **WHEN** the same two queries run on SQL Server +- **THEN** they return the same row order as on Postgres + +#### Scenario: Refinement keeps the policy +- **WHEN** a saved query ordering with `"nulls": "first"` is run with a refinement adding a filter +- **THEN** the refined result keeps NULLs first + +#### Scenario: Invalid policy +- **WHEN** an order item has `"nulls": "middle"` +- **THEN** the query is rejected at construction naming the allowed values diff --git a/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/variables/spec.md b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/variables/spec.md new file mode 100644 index 00000000..1c3e7cd5 --- /dev/null +++ b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/variables/spec.md @@ -0,0 +1,75 @@ +## MODIFIED Requirements + +### Requirement: Every Mode-B formula surface substitutes variables + +`{var}` placeholders in a query's filters, measure formulas, computed-dimension expressions (string and object form) and order expressions SHALL be bound from the query's effective variables as literal values, never as text spliced into the expression. A placeholder inside a string literal, wholly or partly (`'{status}'`, `'%{q}%'`), SHALL contribute its value's text to that literal; a placeholder in a bare literal position SHALL take a number or boolean value, a numeric string reading as its number; a placeholder forming the whole parenthesised list of `IN` SHALL take a list value. `{{` and `}}` inside a string literal SHALL render literal braces. A query carrying placeholders on these surfaces SHALL construct successfully and SHALL execute with the bound values. + +#### Scenario: Measure formula + +- **WHEN** a query on `orders` has measure `sum(amount) * {k} / 100` and `variables={"k": 10}` +- **THEN** it executes and returns the total amount multiplied by 10 and divided by 100, by executed values on SQLite and DuckDB + +#### Scenario: Computed dimension in string and object form + +- **WHEN** a query has dimension `"amount * {k}"`, or dimension `{"expression": "amount * {k}"}`, with `variables={"k": 10}` +- **THEN** both construct and group by `amount * 10` + +#### Scenario: Order expression in colon and call form + +- **WHEN** a query orders by `sum(amount) * {k}` descending, with `variables={"k": -1}` +- **THEN** both construct and the rows come back in ascending total-amount order + +#### Scenario: Placeholder in operand positions + +- **WHEN** a placeholder appears as an aggregation argument (`sum({k})`), a transform argument, a scalar-function argument, a CASE branch, or the whole right-hand side of `in` (`region in ({regions})` with a list value) +- **THEN** the query constructs and executes with the substituted value + +#### Scenario: Quoted string value is escaped + +- **WHEN** a measure contains `'{name}'` inside a quoted literal and `variables={"name": "O'Brien"}` +- **THEN** the literal's value is `O'Brien` and the query executes without a parse error + +### Requirement: Saved measure formulas substitute on the models whose SQL surfaces substitute + +A saved `ModelMeasure.formula` SHALL bind `{var}` placeholders as literal values, by the same rules as query formula surfaces, on exactly the models whose Mode-A surfaces substitute (the query's direct source model and each stage's own source model), using that model's effective variables. Its placeholders SHALL be reported by model variable inspection as required unless the model defaults them. A saved measure reached through a join or a cross-model reference SHALL NOT substitute, and a placeholder in it SHALL raise the unresolved-placeholder error. + +#### Scenario: Saved measure on the source model + +- **WHEN** model `orders` has saved measure `amt_scaled` with formula `sum(amount) * {k}` and a query requests `amt_scaled` with `variables={"k": 10}` +- **THEN** it returns ten times the total amount + +#### Scenario: Saved measure through a join + +- **WHEN** model `customers`, joined from `orders`, has saved measure `spend_scaled` with formula `sum(spend) * {k}` and a query on `orders` requests `customers.spend_scaled` with `variables={"k": 10}` +- **THEN** the query fails with the unresolved-placeholder error naming `{k}` + +#### Scenario: Inspection lists saved-measure variables + +- **WHEN** model `orders` has saved measure formula `sum(amount) * {k}` and no default for `k` +- **THEN** model variable inspection reports `k` as required + +## ADDED Requirements + +### Requirement: Variables supply literal values only + +A placeholder in any other position — an identifier or path segment (`{col}:sum`, `t.{c}`), a function name (`{f}(x)`), a keyword argument's name, or a value that must be a reference (`partition_by={x}`) — SHALL be rejected with an error stating that variables supply literal values. A string value bound into a bare literal position SHALL be rejected with the same error plus a hint to put the placeholder inside quotes. No variable value SHALL ever change the structure of the expression it is bound into. + +#### Scenario: Identifier injection rejected + +- **WHEN** a query on `orders` has measure `sum({col})` with `variables={"col": "amount"}` +- **THEN** it fails with the literal-values error and the hint to quote the placeholder + +#### Scenario: Function and path positions rejected + +- **WHEN** a measure is `{f}(amount)`, or a dimension is `customers.{c}`, or a measure is `sum(amount, partition_by={x})` +- **THEN** each fails with the literal-values error + +#### Scenario: Numeric string in a bare position + +- **WHEN** a filter is `amount > {min}` with `variables={"min": "10"}` +- **THEN** it keeps the rows with amount above 10 + +#### Scenario: Hostile string stays a literal + +- **WHEN** a filter is `status = '{s}'` with `variables={"s": "x' OR '1' = '1"}` +- **THEN** it keeps only the rows whose status equals that whole string — none, on data without it diff --git a/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/tasks.md b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/tasks.md new file mode 100644 index 00000000..97f65651 --- /dev/null +++ b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/tasks.md @@ -0,0 +1,80 @@ +## 1. Tracker records (pr-plan) + +- [x] 1.1 DEV-2101 body: point to this change and list the scope added in planning (casts with `TRY_CAST`, `FILTER` row semantics, placeholder binding, DEV-1831 absorption, split into DEV-2119 / DEV-2120); verify by reading the issue +- [x] 1.2 Comments (not body edits): DEV-2021 (re-scope: remove query-time Mode A for every caller after DEV-2102; `ModelExtension.columns` deleted outright, replaced by Mode-B expressions via DEV-2119 / DEV-2120; inline-model removal waits on DEV-1768 stages), DEV-1768 (EXTEND column items die with DEV-2021; `MODEL … SOURCE (raw SQL)` waits on it; EXTEND / MODEL items stay Mode A until then), DEV-2044 (Mode-B half absorbed here, Mode-A half stays), DEV-1831 (absorbed here), DEV-1320 (absorbed by DEV-2120), DEV-2034 (Storyline must port `storyline/api/adapters/models.py` from `parse_formula` / `FieldSpec` to `parse_expr` + `walk_parsed_refs`), DEV-2043 (eventual replacement for Storyline-side formula validation), DEV-1370 (superseded: one parser, no `FieldSpec`); verify each comment exists + +## 2. Failing tests (pr-tests) + +- [ ] 2.1 Grammar tests for every `queries/expression-grammar` scenario, plus the ambiguity table: `f(x=1)` kwarg vs `=` in scalar calls, `x::t` vs `x:sum`, `a::int::text`, predicates inside transform arguments, `"x"` identifier, `CASE` only before `WHEN`, `date '…'` vs a column named `date`, `**` cases `2**3**2`, `-2**2`, `(-2)**2`, `2**-2`; verify they fail on the current parser +- [ ] 2.2 SQL-form-in-every-position matrix: each form of the proposal as measure, dimension, filter and order target, executed on SQLite and DuckDB, emission-checked on every Tier-1 dialect; verify failing +- [ ] 2.3 `queries/casts` scenarios, including the Postgres 15 / 16 / unknown-version emissions (fake server profile), SQLite `'12abc'`, MySQL emission, and a concrete type a dialect cannot express; verify failing +- [ ] 2.4 `aggregations/functional-form` FILTER scenarios (count, sum ≡ masked sum, weighted_avg numerator and denominator, `last` newest passing row, `corr` / `covar_*` pairs, custom aggregation, rejected aggregate inside `c`, with `partition_by` and `window`); verify failing +- [ ] 2.5 `queries/variables` scenarios incl. injection rejections, numeric-string binding, hostile string, template-derived names for measures and computed dimensions with different runtime values, `date_range` bounds (`{start}`, `{end}`, one-sided, missing, non-string); verify failing +- [ ] 2.6 `queries/date-functions` scenarios (interval literal forms, multi-unit rejection, typed literals, clock keywords and cache bypass); verify failing +- [ ] 2.7 `queries/positions` NULL-placement scenarios on every Tier-1 dialect (SQL Server and Postgres in their integration suites), incl. hidden targets, bracketed aliases, refinement and stage rewrites; verify failing +- [ ] 2.8 `models/stored-upgrade` scenarios: tokenizer migration cases (embedded `"` and `'`, backslashes, control escapes, Unicode, adjacent literals, malformed text, `|`), Mode-A untouched, unversioned client query untouched, 0.10.2 corpus still green; verify failing +- [ ] 2.9 Importer dependency check over `ParsedExpr`: transitive dangling reference and cyclic saved measures for dbt, Cube and OSI; verify failing where behaviour is new, green where it is preserved +- [ ] 2.10 Regression: `sum(a) ** 2` executes (today `NotImplementedError`); `%` sign per dialect; verify failing +- [ ] 2.11 Parser benchmark over the longest formulas in the test suite and docs vs the old parser (recorded, not gating); verify it runs +- [ ] 2.12 Retarget, with the user's approval already given, the tests asserting on deleted helpers to equivalent `parse_expr` / `ParsedExpr` assertions: `tests/test_dev1833_keyword_lexing.py`, `tests/test_dev1450fix_group2_correctness.py`, `tests/test_dev1740_case_rewrite.py`, `tests/test_dev1743_validators.py` (pre-pass helpers `_rewrite_case_when`, `_rewrite_sql_like`, `_preprocess_colons`, `_normalize_sql_filter_operators`), and `tests/test_formula.py`, `tests/test_functional_remedies.py`, `tests/test_dev1576_heals.py`, `tests/test_dev1737_scalar_specs.py`, `tests/test_rank_direction.py` (`parse_formula` / `FieldSpec`); flip tests asserting `"abc"` is a string to the identifier reading; list each changed test in the commit message +- [ ] 2.13 Codex review of the tests against this change (when `reviewers.codex` is on) + +## 3. Architecture move + +- [ ] 3.1 Move `slayer/engine/syntax.py` → `slayer/core/syntax.py` with `la:deterministic-refactor`; move `ALL_TRANSFORMS` / `TIME_TRANSFORMS` / `TIMELESS_TRANSFORMS` to `slayer/core/enums.py`; verify `la-typecheck`, `dr-mock-lint` and the full unit suite +- [ ] 3.2 Apply the approved harness edits H1–H6 exactly as in the "Approved harness edits" section below, then `la-arch-diagrams` (H7); verify `la-arch-check` and `npx -y likec4@1.47.0 validate architecture` + +## 4. Grammar + +- [ ] 4.1 Add `pyparsing` to `pyproject.toml` / `poetry.lock`; verify `poetry install -E all` +- [ ] 4.2 Implement the grammar (design D2) producing the existing `ParsedExpr` nodes plus `Cast` and the `FILTER` predicate; delete `ast.parse`, every pre-pass and `parse_filter_expr`; repoint every caller (`core/query.py`, `engine/bind_inputs.py`, `binding.py`, `population.py`, `model_reads.py`, `schema_drift.py`, `unrelated.py`, `query_engine.py`, `memories/resolver.py`); verify 2.1 / 2.2 parse-level tests and the SQL golden baselines byte-identical +- [ ] 4.3 Quoted identifiers end to end and the single-quote hint in the unresolved-reference error; verify the expression-grammar identifier scenarios +- [ ] 4.4 Rejections with hints (`|`, `//`, `TRIM(BOTH…)`, two-argument scalar `min` / `max`, `TIME '…'`, `SIMILAR TO`, `REGEXP` / `RLIKE`, raw `OVER`) and syntax-error positions; verify their scenarios + +## 5. Keys, binding, rendering + +- [ ] 5.1 `CastKey` (design D3): traversal protocol, binding, checker typing, render via `exp.Cast` with the author's type and the dialect-expressibility check; verify the CAST scenarios +- [ ] 5.2 `TRY_CAST` lowering per dialect, SQLite `slayer_try_cast` registration, MySQL guard, Postgres server profile (`server_profile_sql`, `parse_server_profile`) and render-time profile plumbing; verify 2.3 +- [ ] 5.3 `FILTER (WHERE c)` binding (row-level check) and lowering by masking every row-level input incl. the `first` / `last` ranking key; verify 2.4 +- [ ] 5.4 `BETWEEN`, `IS [NOT] DISTINCT FROM`, `IS [NOT] TRUE|FALSE`, `ILIKE`, `||`, `POSITION`, `SUBSTRING FROM FOR`, interval literals, typed literals, clock keywords; `**` → `pow`; verify 2.2, 2.6, 2.10 +- [ ] 5.5 `OrderItem.nulls` plumbing (design D7) and the multi-term `build_ordered` with the T-SQL emulation over the resolved expression; verify 2.7 + +## 6. Variables + +- [ ] 6.1 Offset-preserving placeholder masking, the three grammar slots, binding into the tree at the existing substitution points, names from the written text, `date_range` literal-only substitution; delete the Mode-B `escape="python"` path and `ComputedDimension.substituted`'s text splice; verify 2.5 + +## 7. Migration + +- [ ] 7.1 Stored-only steps SlayerModel 15 → 16, SlayerQuery 6 → 7, Memory 4 → 5 (design D6), ordered before the legacy load-path repairs; verify 2.8 and the 0.10.2 corpus + +## 8. One parser (DEV-1831) + +- [ ] 8.1 Port the dbt, Cube and OSI importers to `parse_expr` with the shared namespace-aware dependency walk; delete `parse_formula`, `FieldSpec`, `_rewrite_funcstyle_aggregations`, `_preprocess_agg_refs`, `_expand_named_measures`; verify 2.9 and the importer fixture suites +- [ ] 8.2 Check the Mode-B text the facade translator and the importers emit against the new grammar (single quotes, no backslash escapes, no `|`); fix emitters; verify the facade and importer suites + +## 9. Docs and agent surfaces + +- [ ] 9.1 `docs/concepts/formulas.md` (Parsing Internals, operators incl. `**` / `%`, FILTER, the CASE section), `references.md` (two-mode table, string and identifier rules, casts and `TRY_CAST`, interval and typed literals), `queries.md` (filters operator table, Variables / "Trusted input", OrderItem `nulls`), `time.md` if touched; `zensical.toml` only if a page is added; verify `grep` finds no "Python AST" / "Python-AST" claims and no double-quoted string examples in Mode-B docs +- [ ] 9.2 MCP `Field` descriptions in `slayer/core/query.py` / `slayer/mcp/server.py` and `slayer/memories/help_content/*` (strings single-quoted, SQL forms, `nulls`); verify `tests/test_mcp_surface_budget.py` and the documented-idioms tests +- [ ] 9.3 Release-notes draft (untracked, never staged): double-quoted strings, backslash escapes, `|` / `//`, `{var}` literal-only, `parse_formula` removal, `CAST` / `TRY_CAST`, Postgres < 16 `TRY_CAST` + +## 10. Gates + +- [ ] 10.1 `poetry run pytest -m "not integration"`, the CI integration invocation, `poetry run ruff check slayer/ tests/`, `poetry run basedpyright` (no new errors vs baseline), `la-arch-check`; verify all green + +## Approved harness edits (exact text, approved 2026-10-20) + +H1 `architecture/model.c4`: add `syntax = node 'Syntax'` under `core` after `models = node 'Models'`; remove `syntax = node 'Syntax'` from `engine`; remove `core.query -> engine.syntax #legacy`; add `core.query -> core.syntax` after `core.query -> core.models`; `engine.binding -> engine.syntax` → `engine.binding -> core.syntax`; `engine.bind_inputs -> engine.syntax` → `engine.bind_inputs -> core.syntax`. + +H2 `architecture/index.yaml`: `legacy_arrows: {baseline: 8}` → `legacy_arrows: {baseline: 7}`. + +H3 `architecture/system.arc42.md` principle 2: the door list `(`core.query`/`core.models` → `engine.syntax`, `sql.*`, `storage.migrations`)` → `(`core.query`/`core.models` → `sql.*`, `storage.migrations`)`. + +H4 `architecture/system.arc42.md` principle 10 becomes: +"10. **Two expression layers**: Mode A free SQL (`Column.sql`, model `filters`) vs Mode B, a SQL-compatible grammar (formulas, query fields) — one canonical function library, extended never forked. [review] Every Mode B construct means the same on every dialect. [target: DEV-2120]" + +H5 `architecture/core.arc42.md`: §1 door list as in H3; §2 "Pydantic domain models alongside." → "Pydantic domain models alongside; the Mode B grammar: `syntax.py`." + +H6 `architecture/engine.arc42.md`: §2 "`normalization.py` → `syntax.py` (parse)" → "`normalization.py` → `core/syntax.py` (parse)"; principle 9 becomes "**The term-interface boundary**: `binding`, `bind_inputs`, `elaborate`, `elaborate_env` (THE checker), `compile` and `plan` are declared children with only the modeled arrows among them — the compiler consults the checker, never `core.syntax` or binding; every user-facing algebra type error raises in the checker, as a `QueryTypeError`." (status tags unchanged). + +H7: regenerate the embedded mermaid with `la-arch-diagrams`. From abe10ba5a6122e36d26a1cccf6fb2dfe80752fc6 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Wed, 21 Oct 2026 12:36:46 +0200 Subject: [PATCH 02/17] Add the failing test suite for the Mode-B SQL-compatible grammar New tests (fail until implementation): grammar parse, SQL forms in every position (SQLite/DuckDB executed, Tier-1 emission), CAST/TRY_CAST incl. Postgres 15/16/unknown emissions, cast typing, FILTER (WHERE c), variables as literals, interval/typed literals and clock keywords, NULLS FIRST/LAST, stored-text migration, importer dependency checks, parser benchmark, and Postgres / MySQL / SQL Server integration cases. Retargeted off deleted helpers / flipped to the new string rules: - test_dev1833_keyword_lexing: _rewrite_sql_like checks -> parse_expr; SQL string escapes; "p%" is an identifier; fused-Unicode LIKE operands parse whole - test_dev1740_case_rewrite: _rewrite_case_when -> parse_expr equivalences - test_dev1450fix_group2_correctness: double-quoted/backslash literals -> '' - test_dev1743_validators: stale _preprocess_colons comments trimmed - test_formula: _rewrite_funcstyle_aggregations / parse_formula -> parse_expr - test_functional_remedies: remedy asserted through the dbt dependency check - test_dev1576_heals: colon healing and arity at binding; ndigits rejection and function-style alias healing at binding (ruling 2026-10-21) - test_dev1737_scalar_specs, test_rank_direction: parse_formula -> parse_expr / binder - test_syntax: "hello" is an identifier - test_variables_planner: SQL quoting of substituted text (ruling 2026-10-21) - test_mode_a_variable_substitution: escape="python" tests deleted - parse_filter_expr -> parse_expr in test_translator, test_dev1475, test_dev1737_date_functions_parse, test_heal_date_parts, test_heal_functional_aggregations, test_mode_b_placeholder_parse, test_schema_drift_typed, test_sql_boolean_literal_filters - test_osi_converter: emitted formula validated by the new grammar --- .../tasks.md | 27 +- tests/_dev2101_fixtures.py | 427 ++++++++++++++++++ tests/facade/test_translator.py | 4 +- tests/integration/test_dev2101_postgres.py | 86 ++++ tests/integration/test_integration_mysql.py | 46 ++ .../integration/test_integration_sqlserver.py | 47 ++ tests/perf/dev2101_old_parser_baseline.json | 14 + tests/perf/test_dev2101_parser_bench.py | 23 + tests/test_dev1450fix_group2_correctness.py | 39 +- tests/test_dev1475_in_tuple_filters.py | 24 +- tests/test_dev1576_heals.py | 212 ++++----- tests/test_dev1737_date_functions_parse.py | 21 +- tests/test_dev1737_scalar_specs.py | 21 +- tests/test_dev1740_case_rewrite.py | 43 +- tests/test_dev1743_validators.py | 4 - tests/test_dev1833_keyword_lexing.py | 70 ++- tests/test_dev2101_agg_filter.py | 171 +++++++ tests/test_dev2101_cast_typing.py | 76 ++++ tests/test_dev2101_casts.py | 243 ++++++++++ tests/test_dev2101_date_literals.py | 134 ++++++ tests/test_dev2101_grammar_parse.py | 386 ++++++++++++++++ tests/test_dev2101_importer_dependencies.py | 100 ++++ tests/test_dev2101_null_ordering.py | 125 +++++ tests/test_dev2101_sql_forms.py | 254 +++++++++++ tests/test_dev2101_stored_upgrade.py | 198 ++++++++ tests/test_dev2101_variables.py | 156 +++++++ tests/test_formula.py | 259 +++-------- tests/test_functional_remedies.py | 19 +- tests/test_heal_date_parts.py | 12 +- tests/test_heal_functional_aggregations.py | 8 +- tests/test_mode_a_variable_substitution.py | 335 +------------- tests/test_mode_b_placeholder_parse.py | 3 +- tests/test_osi_converter.py | 14 + tests/test_rank_direction.py | 28 +- tests/test_schema_drift_typed.py | 4 +- tests/test_sql_boolean_literal_filters.py | 8 +- tests/test_syntax.py | 63 ++- tests/test_variables_planner.py | 27 +- 38 files changed, 2864 insertions(+), 867 deletions(-) create mode 100644 tests/_dev2101_fixtures.py create mode 100644 tests/integration/test_dev2101_postgres.py create mode 100644 tests/perf/dev2101_old_parser_baseline.json create mode 100644 tests/perf/test_dev2101_parser_bench.py create mode 100644 tests/test_dev2101_agg_filter.py create mode 100644 tests/test_dev2101_cast_typing.py create mode 100644 tests/test_dev2101_casts.py create mode 100644 tests/test_dev2101_date_literals.py create mode 100644 tests/test_dev2101_grammar_parse.py create mode 100644 tests/test_dev2101_importer_dependencies.py create mode 100644 tests/test_dev2101_null_ordering.py create mode 100644 tests/test_dev2101_sql_forms.py create mode 100644 tests/test_dev2101_stored_upgrade.py create mode 100644 tests/test_dev2101_variables.py diff --git a/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/tasks.md b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/tasks.md index 97f65651..497269be 100644 --- a/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/tasks.md +++ b/openspec/changes/dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/tasks.md @@ -5,19 +5,19 @@ ## 2. Failing tests (pr-tests) -- [ ] 2.1 Grammar tests for every `queries/expression-grammar` scenario, plus the ambiguity table: `f(x=1)` kwarg vs `=` in scalar calls, `x::t` vs `x:sum`, `a::int::text`, predicates inside transform arguments, `"x"` identifier, `CASE` only before `WHEN`, `date '…'` vs a column named `date`, `**` cases `2**3**2`, `-2**2`, `(-2)**2`, `2**-2`; verify they fail on the current parser -- [ ] 2.2 SQL-form-in-every-position matrix: each form of the proposal as measure, dimension, filter and order target, executed on SQLite and DuckDB, emission-checked on every Tier-1 dialect; verify failing -- [ ] 2.3 `queries/casts` scenarios, including the Postgres 15 / 16 / unknown-version emissions (fake server profile), SQLite `'12abc'`, MySQL emission, and a concrete type a dialect cannot express; verify failing -- [ ] 2.4 `aggregations/functional-form` FILTER scenarios (count, sum ≡ masked sum, weighted_avg numerator and denominator, `last` newest passing row, `corr` / `covar_*` pairs, custom aggregation, rejected aggregate inside `c`, with `partition_by` and `window`); verify failing -- [ ] 2.5 `queries/variables` scenarios incl. injection rejections, numeric-string binding, hostile string, template-derived names for measures and computed dimensions with different runtime values, `date_range` bounds (`{start}`, `{end}`, one-sided, missing, non-string); verify failing -- [ ] 2.6 `queries/date-functions` scenarios (interval literal forms, multi-unit rejection, typed literals, clock keywords and cache bypass); verify failing -- [ ] 2.7 `queries/positions` NULL-placement scenarios on every Tier-1 dialect (SQL Server and Postgres in their integration suites), incl. hidden targets, bracketed aliases, refinement and stage rewrites; verify failing -- [ ] 2.8 `models/stored-upgrade` scenarios: tokenizer migration cases (embedded `"` and `'`, backslashes, control escapes, Unicode, adjacent literals, malformed text, `|`), Mode-A untouched, unversioned client query untouched, 0.10.2 corpus still green; verify failing -- [ ] 2.9 Importer dependency check over `ParsedExpr`: transitive dangling reference and cyclic saved measures for dbt, Cube and OSI; verify failing where behaviour is new, green where it is preserved -- [ ] 2.10 Regression: `sum(a) ** 2` executes (today `NotImplementedError`); `%` sign per dialect; verify failing -- [ ] 2.11 Parser benchmark over the longest formulas in the test suite and docs vs the old parser (recorded, not gating); verify it runs -- [ ] 2.12 Retarget, with the user's approval already given, the tests asserting on deleted helpers to equivalent `parse_expr` / `ParsedExpr` assertions: `tests/test_dev1833_keyword_lexing.py`, `tests/test_dev1450fix_group2_correctness.py`, `tests/test_dev1740_case_rewrite.py`, `tests/test_dev1743_validators.py` (pre-pass helpers `_rewrite_case_when`, `_rewrite_sql_like`, `_preprocess_colons`, `_normalize_sql_filter_operators`), and `tests/test_formula.py`, `tests/test_functional_remedies.py`, `tests/test_dev1576_heals.py`, `tests/test_dev1737_scalar_specs.py`, `tests/test_rank_direction.py` (`parse_formula` / `FieldSpec`); flip tests asserting `"abc"` is a string to the identifier reading; list each changed test in the commit message -- [ ] 2.13 Codex review of the tests against this change (when `reviewers.codex` is on) +- [x] 2.1 Grammar tests for every `queries/expression-grammar` scenario, plus the ambiguity table: `f(x=1)` kwarg vs `=` in scalar calls, `x::t` vs `x:sum`, `a::int::text`, predicates inside transform arguments, `"x"` identifier, `CASE` only before `WHEN`, `date '…'` vs a column named `date`, `**` cases `2**3**2`, `-2**2`, `(-2)**2`, `2**-2`; verify they fail on the current parser +- [x] 2.2 SQL-form-in-every-position matrix: each form of the proposal as measure, dimension, filter and order target, executed on SQLite and DuckDB, emission-checked on every Tier-1 dialect; verify failing +- [x] 2.3 `queries/casts` scenarios, including the Postgres 15 / 16 / unknown-version emissions (fake server profile), SQLite `'12abc'`, MySQL emission, and a concrete type a dialect cannot express; verify failing +- [x] 2.4 `aggregations/functional-form` FILTER scenarios (count, sum ≡ masked sum, weighted_avg numerator and denominator, `last` newest passing row, `corr` / `covar_*` pairs, custom aggregation, rejected aggregate inside `c`, with `partition_by` and `window`); verify failing +- [x] 2.5 `queries/variables` scenarios incl. injection rejections, numeric-string binding, hostile string, template-derived names for measures and computed dimensions with different runtime values, `date_range` bounds (`{start}`, `{end}`, one-sided, missing, non-string); verify failing +- [x] 2.6 `queries/date-functions` scenarios (interval literal forms, multi-unit rejection, typed literals, clock keywords and cache bypass); verify failing +- [x] 2.7 `queries/positions` NULL-placement scenarios on every Tier-1 dialect (SQL Server and Postgres in their integration suites), incl. hidden targets, bracketed aliases, refinement and stage rewrites; verify failing +- [x] 2.8 `models/stored-upgrade` scenarios: tokenizer migration cases (embedded `"` and `'`, backslashes, control escapes, Unicode, adjacent literals, malformed text, `|`), Mode-A untouched, unversioned client query untouched, 0.10.2 corpus still green; verify failing +- [x] 2.9 Importer dependency check over `ParsedExpr`: transitive dangling reference and cyclic saved measures for dbt and Cube; for OSI, the emitted formula is validated by the new grammar; verify failing where behaviour is new, green where it is preserved +- [x] 2.10 Regression: `sum(a) ** 2` executes (today `NotImplementedError`); `%` sign per dialect; verify failing +- [x] 2.11 Parser benchmark over the longest formulas in the test suite and docs vs the old parser (recorded, not gating); verify it runs +- [x] 2.12 Retarget, with the user's approval already given, the tests asserting on deleted helpers to equivalent `parse_expr` / `ParsedExpr` assertions: `tests/test_dev1833_keyword_lexing.py`, `tests/test_dev1450fix_group2_correctness.py`, `tests/test_dev1740_case_rewrite.py`, `tests/test_dev1743_validators.py` (pre-pass helpers `_rewrite_case_when`, `_rewrite_sql_like`, `_preprocess_colons`, `_normalize_sql_filter_operators`), and `tests/test_formula.py`, `tests/test_functional_remedies.py`, `tests/test_dev1576_heals.py`, `tests/test_dev1737_scalar_specs.py`, `tests/test_rank_direction.py` (`parse_formula` / `FieldSpec`); flip tests asserting `"abc"` is a string to the identifier reading; ruling 2026-10-21: `apply_variables_to_query` keeps returning substituted text, rendered from the bound tree with SQL quoting (`'O''Brien'`, `('a', 'b')`) — `tests/test_variables_planner.py` expectations switch to SQL quoting and the `escape="python"` tests in `tests/test_mode_a_variable_substitution.py` are deleted; list each changed test in the commit message +- [x] 2.13 Codex review of the tests against this change (when `reviewers.codex` is on) ## 3. Architecture move @@ -38,6 +38,7 @@ - [ ] 5.3 `FILTER (WHERE c)` binding (row-level check) and lowering by masking every row-level input incl. the `first` / `last` ranking key; verify 2.4 - [ ] 5.4 `BETWEEN`, `IS [NOT] DISTINCT FROM`, `IS [NOT] TRUE|FALSE`, `ILIKE`, `||`, `POSITION`, `SUBSTRING FROM FOR`, interval literals, typed literals, clock keywords; `**` → `pow`; verify 2.2, 2.6, 2.10 - [ ] 5.5 `OrderItem.nulls` plumbing (design D7) and the multi-term `build_ordered` with the T-SQL emulation over the resolved expression; verify 2.7 +- [ ] 5.6 The binder rejects a `round` `ndigits` that is not an integer literal (`round(x, 2.5)`, `round(x, col)`) with an error naming `round` (ruling 2026-10-21: the check moves from the deleted importer parser); verify the `tests/test_dev1576_heals.py` arity cases ## 6. Variables diff --git a/tests/_dev2101_fixtures.py b/tests/_dev2101_fixtures.py new file mode 100644 index 00000000..1c0fc282 --- /dev/null +++ b/tests/_dev2101_fixtures.py @@ -0,0 +1,427 @@ +"""Shared fixtures for the Mode-B SQL-compatible grammar (spec: queries/expression-grammar and siblings). + +Underscore-prefixed so pytest skips collection. ``orders.customer_id → customers.id`` is many-to-one. + +orders (id, customer_id, region, city, status, amount, discount, price, quantity, + created_at, order_date, created_txt, code, customer_name, note, path, "order total"): + 1 1 EU Paris paid 10 NULL 2 1 2024-01-05 10:00 2024-01-05 2024-12-25 '12' O'Brien we like cats C:\\temp 100 + 2 1 EU Berlin paid 20 5 3 2 2024-01-20 11:00 2024-01-20 2024-06-01 'abc' Smith we LIKE dogs C:/temp 200 + 3 2 US NYC Pending 30 1 4 3 2024-02-10 12:00 2024-02-10 2024-02-29 '12abc' Jones n/a D:\\x 300 + 4 2 US LA paid 40 NULL 5 2 2024-02-25 09:00 2024-02-25 2023-01-01 '7' Brown NULL NULL 400 + 5 3 NULL Rome void NULL 2 1 4 2024-03-15 08:00 2024-03-15 NULL NULL Lee x y NULL +customers (id, name, "Region", tier): 1 A North gold | 2 B South silver | 3 C NULL gold +balances (id, customer_id, balance, updated_at, status): + 1 1 100 2024-01-01 active | 2 1 150 2024-02-01 active | 3 1 999 2024-03-01 inactive + 4 2 50 2024-01-15 active | 5 2 70 2024-02-15 inactive | 6 2 80 2024-03-15 active + +Paid rows (1, 2, 4): count by region EU 2 / US 1 / NULL 0; sum(amount) EU 30 / US 40 / NULL NULL. +weighted_avg(price, weight=quantity) over paid rows = (2 + 6 + 10) / (1 + 2 + 2) = 3.6. +Newest active balance: customer 1 → 150, customer 2 → 80. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator, Iterable +from typing import Any + +import pytest + +from slayer.core.enums import DataType +from slayer.core.models import Aggregation, AggregationParam, Column, ModelJoin, ModelMeasure, SlayerModel +from slayer.core.query import SlayerQuery +from slayer.engine.query_engine import SlayerQueryEngine, SlayerResponse +from slayer.storage.sqlite_conn import transaction + +from tests._engine_helpers import _engine_generate, seeded_exec_engine + +#: Tier-1 dialects whose emission every SQL form must survive. +TIER1 = ["sqlite", "postgres", "duckdb", "mysql", "clickhouse", "tsql", "trino", "snowflake", "bigquery"] +EXEC_BACKENDS = ["sqlite", "duckdb"] +ALL_IDS = frozenset({1, 2, 3, 4, 5}) + + +# --------------------------------------------------------------------------- # +# Models +# --------------------------------------------------------------------------- # +def orders_model(*, data_source: str = "test") -> SlayerModel: + return SlayerModel( + name="orders", sql_table="orders", data_source=data_source, + default_time_dimension="created_at", + columns=[ + Column(name="id", type=DataType.INT, primary_key=True), + Column(name="customer_id", type=DataType.INT), + Column(name="region", type=DataType.TEXT), + Column(name="city", type=DataType.TEXT), + Column(name="status", type=DataType.TEXT), + Column(name="amount", type=DataType.DOUBLE), + Column(name="discount", type=DataType.DOUBLE), + Column(name="price", type=DataType.DOUBLE), + Column(name="quantity", type=DataType.INT), + Column(name="created_at", type=DataType.TIMESTAMP), + Column(name="order_date", type=DataType.DATE), + Column(name="created_txt", type=DataType.TEXT), + Column(name="code", type=DataType.TEXT), + Column(name="customer_name", type=DataType.TEXT), + Column(name="note", type=DataType.TEXT), + Column(name="path", type=DataType.TEXT), + Column(name="order total", sql="order_total", type=DataType.DOUBLE), + Column(name="payload", type=DataType.TEXT), + ], + joins=[ModelJoin(target_model="customers", join_pairs=[["customer_id", "id"]])], + aggregations=[ + Aggregation(name="wsum", formula="SUM({value} * {weight})", + params=[AggregationParam(name="weight", sql="quantity")]), + ], + ) + + +def customers_model(*, data_source: str = "test") -> SlayerModel: + return SlayerModel( + name="customers", sql_table="customers", data_source=data_source, + columns=[ + Column(name="id", type=DataType.INT, primary_key=True), + Column(name="name", type=DataType.TEXT), + Column(name="Region", type=DataType.TEXT), + Column(name="tier", type=DataType.TEXT), + ], + ) + + +def balances_model(*, data_source: str = "test") -> SlayerModel: + return SlayerModel( + name="balances", sql_table="balances", data_source=data_source, + default_time_dimension="updated_at", + columns=[ + Column(name="id", type=DataType.INT, primary_key=True), + Column(name="customer_id", type=DataType.INT), + Column(name="balance", type=DataType.DOUBLE), + Column(name="updated_at", type=DataType.TIMESTAMP), + Column(name="status", type=DataType.TEXT), + ], + ) + + +def dev2101_models(*, data_source: str = "test") -> list[SlayerModel]: + return [orders_model(data_source=data_source), customers_model(data_source=data_source), + balances_model(data_source=data_source)] + + +# --------------------------------------------------------------------------- # +# Data +# --------------------------------------------------------------------------- # +ORDER_COLUMNS = ( + "id", "customer_id", "region", "city", "status", "amount", "discount", "price", "quantity", + "created_at", "order_date", "created_txt", "code", "customer_name", "note", "path", "order_total", "payload", +) +ORDER_ROWS: list[tuple[Any, ...]] = [ + (1, 1, "EU", "Paris", "paid", 10.0, None, 2.0, 1, "2024-01-05 10:00:00", "2024-01-05", "2024-12-25", + "12", "O'Brien", "we like cats", r"C:\temp", 100.0, '{"a": 1}'), + (2, 1, "EU", "Berlin", "paid", 20.0, 5.0, 3.0, 2, "2024-01-20 11:00:00", "2024-01-20", "2024-06-01", + "abc", "Smith", "we LIKE dogs", "C:/temp", 200.0, '{"a": 2}'), + (3, 2, "US", "NYC", "Pending", 30.0, 1.0, 4.0, 3, "2024-02-10 12:00:00", "2024-02-10", "2024-02-29", + "12abc", "Jones", "n/a", r"D:\x", 300.0, '{"a": 3}'), + (4, 2, "US", "LA", "paid", 40.0, None, 5.0, 2, "2024-02-25 09:00:00", "2024-02-25", "2023-01-01", + "7", "Brown", None, None, 400.0, None), + (5, 3, None, "Rome", "void", None, 2.0, 1.0, 4, "2024-03-15 08:00:00", "2024-03-15", None, + None, "Lee", "x", "y", None, None), +] +CUSTOMER_COLUMNS = ("id", "name", "Region", "tier") +CUSTOMER_ROWS: list[tuple[Any, ...]] = [(1, "A", "North", "gold"), (2, "B", "South", "silver"), (3, "C", None, "gold")] +BALANCE_COLUMNS = ("id", "customer_id", "balance", "updated_at", "status") +BALANCE_ROWS: list[tuple[Any, ...]] = [ + (1, 1, 100.0, "2024-01-01 00:00:00", "active"), + (2, 1, 150.0, "2024-02-01 00:00:00", "active"), + (3, 1, 999.0, "2024-03-01 00:00:00", "inactive"), + (4, 2, 50.0, "2024-01-15 00:00:00", "active"), + (5, 2, 70.0, "2024-02-15 00:00:00", "inactive"), + (6, 2, 80.0, "2024-03-15 00:00:00", "active"), +] +TABLES = { + "orders": (ORDER_COLUMNS, ORDER_ROWS), + "customers": (CUSTOMER_COLUMNS, CUSTOMER_ROWS), + "balances": (BALANCE_COLUMNS, BALANCE_ROWS), +} + +#: Per-dialect (text, double, timestamp, identifier quote) spellings for the DDL. +_DDL_TYPES = { + "sqlite": ("TEXT", "REAL", "TIMESTAMP", '"'), + "duckdb": ("VARCHAR", "DOUBLE", "TIMESTAMP", '"'), + "postgres": ("TEXT", "DOUBLE PRECISION", "TIMESTAMP", '"'), + "tsql": ("NVARCHAR(100)", "FLOAT", "DATETIME2", '"'), + "mysql": ("VARCHAR(100)", "DOUBLE", "DATETIME", "`"), +} + + +def ddl(dialect: str) -> list[str]: + """CREATE TABLE statements for the three tables, spelled for ``dialect``.""" + text, double, ts, q = _DDL_TYPES[dialect] + return [ + "CREATE TABLE orders (id INTEGER PRIMARY KEY, customer_id INTEGER, region {t}, city {t}, status {t}, " + "amount {d}, discount {d}, price {d}, quantity INTEGER, created_at {ts}, order_date DATE, " + "created_txt {t}, code {t}, customer_name {t}, note {t}, path {t}, order_total {d}, payload {t})".format( + t=text, d=double, ts=ts), + f"CREATE TABLE customers (id INTEGER PRIMARY KEY, name {text}, {q}Region{q} {text}, tier {text})", + f"CREATE TABLE balances (id INTEGER PRIMARY KEY, customer_id INTEGER, balance {double}, " + f"updated_at {ts}, status {text})", + ] + + +def insert_sql(table: str, *, placeholder: str = "?", quote: str = '"') -> str: + columns, _ = TABLES[table] + cols = ", ".join(f"{quote}{c}{quote}" for c in columns) + return f"INSERT INTO {table} ({cols}) VALUES ({', '.join(placeholder for _ in columns)})" + + +def seed_sqlite(db_path: str) -> None: + with transaction(db_path) as con: + for stmt in ddl("sqlite"): + con.execute(stmt) + for table, (_, rows) in TABLES.items(): + con.executemany(insert_sql(table), rows) + + +def seed_duckdb(db_path: str) -> None: + duckdb = pytest.importorskip("duckdb") + con = duckdb.connect(db_path) + try: + for stmt in ddl("duckdb"): + con.execute(stmt) + for table, (_, rows) in TABLES.items(): + con.executemany(insert_sql(table), rows) + finally: + con.close() + + +async def engine_for(dialect: str) -> AsyncIterator[SlayerQueryEngine]: + """An executing engine over the seeded dataset on ``sqlite`` or ``duckdb`` (fixture body).""" + if dialect == "duckdb": + pytest.importorskip("duckdb") + seed = seed_duckdb if dialect == "duckdb" else seed_sqlite + async with seeded_exec_engine(dialect=dialect, seed=seed, models=dev2101_models()) as (engine, _): + yield engine + + +async def make_exec_engine(request) -> AsyncIterator[SlayerQueryEngine]: + """Body for a ``params=EXEC_BACKENDS`` fixture.""" + async for engine in engine_for(request.param): + yield engine + + +# --------------------------------------------------------------------------- # +# Oracles: SQL forms in every position +# --------------------------------------------------------------------------- # +#: Row-level predicates and the order ids each keeps. +PREDICATES: list[tuple[str, frozenset[int]]] = [ + ("status = 'paid'", frozenset({1, 2, 4})), + ("status <> 'paid'", frozenset({3, 5})), + ("amount != 10", frozenset({2, 3, 4})), + ("discount IS NULL", frozenset({1, 4})), + ("discount IS NOT NULL", frozenset({2, 3, 5})), + ("region is None", frozenset({5})), + ("region IN ('EU', 'XX')", frozenset({1, 2})), + ("region NOT IN ('EU')", frozenset({3, 4})), + ("status = 'paid' AND amount > 15", frozenset({2, 4})), + ("status = 'void' OR amount < 15", frozenset({1, 5})), + ("NOT status = 'paid'", frozenset({3, 5})), + ("note LIKE 'we%'", frozenset({1, 2})), + ("note NOT LIKE 'we%'", frozenset({3, 5})), + ("lower(trim(note)) LIKE 'we like%'", frozenset({1, 2})), + ("note ILIKE 'WE LIKE%'", frozenset({1, 2})), + ("note NOT ILIKE 'we like%'", frozenset({3, 5})), + ("amount BETWEEN 10 AND 20", frozenset({1, 2})), + ("amount NOT BETWEEN 10 AND 20", frozenset({3, 4})), + ("region IS DISTINCT FROM 'EU'", frozenset({3, 4, 5})), + ("region IS NOT DISTINCT FROM 'EU'", frozenset({1, 2})), + ("region IS NOT DISTINCT FROM NULL", frozenset({5})), + ("(amount > 15) IS TRUE", frozenset({2, 3, 4})), + ("(amount > 15) IS NOT TRUE", frozenset({1, 5})), + ("(amount > 15) IS FALSE", frozenset({1})), + ("(amount > 15) IS NOT FALSE", frozenset({2, 3, 4, 5})), + ("status || '-' || region = 'paid-EU'", frozenset({1, 2})), + ("CAST(quantity AS TEXT) = '2'", frozenset({2, 4})), + ("quantity::text = '2'", frozenset({2, 4})), + ("EXTRACT(month FROM created_at) = 2", frozenset({3, 4})), + ("created_at > TIMESTAMP '2024-02-01 00:00:00'", frozenset({3, 4, 5})), + ("order_date >= DATE '2024-02-25'", frozenset({4, 5})), + ("order_date + INTERVAL '7 days' > DATE '2024-03-01'", frozenset({4, 5})), + ("order_date + INTERVAL '7' DAY > DATE '2024-03-01'", frozenset({4, 5})), + ("order_date + INTERVAL 7 DAY > DATE '2024-03-01'", frozenset({4, 5})), + ("order_date < CURRENT_DATE", ALL_IDS), + ("created_at < CURRENT_TIMESTAMP", ALL_IDS), + ("SUBSTRING(status FROM 1 FOR 3) = 'pai'", frozenset({1, 2, 4})), + ("POSITION('n' IN status) > 0", frozenset({3})), + ("customer_name = 'O''Brien'", frozenset({1})), + (r"path = 'C:\temp'", frozenset({1})), + ('"order total" >= 300', frozenset({3, 4})), + ("-amount < -25", frozenset({3, 4})), + ("amount % 20 = 10", frozenset({1, 3})), + ("amount ** 2 > 500", frozenset({3, 4})), + ("CASE WHEN status = 'paid' THEN amount ELSE 0 END > 15", frozenset({2, 4})), + ("CASE status WHEN 'void' THEN 1 ELSE 0 END = 1", frozenset({5})), +] +PRED_IDS = [p for p, _ in PREDICATES] + +#: Row-level value forms and each order's value. +VALUES: list[tuple[str, dict[int, Any]]] = [ + ("status || '-' || region", {1: "paid-EU", 2: "paid-EU", 3: "Pending-US", 4: "paid-US", 5: None}), + ("CAST(quantity AS TEXT)", {1: "1", 2: "2", 3: "3", 4: "2", 5: "4"}), + ("quantity::text", {1: "1", 2: "2", 3: "3", 4: "2", 5: "4"}), + ("EXTRACT(month FROM created_at)", {1: 1, 2: 1, 3: 2, 4: 2, 5: 3}), + ("SUBSTRING(status FROM 2 FOR 3)", {1: "aid", 2: "aid", 3: "end", 4: "aid", 5: "oid"}), + ("POSITION('a' IN status)", {1: 2, 2: 2, 3: 0, 4: 2, 5: 0}), + ("amount % 7", {1: 3, 2: 6, 3: 2, 4: 5, 5: None}), + ("amount ** 2", {1: 100, 2: 400, 3: 900, 4: 1600, 5: None}), + ("CASE WHEN amount > 15 THEN 'big' ELSE 'small' END", {1: "small", 2: "big", 3: "big", 4: "big", 5: "small"}), + ("date_diff('day', DATE '2024-01-01', order_date)", {1: 4, 2: 19, 3: 40, 4: 55, 5: 74}), +] +VALUE_IDS = [v for v, _ in VALUES] +DATE_VALUES: list[tuple[str, dict[int, str]]] = [ + ("order_date + INTERVAL '1 month'", {1: "2024-02-05", 2: "2024-02-20", 3: "2024-03-10", 4: "2024-03-25", + 5: "2024-04-15"}), + ("order_date - INTERVAL 2 DAY", {1: "2024-01-03", 2: "2024-01-18", 3: "2024-02-08", 4: "2024-02-23", + 5: "2024-03-13"}), +] + + +# --------------------------------------------------------------------------- # +# Query shorthands and readers +# --------------------------------------------------------------------------- # +def q(**kw: Any) -> SlayerQuery: + return SlayerQuery.model_validate({"source_model": "orders", **kw}) + + +def m(formula: str, name: str = "v") -> ModelMeasure: + return ModelMeasure(formula=formula, name=name) + + +def ids(resp: SlayerResponse, *, model: str = "orders") -> set[int]: + return {int(r[f"{model}.id"]) for r in resp.data} + + +def ordered_ids(resp: SlayerResponse, *, model: str = "orders") -> list[int]: + return [int(r[f"{model}.id"]) for r in resp.data] + + +def by_key(resp: SlayerResponse, key: str, value: str = "v", *, model: str = "orders") -> dict: + out = {r[f"{model}.{key}"]: r[f"{model}.{value}"] for r in resp.data} + assert len(out) == len(resp.data), f"duplicate result rows for one {key}" + return out + + +def single(resp: SlayerResponse, name: str = "v", *, model: str = "orders") -> Any: + assert len(resp.data) == 1, resp.data + return resp.data[0][f"{model}.{name}"] + + +def num(value: Any) -> float | None: + return None if value is None else float(value) + + +def day(value: Any) -> str | None: + """A date / timestamp cell as ``YYYY-MM-DD`` across SQLite text and DuckDB values.""" + return None if value is None else str(value)[:10] + + +async def emit(query: SlayerQuery, *, dialect: str, models: Iterable[SlayerModel] | None = None, + datasource_fields: dict | None = None) -> str: + """Dry-run SQL for ``query`` over the dev2101 models on ``dialect`` (asserts it parses there).""" + first, *rest = list(models) if models is not None else dev2101_models() + return await _engine_generate(query=query, model=first, extra_models=rest, dialect=dialect, + datasource_fields=datasource_fields) + + +# --------------------------------------------------------------------------- # +# NULL ordering (spec: queries/positions › Order items carry an explicit NULL placement) +# --------------------------------------------------------------------------- # +#: (direction, nulls) → raw-rows order of ids by ``discount`` (NULL on 1 and 4), ties by id. +DISCOUNT_ORDER = { + ("desc", "first"): [1, 4, 2, 5, 3], + ("desc", "last"): [2, 5, 3, 1, 4], + ("asc", "first"): [1, 4, 3, 5, 2], + ("asc", "last"): [3, 5, 2, 1, 4], +} +#: (direction, nulls) → cities ordered by the hidden ``max(discount)`` (NULL for Paris and LA), ties by city. +CITY_ORDER = { + ("asc", "last"): ["NYC", "Rome", "Berlin", "LA", "Paris"], + ("asc", "first"): ["LA", "Paris", "NYC", "Rome", "Berlin"], + ("desc", "first"): ["LA", "Paris", "Berlin", "Rome", "NYC"], + ("desc", "last"): ["Berlin", "Rome", "NYC", "LA", "Paris"], +} + + +def discount_rows_query(*, direction: str, nulls: str) -> SlayerQuery: + return SlayerQuery.model_validate({ + "source_model": "orders", "dimensions": ["id"], "distinct_dimension_values": False, + "order": [{"column": "discount", "direction": direction, "nulls": nulls}, {"column": "id"}], + }) + + +def hidden_max_query(*, direction: str, nulls: str) -> SlayerQuery: + return q(dimensions=["city"], order=[ + {"column": "max(discount)", "direction": direction, "nulls": nulls}, {"column": "city"}, + ]) + + +async def check_null_ordering(engine: SlayerQueryEngine) -> None: + """Every NULL-placement case on a live engine (shared by the server suites).""" + for (direction, nulls), expected in DISCOUNT_ORDER.items(): + resp = await engine.execute(discount_rows_query(direction=direction, nulls=nulls)) + assert ordered_ids(resp) == expected, (direction, nulls, resp.sql) + for (direction, nulls), expected in CITY_ORDER.items(): + resp = await engine.execute(hidden_max_query(direction=direction, nulls=nulls)) + assert [r["orders.city"] for r in resp.data] == expected, (direction, nulls, resp.sql) + assert resp.columns == ["orders.city"] + + +# --------------------------------------------------------------------------- # +# Server suites (Postgres, MySQL, SQL Server) +# --------------------------------------------------------------------------- # +_SERVER_PARAMS = {"postgres": ("%s", '"'), "mysql": ("%s", "`"), "tsql": ("?", '"')} + + +def seed_server(cursor: Any, *, dialect: str) -> None: + """Create and fill the three tables through a DB-API ``cursor`` on ``dialect``.""" + placeholder, quote = _SERVER_PARAMS[dialect] + for stmt in ddl(dialect): + cursor.execute(stmt) + for table, (_, rows) in TABLES.items(): + cursor.executemany(insert_sql(table, placeholder=placeholder, quote=quote), rows) + + +async def check_server_forms(engine: SlayerQueryEngine) -> None: + """Every SQL-form predicate as a filter, and the forms the spec pins on every Tier-1 dialect.""" + for pred, expected in PREDICATES: + resp = await engine.execute(q(dimensions=["id"], filters=[pred])) + assert ids(resp) == expected, (pred, resp.sql) + for expr, values in VALUES: + resp = await engine.execute(q(dimensions=["id", {"expression": expr, "name": "x"}])) + got = {int(k): (v if v is None or isinstance(v, str) else float(v)) for k, v in by_key(resp, "id", "x").items()} + want = {k: (v if v is None or isinstance(v, str) else float(v)) for k, v in values.items()} + assert got == want, (expr, resp.sql) + assert num(single(await engine.execute(q(measures=[m("min(-7 % 3)")])))) == -1 + + +async def check_try_cast(engine: SlayerQueryEngine, *, nulls_invalid: bool = True) -> None: + """``TRY_CAST(code AS INT)``: 12 / NULL / NULL / 7 / NULL; ``nulls_invalid=False`` expects the database's error.""" + query = q(dimensions=["id", {"expression": "TRY_CAST(code AS INT)", "name": "x"}]) + if not nulls_invalid: + sql = (await engine.execute(query, dry_run=True)).sql or "" + assert "PG_INPUT_IS_VALID" not in sql.upper(), sql + with pytest.raises(Exception, match="(?i)invalid input syntax"): # noqa: B017 — the database's own error + await engine.execute(query) + return + resp = await engine.execute(query) + got = {int(k): (None if v is None else int(v)) for k, v in by_key(resp, "id", "x").items()} + assert got == {1: 12, 2: None, 3: None, 4: 7, 5: None}, resp.sql + + +async def check_filtered_aggregations(engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["region"], measures=[m("count(*) FILTER (WHERE status = 'paid')")])) + assert {k: num(v) for k, v in by_key(resp, "region").items()} == {"EU": 2, "US": 1, None: 0}, resp.sql + resp = await engine.execute(q(measures=[m("weighted_avg(price, weight=quantity) FILTER (WHERE status = 'paid')")])) + assert num(single(resp)) == pytest.approx(3.6), resp.sql + resp = await engine.execute(SlayerQuery.model_validate({ + "source_model": "balances", "dimensions": ["customer_id"], + "measures": [{"formula": "last(balance, updated_at) FILTER (WHERE status = 'active')", "name": "v"}], + })) + assert {int(k): num(v) for k, v in by_key(resp, "customer_id", model="balances").items()} == {1: 150.0, 2: 80.0} diff --git a/tests/facade/test_translator.py b/tests/facade/test_translator.py index 84886ac9..117d1615 100644 --- a/tests/facade/test_translator.py +++ b/tests/facade/test_translator.py @@ -14,7 +14,7 @@ from slayer.core.enums import DataType, JoinType, TimeGranularity from slayer.core.models import Column, ModelJoin, ModelMeasure, SlayerModel from slayer.core.query import ModelExtension -from slayer.engine.syntax import Cmp, Ref, parse_filter_expr +from slayer.engine.syntax import Cmp, Ref, parse_expr from slayer.facade.catalog import FacadeCatalog, build_catalog from slayer.facade.rows import FacadeColumn, RowBatch from slayer.facade.translator import ( @@ -926,7 +926,7 @@ def test_double_quoted_column_in_where_becomes_column_not_string_literal(dialect assert len(filters) == 1 assert filters[0] == "status = 'paid'" # The Mode B DSL must read ``status`` as a column, not a literal. - parsed = parse_filter_expr(filters[0]) + parsed = parse_expr(filters[0]) assert isinstance(parsed, Cmp) assert parsed.left == Ref(name="status") diff --git a/tests/integration/test_dev2101_postgres.py b/tests/integration/test_dev2101_postgres.py new file mode 100644 index 00000000..7eb26873 --- /dev/null +++ b/tests/integration/test_dev2101_postgres.py @@ -0,0 +1,86 @@ +"""The Mode-B SQL grammar on a real Postgres (spec: queries/expression-grammar, queries/casts, queries/positions).""" + +from decimal import Decimal + +import pytest + +from slayer.core.models import DatasourceConfig +from slayer.engine.query_engine import SlayerQueryEngine +from slayer.storage.yaml_storage import YAMLStorage +from tests._dev2101_fixtures import ( + by_key, + check_filtered_aggregations, + check_null_ordering, + check_server_forms, + check_try_cast, + dev2101_models, + m, + q, + seed_server, + single, +) + +pytest.importorskip("pytest_postgresql") + +from pytest_postgresql import factories # ALLOW(import-not-top): optional DB driver, gated by pytest.importorskip above + +postgresql_proc = factories.postgresql_proc(port=None) +postgresql = factories.postgresql("postgresql_proc") + + +@pytest.fixture +async def pg(postgresql, tmp_path) -> tuple[SlayerQueryEngine, int]: + cur = postgresql.cursor() + seed_server(cur, dialect="postgres") + cur.execute("SELECT current_setting('server_version_num')") + version_num = int(cur.fetchone()[0]) + postgresql.commit() + storage = YAMLStorage(base_dir=str(tmp_path)) + info = postgresql.info + await storage.save_datasource(DatasourceConfig( + name="pg", type="postgres", host=info.host, port=info.port, + database=info.dbname, username=info.user, password="", + )) + for model in dev2101_models(data_source="pg"): + await storage.save_model(model) + return SlayerQueryEngine(storage=storage), version_num + + +@pytest.mark.integration +async def test_sql_forms(pg) -> None: + await check_server_forms(pg[0]) + + +@pytest.mark.integration +async def test_null_ordering(pg) -> None: + await check_null_ordering(pg[0]) + + +@pytest.mark.integration +async def test_filtered_aggregations(pg) -> None: + await check_filtered_aggregations(pg[0]) + + +@pytest.mark.integration +async def test_try_cast_follows_the_server_version(pg) -> None: + engine, version_num = pg + await check_try_cast(engine, nulls_invalid=version_num >= 160000) + + +@pytest.mark.integration +async def test_plain_cast_raises_on_invalid_input(pg) -> None: + with pytest.raises(Exception, match="(?i)invalid input syntax"): # noqa: B017 — Postgres's own error + await pg[0].execute(q(dimensions=["id", {"expression": "CAST(code AS INT)", "name": "x"}])) + + +@pytest.mark.integration +async def test_parameterised_cast_is_postgres_numeric(pg) -> None: + value = single(await pg[0].execute(q(measures=[m("sum(CAST(amount AS DECIMAL(10,2)))")]))) + assert isinstance(value, Decimal) and value == Decimal("100.00") + + +@pytest.mark.integration +async def test_cast_to_json_executes(pg) -> None: + resp = await pg[0].execute(q(dimensions=["id", {"expression": "CAST(payload AS JSON)", "name": "x"}], + filters=["id = 1"])) + assert by_key(resp, "id", "x")[1] in ({"a": 1}, '{"a": 1}') diff --git a/tests/integration/test_integration_mysql.py b/tests/integration/test_integration_mysql.py index 13aee2b4..32bfe35e 100644 --- a/tests/integration/test_integration_mysql.py +++ b/tests/integration/test_integration_mysql.py @@ -56,6 +56,14 @@ server_statements, with_granularities, ) +from tests._dev2101_fixtures import ( + check_filtered_aggregations, + check_null_ordering, + check_server_forms, + check_try_cast, + dev2101_models, + seed_server as dev2101_seed_server, +) from tests.integration._consecutive_periods_calendar import ( CALENDAR_CASES, assert_calendar_streak, @@ -1581,3 +1589,41 @@ def fails(sql: str) -> bool: finally: conn.close() _drop_module_db(mysql_container, db_name) + + +# --------------------------------------------------------------------------- +# Mode-B SQL grammar (spec: queries/expression-grammar, queries/casts, queries/positions) +# --------------------------------------------------------------------------- + +@pytest.fixture(scope="module") +def _mysql_dev2101_storage(mysql_container, tmp_path_factory): + db_name = _create_module_db(mysql_container) + try: + conn = _admin_connect(mysql_container, dbname=db_name) + try: + with conn.cursor() as cur: + dev2101_seed_server(cur, dialect="mysql") + finally: + conn.close() + storage = YAMLStorage(base_dir=str(tmp_path_factory.mktemp("mysql_dev2101"))) + run_sync(storage.save_datasource(_ds_config(mysql_container, db_name))) + for model in dev2101_models(data_source="testmysql"): + run_sync(storage.save_model(model)) + yield storage + finally: + _drop_module_db(mysql_container, db_name) + + +@pytest.mark.integration +class TestMySQLModeBGrammar: + async def test_sql_forms(self, _mysql_dev2101_storage) -> None: + await check_server_forms(SlayerQueryEngine(storage=_mysql_dev2101_storage)) + + async def test_null_ordering(self, _mysql_dev2101_storage) -> None: + await check_null_ordering(SlayerQueryEngine(storage=_mysql_dev2101_storage)) + + async def test_try_cast(self, _mysql_dev2101_storage) -> None: + await check_try_cast(SlayerQueryEngine(storage=_mysql_dev2101_storage)) + + async def test_filtered_aggregations(self, _mysql_dev2101_storage) -> None: + await check_filtered_aggregations(SlayerQueryEngine(storage=_mysql_dev2101_storage)) diff --git a/tests/integration/test_integration_sqlserver.py b/tests/integration/test_integration_sqlserver.py index 29a18184..e55af9b9 100644 --- a/tests/integration/test_integration_sqlserver.py +++ b/tests/integration/test_integration_sqlserver.py @@ -95,6 +95,14 @@ orders_q, seed_statements as dev2046_seed_statements, ) +from tests._dev2101_fixtures import ( + check_filtered_aggregations, + check_null_ordering, + check_server_forms, + check_try_cast, + dev2101_models, + seed_server as dev2101_seed_server, +) from tests.integration._consecutive_periods_calendar import ( CALENDAR_CASES, assert_calendar_streak, @@ -1604,3 +1612,42 @@ def test_every_sqlserver_alias_failure_is_quoted(sqlserver_container) -> None: ) == [] finally: _drop_module_db(sqlserver_container, db_name) + + +# --------------------------------------------------------------------------- +# Mode-B SQL grammar (spec: queries/expression-grammar, queries/casts, queries/positions) +# --------------------------------------------------------------------------- + +@pytest.fixture(scope="module") +def _sqlserver_dev2101_storage(sqlserver_container, tmp_path_factory): + db_name = _create_module_db(sqlserver_container) + try: + with disposable_engine(_db_url(sqlserver_container, db_name)) as engine: + raw = engine.raw_connection() + try: + dev2101_seed_server(raw.cursor(), dialect="tsql") + raw.commit() + finally: + raw.close() + storage = YAMLStorage(base_dir=str(tmp_path_factory.mktemp("sqlserver_dev2101"))) + run_sync(storage.save_datasource(_ds_config(sqlserver_container, db_name))) + for model in dev2101_models(data_source="testmssql"): + run_sync(storage.save_model(model)) + yield storage + finally: + _drop_module_db(sqlserver_container, db_name) + + +@pytest.mark.integration +class TestSQLServerModeBGrammar: + async def test_sql_forms(self, _sqlserver_dev2101_storage) -> None: + await check_server_forms(SlayerQueryEngine(storage=_sqlserver_dev2101_storage)) + + async def test_null_ordering_over_aliased_targets(self, _sqlserver_dev2101_storage) -> None: + await check_null_ordering(SlayerQueryEngine(storage=_sqlserver_dev2101_storage)) + + async def test_try_cast(self, _sqlserver_dev2101_storage) -> None: + await check_try_cast(SlayerQueryEngine(storage=_sqlserver_dev2101_storage)) + + async def test_filtered_aggregations(self, _sqlserver_dev2101_storage) -> None: + await check_filtered_aggregations(SlayerQueryEngine(storage=_sqlserver_dev2101_storage)) diff --git a/tests/perf/dev2101_old_parser_baseline.json b/tests/perf/dev2101_old_parser_baseline.json new file mode 100644 index 00000000..fcfe34dc --- /dev/null +++ b/tests/perf/dev2101_old_parser_baseline.json @@ -0,0 +1,14 @@ +{ + "weighted_avg(amount, weight=rank(sum(amount, partition_by=spend_band), direction='desc'))": 48.2, + "sum(quantity * rank(avg(unit_price, partition_by=product), direction='desc'))": 57.8, + "sum(rank(amount:sum(window='90d', partition_by=region), direction='desc'))": 46.8, + "customers_v2.lifetime_value:weighted_avg(weight=customers_v2.deep_weight)": 26.2, + "rank(revenue:sum, partition_by=[status, customer_id], direction='desc')": 35.9, + "time_shift(amount:sum / amount:sum(partition_by=[customers.tier]), -1)": 43.1, + "sum(quantity * sum(amount, partition_by=product), partition_by=region)": 48.5, + "avg(sum(amount, partition_by=city) + sum(amount, partition_by=region))": 43.8, + "sum(cumsum(sum(amount, partition_by=[region, ordered_at])) - 1)": 43.3, + "sum(CASE WHEN status == 'paid' THEN amount WHEN status == 'refunded' THEN -amount WHEN status == 'pending' THEN 0 ELSE NULL END) / count(*)": 146.5, + "(c0 > 0 or c0 is None) and (c1 > 1 or c1 is None) and (c2 > 2 or c2 is None) and (c3 > 3 or c3 is None) and (c4 > 4 or c4 is None) and (c5 > 5 or c5 is None) and (c6 > 6 or c6 is None) and (c7 > 7 or c7 is None) and (c8 > 8 or c8 is None) and (c9 > 9 or c9 is None) and (c10 > 10 or c10 is None) and (c11 > 11 or c11 is None)": 184.4, + "sum(m0) * 0 + sum(m1) * 1 + sum(m2) * 2 + sum(m3) * 3 + sum(m4) * 4 + sum(m5) * 5 + sum(m6) * 6 + sum(m7) * 7 + sum(m8) * 8 + sum(m9) * 9 + sum(m10) * 10 + sum(m11) * 11 + sum(m12) * 12 + sum(m13) * 13 + sum(m14) * 14": 224.6 +} diff --git a/tests/perf/test_dev2101_parser_bench.py b/tests/perf/test_dev2101_parser_bench.py new file mode 100644 index 00000000..552b48e6 --- /dev/null +++ b/tests/perf/test_dev2101_parser_bench.py @@ -0,0 +1,23 @@ +"""Mode-B parser benchmark over the longest formulas in the tests and docs (recorded, not gating). + +Run with: poetry run pytest tests/perf/test_dev2101_parser_bench.py --benchmark-only +``dev2101_old_parser_baseline.json`` holds the pre-grammar parser's µs per parse on the same formulas. +""" + +import json +from pathlib import Path + +import pytest + +from slayer.core.syntax import parse_expr + +BASELINE_US: dict[str, float] = json.loads((Path(__file__).parent / "dev2101_old_parser_baseline.json").read_text()) +FORMULAS = list(BASELINE_US) + + +@pytest.mark.parametrize("formula", FORMULAS, ids=[f"f{i}" for i in range(len(FORMULAS))]) +def test_parse(benchmark, formula: str) -> None: + uncached = getattr(parse_expr, "__wrapped__", parse_expr) + benchmark(uncached, formula) + benchmark.extra_info["old_parser_us"] = BASELINE_US[formula] + benchmark.extra_info["ratio_vs_old"] = benchmark.stats.stats.mean * 1e6 / BASELINE_US[formula] diff --git a/tests/test_dev1450fix_group2_correctness.py b/tests/test_dev1450fix_group2_correctness.py index c6243ef9..701d81c7 100644 --- a/tests/test_dev1450fix_group2_correctness.py +++ b/tests/test_dev1450fix_group2_correctness.py @@ -30,7 +30,7 @@ from slayer.engine.schema_drift import _measure_formula_refs from slayer.storage.sqlite_conn import transaction from slayer.ir.source_bundle import ResolvedSourceBundle -from slayer.engine.syntax import parse_expr, parse_expr as _parse, parse_filter_expr +from slayer.engine.syntax import parse_expr, parse_expr as _parse from slayer.storage.yaml_storage import YAMLStorage @@ -42,37 +42,22 @@ def test_over_inside_string_literal_is_not_a_window_clause(): # No raise — the OVER( is inside a quoted literal. parse_expr("status == 'OVER('") - parse_expr('label == "x OVER (y)"') + parse_expr("label == 'x OVER (y)'") def test_over_inside_escaped_string_literal_not_window(): - # OVER( inside a Python-escaped string literal must not trip the window - # guard (Codex round 2): the value is `x " OVER(`. - parse_expr(r'status == "x \" OVER("') + # The value is `x ' OVER(`: a doubled quote must not end the literal early. + parse_expr("status == 'x '' OVER('") def test_in_keyword_inside_escaped_string_literal_not_rewritten(): - """CR review: ``_normalize_sql_filter_operators`` previously used the - SQL-style ('' / "") matcher, so a backslash-escaped quote left the - string body unprotected and ``IN`` / ``IS`` / ``AND`` rewrites leaked - into the literal. Now backed by ``_PY_STRING_LITERAL_RE``. - - Pin: a literal containing an embedded ``IN`` keyword must round-trip - cleanly when the literal is bounded by escaped quotes.""" - # The value of the literal is `x " IN (foo)` — IN must NOT lowercase - # inside the string. The right operand is a literal compared via ==, - # not a SQL-style IN clause. - parse_filter_expr(r'label == "x \" IN (foo)"') + """An embedded ``IN`` inside a literal bounded by a doubled quote stays text.""" + parse_expr("label == 'x '' IN (foo)'") def test_colon_inside_escaped_string_literal_not_preprocessed(): - """CR review: ``_preprocess_colons`` previously used the SQL-style - matcher; an escaped quote leaked colon-syntax aggregation rewrites - into string bodies. Pin: a literal containing ``revenue:sum`` is - not rewritten to an aggregation.""" - # Inner literal is `x " revenue:sum`. The colon there is text, not - # a colon-syntax aggregation source. - parse_expr(r'label == "x \" revenue:sum"') + """A literal containing ``revenue:sum`` after a doubled quote is not an aggregation.""" + parse_expr("label == 'x '' revenue:sum'") # --------------------------------------------------------------------------- @@ -86,11 +71,11 @@ def test_is_null_filter_parses_and_renders(): resulting ``ast.Is`` / ``ast.IsNot`` nodes. Pins both the parse and a downstream render through ``_compose_arithmetic_op``.""" # Parse must accept the SQL-style spelling. - parse_filter_expr("deleted_at IS NULL") - parse_filter_expr("deleted_at IS NOT NULL") + parse_expr("deleted_at IS NULL") + parse_expr("deleted_at IS NOT NULL") # And the lower-cased Python form should also work. - parse_filter_expr("deleted_at is None") - parse_filter_expr("deleted_at is not None") + parse_expr("deleted_at is None") + parse_expr("deleted_at is not None") def test_partition_by_multi_column_parses_and_binds(): diff --git a/tests/test_dev1475_in_tuple_filters.py b/tests/test_dev1475_in_tuple_filters.py index 596edf5b..86b27601 100644 --- a/tests/test_dev1475_in_tuple_filters.py +++ b/tests/test_dev1475_in_tuple_filters.py @@ -45,7 +45,7 @@ Literal, Ref, TupleLit, - parse_filter_expr, + parse_expr, ) import asyncio from slayer.core.models import DatasourceConfig @@ -65,7 +65,7 @@ class TestParserShape: def test_in_with_two_string_literals(self): - result = parse_filter_expr("status in ('a', 'b')") + result = parse_expr("status in ('a', 'b')") assert result == Cmp( op="in", left=Ref(name="status"), @@ -76,7 +76,7 @@ def test_in_with_two_string_literals(self): ) def test_not_in_with_two_string_literals(self): - result = parse_filter_expr("status not in ('a', 'b')") + result = parse_expr("status not in ('a', 'b')") assert result == Cmp( op="not in", left=Ref(name="status"), @@ -87,7 +87,7 @@ def test_not_in_with_two_string_literals(self): ) def test_in_with_numeric_literals(self): - result = parse_filter_expr("amount in (10, 20, 30)") + result = parse_expr("amount in (10, 20, 30)") assert result == Cmp( op="in", left=Ref(name="amount"), @@ -103,7 +103,7 @@ def test_in_with_signed_numeric_literals(self): # so a literal-only RHS guard that strictly checks ``isinstance # Literal`` would reject this. Codex review: collapse the sign # onto the inner numeric. - result = parse_filter_expr("amount in (-1, -2.5, +3)") + result = parse_expr("amount in (-1, -2.5, +3)") assert result == Cmp( op="in", left=Ref(name="amount"), @@ -115,7 +115,7 @@ def test_in_with_signed_numeric_literals(self): ) def test_in_with_dotted_left(self): - result = parse_filter_expr("stores.name in ('Brooklyn', 'Philadelphia')") + result = parse_expr("stores.name in ('Brooklyn', 'Philadelphia')") assert result == Cmp( op="in", left=DottedRef(parts=("stores", "name")), @@ -128,7 +128,7 @@ def test_in_with_dotted_left(self): def test_in_accepts_python_list_syntax(self): # ``[…]`` and ``(…)`` should be interchangeable on the RHS — # both produce a Tuple node. - result = parse_filter_expr("status in ['a', 'b']") + result = parse_expr("status in ['a', 'b']") assert result == Cmp( op="in", left=Ref(name="status"), @@ -147,12 +147,12 @@ def test_in_accepts_python_list_syntax(self): class TestParserGuards: def test_empty_rhs_tuple_rejected(self): with pytest.raises(ValueError, match="empty"): - parse_filter_expr("status in ()") + parse_expr("status in ()") def test_scalar_rhs_is_a_time_point_membership(self): # ``col in 'a'`` is a single-string time-point membership: it parses, and the # checker rejects it against a non-temporal column with the typed error. - assert parse_filter_expr("status in 'a'") == Cmp( + assert parse_expr("status in 'a'") == Cmp( op="in", left=Ref(name="status"), right=Literal(value="a"), ) engine, _ = _make_engine_with_orders() @@ -166,7 +166,7 @@ def test_non_literal_in_rhs_rejected(self): # ``status in (other_col, 'b')`` — the RHS may not reference a # column (no schema-aware binding for dynamic RHS; literals only). with pytest.raises(ValueError, match="literal"): - parse_filter_expr("status in (other_col, 'b')") + parse_expr("status in (other_col, 'b')") def test_direct_in_key_empty_values_rejected(self): # Defense in depth (Codex review): even if a caller bypasses @@ -229,7 +229,7 @@ class TestBinder: def test_in_filter_binds_to_in_key(self): model = _make_orders_with_status() - parsed = parse_filter_expr("status in ('completed', 'pending')") + parsed = parse_expr("status in ('completed', 'pending')") bundle = ResolvedSourceBundle(dialect="postgres", source_model=model, referenced_models=[]) scope = ModelScope(source_model=model) bound = bind_filter(parsed, scope=scope, bundle=bundle) @@ -246,7 +246,7 @@ def test_in_filter_binds_to_in_key(self): def test_not_in_filter_binds_negated(self): model = _make_orders_with_status() - parsed = parse_filter_expr("status not in ('cancelled',)") + parsed = parse_expr("status not in ('cancelled',)") bundle = ResolvedSourceBundle(dialect="postgres", source_model=model, referenced_models=[]) scope = ModelScope(source_model=model) bound = bind_filter(parsed, scope=scope, bundle=bundle) diff --git a/tests/test_dev1576_heals.py b/tests/test_dev1576_heals.py index d6ddd141..75c55769 100644 --- a/tests/test_dev1576_heals.py +++ b/tests/test_dev1576_heals.py @@ -1,10 +1,9 @@ """DEV-1576 — parse-level coverage for the three SlayerQuery heals. -1. Aggregation-name alias / casing normalization (``normalize_aggregation_name`` - + colon-syntax healing in ``parse_formula``; the typed pipeline heals at - binding — see ``tests/test_aggregation_gating.py``). -2. ``round()`` / ``abs()`` as top-level formula functions (parse into a - ``MixedArithmeticField`` passthrough; arity validation). +1. Aggregation-name alias / casing normalization (``normalize_aggregation_name``; + colon-syntax healing happens at binding). +2. ``round()`` / ``abs()`` as top-level formula functions (scalar calls over + aggregations; arity validated at binding). The §3 error-message split and end-to-end execution live in ``tests/test_aggregation_gating.py`` and the integration suites respectively. @@ -17,11 +16,37 @@ BUILTIN_AGGREGATIONS, normalize_aggregation_name, ) -from slayer.core.formula import ( - AggregatedMeasureRef, - MixedArithmeticField, - parse_formula, -) +from slayer.core.enums import DataType +from slayer.core.errors import UnknownFunctionError +from slayer.core.keys import AggregateKey, ValueKey +from slayer.core.models import Aggregation, Column, SlayerModel +from slayer.core.scope import ModelScope +from slayer.engine.binding import bind_expr +from slayer.engine.syntax import AggCall, Literal, Ref, ScalarCall, TransformCall, parse_expr +from slayer.ir.source_bundle import ResolvedSourceBundle + + +def _bind(text: str, *, custom: tuple[str, ...] = ()) -> ValueKey: + model = SlayerModel( + name="orders", sql_table="orders", data_source="t", + columns=[Column(name=n, type=DataType.DOUBLE) for n in ("revenue", "cost", "other", "x")], + aggregations=[Aggregation(name=n, formula="COUNT(DISTINCT {value})") for n in custom], + ) + bundle = ResolvedSourceBundle(dialect="postgres", source_model=model, referenced_models=[]) + return bind_expr(parse_expr(text), scope=ModelScope(source_model=model), bundle=bundle, + allow_measures=True).value_key + + +def _nodes(node): + yield node + for value in getattr(node, "__dict__", {}).values(): + for item in value if isinstance(value, tuple) else (value,): + if hasattr(item, "__dict__"): + yield from _nodes(item) + elif isinstance(item, tuple): + for sub in item: + if hasattr(sub, "__dict__"): + yield from _nodes(sub) # --------------------------------------------------------------------------- @@ -88,7 +113,7 @@ def test_alias_table_only_targets_builtins(self) -> None: # --------------------------------------------------------------------------- -# §1 — colon-syntax healing through parse_formula +# §1 — colon-syntax healing at binding # --------------------------------------------------------------------------- @@ -106,35 +131,28 @@ class TestColonSyntaxAliasHealing: ], ) def test_formula_colon_alias_heals(self, raw_agg: str, expected: str) -> None: - result = parse_formula(f"revenue:{raw_agg}") - assert isinstance(result, AggregatedMeasureRef) - assert result.aggregation_name == expected - assert result.measure_name == "revenue" + key = _bind(f"revenue:{raw_agg}") + assert isinstance(key, AggregateKey) + assert key.agg == expected + assert getattr(key.source, "leaf", None) == "revenue" def test_star_count_alias_unaffected(self) -> None: - result = parse_formula("*:count") - assert isinstance(result, AggregatedMeasureRef) - assert result.aggregation_name == "count" + key = _bind("*:count") + assert isinstance(key, AggregateKey) + assert key.agg == "count" def test_unknown_agg_in_formula_left_for_enrichment(self) -> None: - # parse_formula does not validate the vocabulary — an unknown agg - # parses through (heal leaves it unchanged); enrichment raises later. - result = parse_formula("revenue:bogus") - assert isinstance(result, AggregatedMeasureRef) - assert result.aggregation_name == "bogus" + # The parser does not validate the vocabulary — an unknown agg parses through. + assert parse_expr("revenue:bogus") == AggCall(source=Ref(name="revenue"), agg="bogus") def test_custom_agg_named_like_alias_not_healed(self) -> None: - # A model custom aggregation named like an alias key takes precedence — - # an exact custom-name match is NOT rewritten to the builtin. - result = parse_formula("revenue:countd", extra_agg_names=frozenset({"countd"})) - assert isinstance(result, AggregatedMeasureRef) - assert result.aggregation_name == "countd" + # An exact custom-name match takes precedence over the alias. + key = _bind("revenue:countd", custom=("countd",)) + assert isinstance(key, AggregateKey) and key.agg == "countd" def test_alias_still_heals_when_custom_name_differs(self) -> None: - result = parse_formula( - "revenue:countd", extra_agg_names=frozenset({"some_other_agg"}) - ) - assert result.aggregation_name == "count_distinct" + key = _bind("revenue:countd", custom=("some_other_agg",)) + assert isinstance(key, AggregateKey) and key.agg == "count_distinct" # --------------------------------------------------------------------------- @@ -142,56 +160,43 @@ def test_alias_still_heals_when_custom_name_differs(self) -> None: # --------------------------------------------------------------------------- -def _agg_names(field: MixedArithmeticField) -> set[str]: - return {ref.aggregation_name for ref in field.agg_refs.values()} +def _agg_names(parsed) -> set[str]: + return {n.agg for n in _nodes(parsed) if isinstance(n, AggCall)} + + +def _scalar(parsed, name: str) -> ScalarCall: + assert isinstance(parsed, ScalarCall) and parsed.name.lower() == name, parsed + return parsed class TestScalarFunctionsParse: def test_round_two_args(self) -> None: - result = parse_formula("round(revenue:sum, 2)") - assert isinstance(result, MixedArithmeticField) - assert "round" in result.sql.lower() - assert "2" in result.sql + result = _scalar(parse_expr("round(revenue:sum, 2)"), "round") + assert result.args[1] == Literal(value=2) assert _agg_names(result) == {"sum"} - assert result.sub_transforms == [] + assert not any(isinstance(n, TransformCall) for n in _nodes(result)) def test_round_one_arg(self) -> None: - result = parse_formula("round(revenue:sum)") - assert isinstance(result, MixedArithmeticField) - assert "round" in result.sql.lower() - assert _agg_names(result) == {"sum"} + assert _agg_names(_scalar(parse_expr("round(revenue:sum)"), "round")) == {"sum"} def test_abs_one_arg(self) -> None: - result = parse_formula("abs(revenue:sum)") - assert isinstance(result, MixedArithmeticField) - assert "abs" in result.sql.lower() - assert _agg_names(result) == {"sum"} + assert _agg_names(_scalar(parse_expr("abs(revenue:sum)"), "abs")) == {"sum"} def test_abs_over_arithmetic(self) -> None: - result = parse_formula("abs(revenue:sum - cost:sum)") - assert isinstance(result, MixedArithmeticField) - assert "abs" in result.sql.lower() - assert _agg_names(result) == {"sum"} + assert _agg_names(_scalar(parse_expr("abs(revenue:sum - cost:sum)"), "abs")) == {"sum"} def test_round_over_arithmetic(self) -> None: - result = parse_formula("round(revenue:sum / *:count, 2)") - assert isinstance(result, MixedArithmeticField) - assert "round" in result.sql.lower() - assert _agg_names(result) == {"sum", "count"} + assert _agg_names(_scalar(parse_expr("round(revenue:sum / *:count, 2)"), "round")) == {"sum", "count"} def test_round_negative_ndigits(self) -> None: - # round(x, -2) is valid SQL (round to hundreds); negative integer - # literals must be accepted. - result = parse_formula("round(revenue:sum, -2)") - assert isinstance(result, MixedArithmeticField) - assert "round" in result.sql.lower() + # round(x, -2) is valid SQL (round to hundreds); negative integer literals are accepted. + _scalar(parse_expr("round(revenue:sum, -2)"), "round") + _bind("round(revenue:sum, -2)") def test_round_wraps_nested_transform(self) -> None: - # A transform nested inside round is still extracted as a sub-transform. - result = parse_formula("round(cumsum(revenue:sum), 2)") - assert isinstance(result, MixedArithmeticField) - assert result.sub_transforms, "nested cumsum should become a sub-transform" - assert result.sub_transforms[0][1].transform == "cumsum" + result = _scalar(parse_expr("round(cumsum(revenue:sum), 2)"), "round") + transforms = [n for n in _nodes(result) if isinstance(n, TransformCall)] + assert [t.op for t in transforms] == ["cumsum"] @pytest.mark.parametrize( "raw", @@ -203,71 +208,44 @@ def test_round_wraps_nested_transform(self) -> None: ], ) def test_scalar_functions_case_insensitive(self, raw: str) -> None: - result = parse_formula(raw) - assert isinstance(result, MixedArithmeticField) - assert _agg_names(result) == {"sum"} + assert parse_expr(raw) == parse_expr(raw.split("(", 1)[0].lower() + "(" + raw.split("(", 1)[1]) + assert _agg_names(parse_expr(raw)) == {"sum"} class TestScalarFunctionArity: - def test_round_no_args_raises(self) -> None: - with pytest.raises(ValueError, match="round"): - parse_formula("round()") - - def test_round_three_args_raises(self) -> None: - with pytest.raises(ValueError, match="round"): - parse_formula("round(revenue:sum, 2, 3)") - - def test_round_non_integer_ndigits_raises(self) -> None: - with pytest.raises(ValueError, match="round"): - parse_formula("round(revenue:sum, 2.5)") - - def test_round_non_literal_ndigits_raises(self) -> None: - # ndigits must be a literal, not another measure / expression. - with pytest.raises(ValueError, match="round"): - parse_formula("round(revenue:sum, x)") - - def test_round_kwarg_raises(self) -> None: - with pytest.raises(ValueError, match="round"): - parse_formula("round(revenue:sum, ndigits=2)") - - def test_abs_two_args_raises(self) -> None: - with pytest.raises(ValueError, match="abs"): - parse_formula("abs(revenue:sum, cost:sum)") - - def test_abs_no_args_raises(self) -> None: - with pytest.raises(ValueError, match="abs"): - parse_formula("abs()") - - def test_abs_kwarg_raises(self) -> None: - with pytest.raises(ValueError, match="abs"): - parse_formula("abs(revenue:sum, foo=1)") + @pytest.mark.parametrize(("formula", "fn"), [ + ("round()", "round"), + ("round(revenue:sum, 2, 3)", "round"), + ("round(revenue:sum, ndigits=2)", "round"), + ("abs(revenue:sum, cost:sum)", "abs"), + ("abs()", "abs"), + ("abs(revenue:sum, foo=1)", "abs"), + ("round(revenue:sum, 2.5)", "round"), + ("round(revenue:sum, x)", "round"), + ]) + def test_arity_rejected(self, formula: str, fn: str) -> None: + with pytest.raises(ValueError, match=fn): + _bind(formula) class TestScalarPassthroughPolicy: """Mode B has one canonical scalar-function allowlist (SCALAR_PASSTHROUGH) - applied uniformly at top-level and inside arithmetic. Replaces the old - narrow {round, abs} gate from DEV-1576.""" + applied uniformly at top-level and inside arithmetic.""" @pytest.mark.parametrize("fn", ["ceil", "floor", "sqrt", "ln", "nullif", "coalesce"]) def test_common_scalars_accepted_at_top_level(self, fn: str) -> None: - # All of these landed in the unified SCALAR_PASSTHROUGH set. - result = parse_formula(f"{fn}(revenue:sum)") - assert result is not None + _scalar(parse_expr(f"{fn}(revenue:sum)"), fn) def test_inside_arithmetic_passthrough_still_works(self) -> None: - result = parse_formula("*:count / nullif(revenue:max, 0)") - assert isinstance(result, MixedArithmeticField) - assert "nullif" in result.sql.lower() + parsed = parse_expr("*:count / nullif(revenue:max, 0)") + assert any(isinstance(n, ScalarCall) and n.name == "nullif" for n in _nodes(parsed)) + _bind("*:count / nullif(revenue:max, 0)") def test_unknown_call_inside_arithmetic_now_rejected(self) -> None: - # The unification closed the accidental side door: unknown calls - # inside arithmetic used to silently pass through. - with pytest.raises(ValueError, match="Unknown function"): - parse_formula("revenue:sum + bogus_fn(other:sum)") + with pytest.raises(UnknownFunctionError, match="Unknown function"): + _bind("revenue:sum + bogus_fn(other:sum)") @pytest.mark.parametrize("raw", ["countd(revenue)", "countDistinct(revenue)"]) - def test_funcstyle_aggregation_alias_out_of_scope(self, raw: str) -> None: - # §1 heals colon syntax only. Function-style alias calls are NOT - # rewritten — they fall through to the parser as unknown calls. - with pytest.raises(ValueError, match="Unknown function"): - parse_formula(raw) + def test_funcstyle_aggregation_alias_heals(self, raw: str) -> None: + key = _bind(raw) + assert isinstance(key, AggregateKey) and key.agg == "count_distinct" diff --git a/tests/test_dev1737_date_functions_parse.py b/tests/test_dev1737_date_functions_parse.py index 47336651..3d5cc0e6 100644 --- a/tests/test_dev1737_date_functions_parse.py +++ b/tests/test_dev1737_date_functions_parse.py @@ -16,7 +16,6 @@ ScalarCall, UnaryOp, parse_expr, - parse_filter_expr, ) from slayer.core.scope import ModelScope from slayer.ir.source_bundle import ResolvedSourceBundle @@ -98,7 +97,7 @@ def test_add_unit_case_folds(self) -> None: ]) def test_unknown_unit_rejected_listing_accepted(self, expr: str, listed: str) -> None: with pytest.raises(ValueError, match=listed): - parse_filter_expr(expr) + parse_expr(expr) @pytest.mark.parametrize("expr", [ "date_part(status, created_at) = 1", @@ -110,7 +109,7 @@ def test_unknown_unit_rejected_listing_accepted(self, expr: str, listed: str) -> ]) def test_non_literal_unit_rejected(self, expr: str) -> None: with pytest.raises(ValueError, match="string literal"): - parse_filter_expr(expr) + parse_expr(expr) def test_bucket_form_is_not_a_date_function(self) -> None: assert not isinstance(parse_expr("month(created_at)"), ScalarCall) @@ -124,7 +123,7 @@ def test_zero_arg_calls(self, text: str, name: str) -> None: assert parse_expr(text) == ScalarCall(name=name, args=()) def test_clock_in_filter(self) -> None: - assert parse_filter_expr("created_at <= now()") == Cmp( + assert parse_expr("created_at <= now()") == Cmp( op="<=", left=Ref(name="created_at"), right=ScalarCall(name="now", args=())) @@ -168,12 +167,12 @@ def test_chain_with_minus(self) -> None: assert parse_expr("created_at + interval(1, 'month') - interval(2, 'hour')") == _date_add(inner, _lit(-2), "hour") def test_chained_filter_binds_like_nested_date_add(self) -> None: - assert parse_filter_expr( + assert parse_expr( "created_at + interval(1, 'month') + interval(2, 'hour') < now()" - ) == parse_filter_expr("date_add(date_add(created_at, 1, 'month'), 2, 'hour') < now()") + ) == parse_expr("date_add(date_add(created_at, 1, 'month'), 2, 'hour') < now()") def test_commuted_filter(self) -> None: - assert parse_filter_expr("interval(7, 'day') + order_date > current_date()") == parse_filter_expr( + assert parse_expr("interval(7, 'day') + order_date > current_date()") == parse_expr( "date_add(order_date, 7, 'day') > current_date()") def test_rewrite_happens_inside_an_aggregation(self) -> None: @@ -196,7 +195,7 @@ def test_bound_filter_interns(self) -> None: scope = ModelScope(source_model=model) def bound(text: str): - return bind_filter(parse_filter_expr(text), scope=scope, bundle=bundle).value_key + return bind_filter(parse_expr(text), scope=scope, bundle=bundle).value_key assert bound("created_at + interval(1, 'month') < now()") == bound("date_add(created_at, 1, 'month') < now()") @@ -219,12 +218,12 @@ class TestStrayIntervalRejected: ]) def test_rejected_pointing_to_date_add(self, expr: str) -> None: with pytest.raises(ValueError, match="date_add"): - parse_filter_expr(expr) + parse_expr(expr) def test_unknown_interval_unit_rejected(self) -> None: with pytest.raises(ValueError, match="week_sunday"): - parse_filter_expr("order_date + interval(1, 'fortnight') > order_date") + parse_expr("order_date + interval(1, 'fortnight') > order_date") def test_non_literal_interval_unit_rejected(self) -> None: with pytest.raises(ValueError, match="string literal"): - parse_filter_expr("order_date + interval(1, status) > order_date") + parse_expr("order_date + interval(1, status) > order_date") diff --git a/tests/test_dev1737_scalar_specs.py b/tests/test_dev1737_scalar_specs.py index 69c1656e..a1bdf1c6 100644 --- a/tests/test_dev1737_scalar_specs.py +++ b/tests/test_dev1737_scalar_specs.py @@ -5,9 +5,7 @@ import pydantic import pytest -from slayer.core import formula as legacy_formula from slayer.core import keys -from slayer.core.formula import MixedArithmeticField, parse_formula from slayer.core.keys import ( SCALAR_FUNCTION_ARITY, SCALAR_FUNCTIONS, @@ -18,6 +16,12 @@ from slayer.core.models import Aggregation, SlayerModel from slayer.osi import expression as osi_expression from slayer.osi.expression import convert_expression +from slayer.core.enums import DataType +from slayer.core.models import Column +from slayer.core.scope import ModelScope +from slayer.engine.binding import bind_expr +from slayer.engine.syntax import ScalarCall, parse_expr +from slayer.ir.source_bundle import ResolvedSourceBundle DATE_NAMES = {"date_part", "date_diff", "date_add", "interval", "current_date", "now"} @@ -51,11 +55,10 @@ def test_spec_is_a_pydantic_model(self) -> None: assert issubclass(ScalarSpec, pydantic.BaseModel) def test_one_canonical_passthrough_set(self) -> None: - assert getattr(legacy_formula, "SCALAR_PASSTHROUGH", keys.SCALAR_PASSTHROUGH) is keys.SCALAR_PASSTHROUGH assert osi_expression.SCALAR_PASSTHROUGH is keys.SCALAR_PASSTHROUGH -class TestLegacyFormulaParser: +class TestFormulaParser: @pytest.mark.parametrize("text", [ "date_diff('day', created_at:min, created_at:max)", "revenue:sum / date_part('day', created_at:max)", @@ -64,11 +67,17 @@ class TestLegacyFormulaParser: "DATE_DIFF('day', created_at:min, current_date())", ]) def test_recognises_date_functions(self, text: str) -> None: - assert isinstance(parse_formula(text), MixedArithmeticField) + parsed = parse_expr(text) + top = parsed if isinstance(parsed, ScalarCall) else parsed.right + assert isinstance(top, ScalarCall) and top.name.lower() in DATE_NAMES, parsed def test_unknown_call_lists_date_functions(self) -> None: + model = SlayerModel(name="orders", sql_table="orders", data_source="t", + columns=[Column(name="revenue", type=DataType.DOUBLE)]) + bundle = ResolvedSourceBundle(dialect="postgres", source_model=model, referenced_models=[]) with pytest.raises(ValueError, match="date_diff"): - parse_formula("bogus_fn(revenue:sum)") + bind_expr(parse_expr("bogus_fn(revenue:sum)"), scope=ModelScope(source_model=model), bundle=bundle, + allow_measures=True) class TestOsiDoesNotCarryDateFunctions: diff --git a/tests/test_dev1740_case_rewrite.py b/tests/test_dev1740_case_rewrite.py index 454fa53f..256838bb 100644 --- a/tests/test_dev1740_case_rewrite.py +++ b/tests/test_dev1740_case_rewrite.py @@ -1,6 +1,6 @@ """DEV-1740 Part A — the CASE/iif surface at parse time. -Pins the ``_rewrite_case_when`` preprocessor and the reserved-name +Pins CASE lowering and the reserved-name ``ScalarCall(name="iif")`` it lowers to: searched + simple CASE, nesting, multi-WHEN, missing ELSE, case-insensitive keywords, keywords inside string literals, malformed rejection, ``iif`` arity, and the deliberate rejection of @@ -24,9 +24,7 @@ Literal, Ref, ScalarCall, - _rewrite_case_when, parse_expr, - parse_filter_expr, ) @@ -38,35 +36,26 @@ def _iif(node) -> ScalarCall: # --------------------------------------------------------------------------- # -# The text rewrite: CASE ... END -> nested iif(...) +# CASE ... END lowers to nested iif(...) # --------------------------------------------------------------------------- # class TestRewriteToIif: - def test_searched_single_branch(self) -> None: - out = _rewrite_case_when("CASE WHEN amount > 5 THEN 1 ELSE 0 END") - assert "CASE" not in out.upper() - assert out.strip() == "iif(amount > 5, 1, 0)" - - def test_missing_else_becomes_null(self) -> None: - out = _rewrite_case_when("CASE WHEN amount > 5 THEN 1 END") - assert out.strip() == "iif(amount > 5, 1, None)" - - def test_multi_when_nests_right(self) -> None: - out = _rewrite_case_when( - "CASE WHEN a > 2 THEN 10 WHEN a > 1 THEN 5 ELSE 0 END" - ) - assert out.strip() == "iif(a > 2, 10, iif(a > 1, 5, 0))" - - def test_simple_form_synthesizes_equality(self) -> None: - out = _rewrite_case_when("CASE region WHEN 'EU' THEN 1 ELSE 0 END") - assert out.strip() == "iif(region == 'EU', 1, 0)" + @pytest.mark.parametrize(("case", "iif"), [ + ("CASE WHEN amount > 5 THEN 1 ELSE 0 END", "iif(amount > 5, 1, 0)"), + ("CASE WHEN amount > 5 THEN 1 END", "iif(amount > 5, 1, None)"), + ("CASE WHEN a > 2 THEN 10 WHEN a > 1 THEN 5 ELSE 0 END", "iif(a > 2, 10, iif(a > 1, 5, 0))"), + ("CASE region WHEN 'EU' THEN 1 ELSE 0 END", "iif(region == 'EU', 1, 0)"), + ]) + def test_lowers_to_iif(self, case: str, iif: str) -> None: + assert parse_expr(case) == parse_expr(iif) - def test_non_case_text_is_untouched(self) -> None: - assert _rewrite_case_when("amount + 1") == "amount + 1" + def test_non_case_text_is_not_a_conditional(self) -> None: + assert not isinstance(parse_expr("amount + 1"), ScalarCall) def test_keyword_inside_string_literal_is_not_syntax(self) -> None: # A value that merely contains CASE/END words must survive verbatim. - text = "status == 'END of the CASE when done'" - assert _rewrite_case_when(text) == text + assert parse_expr("status == 'END of the CASE when done'") == Cmp( + op="==", left=Ref(name="status"), right=Literal(value="END of the CASE when done"), + ) # --------------------------------------------------------------------------- # @@ -228,7 +217,7 @@ def test_wrong_arity_message_names_iif(self, argc: int) -> None: # --------------------------------------------------------------------------- # class TestFilterContext: def test_case_in_filter_predicate(self) -> None: - node = parse_filter_expr( + node = parse_expr( "CASE WHEN region = 'EU' THEN 1 ELSE 0 END = 1" ) assert isinstance(node, Cmp) diff --git a/tests/test_dev1743_validators.py b/tests/test_dev1743_validators.py index d5a7e092..cc17b4b0 100644 --- a/tests/test_dev1743_validators.py +++ b/tests/test_dev1743_validators.py @@ -100,8 +100,6 @@ def test_measure_name_reserved(self) -> None: # --------------------------------------------------------------------------- # # P3 — the parser rejects a literal __slayer_ spoof but keeps colon-aggs working. -# The spoof scan runs on the RAW input, BEFORE _preprocess_colons substitutes -# its own __slayer_agg_N__ placeholders [C3]. # --------------------------------------------------------------------------- # class TestParserSlayerPrefixReservation: def test_literal_placeholder_spoof_rejected(self) -> None: @@ -132,8 +130,6 @@ def test_unicode_embedded_slayer_run_still_parses(self) -> None: assert parse_expr("变量__slayer_bar") is not None def test_colon_aggregation_still_parses(self) -> None: - # revenue:sum is rewritten to a __slayer_agg_ placeholder INTERNALLY; - # the raw-input scan must not mistake that for a spoof. parsed = parse_expr("revenue:sum") assert parsed is not None diff --git a/tests/test_dev1833_keyword_lexing.py b/tests/test_dev1833_keyword_lexing.py index c3c7a364..92ec1bf8 100644 --- a/tests/test_dev1833_keyword_lexing.py +++ b/tests/test_dev1833_keyword_lexing.py @@ -8,6 +8,7 @@ from __future__ import annotations +import unicodedata from decimal import Decimal import pytest @@ -23,9 +24,7 @@ Ref, ScalarCall, UnaryOp, - _rewrite_sql_like, parse_expr, - parse_filter_expr, ) @@ -58,7 +57,7 @@ def test_case_in_scalar_call(self) -> None: ) def test_case_in_filter_predicate(self) -> None: - assert parse_filter_expr("case = 1") == Cmp( + assert parse_expr("case = 1") == Cmp( op="==", left=Ref(name="case"), right=Literal(value=Decimal(1)), ) @@ -220,31 +219,31 @@ def test_missing_then_still_specific(self) -> None: # --------------------------------------------------------------------------- # class TestLikeHardening: def test_escaped_quote_pattern(self) -> None: - assert parse_filter_expr(r"col LIKE 'It\'s%'") == ScalarCall( + assert parse_expr("col LIKE 'It''s%'") == ScalarCall( name="like", args=(Ref(name="col"), Literal(value="It's%")), ) def test_like_inside_string_literal_untouched(self) -> None: - node = parse_filter_expr("note == \"we like 'cats'\"") + node = parse_expr("note == 'we like ''cats'''") assert node == Cmp( op="==", left=Ref(name="note"), right=Literal(value="we like 'cats'"), ) def test_dotless_i_is_not_like(self) -> None: - text = "x lıke 'p%'" - assert _rewrite_sql_like(text) == text + with pytest.raises(ValueError): + parse_expr("x lıke 'p%'") def test_kelvin_sign_is_not_like(self) -> None: - text = "x liKe 'p%'" - assert _rewrite_sql_like(text) == text + with pytest.raises(ValueError): + parse_expr("x liKe 'p%'") def test_spoofed_like_fails_loudly(self) -> None: with pytest.raises(ValueError, match=r"Invalid Mode-B expression"): - parse_filter_expr("x lıke 'p%'") + parse_expr("x lıke 'p%'") def test_spoofed_is_keyword_not_normalized(self) -> None: with pytest.raises(ValueError, match=r"Invalid Mode-B expression"): - parse_filter_expr("x ıs None") + parse_expr("x ıs None") # --------------------------------------------------------------------------- # @@ -252,19 +251,19 @@ def test_spoofed_is_keyword_not_normalized(self) -> None: # --------------------------------------------------------------------------- # class TestLikeStillWorks: def test_basic_like(self) -> None: - assert parse_filter_expr("name LIKE 'a%'") == ScalarCall( + assert parse_expr("name LIKE 'a%'") == ScalarCall( name="like", args=(Ref(name="name"), Literal(value="a%")), ) def test_not_like(self) -> None: - node = parse_filter_expr("name NOT LIKE 'a%'") + node = parse_expr("name NOT LIKE 'a%'") assert isinstance(node, UnaryOp) assert node.op == "not" assert isinstance(node.operand, ScalarCall) assert node.operand.name == "like" def test_scalar_call_lhs(self) -> None: - node = parse_filter_expr("lower(customers.email) like '%@x.io'") + node = parse_expr("lower(customers.email) like '%@x.io'") assert isinstance(node, ScalarCall) assert node.name == "like" assert node.args[0] == ScalarCall( @@ -272,18 +271,19 @@ def test_scalar_call_lhs(self) -> None: ) def test_unicode_lhs_still_rewrites(self) -> None: - assert parse_filter_expr("é_col LIKE 'p%'") == ScalarCall( + assert parse_expr("é_col LIKE 'p%'") == ScalarCall( name="like", args=(Ref(name="é_col"), Literal(value="p%")), ) def test_ascii_mixed_case_still_rewrites(self) -> None: - node = parse_filter_expr("x LiKe 'p%'") + node = parse_expr("x LiKe 'p%'") assert isinstance(node, ScalarCall) assert node.name == "like" - def test_double_quoted_pattern_rejected(self) -> None: - with pytest.raises(ValueError, match=r"Invalid Mode-B expression"): - parse_filter_expr('col like "p%"') + def test_double_quoted_pattern_is_an_identifier(self) -> None: + assert parse_expr('col like "p%"') == ScalarCall( + name="like", args=(Ref(name="col"), Ref(name="p%")), + ) # --------------------------------------------------------------------------- # @@ -298,22 +298,22 @@ class TestSiblingRewriterUnicode: def test_null_keyword_fused_into_other_id_start(self) -> None: # ``℘NULL`` is one identifier; without the guard NULL → None rebinds it. - assert parse_filter_expr("℘NULL is None") == Cmp( + assert parse_expr("℘NULL is None") == Cmp( op="is", left=Ref(name="℘NULL"), right=Literal(value=None), ) def test_null_keyword_fused_after_combining_mark(self) -> None: # e + COMBINING ACUTE then NULL; NFKC composes the ref back to ``éNULL``. - assert parse_filter_expr("éNULL is None") == Cmp( + assert parse_expr("éNULL is None") == Cmp( op="is", left=Ref(name="éNULL"), right=Literal(value=None), ) def test_operator_keyword_fused_not_lowercased(self) -> None: # ``℘IS`` is a name; case-normalisation must not fold it to ``℘is``. - assert parse_filter_expr("℘IS") == Ref(name="℘IS") + assert parse_expr("℘IS") == Ref(name="℘IS") def test_null_still_normalizes_when_standalone(self) -> None: - assert parse_filter_expr("amount is NULL") == Cmp( + assert parse_expr("amount is NULL") == Cmp( op="is", left=Ref(name="amount"), right=Literal(value=None), ) @@ -330,26 +330,20 @@ def test_real_over_still_rejected(self) -> None: parse_expr("amount OVER (partition_by)") def test_like_lhs_fused_never_corrupts(self) -> None: - # A fused LHS can't lex as a LIKE operand; rather than the old - # ``℘like(name, …)`` corruption it now errors cleanly. - for expr in ("℘name LIKE 'x%'", "éname LIKE 'x%'"): - with pytest.raises(ValueError): - parse_filter_expr(expr) - - def test_like_unit_leaves_fused_lhs_untouched(self) -> None: - for text in ("℘name LIKE 'x%'", "éname LIKE 'x%'"): - assert _rewrite_sql_like(text) == text + # A fused LHS is one whole LIKE operand, never the old ``℘like(name, …)`` corruption. + for expr in ("℘name LIKE 'x%'", "éname LIKE 'x%'"): + name = unicodedata.normalize("NFKC", expr.split()[0]) + assert parse_expr(expr) == ScalarCall(name="like", args=(Ref(name=name), Literal(value="x%"))) def test_keyword_named_dotted_leaf_preserved(self) -> None: # ``a.NULL`` is a dotted ref; the NULL sub must not rewrite the leaf to # ``a.None`` (parity with the CASE lexer keeping ``customers.end``). - assert parse_filter_expr("a.NULL is None") == Cmp( + assert parse_expr("a.NULL is None") == Cmp( op="is", left=DottedRef(parts=("a", "NULL")), right=Literal(value=None), ) def test_dotted_root_other_id_start_not_a_like_operand(self) -> None: - # ``\u2118.name`` roots at an Other_ID_Start component the ``\w`` LHS can't - # lex whole; LIKE must not slice the leaf off into ``\u2118.like(name, \u2026)``. - assert _rewrite_sql_like("\u2118.name LIKE 'x%'") == "\u2118.name LIKE 'x%'" - with pytest.raises(ValueError): - parse_filter_expr("\u2118.name LIKE 'x%'") + # LIKE must not slice the leaf off into ``\u2118.like(name, \u2026)``. + assert parse_expr("\u2118.name LIKE 'x%'") == ScalarCall( + name="like", args=(DottedRef(parts=("\u2118", "name")), Literal(value="x%")), + ) diff --git a/tests/test_dev2101_agg_filter.py b/tests/test_dev2101_agg_filter.py new file mode 100644 index 00000000..290e2471 --- /dev/null +++ b/tests/test_dev2101_agg_filter.py @@ -0,0 +1,171 @@ +"""``agg(…) FILTER (WHERE c)`` (spec: aggregations/functional-form › An aggregation accepts a FILTER clause).""" + +from __future__ import annotations + +import pytest + +from slayer.core.errors import QueryTypeError +from slayer.core.query import SlayerQuery +from slayer.engine.query_engine import SlayerQueryEngine +from tests._dev2101_fixtures import ( + EXEC_BACKENDS, + TIER1, + by_key, + emit, + m, + make_exec_engine, + num, + q, + single, +) + +PAID = "status = 'paid'" +MONTH = [{"dimension": "created_at", "granularity": "month"}] + + +@pytest.fixture(params=EXEC_BACKENDS) +async def engine(request): + async for e in make_exec_engine(request): + yield e + + +async def _filtered_equals_restricted(engine: SlayerQueryEngine, formula: str, **kw) -> None: + """``formula FILTER (WHERE paid)`` equals ``formula`` over a query restricted to the paid rows.""" + filtered = await engine.execute(q(measures=[m(f"{formula} FILTER (WHERE {PAID})")], **kw)) + restricted = await engine.execute(q(measures=[m(formula)], filters=[PAID], **kw)) + assert len(filtered.data) == len(restricted.data) + key = lambda r: tuple(str(v) for k, v in sorted(r.items()) if k != "orders.v") # noqa: E731 + got = {key(r): num(r["orders.v"]) for r in filtered.data} + want = {key(r): num(r["orders.v"]) for r in restricted.data} + assert got.keys() >= want.keys() + for k, v in want.items(): + assert got[k] == pytest.approx(v), (formula, k) + + +class TestFilterValues: + async def test_filtered_count_by_region(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["region"], measures=[m(f"count(*) FILTER (WHERE {PAID})")])) + assert {k: num(v) for k, v in by_key(resp, "region").items()} == {"EU": 2, "US": 1, None: 0} + + async def test_filtered_sum_equals_masked_sum(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["region"], measures=[ + m(f"sum(amount) FILTER (WHERE {PAID})", "f"), m(f"sum(iif({PAID}, amount, NULL))", "masked"), + ])) + filtered = {r["orders.region"]: num(r["orders.f"]) for r in resp.data} + masked = {r["orders.region"]: num(r["orders.masked"]) for r in resp.data} + assert filtered == masked == {"EU": 30.0, "US": 40.0, None: None} + + @pytest.mark.parametrize(("formula", "expected"), [ + ("count(discount)", 1), ("count(distinct region)", 2), ("avg(amount)", 70 / 3), ("min(amount)", 10.0), + ("max(amount)", 40.0), ("sum(amount)", 70.0), + ]) + async def test_builtin_aggregations(self, engine: SlayerQueryEngine, formula: str, expected: float) -> None: + resp = await engine.execute(q(measures=[m(f"{formula} FILTER (WHERE {PAID})")])) + assert num(single(resp)) == pytest.approx(expected) + + async def test_weighted_average_masks_numerator_and_denominator(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(measures=[m(f"weighted_avg(price, weight=quantity) FILTER (WHERE {PAID})")])) + assert num(single(resp)) == pytest.approx(3.6) + await _filtered_equals_restricted(engine, "weighted_avg(price, weight=quantity)") + + @pytest.mark.parametrize("formula", [ + "corr(price, other=quantity)", "covar_samp(price, other=quantity)", "covar_pop(price, other=quantity)", + ]) + async def test_paired_statistics_mask_both_inputs(self, engine: SlayerQueryEngine, formula: str) -> None: + await _filtered_equals_restricted(engine, formula) + + async def test_custom_aggregation(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(measures=[m(f"wsum(price) FILTER (WHERE {PAID})")])) + assert num(single(resp)) == pytest.approx(18.0) + await _filtered_equals_restricted(engine, "wsum(price, weight=quantity)") + + async def test_filtered_last_picks_the_newest_passing_row(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(SlayerQuery.model_validate({ + "source_model": "balances", "dimensions": ["customer_id"], + "measures": [{"formula": "last(balance, updated_at) FILTER (WHERE status = 'active')", "name": "v"}], + })) + assert {int(k): num(v) for k, v in by_key(resp, "customer_id", model="balances").items()} == {1: 150.0, 2: 80.0} + + async def test_filtered_first_picks_the_oldest_passing_row(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(SlayerQuery.model_validate({ + "source_model": "balances", "dimensions": ["customer_id"], + "measures": [{"formula": "first(balance, updated_at) FILTER (WHERE status = 'inactive')", "name": "v"}], + })) + assert {int(k): num(v) for k, v in by_key(resp, "customer_id", model="balances").items()} == {1: 999.0, 2: 70.0} + + async def test_with_partition_by(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["region", "city"], measures=[ + m(f"sum(amount, partition_by=region) FILTER (WHERE {PAID})"), + ])) + got = {(r["orders.region"], r["orders.city"]): num(r["orders.v"]) for r in resp.data} + assert got == {("EU", "Paris"): 30.0, ("EU", "Berlin"): 30.0, ("US", "NYC"): 40.0, ("US", "LA"): 40.0, + (None, "Rome"): None} + + async def test_with_window(self, engine: SlayerQueryEngine) -> None: + await _filtered_equals_restricted(engine, "sum(amount, window='30d')", time_dimensions=MONTH) + resp = await engine.execute(q(measures=[m(f"sum(amount, window='30d') FILTER (WHERE {PAID})")], + time_dimensions=MONTH)) + got = {str(r["orders.created_at"])[:7]: num(r["orders.v"]) for r in resp.data} + assert got == {"2024-01": 30.0, "2024-02": 40.0, "2024-03": None} + + +class TestFilterPositions: + async def test_unnamed_filtered_aggregation_never_collides(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(measures=["sum(amount)", f"sum(amount) FILTER (WHERE {PAID})"])) + assert len(resp.columns) == 2 + assert sorted(float(v) for v in resp.data[0].values()) == [70.0, 100.0] + + async def test_post_aggregate_filter(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["region"], measures=[m("count(*)")], + filters=[f"count(*) FILTER (WHERE {PAID}) > 1"])) + assert set(by_key(resp, "region")) == {"EU"} + + async def test_order_target(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["region"], measures=[m("count(*)")], + order=[{"column": f"count(*) FILTER (WHERE {PAID})", "direction": "desc"}])) + assert [r["orders.region"] for r in resp.data] == ["EU", "US", None] + + async def test_count_distinct_order_target(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["status"], measures=[m("count(*)")], + order=[{"column": "count(distinct region)", "direction": "desc"}, + {"column": "status"}])) + assert [r["orders.status"] for r in resp.data] == ["paid", "Pending", "void"] + + async def test_inside_arithmetic(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(measures=[m(f"sum(amount) FILTER (WHERE {PAID}) / sum(amount)")])) + assert num(single(resp)) == pytest.approx(0.7) + + async def test_inside_a_transform(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(time_dimensions=MONTH, measures=[m(f"cumsum(count(*) FILTER (WHERE {PAID}))")])) + assert sorted(float(r["orders.v"]) for r in resp.data) == [2.0, 3.0, 3.0] + + async def test_inline_model_measure(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q( + source_model={"source_name": "orders", "measures": [ + {"name": "paid_total", "formula": f"sum(amount) FILTER (WHERE {PAID})"}, + ]}, + measures=["paid_total"], + )) + assert num(single(resp, "paid_total")) == 70.0 + + +class TestFilterConditionIsRowLevel: + @pytest.mark.parametrize("measures", [ + [m("sum(amount) FILTER (WHERE amount > avg(amount))")], + [m("sum(amount) FILTER (WHERE cumsum(sum(amount)) > 1)")], + [m("sum(amount) FILTER (WHERE amount > avg(amount, partition_by=region))")], + ], ids=["aggregate", "transform", "partitioned-aggregate"]) + async def test_non_row_level_condition_rejected(self, engine: SlayerQueryEngine, measures) -> None: + with pytest.raises(QueryTypeError) as info: + await engine.execute(q(measures=measures)) + assert "FILTER" in str(info.value), str(info.value) + + +class TestFilterEmission: + @pytest.mark.parametrize("dialect", TIER1) + @pytest.mark.parametrize("formula", [ + "count(*)", "sum(amount)", "count(distinct region)", "weighted_avg(price, weight=quantity)", + "last(amount, created_at)", "corr(price, other=quantity)", + ]) + async def test_emits(self, dialect: str, formula: str) -> None: + await emit(q(dimensions=["region"], measures=[m(f"{formula} FILTER (WHERE {PAID})")]), dialect=dialect) diff --git a/tests/test_dev2101_cast_typing.py b/tests/test_dev2101_cast_typing.py new file mode 100644 index 00000000..e3730cff --- /dev/null +++ b/tests/test_dev2101_cast_typing.py @@ -0,0 +1,76 @@ +"""A cast's inferred result type (spec: queries/casts › A cast's result type is inferred).""" + +from __future__ import annotations + +import pytest + +from slayer.core.enums import DataType +from slayer.core.keys import CastKey, ValueKey, walk_value_keys +from slayer.core.scope import ModelScope +from slayer.core.syntax import parse_expr +from slayer.engine.binding import bind_expr +from slayer.engine.key_metadata import dimension_key_metadata, measure_key_type +from slayer.ir.source_bundle import ResolvedSourceBundle +from tests._dev2101_fixtures import customers_model, orders_model + +ORDERS = orders_model() +BUNDLE = ResolvedSourceBundle(dialect="postgres", source_model=ORDERS, referenced_models=[customers_model()]) + +TYPE_CASES = [ + ("INT", DataType.INT), ("INTEGER", DataType.INT), ("BIGINT", DataType.INT), ("SMALLINT", DataType.INT), + ("TINYINT", DataType.INT), + ("FLOAT", DataType.DOUBLE), ("DOUBLE", DataType.DOUBLE), ("DOUBLE PRECISION", DataType.DOUBLE), + ("REAL", DataType.DOUBLE), ("DECIMAL(10,2)", DataType.DOUBLE), ("NUMERIC", DataType.DOUBLE), + ("TEXT", DataType.TEXT), ("VARCHAR(3)", DataType.TEXT), ("CHAR(2)", DataType.TEXT), + ("BOOLEAN", DataType.BOOLEAN), + ("DATE", DataType.DATE), + ("TIMESTAMP", DataType.TIMESTAMP), ("DATETIME", DataType.TIMESTAMP), + ("JSON", DataType.UNKNOWN), ("UUID", DataType.UNKNOWN), ("BYTEA", DataType.UNKNOWN), +] + + +def _bind(text: str) -> ValueKey: + return bind_expr(parse_expr(text), scope=ModelScope(source_model=ORDERS), bundle=BUNDLE, + allow_measures=True).value_key + + +class TestCastKey: + @pytest.mark.parametrize("text", ["CAST(amount AS INT)", "amount::int", "TRY_CAST(code AS INT)"]) + def test_cast_binds_to_its_own_key_kind(self, text: str) -> None: + key = _bind(text) + assert isinstance(key, CastKey), key + assert key.try_ is text.startswith("TRY_") + + def test_spellings_share_one_key(self) -> None: + assert _bind("CAST(amount AS INT)") == _bind("amount::int") + + def test_try_cast_is_a_different_value(self) -> None: + assert _bind("TRY_CAST(code AS INT)") != _bind("CAST(code AS INT)") + + def test_traversal_reaches_the_source(self) -> None: + keys = list(walk_value_keys(_bind("amount::int::text"))) + assert sum(isinstance(k, CastKey) for k in keys) == 2 + assert any(getattr(k, "leaf", None) == "amount" for k in keys) + + +class TestInferredType: + @pytest.mark.parametrize(("target", "expected"), TYPE_CASES, ids=[t for t, _ in TYPE_CASES]) + def test_dimension_type(self, target: str, expected: DataType) -> None: + dim_type, _fmt, _desc = dimension_key_metadata(model=ORDERS, key=_bind(f"CAST(code AS {target})"), + bundle=BUNDLE) + assert dim_type == expected + + @pytest.mark.parametrize(("target", "expected"), [c for c in TYPE_CASES if c[1] is not DataType.UNKNOWN], + ids=[t for t, e in TYPE_CASES if e is not DataType.UNKNOWN]) + def test_measure_type(self, target: str, expected: DataType) -> None: + assert measure_key_type(model=ORDERS, key=_bind(f"max(CAST(code AS {target}))")) == expected + + def test_try_cast_types_like_cast(self) -> None: + dim_type, _fmt, _desc = dimension_key_metadata(model=ORDERS, key=_bind("TRY_CAST(code AS DATE)"), + bundle=BUNDLE) + assert dim_type == DataType.DATE + + def test_cast_to_date_feeds_a_date_function(self) -> None: + key = _bind("date_diff('day', CAST(created_txt AS DATE), DATE '2025-01-01')") + dim_type, _fmt, _desc = dimension_key_metadata(model=ORDERS, key=key, bundle=BUNDLE) + assert dim_type == DataType.INT diff --git a/tests/test_dev2101_casts.py b/tests/test_dev2101_casts.py new file mode 100644 index 00000000..9117b693 --- /dev/null +++ b/tests/test_dev2101_casts.py @@ -0,0 +1,243 @@ +"""CAST / ``::`` / TRY_CAST in Mode-B expressions (spec: queries/casts).""" + +from __future__ import annotations + +from datetime import date +from typing import Any + +import pytest +import sqlglot +from sqlglot import exp + +from slayer.core.errors import QueryTypeError +from slayer.engine.query_engine import SlayerQueryEngine +from slayer.sql.client import SlayerSQLClient +from slayer.sql.dialects import get_dialect +from slayer.sql.dialects.base import ServerProfile +from tests._dev2101_fixtures import ( + EXEC_BACKENDS, TIER1, by_key, day, emit, engine_for, m, make_exec_engine, num, q, single, +) + +TRY_CODE = {1: 12, 2: None, 3: None, 4: 7, 5: None} + + +@pytest.fixture(params=EXEC_BACKENDS) +async def engine(request): + async for e in make_exec_engine(request): + yield e + + +def _ints(values: dict) -> dict[int, Any]: + return {int(k): (None if v is None else int(v)) for k, v in values.items()} + + +def _casts(sql: str, *, dialect: str) -> list[exp.Cast]: + return list(sqlglot.parse_one(sql, dialect=dialect).find_all(exp.Cast)) + + +class TestCastExecution: + @pytest.mark.parametrize("spelling", ["CAST(amount AS DECIMAL(10,2))", "amount::decimal(10,2)"]) + async def test_parameterised_cast_in_measure(self, engine: SlayerQueryEngine, spelling: str) -> None: + assert num(single(await engine.execute(q(measures=[m(f"sum({spelling})")])))) == pytest.approx(100.0) + + async def test_chained_casts(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id", {"expression": "amount::int::text", "name": "x"}])) + assert {int(k): v for k, v in by_key(resp, "id", "x").items()} == {1: "10", 2: "20", 3: "30", 4: "40", 5: None} + + @pytest.mark.parametrize("position", ["dimension", "filter", "order"]) + async def test_cast_in_every_position(self, engine: SlayerQueryEngine, position: str) -> None: + expr = "CAST(quantity AS TEXT)" + if position == "dimension": + resp = await engine.execute(q(dimensions=["id", {"expression": expr, "name": "x"}])) + assert by_key(resp, "id", "x")[3] == "3" + elif position == "filter": + resp = await engine.execute(q(dimensions=["id"], filters=[f"{expr} = '4'"])) + assert [int(r["orders.id"]) for r in resp.data] == [5] + else: + resp = await engine.execute(q(dimensions=["id"], distinct_dimension_values=False, + order=[{"column": expr, "direction": "desc"}, {"column": "id"}])) + assert [int(r["orders.id"]) for r in resp.data] == [5, 3, 2, 4, 1] + + async def test_cast_to_date_is_a_date_operand(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=[ + "id", {"expression": "date_diff('day', CAST(created_txt AS DATE), DATE '2025-01-01')", "name": "x"}, + ])) + expected = {1: date(2024, 12, 25), 2: date(2024, 6, 1), 3: date(2024, 2, 29), 4: date(2023, 1, 1), 5: None} + assert _ints(by_key(resp, "id", "x")) == { + k: (None if v is None else (date(2025, 1, 1) - v).days) for k, v in expected.items() + } + + +@pytest.fixture +async def sqlite_engine(): + async for e in engine_for("sqlite"): + yield e + + +class TestInvalidInputFollowsTheDatabase: + async def test_sqlite_cast_keeps_its_own_semantics(self, sqlite_engine: SlayerQueryEngine) -> None: + resp = await sqlite_engine.execute(q(dimensions=["id", {"expression": "CAST(code AS INT)", "name": "x"}])) + assert _ints(by_key(resp, "id", "x")) == {1: 12, 2: 0, 3: 12, 4: 7, 5: None} + + +class TestTryCastExecution: + async def test_invalid_text_becomes_null(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id", {"expression": "TRY_CAST(code AS INT)", "name": "x"}])) + assert _ints(by_key(resp, "id", "x")) == TRY_CODE + + async def test_prefix_is_not_truncated(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id"], filters=["TRY_CAST(code AS INT) IS NULL"])) + assert {int(r["orders.id"]) for r in resp.data} == {2, 3, 5} + + async def test_try_cast_order_target(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id"], distinct_dimension_values=False, + filters=["TRY_CAST(code AS INT) IS NOT NULL"], + order=[{"column": "TRY_CAST(code AS INT)", "direction": "desc"}])) + assert [int(r["orders.id"]) for r in resp.data] == [1, 4] + + async def test_try_cast_in_measure_and_filter(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(measures=[m("sum(TRY_CAST(code AS INT))")], + filters=["TRY_CAST(code AS INT) > 0"])) + assert num(single(resp)) == 19 + + @pytest.mark.parametrize(("text", "expected"), [ + ("12", 12.0), (" 12 ", 12.0), ("+3", 3.0), ("-4.5e1", -45.0), ("2.5", 2.5), + ("abc", None), ("12abc", None), ("1e", None), ("", None), ("1 2", None), + ]) + async def test_number_grammar(self, engine: SlayerQueryEngine, text: str, expected: float | None) -> None: + resp = await engine.execute(q(measures=[m(f"max(TRY_CAST('{text}' AS DOUBLE))")])) + assert num(single(resp)) == expected + + @pytest.mark.parametrize(("text", "expected"), [ + ("2024-02-29", "2024-02-29"), ("2024-02-30", None), ("2024-13-01", None), ("not a date", None), + ]) + async def test_date_grammar(self, engine: SlayerQueryEngine, text: str, expected: str | None) -> None: + resp = await engine.execute(q(measures=[m(f"max(TRY_CAST('{text}' AS DATE))")])) + assert day(single(resp)) == expected + + +class TestSqliteTryCastBooleans: + @pytest.mark.parametrize(("text", "expected"), [ + ("true", True), ("FALSE", False), ("t", True), ("F", False), ("yes", True), ("No", False), + ("1", True), ("0", False), ("maybe", None), ("2", None), + ]) + async def test_boolean_words(self, sqlite_engine: SlayerQueryEngine, text: str, expected) -> None: + resp = await sqlite_engine.execute(q(measures=[m(f"max(TRY_CAST('{text}' AS BOOLEAN))")])) + value = single(resp) + assert (None if value is None else bool(value)) == expected + + +class TestCastEmission: + async def test_postgres_keeps_the_parameterised_type(self) -> None: + sql = await emit(q(measures=[m("sum(CAST(amount AS DECIMAL(10,2)))")]), dialect="postgres") + assert "DECIMAL(10, 2)" in sql, sql + + async def test_mysql_text_cast_uses_its_character_type(self) -> None: + sql = await emit(q(dimensions=["id", {"expression": "amount::text", "name": "x"}]), dialect="mysql") + casts = [c for c in _casts(sql, dialect="mysql") if "amount" in c.this.sql().lower()] + assert casts and all("CHAR" in c.to.sql(dialect="mysql").upper() for c in casts), sql + + async def test_chained_casts_nest(self) -> None: + sql = await emit(q(dimensions=["id", {"expression": "amount::int::text", "name": "x"}]), dialect="postgres") + outer = next(c for c in _casts(sql, dialect="postgres") if isinstance(c.this, exp.Cast)) + assert outer.to.is_type(exp.DataType.Type.TEXT) and outer.this.to.is_type(exp.DataType.Type.INT), sql + + @pytest.mark.parametrize("dialect", TIER1) + async def test_cast_emits_everywhere(self, dialect: str) -> None: + await emit(q(dimensions=["id", {"expression": "CAST(quantity AS TEXT)", "name": "x"}], + measures=[m("sum(amount::int)")], filters=["CAST(amount AS DECIMAL(10,2)) > 1"]), + dialect=dialect) + + async def test_unknown_target_type_renders(self) -> None: + sql = await emit(q(dimensions=["id", {"expression": "CAST(payload AS JSON)", "name": "x"}]), dialect="postgres") + assert any(c.to.is_type(exp.DataType.Type.JSON) for c in _casts(sql, dialect="postgres")), sql + + @pytest.mark.parametrize("dialect", ["postgres", "tsql"]) + async def test_dialect_specific_type_accepted_where_expressible(self, dialect: str) -> None: + sql = await emit(q(measures=[m("sum(CAST(amount AS MONEY))")]), dialect=dialect) + assert "MONEY" in sql.upper(), sql + + @pytest.mark.parametrize("dialect", ["bigquery", "mysql", "trino", "snowflake", "clickhouse"]) + async def test_inexpressible_type_rejected_at_plan_time(self, dialect: str) -> None: + with pytest.raises(QueryTypeError) as info: + await emit(q(measures=[m("sum(CAST(amount AS MONEY))")]), dialect=dialect) + msg = str(info.value).lower() + assert "money" in msg and dialect in msg, msg + + +_TRY_CAST_MARKERS = { + "duckdb": "TRY_CAST(", + "tsql": "TRY_CAST(", + "snowflake": "TRY_CAST(", + "trino": "TRY_CAST(", + "bigquery": "SAFE_CAST(", + "clickhouse": "ACCURATECASTORNULL(", + "sqlite": "SLAYER_TRY_CAST(", + "mysql": "REGEXP", +} + + +class TestTryCastEmission: + @pytest.mark.parametrize("dialect", sorted(_TRY_CAST_MARKERS)) + async def test_per_dialect_form(self, dialect: str) -> None: + sql = await emit(q(dimensions=["id", {"expression": "TRY_CAST(code AS INT)", "name": "x"}]), dialect=dialect) + assert _TRY_CAST_MARKERS[dialect] in sql.upper(), sql + + @pytest.mark.parametrize("target", ["INT", "DOUBLE", "DATE", "TIMESTAMP", "BOOLEAN", "DECIMAL(10,2)"]) + @pytest.mark.parametrize("dialect", TIER1) + async def test_every_target_emits(self, dialect: str, target: str) -> None: + await emit(q(dimensions=["id", {"expression": f"TRY_CAST(code AS {target})", "name": "x"}]), dialect=dialect) + + +def _patch_profile(monkeypatch, *, version: tuple[int, int] | None = None, fail: bool = False) -> list[int]: + calls: list[int] = [] + + async def fake(_self) -> ServerProfile: + calls.append(1) + if fail: + raise ConnectionError("probe failed") + return ServerProfile(version=version) + + monkeypatch.setattr(SlayerSQLClient, "server_profile", fake) + return calls + + +TRY_Q = q(dimensions=["id", {"expression": "TRY_CAST(code AS INT)", "name": "x"}]) + + +class TestPostgresTryCast: + def test_profile_probe_statement(self) -> None: + assert get_dialect("postgres").server_profile_sql() == "SELECT current_setting('server_version_num')" + + @pytest.mark.parametrize(("row", "version"), [ + (("160002",), (16, 2)), (("150004",), (15, 4)), ((170000,), (17, 0)), (("garbage",), None), ((None,), None), + ]) + def test_profile_parse(self, row: tuple, version: tuple[int, int] | None) -> None: + assert get_dialect("postgres").parse_server_profile(base_row=row) == ServerProfile(version=version) + + def test_profile_parse_tolerates_no_row(self) -> None: + assert get_dialect("postgres").parse_server_profile(base_row=None) == ServerProfile() + + async def test_postgres_16_guards_with_the_native_check(self, monkeypatch) -> None: + _patch_profile(monkeypatch, version=(16, 2)) + sql = await emit(TRY_Q, dialect="postgres") + assert "PG_INPUT_IS_VALID(" in sql.upper(), sql + + @pytest.mark.parametrize("version", [(15, 4), None]) + async def test_older_or_unknown_server_gets_a_plain_cast(self, monkeypatch, version) -> None: + _patch_profile(monkeypatch, version=version) + sql = await emit(TRY_Q, dialect="postgres") + assert "PG_INPUT_IS_VALID(" not in sql.upper() and "TRY_CAST" not in sql.upper(), sql + assert _casts(sql, dialect="postgres"), sql + + async def test_failed_probe_gets_a_plain_cast(self, monkeypatch) -> None: + _patch_profile(monkeypatch, fail=True) + sql = await emit(TRY_Q, dialect="postgres") + assert "PG_INPUT_IS_VALID(" not in sql.upper(), sql + + async def test_profile_probed_only_for_try_cast(self, monkeypatch) -> None: + calls = _patch_profile(monkeypatch, version=(16, 2)) + await emit(q(dimensions=["id", {"expression": "CAST(code AS INT)", "name": "x"}]), dialect="postgres") + assert calls == [] + await emit(TRY_Q, dialect="postgres") + assert calls == [1] diff --git a/tests/test_dev2101_date_literals.py b/tests/test_dev2101_date_literals.py new file mode 100644 index 00000000..fc82bde6 --- /dev/null +++ b/tests/test_dev2101_date_literals.py @@ -0,0 +1,134 @@ +"""SQL interval literals, typed date literals and clock keywords (spec: queries/date-functions).""" + +from __future__ import annotations + +import pytest + +from slayer.core.errors import TimeLiteralError +from slayer.engine.query_engine import SlayerQueryEngine +from slayer.sql.client import SlayerSQLClient +from tests._dev2101_fixtures import EXEC_BACKENDS, TIER1, by_key, emit, engine_for, ids, make_exec_engine, q + + +@pytest.fixture(params=EXEC_BACKENDS) +async def engine(request): + async for e in make_exec_engine(request): + yield e + + +async def _sql(filters: list[str], **kw) -> str: + return await emit(q(dimensions=["id"], filters=filters, **kw), dialect="postgres") + + +async def _same_sql(a: str, b: str) -> None: + assert await _sql([a]) == await _sql([b]), (a, b) + + +class TestIntervalLiterals: + @pytest.mark.parametrize("literal", ["INTERVAL '7 days'", "INTERVAL '7' DAY", "INTERVAL 7 DAY", + "interval '7 day'", "INTERVAL '7' days"]) + async def test_literal_equals_date_add(self, literal: str) -> None: + await _same_sql(f"order_date + {literal} > CURRENT_DATE", "date_add(order_date, 7, 'day') > current_date()") + + async def test_subtraction(self) -> None: + await _same_sql("created_at - INTERVAL '2 hours' < now()", "date_add(created_at, -2, 'hour') < now()") + + async def test_commuted_addition(self) -> None: + await _same_sql("INTERVAL '7 days' + order_date > CURRENT_DATE", "date_add(order_date, 7, 'day') > current_date()") + + async def test_chain_applies_left_to_right(self) -> None: + await _same_sql("created_at + INTERVAL '1 month' + INTERVAL 2 HOUR < now()", + "date_add(date_add(created_at, 1, 'month'), 2, 'hour') < now()") + + @pytest.mark.parametrize(("literal", "unit"), [ + ("INTERVAL '3 weeks'", "week"), ("INTERVAL '1 MONTH'", "month"), ("INTERVAL 2 years", "year"), + ("INTERVAL '90' MINUTE", "minute"), ("INTERVAL '1 quarter'", "quarter"), + ]) + async def test_units(self, literal: str, unit: str) -> None: + n = literal.split("'")[1].split()[0] if "'" in literal else literal.split()[1] + await _same_sql(f"created_at + {literal} < now()", f"date_add(created_at, {n}, '{unit}') < now()") + + async def test_executes(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id"], filters=["order_date + INTERVAL '7 days' > DATE '2024-03-01'"])) + assert ids(resp) == {4, 5} + + async def test_multi_unit_literal_suggests_a_chain(self, engine: SlayerQueryEngine) -> None: + with pytest.raises(ValueError) as info: + await engine.execute(q(dimensions=["id"], filters=["order_date + INTERVAL '1 day 2 hours' > now()"])) + assert "+ INTERVAL '1 day' + INTERVAL '2 hours'" in str(info.value), str(info.value) + + @pytest.mark.parametrize("filt", [ + "INTERVAL '7 days' > order_date", + "INTERVAL '7 days' - order_date > 0", + "-INTERVAL '7 days' + order_date > order_date", + "order_date + INTERVAL '1 day' * 2 > order_date", + "date_part('day', INTERVAL '1 day') > 0", + "order_date + (INTERVAL '1 day' + INTERVAL '1 day') > order_date", + ]) + async def test_stray_interval_literal_points_to_date_add(self, engine: SlayerQueryEngine, filt: str) -> None: + with pytest.raises(ValueError) as info: + await engine.execute(q(dimensions=["id"], filters=[filt])) + assert "date_add" in str(info.value), str(info.value) + + @pytest.mark.parametrize("literal", ["INTERVAL 'x days'", "INTERVAL '1.5 days'", "INTERVAL '7 fortnights'"]) + async def test_bad_count_or_unit(self, engine: SlayerQueryEngine, literal: str) -> None: + with pytest.raises(ValueError): + await engine.execute(q(dimensions=["id"], filters=[f"order_date + {literal} > order_date"])) + + @pytest.mark.parametrize("dialect", TIER1) + async def test_emits_everywhere(self, dialect: str) -> None: + await emit(q(dimensions=["id", {"expression": "order_date + INTERVAL '1 month'", "name": "x"}], + filters=["created_at - INTERVAL 2 HOUR < CURRENT_TIMESTAMP"]), dialect=dialect) + + +class TestTypedLiterals: + async def test_typed_literals_bind_and_execute(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q( + dimensions=["id", {"expression": "date_diff('day', DATE '2024-01-01', created_at)", "name": "x"}], + filters=["created_at >= TIMESTAMP '2024-01-20 00:00'"], + )) + assert {int(k): int(v) for k, v in by_key(resp, "id", "x").items()} == {2: 19, 3: 40, 4: 55, 5: 74} + + async def test_typed_literal_equals_iso_literal_in_a_date_position(self) -> None: + await _same_sql("order_date >= DATE '2024-02-25'", "order_date >= '2024-02-25'") + + @pytest.mark.parametrize("filt", ["created_at >= DATE '2024-02-30'", "created_at >= TIMESTAMP '2024-01-01 25:00'", + "order_date >= DATE 'tomorrow'"]) + async def test_impossible_or_malformed_literal(self, engine: SlayerQueryEngine, filt: str) -> None: + with pytest.raises(TimeLiteralError) as info: + await engine.execute(q(dimensions=["id"], filters=[filt])) + literal = filt.split("'")[1] + assert literal in str(info.value), str(info.value) + + async def test_typed_literal_outside_a_date_position(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id", {"expression": "DATE '2024-01-01'", "name": "d"}], + filters=["id = 1"])) + assert str(resp.data[0]["orders.d"])[:10] == "2024-01-01" + + +@pytest.fixture +def data_calls(monkeypatch) -> list[str]: + calls: list[str] = [] + orig = SlayerSQLClient.execute + + async def spy(self, sql, timeout_seconds=120): + calls.append(sql) + return await orig(self, sql=sql, timeout_seconds=timeout_seconds) + + monkeypatch.setattr(SlayerSQLClient, "execute", spy) + return calls + + +class TestClockKeywords: + @pytest.mark.parametrize(("keyword", "call"), [("CURRENT_DATE", "current_date()"), ("CURRENT_TIMESTAMP", "now()")]) + async def test_keyword_equals_call(self, keyword: str, call: str) -> None: + await _same_sql(f"created_at >= {keyword}", f"created_at >= {call}") + + @pytest.mark.parametrize("keyword", ["CURRENT_DATE", "current_timestamp"]) + async def test_keyword_bypasses_the_result_cache(self, data_calls: list[str], keyword: str) -> None: + async for eng in engine_for("sqlite"): + query = q(dimensions=["id"], filters=[f"created_at <= {keyword}"]) + await eng.execute(query, cache=True) + await eng.execute(query, cache=True) + assert len([s for s in data_calls if "slayer_rk_" not in s]) == 2 + assert eng.cache_size == 0 diff --git a/tests/test_dev2101_grammar_parse.py b/tests/test_dev2101_grammar_parse.py new file mode 100644 index 00000000..9f69b36e --- /dev/null +++ b/tests/test_dev2101_grammar_parse.py @@ -0,0 +1,386 @@ +"""Parse-level tests for the Mode-B grammar (spec: queries/expression-grammar, queries/casts, aggregations/functional-form).""" + +from __future__ import annotations + +import importlib.util +import re +from decimal import Decimal + +import pytest + +from slayer.core import syntax +from slayer.core.errors import IllegalWindowInFilterError +from slayer.core.syntax import AggCall, Cast, DottedRef, Literal, Ref, ScalarCall, TransformCall, parse_expr + + +def same(a: str, b: str) -> None: + assert parse_expr(a) == parse_expr(b), (a, b) + + +def rejected(text: str, *needles: str, exc: type[Exception] = ValueError) -> str: + with pytest.raises(exc) as info: + parse_expr(text) + msg = str(info.value) + for needle in needles: + assert needle.lower() in msg.lower(), msg + return msg + + +class TestOneParser: + def test_parser_lives_in_core(self) -> None: + assert importlib.util.find_spec("slayer.engine.syntax") is None + + def test_no_filter_only_entry_point(self) -> None: + assert not hasattr(syntax, "parse_filter_expr") + + def test_parse_is_cached(self) -> None: + assert hasattr(parse_expr, "cache_info") + assert parse_expr("sum(amount) + 1") is parse_expr("sum(amount) + 1") + + @pytest.mark.parametrize("helper", [ + "_rewrite_case_when", "_rewrite_sql_like", "_preprocess_colons", "_normalize_sql_filter_operators", + "_rewrite_extract", "_rewrite_count_distinct", "_preprocess_star_args", "_rewrite_comparison_equals", + ]) + def test_text_pre_passes_are_gone(self, helper: str) -> None: + assert not hasattr(syntax, helper) + + +class TestSqlSpellings: + @pytest.mark.parametrize(("sql", "legacy"), [ + ("status = 'paid'", "status == 'paid'"), + ("status <> 'paid'", "status != 'paid'"), + ("amount IS NULL", "amount is None"), + ("amount IS NOT NULL", "amount is not None"), + ("amount is not null", "amount is not None"), + ("a = 1 AND b = 2 OR NOT c = 3", "a == 1 and b == 2 or not c == 3"), + ("region IN ('EU', 'US')", "region in ['EU', 'US']"), + ("region NOT IN ('EU')", "region not in ['EU']"), + ("amount BETWEEN 10 AND 20", "amount >= 10 and amount <= 20"), + ("amount NOT BETWEEN 10 AND 20", "not (amount >= 10 and amount <= 20)"), + ("amount between 1 + 1 and 3 * 2", "amount >= 1 + 1 and amount <= 3 * 2"), + ("count(distinct region)", "count_distinct(region)"), + ("COUNT(DISTINCT region)", "count_distinct(region)"), + ("EXTRACT(month FROM created_at)", "date_part('month', created_at)"), + ("POSITION('a' IN status)", "instr(status, 'a')"), + ("SUBSTRING(status FROM 1 FOR 3)", "substr(status, 1, 3)"), + ("SUBSTRING(status FROM 2)", "substr(status, 2)"), + ("CURRENT_DATE", "current_date()"), + ("CURRENT_TIMESTAMP", "now()"), + ("status LIKE 'pa%'", "like(status, 'pa%')"), + ("status NOT LIKE 'pa%'", "not like(status, 'pa%')"), + ("lower(trim(status)) LIKE 'pa%'", "like(lower(trim(status)), 'pa%')"), + ("amount ** 2", "pow(amount, 2)"), + ]) + def test_spelling_collapses_to_one_node(self, sql: str, legacy: str) -> None: + same(sql, legacy) + + @pytest.mark.parametrize("text", [ + "status ILIKE 'PA%'", "status NOT ILIKE 'pa%'", "status || '-' || region", + "region IS DISTINCT FROM 'EU'", "region IS NOT DISTINCT FROM NULL", + "(amount > 15) IS TRUE", "(amount > 15) IS NOT TRUE", "(amount > 15) IS FALSE", "(amount > 15) IS NOT FALSE", + "-7 % 3", "DATE '2024-01-01'", "TIMESTAMP '2024-01-01 00:00'", + ]) + def test_sql_form_parses(self, text: str) -> None: + parse_expr(text) + + def test_like_inside_string_is_text(self) -> None: + assert parse_expr("note = 'we like cats'") == parse_expr("note == 'we like cats'") + + def test_ilike_is_not_like(self) -> None: + assert parse_expr("status ILIKE 'pa%'") != parse_expr("status LIKE 'pa%'") + + +class TestCase: + @pytest.mark.parametrize(("case", "iif"), [ + ("CASE WHEN amount > 100 THEN 'big' ELSE 'small' END", "iif(amount > 100, 'big', 'small')"), + ("CASE WHEN amount > 100 THEN 'big' END", "iif(amount > 100, 'big', None)"), + ("CASE status WHEN 'a' THEN 1 WHEN 'b' THEN 2 END", "iif(status == 'a', 1, iif(status == 'b', 2, None))"), + ("CASE WHEN region = 'EU' AND amount IS NOT NULL THEN 1 ELSE 0 END", + "iif(region == 'EU' and amount is not None, 1, 0)"), + ("case when a > 1 then case when b > 1 then 1 else 2 end else 3 end", "iif(a > 1, iif(b > 1, 1, 2), 3)"), + ("sum(CASE WHEN status = 'paid' THEN amount ELSE 0 END)", "sum(iif(status == 'paid', amount, 0))"), + ]) + def test_case_lowers_to_iif(self, case: str, iif: str) -> None: + same(case, iif) + + @pytest.mark.parametrize(("text", "missing"), [ + ("CASE WHEN a THEN 1", "END"), + ("CASE WHEN a 1 END", "THEN"), + ]) + def test_malformed_case_names_the_keyword(self, text: str, missing: str) -> None: + msg = rejected(text, "CASE") + assert missing in msg, msg + + +class TestKeywordsOnlyInTheirSlot: + @pytest.mark.parametrize("name", ["case", "end", "with", "model", "date", "filter", "timestamp", "between"]) + def test_keyword_word_is_an_identifier(self, name: str) -> None: + assert parse_expr(name) == Ref(name=name) + assert parse_expr(f"sum({name})") == AggCall(source=Ref(name=name), agg="sum") + + def test_case_is_conditional_only_before_when(self) -> None: + assert parse_expr("case + 1").left == Ref(name="case") + assert isinstance(parse_expr("CASE WHEN a THEN 1 END"), ScalarCall) + + def test_date_literal_vs_date_column(self) -> None: + assert parse_expr("date + 1") != parse_expr("DATE '2024-01-01'") + assert parse_expr("DATE '2024-01-01'") != parse_expr("'2024-01-01'") + same("date '2024-01-01'", "DATE '2024-01-01'") + + @pytest.mark.parametrize("text", ["amount \u0131s NULL", "a \u0430nd b"]) + def test_non_ascii_keyword_spoof_is_not_a_keyword(self, text: str) -> None: + with pytest.raises(ValueError): + parse_expr(text) + + +class TestPrecedence: + @pytest.mark.parametrize(("text", "explicit"), [ + ("NOT amount > 100 AND status = 'paid'", "(not (amount > 100)) and (status == 'paid')"), + ("a OR b AND c", "a or (b and c)"), + ("a = 1 OR b = 2 AND c = 3", "(a == 1) or ((b == 2) and (c == 3))"), + ("status || 'x' = 'paidx'", "(status || 'x') = 'paidx'"), + ("a || b + c", "a || (b + c)"), + ("a + b * c", "a + (b * c)"), + ("a - b - c", "(a - b) - c"), + ("a / b % c", "(a / b) % c"), + ("2 ** 3 ** 2", "2 ** (3 ** 2)"), + ("-2 ** 2", "-(2 ** 2)"), + ("2 ** -2", "2 ** (-2)"), + ("amount::int + 1", "(amount::int) + 1"), + ("-amount::int", "-(amount::int)"), + ("amount::int ** 2", "(amount::int) ** 2"), + ]) + def test_binding(self, text: str, explicit: str) -> None: + same(text, explicit) + + def test_parenthesised_negative_base(self) -> None: + assert parse_expr("(-2) ** 2") != parse_expr("-2 ** 2") + + +class TestKeywordArguments: + def test_kwarg_in_aggregation(self) -> None: + parsed = parse_expr("percentile(amount, p=0.9)") + assert isinstance(parsed, AggCall) + assert dict(parsed.kwargs) == {"p": Literal(value=Decimal("0.9"))} + + def test_kwarg_in_custom_aggregation(self) -> None: + parsed = parse_expr("wsum(price, weight = quantity)") + assert isinstance(parsed, AggCall) + assert dict(parsed.kwargs) == {"weight": Ref(name="quantity")} + + def test_kwarg_in_transform(self) -> None: + parsed = parse_expr("rank(sum(amount), direction = 'desc')") + assert isinstance(parsed, TransformCall) + assert dict(parsed.kwargs)["direction"] == Literal(value="desc") + + def test_comparison_inside_scalar_call(self) -> None: + parsed = parse_expr("sum(iif(status = 'paid', amount, 0))") + assert isinstance(parsed, AggCall) and isinstance(parsed.source, ScalarCall) + assert parsed.source.args[0] == parse_expr("status == 'paid'") + + def test_comparison_first_argument_of_coalesce(self) -> None: + parsed = parse_expr("coalesce(a = 1, b)") + assert isinstance(parsed, ScalarCall) + assert parsed.args[0] == parse_expr("a == 1") + + def test_predicate_inside_transform_argument(self) -> None: + parsed = parse_expr("cumsum(sum(amount) = 10)") + assert isinstance(parsed, TransformCall) + assert parsed.input == parse_expr("sum(amount) == 10") + + def test_predicate_inside_aggregation_source(self) -> None: + same("sum(amount = 10)", "sum(amount == 10)") + + +class TestColonAndCast: + def test_colon_aggregation_kept(self) -> None: + same("amount:sum", "sum(amount)") + + def test_double_colon_is_a_cast(self) -> None: + parsed = parse_expr("amount::int") + assert isinstance(parsed, Cast) + assert parsed.source == Ref(name="amount") + assert parsed.type_text.upper() == "INT" + assert parsed.try_ is False + + def test_chained_casts(self) -> None: + parsed = parse_expr("amount::int::text") + assert isinstance(parsed, Cast) and parsed.type_text.upper() == "TEXT" + assert isinstance(parsed.source, Cast) and parsed.source.type_text.upper() == "INT" + assert parsed.source.source == Ref(name="amount") + + def test_cast_call_equals_double_colon(self) -> None: + same("CAST(amount AS int)", "amount::int") + same("cast(amount as int)", "amount::int") + + def test_parameterised_type_kept(self) -> None: + parsed = parse_expr("CAST(amount AS DECIMAL(10,2))") + assert isinstance(parsed, Cast) + assert re.fullmatch(r"DECIMAL\(\s*10\s*,\s*2\s*\)", parsed.type_text.upper()), parsed.type_text + + def test_try_cast(self) -> None: + parsed = parse_expr("TRY_CAST(code AS INT)") + assert isinstance(parsed, Cast) and parsed.try_ is True + assert parsed != parse_expr("CAST(code AS INT)") + + def test_cast_inside_aggregation(self) -> None: + parsed = parse_expr("sum(CAST(amount AS DECIMAL(10,2)))") + assert isinstance(parsed, AggCall) and isinstance(parsed.source, Cast) + + def test_colon_aggregation_over_cast_free_text(self) -> None: + assert isinstance(parse_expr("revenue:sum"), AggCall) + assert isinstance(parse_expr("revenue::text"), Cast) + + +class TestIdentifiers: + def test_quoted_identifier(self) -> None: + assert parse_expr('"order total"') == Ref(name="order total") + + def test_quoted_plain_name_equals_bare(self) -> None: + same('"status"', "status") + + def test_quoted_segment_in_dotted_path(self) -> None: + assert parse_expr('customers."Region"') == DottedRef(parts=("customers", "Region")) + + def test_doubled_quote_inside_identifier(self) -> None: + assert parse_expr('"a""b"') == Ref(name='a"b') + + def test_quoted_identifier_as_aggregation_source(self) -> None: + assert parse_expr('sum("order total")') == AggCall(source=Ref(name="order total"), agg="sum") + + @pytest.mark.parametrize("text", ['"__slayer_x"', "__slayer_x", 'customers."__slayer_x"']) + def test_reserved_prefix_rejected_quoted_or_not(self, text: str) -> None: + with pytest.raises(ValueError): + parse_expr(text) + + +class TestStrings: + @pytest.mark.parametrize(("text", "value"), [ + ("'O''Brien'", "O'Brien"), + (r"'C:\temp'", r"C:\temp"), + (r"'a\nb'", r"a\nb"), + ("''", ""), + ("'say \"hi\"'", 'say "hi"'), + ("'OVER('", "OVER("), + ]) + def test_single_quoted_literal(self, text: str, value: str) -> None: + assert parse_expr(text) == Literal(value=value) + + def test_double_quotes_never_make_a_string(self) -> None: + assert parse_expr('status = "paid"') == parse_expr("status == paid") + + +class TestFilterClause: + def test_filter_attaches_to_the_aggregation(self) -> None: + parsed = parse_expr("sum(amount) FILTER (WHERE status = 'paid')") + assert isinstance(parsed, AggCall) + assert parsed.filter == parse_expr("status == 'paid'") + assert parsed.model_copy(update={"filter": None}) == parse_expr("sum(amount)") + + def test_unfiltered_aggregation_has_no_filter(self) -> None: + parsed = parse_expr("sum(amount)") + assert isinstance(parsed, AggCall) and parsed.filter is None + + def test_keywords_case_insensitive(self) -> None: + same("count(*) filter (where status = 'paid')", "count(*) FILTER (WHERE status = 'paid')") + + def test_filter_with_kwargs(self) -> None: + parsed = parse_expr("sum(amount, partition_by=region) FILTER (WHERE status = 'paid')") + assert isinstance(parsed, AggCall) and parsed.filter is not None + assert dict(parsed.kwargs)["partition_by"] == Ref(name="region") + + def test_filtered_aggregation_in_arithmetic(self) -> None: + parsed = parse_expr("sum(amount) FILTER (WHERE status = 'paid') / sum(amount)") + assert isinstance(parsed.left, AggCall) and parsed.left.filter is not None + + @pytest.mark.parametrize("text", [ + "sum(amount) FILTER (status = 'paid')", + "amount FILTER (WHERE status = 'paid')", + "sum(amount) FILTER (WHERE)", + ]) + def test_malformed_filter_rejected(self, text: str) -> None: + rejected(text, "FILTER") + + +class TestPlaceholders: + @pytest.mark.parametrize("text", [ + "amount > {min}", "status = '{s}'", "note LIKE '%{q}%'", "region IN ({regions})", + "sum(amount) * {k} / 100", "sum({k})", "round(sum(amount), {p})", + ]) + def test_literal_slots_parse(self, text: str) -> None: + parse_expr(text) + + @pytest.mark.parametrize("text", [ + "{f}(amount)", "customers.{c}", "{col}:sum", "t.{c} > 1", "sum(amount, {k}=1)", + ]) + def test_non_literal_slots_rejected(self, text: str) -> None: + rejected(text, "literal values") + + def test_error_positions_survive_masking(self) -> None: + msg = rejected("{abcdef} > > 5") + assert re.search(r"\b(11|12)\b", msg), msg + + +class TestRejectionsWithHints: + def test_bitwise_or(self) -> None: + rejected("min(status | region)", "|", "||") + + def test_floor_division(self) -> None: + rejected("sum(amount) // 2", "div(", "floor(") + + def test_two_argument_scalar_max(self) -> None: + rejected("max(amount, 10)", "greatest(amount, 10)") + + def test_two_argument_scalar_min(self) -> None: + rejected("min(amount, 10)", "least(amount, 10)") + + @pytest.mark.parametrize("text", ["TRIM(BOTH ' ' FROM status)", "trim(leading 'x' from status)", + "TRIM(TRAILING FROM status)"]) + def test_trim_keyword_forms(self, text: str) -> None: + rejected(text, "TRIM") + + def test_time_literal(self) -> None: + rejected("created_at > TIME '10:00'", "TIME") + + def test_similar_to(self) -> None: + rejected("status SIMILAR TO 'p%'", "SIMILAR TO") + + @pytest.mark.parametrize("op", ["REGEXP", "RLIKE"]) + def test_regex_operators(self, op: str) -> None: + rejected(f"status {op} '^p'", op) + + def test_raw_over(self) -> None: + with pytest.raises(IllegalWindowInFilterError): + parse_expr("sum(amount) OVER (PARTITION BY region)") + + def test_over_inside_string_is_text(self) -> None: + same("status = 'OVER('", "status == 'OVER('") + + def test_syntax_error_position(self) -> None: + msg = rejected("amount > > 5") + assert re.search(r"\b(9|10)\b", msg), msg + + def test_chained_comparison(self) -> None: + with pytest.raises(ValueError): + parse_expr("a < b < c") + + def test_empty(self) -> None: + with pytest.raises(ValueError): + parse_expr(" ") + + +class TestLegacySpellingsKept: + @pytest.mark.parametrize(("legacy", "canonical"), [ + ("customers.sum(revenue)", "sum(customers.revenue)"), + ("revenue:sum", "sum(revenue)"), + ("*:count", "count(*)"), + ("status == 'paid' and region is not None", "status = 'paid' AND region IS NOT NULL"), + ("region in ['EU', 'US']", "region IN ('EU', 'US')"), + ]) + def test_legacy_equals_canonical(self, legacy: str, canonical: str) -> None: + same(legacy, canonical) + + @pytest.mark.parametrize(("text", "value"), [("True", True), ("False", False), ("None", None), ("NULL", None), + ("TRUE", True), ("false", False)]) + def test_constant_words(self, text: str, value) -> None: + assert parse_expr(text) == Literal(value=value) diff --git a/tests/test_dev2101_importer_dependencies.py b/tests/test_dev2101_importer_dependencies.py new file mode 100644 index 00000000..6d37b512 --- /dev/null +++ b/tests/test_dev2101_importer_dependencies.py @@ -0,0 +1,100 @@ +"""Importers check saved-measure dependencies over the one Mode-B parser (design D8).""" + +from __future__ import annotations + +import importlib +import importlib.util + +import pytest + +from slayer.core.enums import DataType +from slayer.core.models import Aggregation, Column, ModelMeasure, SlayerModel +from slayer.cube.converter import CubeToSlayerConverter +from slayer.cube.report import CubeConversionReport +from slayer.dbt.converter import DbtToSlayerConverter +from slayer.dbt.models import DbtProject + +MEASURES = { + "a": "sum(amount)", + "b": "a + missing", + "c": "b * 2", + "d": "e + 1", + "e": "d + 1", + "f": "a / count(*)", + "g": "sum(amount) FILTER (WHERE status = 'paid')", + "h": "a + CAST(sum(quantity) AS DOUBLE)", + "k": "count(distinct status) + a", +} +SURVIVORS = {"a", "f", "g", "h", "k"} +DANGLING = {"b", "c"} +CYCLIC = {"d", "e"} + + +def _measures(extra: dict[str, str] | None = None) -> list[ModelMeasure]: + return [ModelMeasure(name=n, formula=f) for n, f in {**MEASURES, **(extra or {})}.items()] + + +class TestDbtDependencyCheck: + def _prune(self, measures: list[ModelMeasure]) -> tuple[set[str], set[str]]: + model = SlayerModel( + name="orders", sql_table="orders", data_source="test", + columns=[Column(name=c, type=DataType.DOUBLE) for c in ("id", "amount", "quantity", "price")] + + [Column(name="status", type=DataType.TEXT)], + aggregations=[Aggregation(name="wsum", formula="SUM({value} * {weight})")], + measures=measures, + ) + conv = DbtToSlayerConverter(project=DbtProject(semantic_models=[], metrics=[]), data_source="test") + conv._prune_model_measures(model) + dropped = {w.metric_name for w in conv._unconverted if w.category == "dangling_reference"} + return {m.name for m in model.measures if m.name}, {d for d in dropped if d} + + def test_survivors_and_drops(self) -> None: + kept, dropped = self._prune(_measures()) + assert kept == SURVIVORS + assert dropped == DANGLING | CYCLIC + + def test_bare_column_outside_an_aggregation_is_dangling(self) -> None: + kept, dropped = self._prune(_measures({"i": "amount + a"})) + assert "i" in dropped and "i" not in kept + + def test_custom_aggregation_name_is_in_the_namespace(self) -> None: + kept, _ = self._prune(_measures({"j": "wsum(price, weight=quantity) + a"})) + assert "j" in kept + + def test_self_reference_is_cyclic(self) -> None: + _, dropped = self._prune(_measures({"s": "s + 1"})) + assert "s" in dropped + + +class TestCubeDependencyCheck: + def _drop(self, measures: list[ModelMeasure]) -> set[str]: + conv = CubeToSlayerConverter(project=None, data_source="test") + kept = conv._drop_unparseable_measures("orders", measures, CubeConversionReport()) + return {m.name for m in kept if m.name} + + def test_survivors(self) -> None: + assert self._drop(_measures()) == SURVIVORS + + def test_transitive_dangling_reference_dropped(self) -> None: + assert not self._drop(_measures()) & DANGLING + + def test_cyclic_measures_dropped(self) -> None: + assert not self._drop(_measures()) & CYCLIC + + +class TestOneParser: + @pytest.mark.parametrize("name", [ + "parse_formula", "FieldSpec", "AggregatedMeasureRef", "ArithmeticField", "TransformField", + "MixedArithmeticField", "_rewrite_funcstyle_aggregations", "_preprocess_agg_refs", "_expand_named_measures", + ]) + def test_legacy_parser_deleted(self, name: str) -> None: + if importlib.util.find_spec("slayer.core.formula") is not None: + assert not hasattr(importlib.import_module("slayer.core.formula"), name) + + @pytest.mark.parametrize("module", ["slayer.dbt.converter", "slayer.cube.converter", "slayer.osi.converter"]) + def test_importers_do_not_use_the_legacy_parser(self, module: str) -> None: + assert not hasattr(importlib.import_module(module), "parse_formula") + + @pytest.mark.parametrize("name", ["ALL_TRANSFORMS", "TIME_TRANSFORMS", "TIMELESS_TRANSFORMS"]) + def test_transform_tables_live_in_enums(self, name: str) -> None: + assert hasattr(importlib.import_module("slayer.core.enums"), name) diff --git a/tests/test_dev2101_null_ordering.py b/tests/test_dev2101_null_ordering.py new file mode 100644 index 00000000..d01f82dc --- /dev/null +++ b/tests/test_dev2101_null_ordering.py @@ -0,0 +1,125 @@ +"""``OrderItem.nulls`` — explicit NULL placement (spec: queries/positions).""" + +from __future__ import annotations + +import pytest +import sqlglot +from pydantic import ValidationError +from sqlglot import exp + +from slayer.core.query import OrderItem, SlayerQuery +from slayer.engine.query_engine import SlayerQueryEngine +from tests._dev2101_fixtures import ( + CITY_ORDER, + DISCOUNT_ORDER, + EXEC_BACKENDS, + TIER1, + check_null_ordering, + discount_rows_query, + emit, + hidden_max_query, + m, + make_exec_engine, + ordered_ids, + q, +) + + +@pytest.fixture(params=EXEC_BACKENDS) +async def engine(request): + async for e in make_exec_engine(request): + yield e + + +class TestOrderItemField: + @pytest.mark.parametrize("nulls", ["first", "last"]) + def test_accepted_and_serialised(self, nulls: str) -> None: + item = OrderItem.model_validate({"column": "discount", "direction": "desc", "nulls": nulls}) + assert item.nulls == nulls + assert OrderItem.model_validate(item.model_dump()).nulls == nulls + + def test_default_is_unset(self) -> None: + assert OrderItem.model_validate({"column": "discount"}).nulls is None + + def test_invalid_policy_names_the_allowed_values(self) -> None: + with pytest.raises(ValidationError) as info: + SlayerQuery.model_validate({"source_model": "orders", "dimensions": ["id"], + "order": [{"column": "discount", "nulls": "middle"}]}) + msg = str(info.value) + assert "first" in msg and "last" in msg, msg + + def test_survives_query_round_trip(self) -> None: + query = discount_rows_query(direction="desc", nulls="first") + again = SlayerQuery.model_validate(query.model_dump(mode="json")) + assert again.order is not None and again.order[0].nulls == "first" + + def test_expression_order_item_keeps_the_policy(self) -> None: + item = OrderItem.model_validate({"column": "sum(amount) * 2", "nulls": "last"}) + assert item.nulls == "last" and item.raw_formula == "sum(amount) * 2" + + +class TestExecutedPlacement: + async def test_all_cases(self, engine: SlayerQueryEngine) -> None: + await check_null_ordering(engine) + + @pytest.mark.parametrize(("direction", "nulls"), sorted(DISCOUNT_ORDER)) + async def test_projected_measure(self, engine: SlayerQueryEngine, direction: str, nulls: str) -> None: + resp = await engine.execute(q(dimensions=["city"], measures=[m("max(discount)")], order=[ + {"column": "v", "direction": direction, "nulls": nulls}, {"column": "city"}])) + assert [r["orders.city"] for r in resp.data] == CITY_ORDER[(direction, nulls)] + + @pytest.mark.parametrize(("direction", "nulls"), sorted(DISCOUNT_ORDER)) + async def test_expression_target(self, engine: SlayerQueryEngine, direction: str, nulls: str) -> None: + resp = await engine.execute(q(dimensions=["id"], distinct_dimension_values=False, order=[ + {"column": "discount * 2", "direction": direction, "nulls": nulls}, {"column": "id"}])) + assert ordered_ids(resp) == DISCOUNT_ORDER[(direction, nulls)] + + async def test_hidden_value_is_not_returned(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(hidden_max_query(direction="asc", nulls="last")) + assert resp.columns == ["orders.city"] + + async def test_stage_rewrite_keeps_the_policy(self, engine: SlayerQueryEngine) -> None: + inner = SlayerQuery.model_validate({"name": "by_city", "source_model": "orders", "dimensions": ["city"], + "measures": [{"formula": "max(discount)", "name": "md"}]}) + outer = SlayerQuery.model_validate({"source_model": "by_city", "dimensions": ["city"], + "measures": [{"formula": "max(md)", "name": "v"}], + "order": [{"column": "v", "direction": "asc", "nulls": "first"}, + {"column": "city"}]}) + resp = await engine.execute([inner, outer]) + assert [r["by_city.city"] for r in resp.data] == CITY_ORDER[("asc", "first")] + + async def test_refinement_keeps_the_policy(self, engine: SlayerQueryEngine) -> None: + await engine.create_model_from_query(query=discount_rows_query(direction="desc", nulls="first"), + name="discount_rows") + resp = await engine.execute("discount_rows", refine={"filters": ["id <> 2"]}) + id_key = next(c for c in resp.columns if c.endswith(".id")) + assert [int(r[id_key]) for r in resp.data] == [1, 4, 5, 3] + + +def _null_policy_terms(sql: str, *, dialect: str) -> list[exp.Ordered]: + return list(sqlglot.parse_one(sql, dialect=dialect).find_all(exp.Ordered)) + + +class TestEmission: + @pytest.mark.parametrize("dialect", TIER1) + @pytest.mark.parametrize(("direction", "nulls"), sorted(DISCOUNT_ORDER)) + async def test_policy_reaches_every_dialect(self, dialect: str, direction: str, nulls: str) -> None: + for query in (discount_rows_query(direction=direction, nulls=nulls), + hidden_max_query(direction=direction, nulls=nulls)): + sql = await emit(query, dialect=dialect) + terms = _null_policy_terms(sql, dialect=dialect) + native = any(t.args.get("nulls_first") is (nulls == "first") and "discount" in t.this.sql().lower() + for t in terms) + emulated = any(t.this.find(exp.Is) is not None for t in terms) + assert native or emulated, sql + + async def test_tsql_orders_over_the_resolved_expression(self) -> None: + sql = await emit(hidden_max_query(direction="asc", nulls="last"), dialect="tsql") + null_keys = [t for t in _null_policy_terms(sql, dialect="tsql") if t.this.find(exp.Is) is not None] + assert null_keys and all(t.this.find(exp.Max) is not None for t in null_keys), sql + + @pytest.mark.parametrize("dialect", ["postgres", "duckdb", "sqlite"]) + async def test_no_policy_emits_no_modifier(self, dialect: str) -> None: + sql = await emit(q(dimensions=["id"], distinct_dimension_values=False, + order=[{"column": "discount", "direction": "desc"}]), dialect=dialect) + assert "NULLS" not in sql.upper(), sql diff --git a/tests/test_dev2101_sql_forms.py b/tests/test_dev2101_sql_forms.py new file mode 100644 index 00000000..f3fccc1b --- /dev/null +++ b/tests/test_dev2101_sql_forms.py @@ -0,0 +1,254 @@ +"""SQL forms in every Mode-B position, executed on SQLite / DuckDB and emitted on every Tier-1 dialect. + +Spec: queries/expression-grammar, queries/expression-keywords. Dataset: ``tests/_dev2101_fixtures.py``. +""" + +from __future__ import annotations + +from typing import Any + +import pytest +import sqlglot + +from slayer.core.errors import UnknownReferenceError +from slayer.engine.query_engine import SlayerQueryEngine +from tests._dev2101_fixtures import ( + ALL_IDS, + DATE_VALUES, + EXEC_BACKENDS, + PRED_IDS, + PREDICATES, + TIER1, + VALUE_IDS, + VALUES, + by_key, + day, + emit, + ids, + m, + make_exec_engine, + num, + ordered_ids, + q, + single, +) + +@pytest.fixture(params=EXEC_BACKENDS) +async def engine(request): + async for e in make_exec_engine(request): + yield e + + +def _norm(value: Any) -> Any: + return value if value is None or isinstance(value, str) else float(value) + + +def _expected(values: dict[int, Any]) -> dict[int, Any]: + return {k: _norm(v) for k, v in values.items()} + + +class TestPredicateInEveryPosition: + @pytest.mark.parametrize(("pred", "expected"), PREDICATES, ids=PRED_IDS) + async def test_filter(self, engine: SlayerQueryEngine, pred: str, expected: frozenset[int]) -> None: + resp = await engine.execute(q(dimensions=["id"], filters=[pred])) + assert ids(resp) == expected + + @pytest.mark.parametrize(("pred", "expected"), PREDICATES, ids=PRED_IDS) + async def test_measure(self, engine: SlayerQueryEngine, pred: str, expected: frozenset[int]) -> None: + resp = await engine.execute(q(measures=[m(f"sum(iif({pred}, 1, 0))")])) + assert num(single(resp)) == len(expected) + + @pytest.mark.parametrize(("pred", "expected"), PREDICATES, ids=PRED_IDS) + async def test_dimension(self, engine: SlayerQueryEngine, pred: str, expected: frozenset[int]) -> None: + resp = await engine.execute(q(dimensions=["id", {"expression": pred, "name": "p"}])) + assert {int(r["orders.id"]) for r in resp.data if r["orders.p"]} == expected + + @pytest.mark.parametrize(("pred", "expected"), PREDICATES, ids=PRED_IDS) + async def test_order_target(self, engine: SlayerQueryEngine, pred: str, expected: frozenset[int]) -> None: + resp = await engine.execute(q( + dimensions=["id"], distinct_dimension_values=False, + order=[{"column": f"iif({pred}, 0, 1)"}, {"column": "id"}], + )) + assert ordered_ids(resp) == sorted(expected) + sorted(ALL_IDS - expected) + + +class TestValueInEveryPosition: + @pytest.mark.parametrize(("expr", "values"), VALUES, ids=VALUE_IDS) + async def test_dimension(self, engine: SlayerQueryEngine, expr: str, values: dict[int, Any]) -> None: + resp = await engine.execute(q(dimensions=["id", {"expression": expr, "name": "x"}])) + assert {int(k): _norm(v) for k, v in by_key(resp, "id", "x").items()} == _expected(values) + + @pytest.mark.parametrize(("expr", "values"), VALUES, ids=VALUE_IDS) + async def test_measure(self, engine: SlayerQueryEngine, expr: str, values: dict[int, Any]) -> None: + resp = await engine.execute(q(dimensions=["id"], measures=[m(f"max({expr})")])) + assert {int(k): _norm(v) for k, v in by_key(resp, "id").items()} == _expected(values) + + @pytest.mark.parametrize(("expr", "values"), VALUES, ids=VALUE_IDS) + async def test_filter(self, engine: SlayerQueryEngine, expr: str, values: dict[int, Any]) -> None: + resp = await engine.execute(q(dimensions=["id"], filters=[f"{expr} IS NOT NULL"])) + assert ids(resp) == {k for k, v in values.items() if v is not None} + + @pytest.mark.parametrize(("expr", "values"), VALUES, ids=VALUE_IDS) + async def test_order_target(self, engine: SlayerQueryEngine, expr: str, values: dict[int, Any]) -> None: + resp = await engine.execute(q( + dimensions=["id"], distinct_dimension_values=False, filters=[f"{expr} IS NOT NULL"], + order=[{"column": expr, "direction": "desc"}, {"column": "id"}], + )) + present = {k: _norm(v) for k, v in values.items() if v is not None} + assert ordered_ids(resp) == sorted(sorted(present), key=present.__getitem__, reverse=True) + + @pytest.mark.parametrize(("expr", "values"), DATE_VALUES, ids=[v for v, _ in DATE_VALUES]) + async def test_interval_dimension(self, engine: SlayerQueryEngine, expr: str, values: dict[int, str]) -> None: + resp = await engine.execute(q(dimensions=["id", {"expression": expr, "name": "x"}])) + assert {int(k): day(v) for k, v in by_key(resp, "id", "x").items()} == values + + +class TestAggregateForms: + async def test_count_distinct_measure(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(measures=[m("count(distinct region)"), m("count_distinct(region)", "w")])) + assert resp.data[0]["orders.v"] == resp.data[0]["orders.w"] == 2 + + async def test_count_distinct_in_post_aggregate_filter(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["status"], measures=[m("count(*)")], + filters=["count(distinct region) > 1"])) + assert set(by_key(resp, "status")) == {"paid"} + + async def test_sql_spelled_post_aggregate_filter(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["region"], measures=[m("sum(amount)")], + filters=["sum(amount) BETWEEN 25 AND 75", "region IS NOT NULL"])) + assert {k: num(v) for k, v in by_key(resp, "region").items()} == {"EU": 30.0, "US": 70.0} + + async def test_sql_predicate_in_measure(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(measures=[ + m("sum(iif(status = 'paid' AND amount IS NOT NULL, amount, 0))", "sql"), + m("sum(iif(status == 'paid' and amount is not None, amount, 0))", "py"), + ])) + row = resp.data[0] + assert num(row["orders.sql"]) == num(row["orders.py"]) == 70.0 + + async def test_order_by_sql_spelled_measure(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["status"], measures=[m("count(*)")], + order=[{"column": "sum(iif(status <> 'paid', 1, 0))", "direction": "desc"}, + {"column": "status"}])) + assert [r["orders.status"] for r in resp.data] == ["Pending", "void", "paid"] + + async def test_case_condition_in_measure(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(measures=[m("sum(CASE WHEN region = 'EU' AND amount IS NOT NULL THEN 1 ELSE 0 END)")])) + assert num(single(resp)) == 2 + + +class TestExpressionGrammarScenarios: + async def test_same_between_text_in_filter_measure_and_order(self, engine: SlayerQueryEngine) -> None: + pred = "amount BETWEEN 10 AND 20" + filtered = await engine.execute(q(dimensions=["id"], filters=[pred])) + measured = await engine.execute(q(measures=[m(f"sum(iif({pred}, 1, 0))")])) + ordered = await engine.execute(q(dimensions=["id"], distinct_dimension_values=False, + order=[{"column": f"iif({pred}, 0, 1)"}, {"column": "id"}])) + assert ids(filtered) == {1, 2} + assert num(single(measured)) == 2 + assert ordered_ids(ordered)[:2] == [1, 2] + + async def test_boolean_tests_partition_the_rows(self, engine: SlayerQueryEngine) -> None: + kept = ids(await engine.execute(q(dimensions=["id"], filters=["(amount > 15) IS TRUE"]))) + rest = ids(await engine.execute(q(dimensions=["id"], filters=["(amount > 15) IS NOT TRUE"]))) + assert kept | rest == ALL_IDS and not kept & rest + assert 5 in rest + + async def test_concatenation_dimension(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=[{"expression": "status || '-' || region", "name": "tag"}], + measures=[m("count(*)")])) + assert by_key(resp, "tag") == {"paid-EU": 2, "Pending-US": 1, "paid-US": 1, None: 1} + + async def test_modulo_sign(self, engine: SlayerQueryEngine) -> None: + assert num(single(await engine.execute(q(measures=[m("min(-7 % 3)")])))) == -1 + + @pytest.mark.parametrize(("formula", "expected"), [ + ("min(2 ** 3 ** 2)", 512), ("min(-2 ** 2)", -4), ("min((-2) ** 2)", 4), ("min(2 ** -2)", 0.25), + ]) + async def test_power(self, engine: SlayerQueryEngine, formula: str, expected: float) -> None: + assert num(single(await engine.execute(q(measures=[m(formula)])))) == pytest.approx(expected) + + async def test_power_over_an_aggregate(self, engine: SlayerQueryEngine) -> None: + assert num(single(await engine.execute(q(measures=[m("sum(amount) ** 2")])))) == pytest.approx(10000.0) + + async def test_not_binds_looser_than_comparison(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id"], filters=["NOT amount > 15 AND status = 'paid'"])) + assert ids(resp) == {1} + + async def test_concatenation_binds_tighter_than_comparison(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id"], filters=["status || 'x' = 'paidx'"])) + assert ids(resp) == {1, 2, 4} + + async def test_like_over_nested_call(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id"], filters=["lower(trim(status)) LIKE 'pa%'"])) + assert ids(resp) == {1, 2, 4} + + async def test_ilike_case_insensitive(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id"], filters=["status ILIKE 'PA%'"])) + assert ids(resp) == {1, 2, 4} + + async def test_like_inside_a_string_is_text(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id"], filters=["note = 'we like cats'"])) + assert ids(resp) == {1} + + async def test_substring_and_position_equal_their_calls(self, engine: SlayerQueryEngine) -> None: + sql_form = await engine.execute(q(dimensions=["id", "SUBSTRING(status FROM 1 FOR 3)", "POSITION('a' IN status)"])) + calls = await engine.execute(q(dimensions=["id", "substr(status, 1, 3)", "instr(status, 'a')"])) + assert sql_form.columns == calls.columns + assert sql_form.data == calls.data + assert sql_form.sql == calls.sql + + async def test_quoted_column_name(self, engine: SlayerQueryEngine) -> None: + assert num(single(await engine.execute(q(measures=[m('sum("order total")')])))) == 1000.0 + + async def test_quoted_segment_in_dotted_path(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=['customers."Region"'], measures=[m("count(*)")])) + assert by_key(resp, "customers.Region") == {"North": 2, "South": 2, None: 1} + + async def test_double_quoted_string_hint(self, engine: SlayerQueryEngine) -> None: + with pytest.raises(UnknownReferenceError) as info: + await engine.execute(q(dimensions=["id"], filters=['status = "paid"'])) + msg = str(info.value) + assert "paid" in msg and "single quote" in msg.lower(), msg + + async def test_doubled_quote(self, engine: SlayerQueryEngine) -> None: + assert ids(await engine.execute(q(dimensions=["id"], filters=["customer_name = 'O''Brien'"]))) == {1} + + async def test_backslash_is_literal(self, engine: SlayerQueryEngine) -> None: + assert ids(await engine.execute(q(dimensions=["id"], filters=[r"path = 'C:\temp'"]))) == {1} + + async def test_python_and_sql_spellings_render_identical_sql(self, engine: SlayerQueryEngine) -> None: + py = await engine.execute(q(dimensions=["id"], filters=["status == 'paid' and region is not None"]), dry_run=True) + sql = await engine.execute(q(dimensions=["id"], filters=["status = 'paid' AND region IS NOT NULL"]), dry_run=True) + assert py.sql == sql.sql + + async def test_comparison_inside_scalar_call(self, engine: SlayerQueryEngine) -> None: + assert num(single(await engine.execute(q(measures=[m("sum(iif(status = 'paid', amount, 0))")])))) == 70.0 + + +class TestEmission: + """Every form renders SQL that parses on every Tier-1 dialect, in a filter and a dimension.""" + + @pytest.mark.parametrize("dialect", TIER1) + @pytest.mark.parametrize("pred", PRED_IDS) + async def test_predicate_emits(self, dialect: str, pred: str) -> None: + await emit(q(dimensions=["id", {"expression": pred, "name": "p"}], filters=[pred]), dialect=dialect) + + @pytest.mark.parametrize("dialect", TIER1) + @pytest.mark.parametrize("expr", VALUE_IDS + [v for v, _ in DATE_VALUES]) + async def test_value_emits(self, dialect: str, expr: str) -> None: + await emit(q(dimensions=["id", {"expression": expr, "name": "x"}], measures=[m(f"max({expr})")]), + dialect=dialect) + + @pytest.mark.parametrize("dialect", TIER1) + async def test_ilike_is_case_insensitive_everywhere(self, dialect: str) -> None: + sql = await emit(q(dimensions=["id"], filters=["status ILIKE 'PA%'"]), dialect=dialect) + tree = sqlglot.parse_one(sql, dialect=dialect) + native = tree.find(sqlglot.exp.ILike) is not None or (dialect == "sqlite" and " LIKE " in sql.upper()) + assert native or "LOWER(" in sql.upper(), sql + + @pytest.mark.parametrize("dialect", TIER1) + async def test_null_safe_comparison_everywhere(self, dialect: str) -> None: + sql = await emit(q(dimensions=["id"], filters=["region IS NOT DISTINCT FROM NULL"]), dialect=dialect) + upper = sql.upper() + assert any(form in upper for form in ("IS NULL", "DISTINCT FROM", "DISTINCTFROM", "<=>", "EQUAL_NULL")), sql diff --git a/tests/test_dev2101_stored_upgrade.py b/tests/test_dev2101_stored_upgrade.py new file mode 100644 index 00000000..a872681c --- /dev/null +++ b/tests/test_dev2101_stored_upgrade.py @@ -0,0 +1,198 @@ +"""Stored Mode-B text is upgraded to the SQL string and identifier rules (spec: models/stored-upgrade).""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from slayer.core.errors import UnknownReferenceError +from slayer.core.models import DatasourceConfig, SlayerModel +from slayer.core.query import SlayerQuery +from slayer.memories.models import Memory +from slayer.sql import engine_factory +from slayer.storage.migrations import CURRENT_VERSIONS +from tests._stored_upgrade_fixtures import ( + BACKENDS, + DS, + customers_v10, + measure_total, + memory_v2, + orders_v10, + query_backed_v10, + raw_store, + rev_query, + run, + seed_shop, + stored_bytes, +) + +#: (stored legacy text, the text it loads as). +TOKEN_CASES = [ + pytest.param('status == "paid"', "status == 'paid'", id="double-quoted"), + pytest.param('name == "O\'Brien"', "name == 'O''Brien'", id="embedded-apostrophe"), + pytest.param(r"name == 'O\'Brien'", "name == 'O''Brien'", id="escaped-apostrophe"), + pytest.param('note == "tab\\there"', "note == 'tab\there'", id="control-escape"), + pytest.param(r"path == 'C:\\temp'", r"path == 'C:\temp'", id="escaped-backslash"), + pytest.param(r'path == r"C:\temp"', r"path == 'C:\temp'", id="raw-string"), + pytest.param("note == 'say \"hi\"'", "note == 'say \"hi\"'", id="embedded-double-quote"), + pytest.param('city == "café"', "city == 'café'", id="unicode"), + pytest.param(r'city == "caf\u00e9"', "city == 'café'", id="unicode-escape"), + pytest.param('note == "a | b"', "note == 'a | b'", id="pipe-inside-string"), + pytest.param("note == \"a\" 'b'", "note == 'ab'", id="adjacent-literals"), + pytest.param('note == """x"""', "note == 'x'", id="triple-quoted"), + pytest.param("status | region", "status || region", id="pipe"), + pytest.param("concat(status | region, \"-\")", "concat(status || region, '-')", id="pipe-and-string"), + pytest.param("status == 'paid'", "status == 'paid'", id="already-sql"), + pytest.param("sum(amount", "sum(amount", id="untokenisable"), + pytest.param('status == "paid', 'status == "paid', id="unterminated"), +] + + +def _model_doc(*, formula: str, version: int = 15, **extra: Any) -> dict: + return {**orders_v10(version=version), "measures": [{"name": "x", "formula": formula}], **extra} + + +def _model_formula(text: str) -> str: + return SlayerModel.model_validate(_model_doc(formula=text)).measures[0].formula + + +def _query(**fields: Any) -> SlayerQuery: + return SlayerQuery.model_validate({"version": 6, "source_model": "orders", **fields}) + + +class TestVersions: + def test_versions_bumped(self) -> None: + assert CURRENT_VERSIONS["SlayerModel"] == 16 + assert CURRENT_VERSIONS["SlayerQuery"] == 7 + assert CURRENT_VERSIONS["Memory"] == 5 + + +class TestTokenRewrite: + @pytest.mark.parametrize(("stored", "loaded"), TOKEN_CASES) + def test_model_measure_formula(self, stored: str, loaded: str) -> None: + assert _model_formula(stored) == loaded + + @pytest.mark.parametrize(("stored", "loaded"), TOKEN_CASES) + def test_query_filter(self, stored: str, loaded: str) -> None: + assert _query(filters=[stored]).filters == [loaded] + + def test_current_version_text_is_never_rewritten(self) -> None: + doc = _model_doc(formula='sum(iif(status == "paid", amount, 0))', version=16) + assert SlayerModel.model_validate(doc).measures[0].formula == 'sum(iif(status == "paid", amount, 0))' + + +class TestEverySurface: + def test_query_fields(self) -> None: + query = _query( + measures=['sum(iif(region == "EU", amount, 0))', {"formula": 'count_distinct(status | "x")', "name": "c"}], + dimensions=['concat(status, "!")', {"expression": 'status | "-"', "name": "tag"}], + filters=['status == "paid"'], + order=[{"column": 'sum(iif(status == "paid", amount, 0))', "direction": "desc"}], + ) + assert [m.formula for m in query.measures or []] == [ + "sum(iif(region == 'EU', amount, 0))", "count_distinct(status || 'x')", + ] + assert [d.expression for d in query.dimensions or []] == ["concat(status, '!')", "status || '-'"] + assert query.filters == ["status == 'paid'"] + assert query.order is not None and query.order[0].raw_formula == "sum(iif(status == 'paid', amount, 0))" + + def test_inline_extension_measures(self) -> None: + query = _query(source_model={"source_name": "orders", "measures": [ + {"name": "paid", "formula": 'sum(iif(status == "paid", amount, 0))'}]}, measures=["paid"]) + source = query.source_model + assert source.measures[0].formula == "sum(iif(status == 'paid', amount, 0))" # type: ignore[union-attr] + + def test_model_source_queries(self) -> None: + doc = query_backed_v10(name="paid_rev", version=15, stages=[rev_query(filters=['status == "ok"'])]) + model = SlayerModel.model_validate(doc) + assert (model.source_queries or [])[0].filters == ["status == 'ok'"] + + def test_memory_query(self) -> None: + mem = Memory.model_validate(memory_v2(memory_id="m", version=4, query=rev_query(filters=['status == "ok"']))) + assert mem.query is not None and mem.query.filters == ["status == 'ok'"] + + def test_mode_a_text_is_untouched(self) -> None: + doc = _model_doc(formula="sum(amount)") + doc["columns"] = [*doc["columns"], {"name": "total2", "sql": '"Order Total" * 2', "type": "DOUBLE"}] + doc["filters"] = ['status = "x"'] + model = SlayerModel.model_validate(doc) + assert next(c for c in model.columns if c.name == "total2").sql == '"Order Total" * 2' + assert model.filters == ['status = "x"'] + + def test_unversioned_client_query_is_untouched(self) -> None: + query = SlayerQuery.model_validate({"source_model": "orders", "filters": ['status = "paid"']}) + assert query.filters == ['status = "paid"'] + + +@pytest.fixture +def shop_ds(tmp_path): + db = str(tmp_path / "shop.duckdb") + seed_shop(db, dialect="duckdb") + ds = DatasourceConfig(name=DS, type="duckdb", database=db) + try: + yield ds + finally: + engine_factory.invalidate_engine(ds) + + +async def _store(*, tmp_path, ds, backend: str, stage: dict, memories: list[dict] | None = None): + qb = query_backed_v10(name="rev_q", stages=[stage]) + return await raw_store(backend=backend, base=str(tmp_path / "store"), datasource=ds, + models=[orders_v10(), customers_v10(), qb], memories=memories or []) + + +class TestStoredResultsUnchanged: + @pytest.mark.parametrize("backend", BACKENDS) + async def test_double_quoted_filter_and_measure(self, tmp_path, shop_ds, backend: str) -> None: + stage = {"source_model": "orders", "filters": ['status == "ok"'], + "measures": [{"formula": 'sum(iif(status == "ok", amount, 0))', "name": "rev"}]} + storage = await _store(tmp_path=tmp_path, ds=shop_ds, backend=backend, stage=stage) + model = await storage.get_model("rev_q", data_source=DS) + assert model is not None + [loaded] = model.source_queries or [] + assert loaded.filters == ["status == 'ok'"] + assert loaded.measures is not None and loaded.measures[0].formula == "sum(iif(status == 'ok', amount, 0))" + resp = await run(storage, {"source_model": "rev_q", "measures": ["sum(rev)"]}) + assert measure_total(resp, measure="rev_sum") == 80.0 + + async def test_rewrite_precedes_the_legacy_time_literal_repair(self, tmp_path, shop_ds) -> None: + stage = rev_query(filters=['ordered_at >= "2024-01-01T00:00:00Z"']) + storage = await _store(tmp_path=tmp_path, ds=shop_ds, backend="yaml", stage=stage) + model = await storage.get_model("rev_q", data_source=DS) + assert model is not None and (model.source_queries or [])[0].filters == ["ordered_at >= '2024-01-01T00:00:00'"] + resp = await run(storage, {"source_model": "rev_q", "measures": ["sum(rev)"]}) + assert measure_total(resp, measure="rev_sum") == 200.0 + + async def test_pipe_concatenation_returns_the_same_values(self, tmp_path, shop_ds) -> None: + stage = {"source_model": "orders", "dimensions": [{"expression": 'status | "#"', "name": "tag"}], + "measures": [{"formula": "sum(amount)", "name": "rev"}]} + storage = await _store(tmp_path=tmp_path, ds=shop_ds, backend="yaml", stage=stage) + resp = await run(storage, {"source_model": "rev_q", "dimensions": ["tag"], "measures": ["sum(rev)"]}) + assert {r["rev_q.tag"]: float(r["rev_q.rev_sum"]) for r in resp.data} == { + "ok#": 80.0, "new#": 70.0, "2024/01/01#": 60.0, + } + + @pytest.mark.parametrize("backend", BACKENDS) + async def test_memory_query(self, tmp_path, shop_ds, backend: str) -> None: + query = rev_query(version=4, filters=['status == "ok"']) + storage = await raw_store(backend=backend, base=str(tmp_path / "store"), datasource=shop_ds, + models=[orders_v10(), customers_v10()], + memories=[memory_v2(memory_id="m_q", query=query)]) + mem = await storage.get_memory("m_q") + assert mem.query is not None and mem.query.filters == ["status == 'ok'"] + assert measure_total(await run(storage, mem.query), measure="rev") == 80.0 + + async def test_resave_writes_the_new_spelling(self, tmp_path, shop_ds) -> None: + stage = rev_query(filters=['status == "ok"']) + storage = await _store(tmp_path=tmp_path, ds=shop_ds, backend="yaml", stage=stage) + model = await storage.get_model("rev_q", data_source=DS) + assert model is not None + await storage.save_model(model) + assert '"ok"' not in stored_bytes(backend="yaml", base=str(tmp_path / "store"), name="rev_q") + + async def test_unversioned_client_query_fails_with_the_hint(self, tmp_path, shop_ds) -> None: + storage = await _store(tmp_path=tmp_path, ds=shop_ds, backend="yaml", stage=rev_query()) + with pytest.raises(UnknownReferenceError) as info: + await run(storage, {"source_model": "orders", "measures": ["sum(amount)"], "filters": ['status = "ok"']}) + assert "single quote" in str(info.value).lower(), str(info.value) diff --git a/tests/test_dev2101_variables.py b/tests/test_dev2101_variables.py new file mode 100644 index 00000000..7447b76b --- /dev/null +++ b/tests/test_dev2101_variables.py @@ -0,0 +1,156 @@ +"""``{var}`` placeholders bind as literal values only (spec: queries/variables).""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from slayer.core.query import ComputedDimension +from slayer.engine.query_engine import SlayerQueryEngine +from tests._dev2101_fixtures import EXEC_BACKENDS, by_key, ids, m, make_exec_engine, num, q, single + +MONTH = {"dimension": "created_at", "granularity": "month"} + + +@pytest.fixture(params=EXEC_BACKENDS) +async def engine(request): + async for e in make_exec_engine(request): + yield e + + +async def _rejected(engine: SlayerQueryEngine, *needles: str, **kw: Any) -> str: + with pytest.raises(ValueError) as info: + await engine.execute(q(**kw)) + msg = str(info.value) + for needle in needles: + assert needle.lower() in msg.lower(), msg + return msg + + +class TestLiteralSlots: + async def test_numeric_string_in_a_bare_position(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id"], filters=["amount > {min}"], variables={"min": "10"})) + assert ids(resp) == {2, 3, 4} + + async def test_number_in_a_bare_position(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id"], filters=["amount > {min}"], variables={"min": 25})) + assert ids(resp) == {3, 4} + + async def test_boolean_in_a_bare_position(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(measures=[m("sum(iif({on}, amount, 0))")], variables={"on": True})) + assert num(single(resp)) == 100.0 + + async def test_whole_string_literal(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id"], filters=["status = '{s}'"], variables={"s": "Pending"})) + assert ids(resp) == {3} + + async def test_partial_string_literal(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id"], filters=["note LIKE '%{q}%'"], variables={"q": "cats"})) + assert ids(resp) == {1} + + async def test_quoted_value_with_an_apostrophe(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id"], filters=["customer_name = '{n}'"], variables={"n": "O'Brien"})) + assert ids(resp) == {1} + + async def test_hostile_string_stays_a_literal(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id"], filters=["status = '{s}'"], + variables={"s": "x' OR '1' = '1"})) + assert ids(resp) == set() + + async def test_backslash_value_stays_literal(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id"], filters=["path = '{p}'"], variables={"p": r"C:\temp"})) + assert ids(resp) == {1} + + async def test_list_in_the_in_slot(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=["id"], filters=["customer_name IN ({names})"], + variables={"names": ["O'Brien", "Lee"]})) + assert ids(resp) == {1, 5} + + async def test_doubled_braces_render_literal_braces(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(dimensions=[{"expression": "'{{' || status || '}}'", "name": "b"}], + measures=[m("count(*)")], variables={"k": 1})) + assert set(by_key(resp, "b")) == {"{paid}", "{Pending}", "{void}"} + + async def test_placeholder_in_a_case_branch_and_scalar_argument(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q( + measures=[m("sum(CASE WHEN status = '{s}' THEN round(amount / {d}, {p}) ELSE 0 END)")], + variables={"s": "paid", "d": 3, "p": 0}, + )) + assert num(single(resp)) == 3 + 7 + 13 + + +class TestStructureIsNeverInjected: + async def test_string_in_a_bare_position_is_rejected_with_the_quote_hint(self, engine: SlayerQueryEngine) -> None: + await _rejected(engine, "literal values", "quote", measures=[m("sum({col})")], variables={"col": "amount"}) + + async def test_non_numeric_string_compared_bare(self, engine: SlayerQueryEngine) -> None: + await _rejected(engine, "literal values", dimensions=["id"], filters=["amount > {min}"], + variables={"min": "1 OR 1 = 1"}) + + async def test_list_outside_the_in_slot(self, engine: SlayerQueryEngine) -> None: + await _rejected(engine, "literal values", dimensions=["id"], filters=["amount > {xs}"], + variables={"xs": [1, 2]}) + + @pytest.mark.parametrize("kw", [ + {"measures": ["{f}(amount)"], "variables": {"f": "sum"}}, + {"dimensions": ["customers.{c}"], "variables": {"c": "name"}}, + {"measures": ["sum(amount, partition_by={x})"], "dimensions": ["region"], "variables": {"x": "region"}}, + {"measures": ["{col}:sum"], "variables": {"col": "amount"}}, + {"measures": ["sum(amount, {k}=1)"], "variables": {"k": "partition_by"}}, + ], ids=["function", "path-segment", "reference-kwarg", "colon-source", "kwarg-name"]) + async def test_non_literal_positions_rejected(self, engine: SlayerQueryEngine, kw: dict) -> None: + await _rejected(engine, "literal values", **kw) + + +class TestTemplateNames: + @pytest.mark.parametrize("k", [2, 3]) + async def test_measure_name_comes_from_the_written_text(self, engine: SlayerQueryEngine, k: int) -> None: + resp = await engine.execute(q(measures=["sum(amount) * {k}"], variables={"k": k})) + assert resp.data == [{"orders.amount_sum_k": pytest.approx(100.0 * k)}] + + @pytest.mark.parametrize("k", [2, 3]) + @pytest.mark.parametrize("dimension", ["quantity * {k}", {"expression": "quantity * {k}"}]) + async def test_dimension_name_comes_from_the_written_text(self, engine: SlayerQueryEngine, dimension, k: int) -> None: + resp = await engine.execute(q(dimensions=[dimension], measures=[m("count(*)")], variables={"k": k})) + assert {float(v) for v in by_key(resp, "quantity_k")} == {float(x * k) for x in (1, 2, 3, 4)} + + def test_no_text_splice_on_computed_dimensions(self) -> None: + assert not hasattr(ComputedDimension, "substituted") + + +class TestDateRangeBounds: + async def test_both_bounds(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(measures=[m("sum(amount)")], time_dimensions=[ + {**MONTH, "date_range": ["{start}", "{end}"]}], variables={"start": "2024-02-01", "end": "2024-02-29"})) + assert [num(r["orders.v"]) for r in resp.data] == [70.0] + + async def test_one_sided_lower(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(measures=[m("sum(amount)")], time_dimensions=[ + {**MONTH, "date_range": ["{start}", None]}], variables={"start": "2024-02-01"})) + assert sorted(str(r["orders.created_at"])[:7] for r in resp.data) == ["2024-02", "2024-03"] + + async def test_one_sided_upper(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(measures=[m("sum(amount)")], time_dimensions=[ + {**MONTH, "date_range": [None, "{end}"]}], variables={"end": "2024-01-31"})) + assert [num(r["orders.v"]) for r in resp.data] == [30.0] + + async def test_single_period(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(measures=[m("sum(amount)")], time_dimensions=[ + {**MONTH, "date_range": "{p}"}], variables={"p": "2024-02"})) + assert [num(r["orders.v"]) for r in resp.data] == [70.0] + + async def test_missing_variable(self, engine: SlayerQueryEngine) -> None: + await _rejected(engine, "end", measures=[m("sum(amount)")], + time_dimensions=[{**MONTH, "date_range": ["{start}", "{end}"]}], + variables={"start": "2024-02-01"}) + + async def test_non_string_year(self, engine: SlayerQueryEngine) -> None: + resp = await engine.execute(q(measures=[m("sum(amount)")], time_dimensions=[ + {**MONTH, "date_range": ["{start}", None]}], variables={"start": 2024})) + assert len(resp.data) == 3 + + async def test_bound_value_is_not_mode_b(self, engine: SlayerQueryEngine) -> None: + await _rejected(engine, measures=[m("sum(amount)")], + time_dimensions=[{**MONTH, "date_range": ["{start}", None]}], + variables={"start": "2024-01-01' OR '1'='1"}) diff --git a/tests/test_formula.py b/tests/test_formula.py index 02c5888a..c2f31d8e 100644 --- a/tests/test_formula.py +++ b/tests/test_formula.py @@ -1,32 +1,17 @@ -"""Tests for the retained formula helpers — ``_rewrite_funcstyle_aggregations`` -(function-style -> colon rewrite + ORDER BY normalization) — plus the typed -filter parser's injection hardening. - -The legacy free-function formula parser, its AST node types, and the legacy -``parse_filter`` were removed when the typed pipeline took over; their -coverage now lives in ``test_syntax.py`` (parse shape, LIKE / ``||`` / scalar -calls in filters), ``test_sql_boolean_literal_filters.py`` (SQL-cased boolean -literals), ``test_binding.py`` / ``test_transforms_planner.py`` (bind-time -validation), ``test_transform_lowerer.py`` (change/change_pct desugar), -``test_named_measures.py`` (named-measure expansion, end-to-end), -``test_model_measure_expansion.py`` (bind-time expansion, cycles, scoping), -``test_measure_expansion.py`` (expansion eligibility), and -``test_schema_drift_typed.py`` / ``test_memories_resolver_typed.py`` (typed -ref-walk path extraction). -""" +"""Formula-level parse checks: functional aggregation spellings, order-column normalization, +filter injection hardening and string hygiene, all over the one Mode-B parser.""" import warnings import pytest -from slayer.core.formula import _rewrite_funcstyle_aggregations, parse_formula from slayer.core.models import Aggregation from slayer.core.query import _FUNCSTYLE_PENDING, OrderItem -from slayer.engine.syntax import BoolOp, Cmp, parse_filter_expr +from slayer.engine.syntax import AggCall, BoolOp, Cmp, Literal, Ref, ScalarCall, parse_expr class TestFilterInjection: - """SQL-injection hardening for ``parse_filter_expr`` — the choke-point for + """SQL-injection hardening for ``parse_expr`` — the choke-point for all user-supplied Mode-B filter expressions. Payloads must be rejected at parse time; literal SQL emission is dialect-owned (sqlglot) and covered by the SQL-generator round-trip suites.""" @@ -48,11 +33,11 @@ class TestFilterInjection: ) def test_payload_rejected_at_parse(self, payload: str) -> None: with pytest.raises(ValueError, match="Invalid Mode-B expression"): - parse_filter_expr(payload) + parse_expr(payload) def test_allows_tautology_with_literal(self) -> None: # ``1 = 1`` is a legal, user-authored tautology — not injection per se. - result = parse_filter_expr("status = 'a' or 1 = 1") + result = parse_expr("status = 'a' or 1 = 1") assert isinstance(result, BoolOp) assert result.op == "or" @@ -60,7 +45,7 @@ def test_deeply_nested_boolean_does_not_crash(self) -> None: # 200 chained ORs must parse bounded or raise cleanly — never crash. payload = " or ".join(["x = 1"] * 200) try: - result = parse_filter_expr(payload) + result = parse_expr(payload) except ValueError: return assert isinstance(result, BoolOp) @@ -73,184 +58,64 @@ def test_deeply_nested_boolean_does_not_crash(self) -> None: class TestFuncStyleRewrite: - """Unit tests for _rewrite_funcstyle_aggregations.""" + """The functional aggregation spelling parses to the same node as the colon spelling.""" + + @pytest.mark.parametrize(("functional", "colon"), [ + ("sum(revenue)", "revenue:sum"), + ("avg(amount)", "amount:avg"), + ("min(price)", "price:min"), + ("max(price)", "price:max"), + ("count(*)", "*:count"), + ("count(customer_id)", "customer_id:count"), + ("count_distinct(id)", "id:count_distinct"), + ("median(price)", "price:median"), + ("sum(customers.revenue)", "customers.revenue:sum"), + ("sum(a.b.c.d)", "a.b.c.d:sum"), + ("weighted_avg(price, weight=qty)", "price:weighted_avg(weight=qty)"), + ("last(revenue, ordered_at)", "revenue:last(ordered_at)"), + ("first(revenue, ordered_at)", "revenue:first(ordered_at)"), + ("percentile(revenue, p=0.95)", "revenue:percentile(p=0.95)"), + ("sum(revenue) / count(*)", "revenue:sum / *:count"), + ("sum(revenue) + avg(amount)", "revenue:sum + amount:avg"), + ("cumsum(sum(revenue))", "cumsum(revenue:sum)"), + ("change(sum(revenue))", "change(revenue:sum)"), + ("last(revenue:sum) + sum(amount)", "last(revenue:sum) + amount:sum"), + ("rolling_avg(revenue)", "revenue:rolling_avg"), + ("count(customers.*)", "customers.*:count"), + ("sum(revenue) > 0 and name = 'count(x)'", "revenue:sum > 0 and name == 'count(x)'"), + ("sum(revenue) > 100", "revenue:sum > 100"), + ("stddev_samp(latency)", "latency:stddev_samp"), + ("stddev_pop(latency)", "latency:stddev_pop"), + ("var_samp(latency)", "latency:var_samp"), + ("var_pop(latency)", "latency:var_pop"), + ("corr(price, other=quantity)", "price:corr(other=quantity)"), + ("covar_samp(price, other=quantity)", "price:covar_samp(other=quantity)"), + ("covar_pop(price, other=quantity)", "price:covar_pop(other=quantity)"), + ]) + def test_functional_equals_colon(self, functional: str, colon: str) -> None: + assert parse_expr(functional) == parse_expr(colon) - def test_sum(self) -> None: - assert _rewrite_funcstyle_aggregations("sum(revenue)") == "revenue:sum" - - def test_avg(self) -> None: - assert _rewrite_funcstyle_aggregations("avg(amount)") == "amount:avg" - - def test_min(self) -> None: - assert _rewrite_funcstyle_aggregations("min(price)") == "price:min" - - def test_max(self) -> None: - assert _rewrite_funcstyle_aggregations("max(price)") == "price:max" - - def test_count_star(self) -> None: - assert _rewrite_funcstyle_aggregations("count(*)") == "*:count" - - def test_count_column(self) -> None: - assert _rewrite_funcstyle_aggregations("count(customer_id)") == "customer_id:count" - - def test_count_distinct(self) -> None: - assert _rewrite_funcstyle_aggregations("count_distinct(id)") == "id:count_distinct" - - def test_median(self) -> None: - assert _rewrite_funcstyle_aggregations("median(price)") == "price:median" - - def test_first_bare(self) -> None: - assert _rewrite_funcstyle_aggregations("first(revenue)") == "revenue:first" - - def test_last_bare(self) -> None: - assert _rewrite_funcstyle_aggregations("last(revenue)") == "revenue:last" - - def test_cross_model(self) -> None: - assert _rewrite_funcstyle_aggregations("sum(customers.revenue)") == "customers.revenue:sum" - - def test_multi_hop(self) -> None: - assert _rewrite_funcstyle_aggregations("sum(a.b.c.d)") == "a.b.c.d:sum" - - def test_weighted_avg_kwargs(self) -> None: - assert _rewrite_funcstyle_aggregations("weighted_avg(price, weight=qty)") == "price:weighted_avg(weight=qty)" - - def test_last_with_positional_arg(self) -> None: - assert _rewrite_funcstyle_aggregations("last(revenue, ordered_at)") == "revenue:last(ordered_at)" - - def test_first_with_positional_arg(self) -> None: - assert _rewrite_funcstyle_aggregations("first(revenue, ordered_at)") == "revenue:first(ordered_at)" - - def test_percentile_kwargs(self) -> None: - assert _rewrite_funcstyle_aggregations("percentile(revenue, p=0.95)") == "revenue:percentile(p=0.95)" - - # Compound expressions - def test_arithmetic(self) -> None: - assert _rewrite_funcstyle_aggregations("sum(revenue) / count(*)") == "revenue:sum / *:count" - - def test_addition(self) -> None: - assert _rewrite_funcstyle_aggregations("sum(revenue) + avg(amount)") == "revenue:sum + amount:avg" - - # Nested in transforms - def test_nested_in_transform(self) -> None: - assert _rewrite_funcstyle_aggregations("cumsum(sum(revenue))") == "cumsum(revenue:sum)" - - def test_nested_in_change(self) -> None: - assert _rewrite_funcstyle_aggregations("change(sum(revenue))") == "change(revenue:sum)" - - # Ambiguity: last/first as transform (colon syntax in inner) — leave alone - def test_last_transform_untouched(self) -> None: - assert _rewrite_funcstyle_aggregations("last(revenue:sum)") == "last(revenue:sum)" - - def test_first_transform_untouched(self) -> None: - assert _rewrite_funcstyle_aggregations("first(revenue:sum)") == "first(revenue:sum)" - - # Mixed: transform + function-style aggregation in same expression - def test_transform_and_funcstyle(self) -> None: - result = _rewrite_funcstyle_aggregations("last(revenue:sum) + sum(amount)") - assert "last(revenue:sum)" in result - assert "amount:sum" in result - - # No-op cases - def test_already_colon_syntax(self) -> None: - assert _rewrite_funcstyle_aggregations("revenue:sum") == "revenue:sum" - - def test_plain_transform(self) -> None: - assert _rewrite_funcstyle_aggregations("cumsum(revenue:sum)") == "cumsum(revenue:sum)" - - def test_unknown_function(self) -> None: - assert _rewrite_funcstyle_aggregations("some_func(x)") == "some_func(x)" - - def test_no_args(self) -> None: - assert _rewrite_funcstyle_aggregations("revenue") == "revenue" - - def test_colon_syntax_with_args_in_last(self) -> None: - """revenue:last(ordered_at) should not be touched.""" - assert _rewrite_funcstyle_aggregations("revenue:last(ordered_at)") == "revenue:last(ordered_at)" - - # Custom aggregation names def test_custom_agg_name(self) -> None: - result = _rewrite_funcstyle_aggregations( - "rolling_avg(revenue)", extra_agg_names=frozenset({"rolling_avg"}) - ) - assert result == "revenue:rolling_avg" - - def test_custom_agg_unknown_without_extra(self) -> None: - """Without extra_agg_names, custom agg names are not rewritten.""" - assert _rewrite_funcstyle_aggregations("rolling_avg(revenue)") == "rolling_avg(revenue)" + assert parse_expr("rolling_avg(revenue)") == AggCall(source=Ref(name="revenue"), agg="rolling_avg") def test_emits_no_warning(self) -> None: with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") - _rewrite_funcstyle_aggregations("sum(revenue)") + parse_expr("sum(revenue)") assert w == [] - def test_path_star_count(self) -> None: - assert _rewrite_funcstyle_aggregations("count(customers.*)") == "customers.*:count" - - def test_parse_formula_path_star_count(self) -> None: - assert parse_formula("count(customers.*)") == parse_formula("customers.*:count") - - # Quoted string literals — must not be rewritten - def test_inside_single_quoted_string(self) -> None: - assert _rewrite_funcstyle_aggregations("name = 'sum(revenue)'") == "name = 'sum(revenue)'" - - def test_mixed_quoted_and_unquoted(self) -> None: - result = _rewrite_funcstyle_aggregations("sum(revenue) > 0 and name = 'count(x)'") - assert result == "revenue:sum > 0 and name = 'count(x)'" - - # Escaped quotes inside strings - def test_escaped_quote_in_string(self) -> None: - """Backslash-escaped quote inside a string must not break string tracking.""" - assert _rewrite_funcstyle_aggregations(r"name = 'it\'s sum(x)'") == r"name = 'it\'s sum(x)'" - - # Filter context - def test_in_filter_expression(self) -> None: - result = _rewrite_funcstyle_aggregations("sum(revenue) > 100") - assert result == "revenue:sum > 100" - - # Stat aggregations - def test_stddev_samp_funcstyle(self) -> None: - """`stddev_samp(latency)` must rewrite to colon syntax once the - aggregation name is registered as built-in.""" - assert _rewrite_funcstyle_aggregations("stddev_samp(latency)") == "latency:stddev_samp" - - def test_stddev_pop_funcstyle(self) -> None: - assert _rewrite_funcstyle_aggregations("stddev_pop(latency)") == "latency:stddev_pop" - - def test_var_samp_funcstyle(self) -> None: - assert _rewrite_funcstyle_aggregations("var_samp(latency)") == "latency:var_samp" - - def test_var_pop_funcstyle(self) -> None: - assert _rewrite_funcstyle_aggregations("var_pop(latency)") == "latency:var_pop" - - @pytest.mark.parametrize("agg", ["corr", "covar_samp", "covar_pop"]) - def test_two_arg_stat_funcstyle_with_other_kwarg(self, agg: str) -> None: - """`corr(price, other=quantity)` and `covar_*(price, other=quantity)` - all mirror `weighted_avg(price, weight=qty)` — first positional arg - becomes the LHS column, named kwarg(s) become agg_kwargs.""" - assert ( - _rewrite_funcstyle_aggregations(f"{agg}(price, other=quantity)") - == f"price:{agg}(other=quantity)" - ) - - # Scalar math functions must NOT be rewritten — they are plain SQL - # passthrough used inside Column.sql / formula expressions. - @pytest.mark.parametrize( - "scalar_call", - [ - "ln(amount)", - "log10(amount)", - "log(10, amount)", - "exp(rate)", - "sqrt(price)", - "pow(2, 10)", - "power(2, 10)", - ], - ) - def test_scalar_math_unchanged(self, scalar_call: str) -> None: - # Scalar math UDF names are not aggregations; the rewrite must - # leave them untouched. - assert _rewrite_funcstyle_aggregations(scalar_call) == scalar_call + @pytest.mark.parametrize(("text", "value"), [ + ("name = 'sum(revenue)'", "sum(revenue)"), + ("name = 'it''s sum(x)'", "it's sum(x)"), + ]) + def test_inside_string_literal(self, text: str, value: str) -> None: + assert parse_expr(text) == Cmp(op="==", left=Ref(name="name"), right=Literal(value=value)) + + @pytest.mark.parametrize("scalar_call", [ + "ln(amount)", "log10(amount)", "log(10, amount)", "exp(rate)", "sqrt(price)", "pow(2, 10)", "power(2, 10)", + ]) + def test_scalar_math_is_not_an_aggregation(self, scalar_call: str) -> None: + assert isinstance(parse_expr(scalar_call), ScalarCall) class TestAggregationNameValidation: @@ -323,29 +188,29 @@ class TestStringHygieneFilters: def test_pipe_pipe_chain_three_operands(self) -> None: # Chained `||` desugars left-associatively to nested concat calls. - result = parse_filter_expr("a || b || c = 'foo'") + result = parse_expr("a || b || c = 'foo'") assert isinstance(result, Cmp) outer = result.left assert outer.name == "concat" assert outer.args[0].name == "concat" def test_pipe_pipe_no_spaces(self) -> None: - result = parse_filter_expr("a||b = 'foo'") + result = parse_expr("a||b = 'foo'") assert isinstance(result, Cmp) assert result.left.name == "concat" def test_pipe_pipe_with_function_call_operands(self) -> None: - result = parse_filter_expr("lower(name) || ' ' || trim(addr) = 'eu london'") + result = parse_expr("lower(name) || ' ' || trim(addr) = 'eu london'") assert isinstance(result, Cmp) assert result.left.name == "concat" def test_pipe_pipe_preserved_in_string_literal(self) -> None: # `||` inside a string literal must NOT be rewritten. - result = parse_filter_expr("note = 'a||b'") + result = parse_expr("note = 'a||b'") assert isinstance(result, Cmp) assert result.right.value == "a||b" def test_function_name_preserved_in_string_literal(self) -> None: - result = parse_filter_expr("note = 'lower(x)'") + result = parse_expr("note = 'lower(x)'") assert isinstance(result, Cmp) assert result.right.value == "lower(x)" diff --git a/tests/test_functional_remedies.py b/tests/test_functional_remedies.py index adf2d64e..1351d9e7 100644 --- a/tests/test_functional_remedies.py +++ b/tests/test_functional_remedies.py @@ -6,8 +6,10 @@ from pydantic import ValidationError from slayer.core.errors import DerivedColumnFanningError -from slayer.core.formula import parse_formula -from slayer.core.models import Column, ModelMeasure +from slayer.core.enums import DataType +from slayer.core.models import Column, ModelJoin, ModelMeasure, SlayerModel +from slayer.dbt.converter import DbtToSlayerConverter +from slayer.dbt.models import DbtProject from slayer.engine.column_dependency import _unproven_arity_message from slayer.engine.syntax import split_entity_agg_ref from slayer.sql.generator import AggRenderSpec, SQLGenerator @@ -84,8 +86,15 @@ def test_colon_in_name_reason() -> None: @pytest.mark.parametrize("formula", ["customers.revenue", "cumsum(customers.revenue)"]) def test_cross_model_bare_measure_remedy(formula: str) -> None: - with pytest.raises(ValueError) as ei: - parse_formula(formula) - msg = str(ei.value) + model = SlayerModel( + name="orders", sql_table="orders", data_source="test", + columns=[Column(name="id", type=DataType.INT, primary_key=True), Column(name="customer_id", type=DataType.INT)], + joins=[ModelJoin(target_model="customers", join_pairs=[["customer_id", "id"]])], + measures=[ModelMeasure(name="m", formula=formula)], + ) + conv = DbtToSlayerConverter(project=DbtProject(semantic_models=[], metrics=[]), data_source="test") + conv._prune_model_measures(model) + [warning] = conv._unconverted + msg = warning.message assert "'sum(customers.revenue)'" in msg _assert_no_colon_agg(msg) diff --git a/tests/test_heal_date_parts.py b/tests/test_heal_date_parts.py index 6cf0b9fa..999ade0e 100644 --- a/tests/test_heal_date_parts.py +++ b/tests/test_heal_date_parts.py @@ -11,7 +11,7 @@ from slayer.core.errors import GranularityDefinitionError, SlayerError from slayer.core.models import Aggregation, DatasourceConfig from slayer.engine.query_engine import SlayerQueryEngine -from slayer.engine.syntax import Arith, Cmp, Literal, Ref, parse_expr, parse_filter_expr +from slayer.engine.syntax import Arith, Cmp, Literal, Ref, parse_expr from tests._dev1737_fixtures import PARTS from tests._query_heal_fixtures import ( BACKENDS, @@ -80,7 +80,7 @@ def test_case_folds(self, spelling: str) -> None: assert parse_expr(spelling) == parse_expr(canonical("day_of_week")) def test_isoweek_part_in_filter(self) -> None: - assert parse_filter_expr("date_part('ISOWEEK', created_at) = 1") == parse_filter_expr( + assert parse_expr("date_part('ISOWEEK', created_at) = 1") == parse_expr( "date_part('week', created_at) = 1") @pytest.mark.parametrize(("expr", "count"), [("isodow(created_at, 1)", "2"), ("isodow()", "0")]) @@ -130,11 +130,11 @@ def test_nested_and_composed(self, healed: str, canonical_text: str) -> None: assert parse_expr(healed) == parse_expr(canonical_text) def test_filter_spelling(self) -> None: - assert parse_filter_expr("EXTRACT(day FROM created_at) = 1") == parse_filter_expr( + assert parse_expr("EXTRACT(day FROM created_at) = 1") == parse_expr( "date_part('day', created_at) = 1") def test_keyword_text_in_string_literal_unchanged(self) -> None: - assert parse_filter_expr("status == 'extract(day from x)'") == Cmp( + assert parse_expr("status == 'extract(day from x)'") == Cmp( op="==", left=Ref(name="status"), right=Literal(value="extract(day from x)")) def test_identifier_named_extract_from_untouched(self) -> None: @@ -157,7 +157,7 @@ def test_malformed_extract_rejected(self, expr: str) -> None: def test_unknown_part_lists_accepted_parts(self) -> None: with pytest.raises(ValueError) as info: - parse_filter_expr("extract(epoch from created_at) > 0") + parse_expr("extract(epoch from created_at) > 0") message = str(info.value) assert "epoch" in message for part in PARTS: @@ -170,7 +170,7 @@ class TestAmbiguousWeekdays: def test_part_names_both_conventions(self, name: str, template: str) -> None: text = template.format(n=name) + " = 1" with pytest.raises(ValueError) as info: - parse_filter_expr(text) + parse_expr(text) for remedy in BOTH_CONVENTIONS: assert remedy in str(info.value) diff --git a/tests/test_heal_functional_aggregations.py b/tests/test_heal_functional_aggregations.py index fd734be5..422acd5a 100644 --- a/tests/test_heal_functional_aggregations.py +++ b/tests/test_heal_functional_aggregations.py @@ -12,7 +12,7 @@ from slayer.core.models import Column from slayer.core.query import SlayerQuery from slayer.engine.query_engine import SlayerQueryEngine -from slayer.engine.syntax import AggCall, Cmp, Literal, Ref, parse_expr, parse_filter_expr +from slayer.engine.syntax import AggCall, Cmp, Literal, Ref, parse_expr from tests._engine_helpers import _engine_generate from tests._query_heal_fixtures import ( BACKENDS, @@ -80,14 +80,14 @@ def test_composed(self, healed: str, canonical_text: str) -> None: assert parse_expr(healed) == parse_expr(canonical_text) def test_filter_position(self) -> None: - assert parse_filter_expr("count(distinct customer_id) > 1") == parse_filter_expr( + assert parse_expr("count(distinct customer_id) > 1") == parse_expr( "count_distinct(customer_id) > 1") def test_identifier_starting_with_distinct_untouched(self) -> None: assert parse_expr("count(distinct_id)") == AggCall(source=Ref(name="distinct_id"), agg="count") def test_quoted_distinct_unchanged(self) -> None: - assert parse_filter_expr("status == 'count(distinct x)'") == Cmp( + assert parse_expr("status == 'count(distinct x)'") == Cmp( op="==", left=Ref(name="status"), right=Literal(value="count(distinct x)")) @pytest.mark.parametrize("expr", [ @@ -149,7 +149,7 @@ def test_equals_functional_twin(self, healed: str, canonical_text: str) -> None: assert parse_expr(healed) == parse_expr(canonical_text) def test_filter_position(self) -> None: - assert parse_filter_expr("order_items.count(*) > 1") == parse_filter_expr("count(order_items.*) > 1") + assert parse_expr("order_items.count(*) > 1") == parse_expr("count(order_items.*) > 1") @pytest.mark.parametrize(("expr", "functional"), [ ("order_items.weighted_avg(price, qty)", "weighted_avg(order_items."), diff --git a/tests/test_mode_a_variable_substitution.py b/tests/test_mode_a_variable_substitution.py index 61422fa1..7b9256d3 100644 --- a/tests/test_mode_a_variable_substitution.py +++ b/tests/test_mode_a_variable_substitution.py @@ -8,7 +8,6 @@ All end-to-end cases run against a real file-backed SQLite datasource and assert on RESULT DATA, not just generated SQL strings. """ -import ast from slayer.storage.sqlite_conn import transaction import tempfile @@ -484,8 +483,7 @@ async def test_backslash_value_sqlite_roundtrip(self) -> None: async def test_mode_b_query_filter_matches_only_its_row( self, value: str, expected: float ) -> None: - # Mode-B: SlayerQuery.filters → escape="python". Quote values round-trip - # (Python-AST parse, no SQL re-parse of the value). + # Mode-B: SlayerQuery.filters bind the value as a literal; quote values round-trip. model = SlayerModel( name="t", sql_table="t", @@ -844,28 +842,6 @@ async def test_inspect_model_shows_literal_var(self) -> None: # 11. Unit: hardened substitute_variables (escaping, scalar-only) # --------------------------------------------------------------------------- -def _python_mode_roundtrips(template: str, value: str) -> bool: - """Substitute ``value`` into ``template`` with ``escape="python"``, parse the - result as a Python expression, and return whether the recovered string - constant equals the original ``value``. This verifies the escaping CONTRACT - (the substituted filter round-trips through SLayer's ast.parse-based Mode-B - parser) rather than just an opaque expected string.""" - - - substituted = substitute_variables( - filter_str=template, variables={"v": value}, escape="python" - ) - # template is `col = `; SLayer's Mode-B parser reads SQL `=` as - # equality, so mirror that (`=` → `==`) to get a parseable expression, then - # pull the RHS constant back out. - expr = ast.parse(substituted.replace(" = ", " == ", 1), mode="eval") - compare = expr.body - assert isinstance(compare, ast.Compare) - rhs = compare.comparators[0] - assert isinstance(rhs, ast.Constant) - return rhs.value == value - - class TestSubstituteVariablesHardened: def test_sql_mode_string_value_doubles_single_quote(self) -> None: @@ -896,90 +872,6 @@ def test_sql_mode_backslash_untouched_standard_dialect(self) -> None: ) assert result == r"path = 'a\b'" - def test_python_mode_string_value_backslash_escapes_quote(self) -> None: - - # Mode-B (Python-AST-parsed) filters backslash-escape the quote — SQL - # quote-doubling would be parsed as adjacent-literal concatenation - # ('O''Brien' → 'OBrien') by the Python AST, silently corrupting it. - result = substitute_variables( - filter_str="status = '{v}'", variables={"v": "O'Brien"}, escape="python" - ) - assert result == "status = 'O\\'Brien'" - - def test_python_mode_double_quote_escaped(self) -> None: - - result = substitute_variables( - filter_str='status = "{v}"', variables={"v": 'say "hi"'}, escape="python" - ) - assert result == 'status = "say \\"hi\\""' - - def test_python_mode_backslash_doubled_first(self) -> None: - - # Backslash must be escaped BEFORE quotes, else a\'b would become - # a\\'b's quote unescaped. Trailing/standalone backslash → doubled. - result = substitute_variables( - filter_str="path = '{v}'", variables={"v": "a\\b"}, escape="python" - ) - assert result == "path = 'a\\\\b'" - - def test_python_mode_backslash_then_quote(self) -> None: - - # Value a\'b (backslash, quote) → a\\\'b (doubled backslash, escaped - # quote) so the Python AST reads it back as the literal 4-char string. - result = substitute_variables( - filter_str="path = '{v}'", variables={"v": "a\\'b"}, escape="python" - ) - assert result == "path = 'a\\\\\\'b'" - - def test_python_mode_trailing_backslash(self) -> None: - - result = substitute_variables( - filter_str="path = '{v}'", variables={"v": "abc\\"}, escape="python" - ) - assert result == "path = 'abc\\\\'" - - def test_python_mode_backslash_before_double_quote(self) -> None: - - # Symmetric to backslash-before-single-quote, in a double-quoted - # template: value a\"b → a\\\"b so the AST recovers the 4-char value. - result = substitute_variables( - filter_str='path = "{v}"', variables={"v": 'a\\"b'}, escape="python" - ) - assert result == 'path = "a\\\\\\"b"' - - @pytest.mark.parametrize( - "template,value", - [ - ("col = '{v}'", "O'Brien"), - ('col = "{v}"', 'say "hi"'), - ("col = '{v}'", "a\\b"), - ("col = '{v}'", "a\\'b"), - ('col = "{v}"', 'a\\"b'), - ("col = '{v}'", "abc\\"), - ("col = '{v}'", "plain"), - # Control chars must be backslash-escaped so a raw newline/CR/tab/NUL - # doesn't make the single-quoted literal a SyntaxError. - ("col = '{v}'", "a\nb"), - ("col = '{v}'", "a\r\nb"), - ("col = '{v}'", "a\tb"), - ("col = '{v}'", "a\x00b"), - ], - ) - def test_python_mode_ast_roundtrip(self, template: str, value: str) -> None: - # Semantic contract: whatever escaping produces, ast.parse of the - # substituted filter must recover the ORIGINAL value (both quote - # delimiters, backslash, backslash-before-quote, and control chars). - assert _python_mode_roundtrips(template, value) - - def test_python_mode_newline_escaped(self) -> None: - - # A real newline becomes the two-char escape \n so the literal stays on - # one line and re-parses to the original value. - result = substitute_variables( - filter_str="note = '{v}'", variables={"v": "a\nb"}, escape="python" - ) - assert result == "note = 'a\\nb'" - def test_sql_mode_newline_left_raw(self) -> None: # SQL string literals permit raw newlines, so sql-mode must NOT escape @@ -991,7 +883,7 @@ def test_sql_mode_newline_left_raw(self) -> None: ) assert result == "note = 'a\nb'" - def test_number_value_not_escaped_either_mode(self) -> None: + def test_number_value_not_escaped(self) -> None: assert ( substitute_variables( @@ -1000,10 +892,6 @@ def test_number_value_not_escaped_either_mode(self) -> None: ) == "amount > 100" ) - assert ( - substitute_variables(filter_str="amount > {n}", variables={"n": 100}, escape="python") - == "amount > 100" - ) def test_float_value_not_escaped(self) -> None: @@ -1025,10 +913,6 @@ def test_bool_value_accepted(self) -> None: ) == "flag = True" ) - assert ( - substitute_variables(filter_str="flag = {v}", variables={"v": False}, escape="python") - == "flag = False" - ) def test_nan_value_raises(self) -> None: @@ -1050,7 +934,8 @@ def test_inf_value_raises(self) -> None: neg_inf = float("-inf") with pytest.raises(ValueError, match="finite"): substitute_variables( - filter_str="x = {v}", variables={"v": neg_inf}, escape="python" + filter_str="x = {v}", variables={"v": neg_inf}, + escape="sql", backslash_escapes=False, ) def test_dict_value_raises(self) -> None: @@ -1091,7 +976,7 @@ def test_invalid_escape_value_raises(self) -> None: # The message must name the VALID modes — so a wrong ordering that hit # the sql-mode ``backslash_escapes`` guard first (whose message also # contains the word "escape") could not masquerade as a pass. - with pytest.raises(ValueError, match="sql.*python|python.*sql"): + with pytest.raises(ValueError, match="sql"): substitute_variables( filter_str="x = '{v}'", variables={"v": "a"}, escape="other" ) @@ -1100,7 +985,7 @@ def test_invalid_escape_takes_precedence_over_missing_flag(self) -> None: # An invalid escape mode is rejected BEFORE the sql-mode # backslash_escapes guard — the error is about the mode, not the flag. - with pytest.raises(ValueError, match="sql.*python|python.*sql") as exc: + with pytest.raises(ValueError, match="sql") as exc: substitute_variables( filter_str="x = '{v}'", variables={"v": "a"}, escape="bogus" ) @@ -1566,71 +1451,6 @@ def test_sql_injection_element_stays_inside_literal(self) -> None: assert result == "region IN ('x'') OR (''1''=''1')" -class TestListValueRenderingPython: - """``escape="python"`` list rendering: comma-joined WITH a trailing comma so - the Mode-B Python-AST parser always reads a tuple, never a bare string - (``x in ('A')`` is string membership; ``x in ('A',)`` is a 1-tuple).""" - - def test_python_list_trailing_comma_single(self) -> None: - - result = substitute_variables( - filter_str="region in ({v})", - variables={"v": ["A"]}, - escape="python", - ) - assert result == "region in ('A',)" - - def test_python_list_trailing_comma_multi(self) -> None: - - result = substitute_variables( - filter_str="region in ({v})", - variables={"v": ["A", "B"]}, - escape="python", - ) - assert result == "region in ('A', 'B',)" - - def test_python_list_numbers_bare_trailing_comma(self) -> None: - - result = substitute_variables( - filter_str="x in ({v})", - variables={"v": [1, 2]}, - escape="python", - ) - assert result == "x in (1, 2,)" - - @pytest.mark.parametrize( - "values", [["A"], ["A", "B"], ["O'Brien", "a\\b"], ["a\nb", "x"]] - ) - def test_python_list_ast_roundtrip_is_tuple(self, values: list) -> None: - # The substituted ``x in (...)`` must parse to an ast.Tuple (never a - # bare Constant) whose elements recover the ORIGINAL string values — - # so single-element lists work and quotes/backslashes round-trip. - - - substituted = substitute_variables( - filter_str="region in ({v})", - variables={"v": values}, - escape="python", - ) - expr = ast.parse(substituted, mode="eval") - compare = expr.body - assert isinstance(compare, ast.Compare) - rhs = compare.comparators[0] - assert isinstance(rhs, ast.Tuple) - recovered = [e.value for e in rhs.elts] - assert recovered == values - - def test_python_tuple_accepted_same_as_list(self) -> None: - - from_list = substitute_variables( - filter_str="x in ({v})", variables={"v": ["A", "B"]}, escape="python" - ) - from_tuple = substitute_variables( - filter_str="x in ({v})", variables={"v": ("A", "B")}, escape="python" - ) - assert from_list == from_tuple == "x in ('A', 'B',)" - - class TestListValueRenderingErrors: def test_empty_list_raises_naming_variable(self) -> None: @@ -1654,7 +1474,7 @@ def test_empty_tuple_raises(self) -> None: substitute_variables( filter_str="region IN ({regions})", variables={"regions": ()}, - escape="python", + escape="sql", backslash_escapes=False, ) assert "empty" in str(exc.value).lower() @@ -1698,7 +1518,7 @@ def test_dict_element_raises(self) -> None: substitute_variables( filter_str="region IN ({regions})", variables={"regions": ["A", {"b": 1}]}, - escape="python", + escape="sql", backslash_escapes=False, ) assert "regions" in str(exc.value) @@ -2032,147 +1852,14 @@ def test_sql_without_flag_raises_even_for_number(self) -> None: filter_str="x > {n}", variables={"n": 1}, escape="sql" ) - def test_python_mode_ignores_backslash_escapes(self) -> None: - # python mode never needs the flag; passing it is accepted and ignored - # (both True and False yield the identical python-escaped result). - base = substitute_variables( - filter_str="p = '{v}'", variables={"v": "O'Brien"}, escape="python" - ) - for be in (True, False, None): - assert substitute_variables( - filter_str="p = '{v}'", variables={"v": "O'Brien"}, - escape="python", backslash_escapes=be, - ) == base - # --------------------------------------------------------------------------- -# 21. Python regime full C0-control pass (Gap 2) +# 21. SQL regime leaves control chars raw # --------------------------------------------------------------------------- -def _python_literal_value(substituted_literal: str): - """Parse a single-quoted python literal string (as produced by the - escaping) and return the recovered value via ast.literal_eval.""" - return ast.literal_eval(substituted_literal) - - -class TestPythonModeControlChars: - """After backslash/quote escaping, ``escape="python"`` encodes every C0 - control char (U+0000–U+001F) so SLayer's ast.parse-based Mode-B parser — - which rejects a raw newline / NUL inside a string literal — recovers the - original value.""" - - def test_newline_named_escape(self) -> None: - result = substitute_variables( - filter_str="'{v}'", variables={"v": "a\nb"}, escape="python" - ) - assert result == "'a\\nb'" - # Recover from the ACTUAL produced literal (not a hard-coded one). - assert _python_literal_value(result) == "a\nb" - - def test_carriage_return_named_escape(self) -> None: - result = substitute_variables( - filter_str="p = '{v}'", variables={"v": "a\rb"}, escape="python" - ) - assert result == "p = 'a\\rb'" - - def test_tab_named_escape(self) -> None: - result = substitute_variables( - filter_str="p = '{v}'", variables={"v": "a\tb"}, escape="python" - ) - assert result == "p = 'a\\tb'" - - def test_nul_hex_escape(self) -> None: - result = substitute_variables( - filter_str="p = '{v}'", variables={"v": "a\x00b"}, escape="python" - ) - assert result == "p = 'a\\x00b'" - - def test_other_c0_hex_escape(self) -> None: - # Vertical tab (0x0b) has no named escape → \x0b. - result = substitute_variables( - filter_str="p = '{v}'", variables={"v": "a\x0bb"}, escape="python" - ) - assert result == "p = 'a\\x0bb'" - - @pytest.mark.parametrize("codepoint", list(range(0x00, 0x20))) - def test_every_c0_char_roundtrips_through_ast(self, codepoint: int) -> None: - # The whole C0 range must round-trip: substitute → the produced literal - # is a valid python string literal recovering the original value. - value = f"x{chr(codepoint)}y" - substituted = substitute_variables( - filter_str="'{v}'", variables={"v": value}, escape="python" - ) - # ast.parse must not raise, and the recovered value must be identical. - assert _python_literal_value(substituted) == value - - @pytest.mark.parametrize("codepoint", list(range(0x00, 0x20))) - def test_every_c0_char_exact_rendering(self, codepoint: int) -> None: - # Pin the EXACT encoding (not merely "some valid escape"): \t\n\r use - # their named escape, every other C0 char uses lowercase \xNN. This - # locks the rendering contract so an implementation that used octal or - # \u would still be caught even though ast.parse would accept it. - char = chr(codepoint) - named = {"\t": "\\t", "\n": "\\n", "\r": "\\r"} - expected_escape = named.get(char, f"\\x{codepoint:02x}") - substituted = substitute_variables( - filter_str="'{v}'", variables={"v": char}, escape="python" - ) - assert substituted == f"'{expected_escape}'" - - @pytest.mark.parametrize("codepoint", list(range(0x00, 0x20))) - def test_every_c0_char_roundtrips_in_list_element(self, codepoint: int) -> None: - # DEV-1730 list elements go through the same escaping; a control char in - # an element must round-trip so the ast tuple recovers it. - value = f"x{chr(codepoint)}y" - substituted = substitute_variables( - filter_str="c in ({v})", variables={"v": [value]}, escape="python" - ) - expr = ast.parse(substituted, mode="eval") - rhs = expr.body.comparators[0] - assert isinstance(rhs, ast.Tuple) - assert [e.value for e in rhs.elts] == [value] - - def test_non_control_unicode_untouched(self) -> None: - # A non-C0 char (accented letter, U+2028 line separator) is NOT a C0 - # control char, so the pass leaves it verbatim. - for value in ("café", "a
b", "emoji😀"): - substituted = substitute_variables( - filter_str="'{v}'", variables={"v": value}, escape="python" - ) - assert _python_literal_value(substituted) == value - - @pytest.mark.parametrize( - "codepoint", - [0x2028, 0x2029, 0x0085, 0x00A0, 0x00E9, 0x1F600], - ids=["line-sep", "para-sep", "nel", "nbsp", "e-acute", "emoji"], - ) - def test_non_c0_char_textually_untouched(self, codepoint: int) -> None: - # Non-C0 chars (incl. the deceptively line-break-looking U+2028/U+2029, - # U+0085 NEL) are NOT encoded — they are left in the output byte-for-byte - # AND still round-trip (ast recovers them). A weaker recovery-only check - # would also pass if we'd wrongly \\u-escaped them, so assert BOTH. - char = chr(codepoint) - value = f"x{char}y" - substituted = substitute_variables( - filter_str="'{v}'", variables={"v": value}, escape="python" - ) - assert char in substituted # textually present, not escaped - assert substituted == f"'{value}'" - assert _python_literal_value(substituted) == value - - def test_backslash_before_control_char(self) -> None: - # Order: backslash doubled first, THEN the control char encoded — a - # value of backslash+newline must recover exactly. - value = "\\\n" - substituted = substitute_variables( - filter_str="'{v}'", variables={"v": value}, escape="python" - ) - assert substituted == "'\\\\\\n'" - assert _python_literal_value(substituted) == value - +class TestSqlModeControlChars: def test_sql_mode_leaves_control_chars_raw(self) -> None: - # The C0 pass is python-only: SQL literals accept raw newlines and - # sqlglot re-emits them, so sql mode must NOT encode control chars. + # SQL literals accept raw newlines and sqlglot re-emits them, so sql mode must NOT encode control chars. result = substitute_variables( filter_str="p = '{v}'", variables={"v": "a\nb"}, escape="sql", backslash_escapes=False, diff --git a/tests/test_mode_b_placeholder_parse.py b/tests/test_mode_b_placeholder_parse.py index 522b9577..58956170 100644 --- a/tests/test_mode_b_placeholder_parse.py +++ b/tests/test_mode_b_placeholder_parse.py @@ -19,7 +19,6 @@ UnaryOp, canonical_measure_text, parse_expr, - parse_filter_expr, ) K = Placeholder(name="k") @@ -59,7 +58,7 @@ def test_placeholder_position(text: str, expected) -> None: def test_filter_parser_admits_placeholder() -> None: - assert parse_filter_expr("amount = {k}") == Cmp(op="==", left=AMOUNT, right=K) + assert parse_expr("amount = {k}") == Cmp(op="==", left=AMOUNT, right=K) @pytest.mark.parametrize("text", ["amount:sum * {k} / 100", "sum(amount) * {k} / 100"]) diff --git a/tests/test_osi_converter.py b/tests/test_osi_converter.py index cf230976..780e823d 100644 --- a/tests/test_osi_converter.py +++ b/tests/test_osi_converter.py @@ -1240,3 +1240,17 @@ def test_relationship_keyed_on_renamed_field_names_the_field(shop_engine): key = orders.get_column("cust_key") assert key is not None assert key.physical_name == "customer_id" + + +def test_emitted_formula_is_validated_by_the_mode_b_grammar(shop_engine): + doc = _mini_doc( + datasets=[OSIDataset(name="orders", source="orders", + fields=[OSIField(name="amount", expression=_expr("amount")), + OSIField(name="status", expression=_expr("status"))])], + metrics=[OSIMetric(name="paid_share", + expression=_expr("SUM(CASE WHEN status = 'paid' THEN amount END) / COUNT(*)"))], + ) + orders = _by_name(_convert(shop_engine, doc))["orders"] + formula = {m.name: m for m in orders.measures}["paid_share"].formula + parse_expr(formula) + assert '"' not in formula and "\\" not in formula and " | " not in formula diff --git a/tests/test_rank_direction.py b/tests/test_rank_direction.py index cd33c1a4..f6a2caca 100644 --- a/tests/test_rank_direction.py +++ b/tests/test_rank_direction.py @@ -7,12 +7,11 @@ import pytest from slayer.core import errors as core_errors -from slayer.core.formula import parse_formula from slayer.core.keys import TransformKey from slayer.core.models import ModelMeasure from slayer.core.scope import ModelScope from slayer.engine.binding import bind_expr -from slayer.engine.syntax import parse_expr +from slayer.engine.syntax import Literal, TransformCall, parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle from tests._dev1847_fixtures import ( @@ -318,11 +317,9 @@ def test_unknown_keyword_lists_direction_where_accepted(self, op, advertised): formula = f"{op}(sum(amount), foo=1)" with pytest.raises(core_errors.TransformArgumentError) as binder: _bind(formula) - with pytest.raises(ValueError) as importer: - parse_formula(formula) - for msg in (str(binder.value), str(importer.value)): - assert "foo" in msg, msg - assert ("direction" in msg) is advertised, msg + msg = str(binder.value) + assert "foo" in msg, msg + assert ("direction" in msg) is advertised, msg def test_unknown_keyword_outside_rank_family_lists_no_rank_keywords(self): with pytest.raises(core_errors.TransformArgumentError) as ei: @@ -341,24 +338,23 @@ class TestImporterParity: "ntile(sum(amount), n=4, direction='asc')", "percent_rank(sum(amount), direction='desc')", ]) - def test_same_error_as_the_binder(self, formula): - with pytest.raises(core_errors.TransformArgumentError) as importer: - parse_formula(formula) - with pytest.raises(core_errors.TransformArgumentError) as binder: + def test_parse_is_pure_syntax_and_the_binder_judges(self, formula): + assert isinstance(parse_expr(formula), TransformCall) + with pytest.raises(core_errors.TransformArgumentError): _bind(formula) - assert str(importer.value) == str(binder.value) @pytest.mark.parametrize(("formula", "want"), [ ("rank(sum(amount), direction='Ascending')", "asc"), ("dense_rank(sum(amount), partition_by=region, direction='DESC')", "desc"), ]) def test_accepts_and_normalises(self, formula, want): - field = parse_formula(formula) - assert field.kwargs["direction"] == want # type: ignore[union-attr] + assert dict(_bind(formula).kwargs)["direction"] == want # type: ignore[union-attr] def test_ntile_keeps_no_direction(self): - field = parse_formula("ntile(sum(amount), n=4)") - assert "direction" not in field.kwargs # type: ignore[union-attr] + parsed = parse_expr("ntile(sum(amount), n=4)") + assert isinstance(parsed, TransformCall) + assert "direction" not in dict(parsed.kwargs) + assert dict(parsed.kwargs)["n"] == Literal(value=4) # --------------------------------------------------------------------------- # diff --git a/tests/test_schema_drift_typed.py b/tests/test_schema_drift_typed.py index 0539b5c9..c88bcd05 100644 --- a/tests/test_schema_drift_typed.py +++ b/tests/test_schema_drift_typed.py @@ -312,7 +312,7 @@ def test_colon_agg_dunder_source_is_extracted(self) -> None: class TestFilterRefsDslParity: """DEV-1833 parity suite pinned while migrating ``_filter_refs_dsl`` onto - ``parse_filter_expr`` (verified green on the legacy implementation first). + ``parse_expr`` (verified green on the legacy implementation first). Set equality only: reference order is expression order.""" def test_colon_agg(self) -> None: @@ -360,7 +360,7 @@ def test_unparseable_returns_empty(self) -> None: class TestFilterRefsDslMigrationGains: - """Refs only the migrated ``parse_filter_expr`` implementation surfaces + """Refs only the migrated ``parse_expr`` implementation surfaces (DEV-1833); red until the migration lands.""" def test_funcstyle_custom_agg(self) -> None: diff --git a/tests/test_sql_boolean_literal_filters.py b/tests/test_sql_boolean_literal_filters.py index c5e94dd9..ae1974e8 100644 --- a/tests/test_sql_boolean_literal_filters.py +++ b/tests/test_sql_boolean_literal_filters.py @@ -19,23 +19,23 @@ from slayer.core.models import Column, DatasourceConfig, SlayerModel from slayer.core.query import SlayerQuery from slayer.engine.query_engine import SlayerQueryEngine -from slayer.engine.syntax import BoolOp, Cmp, Literal, Ref, parse_filter_expr +from slayer.engine.syntax import BoolOp, Cmp, Literal, Ref, parse_expr from slayer.storage.yaml_storage import YAMLStorage class TestParserBooleanLiterals: @pytest.mark.parametrize("spelling", ["true", "TRUE", "True"]) def test_true_any_casing_is_a_literal(self, spelling): - result = parse_filter_expr(f"is_active = {spelling}") + result = parse_expr(f"is_active = {spelling}") assert result == Cmp(op="==", left=Ref(name="is_active"), right=Literal(value=True)) @pytest.mark.parametrize("spelling", ["false", "FALSE", "False"]) def test_false_any_casing_is_a_literal(self, spelling): - result = parse_filter_expr(f"is_active = {spelling}") + result = parse_expr(f"is_active = {spelling}") assert result == Cmp(op="==", left=Ref(name="is_active"), right=Literal(value=False)) def test_bare_boolean_in_disjunction_is_not_a_ref(self): - result = parse_filter_expr("status = 'x' or false") + result = parse_expr("status = 'x' or false") assert isinstance(result, BoolOp) assert Literal(value=False) in result.operands diff --git a/tests/test_syntax.py b/tests/test_syntax.py index 2a0c711a..5f597e8e 100644 --- a/tests/test_syntax.py +++ b/tests/test_syntax.py @@ -35,7 +35,6 @@ TransformCall, UnaryOp, parse_expr, - parse_filter_expr, ) @@ -67,8 +66,8 @@ def test_decimal_literal(self): def test_string_literal_single_quote(self): assert parse_expr("'hello'") == Literal(value="hello") - def test_string_literal_double_quote(self): - assert parse_expr('"hello"') == Literal(value="hello") + def test_double_quotes_are_an_identifier(self): + assert parse_expr('"hello"') == Ref(name="hello") def test_bool_true(self): assert parse_expr("True") == Literal(value=True) @@ -292,7 +291,7 @@ def test_scalar_names_are_case_insensitive(self): assert result.name == "coalesce" def test_uppercase_scalar_in_filter(self): - result = parse_filter_expr("ROUND(revenue, 2) > 5") + result = parse_expr("ROUND(revenue, 2) > 5") assert isinstance(result, Cmp) assert isinstance(result.left, ScalarCall) assert result.left.name == "round" @@ -622,23 +621,23 @@ def test_consecutive_periods_predicate_then_compared(self): # --------------------------------------------------------------------------- -# Filter-operator normalization (parse_filter_expr) +# Filter-operator normalization (parse_expr) # --------------------------------------------------------------------------- class TestFilterOperatorNormalization: - """``parse_filter_expr`` accepts SQL operator spellings (``=`` for + """``parse_expr`` accepts SQL operator spellings (``=`` for equality, ``AND`` / ``IS`` / ``NULL`` ...) on top of the Python DSL.""" def test_sql_equality_normalised(self): - result = parse_filter_expr("status = 'paid'") + result = parse_expr("status = 'paid'") assert isinstance(result, Cmp) assert result.op == "==" assert result.right == Literal(value="paid") def test_transform_no_kwargs_in_filter(self): # The no-kwargs top-N form is unaffected by the operator rewrite. - result = parse_filter_expr("dense_rank(revenue:sum, direction='desc') <= 5") + result = parse_expr("dense_rank(revenue:sum, direction='desc') <= 5") assert isinstance(result, Cmp) assert isinstance(result.left, TransformCall) assert result.left.op == "dense_rank" @@ -648,7 +647,7 @@ def test_sql_concat_pipe_pipe_normalised(self): # normalizer rewrites ``||`` → ``|``, then ``_convert`` desugars the # BitOr to concat). ``|`` binds tighter than ``==`` just as ``||`` # binds tighter than ``=`` in SQL. - result = parse_filter_expr("status || status = 'foo'") + result = parse_expr("status || status = 'foo'") assert isinstance(result, Cmp) assert result.op == "==" assert isinstance(result.left, ScalarCall) @@ -660,13 +659,13 @@ def test_sql_like_operator_normalised(self): # DEV-1704: a SQL ``LIKE`` operator (e.g. from the pg-facade WHERE # translation) normalises to the ``like(col, pattern)`` scalar the DSL # already emits as SQL LIKE. - result = parse_filter_expr("name LIKE 'do%'") + result = parse_expr("name LIKE 'do%'") assert isinstance(result, ScalarCall) assert result.name == "like" assert result.args == (Ref(name="name"), Literal(value="do%")) def test_sql_not_like_operator_normalised(self): - result = parse_filter_expr("name NOT LIKE 'x%'") + result = parse_expr("name NOT LIKE 'x%'") assert isinstance(result, UnaryOp) assert result.op == "not" assert isinstance(result.operand, ScalarCall) @@ -676,7 +675,7 @@ def test_sql_not_like_operator_normalised(self): def test_sql_like_dotted_and_comma_pattern(self): # Dotted LHS (joined column) + a pattern that contains a comma (the # categorical-comma metabase repro) must both round-trip. - result = parse_filter_expr("customers.name LIKE '%,%'") + result = parse_expr("customers.name LIKE '%,%'") assert isinstance(result, ScalarCall) assert result.name == "like" assert result.args == ( @@ -686,7 +685,7 @@ def test_sql_like_dotted_and_comma_pattern(self): def test_like_word_inside_string_literal_not_rewritten(self): # The ``like`` inside a string literal is NOT a LIKE operator: the value # stays a literal, the LHS a column reference. - result = parse_filter_expr("name == 'a like b'") + result = parse_expr("name == 'a like b'") assert isinstance(result, Cmp) assert result.op == "==" assert result.left == Ref(name="name") @@ -696,7 +695,7 @@ def test_transform_kwarg_preserved_in_filter(self): # DEV-1492: ntile(revenue:sum, n=4) <= 1 — the n=4 kwarg must # survive the SQL operator-normalization (the previous '=' -> '==' # rewrite mangled kwargs inside call parens). - result = parse_filter_expr("ntile(revenue:sum, n=4) <= 1") + result = parse_expr("ntile(revenue:sum, n=4) <= 1") assert isinstance(result, Cmp) assert isinstance(result.left, TransformCall) assert result.left.op == "ntile" @@ -706,7 +705,7 @@ def test_transform_kwarg_preserved_in_filter(self): def test_rank_partition_by_kwarg_preserved_in_filter(self): # Both kwargs survive the operator rewrite; binder turns partition_by into a # column ref (covered by SQL-gen tests). - result = parse_filter_expr("rank(revenue:sum, partition_by=region, direction='desc') <= 1") + result = parse_expr("rank(revenue:sum, partition_by=region, direction='desc') <= 1") assert isinstance(result, Cmp) assert result.op == "<=" assert isinstance(result.left, TransformCall) @@ -720,7 +719,7 @@ def test_rank_partition_by_kwarg_preserved_in_filter(self): def test_rank_partition_by_list_kwarg_preserved_in_filter(self): # DEV-1492: list-form partition_by kwarg survives. _convert_kwarg_value # converts the list to a tuple of Refs. - result = parse_filter_expr( + result = parse_expr( "rank(revenue:sum, partition_by=[region, channel], direction='desc') <= 1" ) assert isinstance(result, Cmp) @@ -742,7 +741,7 @@ def test_rank_partition_by_list_kwarg_preserved_in_filter(self): def test_filter_transform_preserves_aggregation_positional_arg(self): # change(revenue:last(ordered_at)) > 0 — the first/last ranking-time # positional arg must survive on the inner AggCall. - result = parse_filter_expr("change(revenue:last(ordered_at)) > 0") + result = parse_expr("change(revenue:last(ordered_at)) > 0") assert isinstance(result, Cmp) inner = result.left assert isinstance(inner, TransformCall) @@ -755,7 +754,7 @@ def test_filter_transform_preserves_aggregation_positional_arg(self): def test_filter_transform_preserves_aggregation_kwarg(self): # change(price:weighted_avg(weight=quantity)) > 0 — the aggregation's # own kwarg must survive, distinct from the transform's kwargs. - result = parse_filter_expr( + result = parse_expr( "change(price:weighted_avg(weight=quantity)) > 0" ) assert isinstance(result, Cmp) @@ -769,7 +768,7 @@ def test_filter_transform_preserves_aggregation_kwarg(self): def test_filter_transform_preserves_mixed_agg_args_and_kwargs(self): # change(price:weighted_avg(col1, weight=quantity)) > 0 — positional # and keyword args on the same aggregation, both preserved. - result = parse_filter_expr( + result = parse_expr( "change(price:weighted_avg(col1, weight=quantity)) > 0" ) inner = result.left @@ -782,7 +781,7 @@ def test_filter_transform_preserves_mixed_agg_args_and_kwargs(self): def test_filter_ntile_preserves_both_n_and_partition_by(self): # DEV-1353 regression shape: dropping partition_by would rank globally # instead of within the partition; dropping n would fail to re-parse. - result = parse_filter_expr( + result = parse_expr( "ntile(revenue:sum, n=4, partition_by=cohort) <= 1" ) assert isinstance(result, Cmp) @@ -798,7 +797,7 @@ def test_grouping_paren_equality_still_rewritten(self): # DEV-1492 regression guard: a `=` inside a *grouping* paren (not a # call) must still be rewritten to `==`. The kwarg-preservation rule # keys off CALL-paren classification, not bare paren depth. - result = parse_filter_expr("(status = 'x' or amount = 5)") + result = parse_expr("(status = 'x' or amount = 5)") assert isinstance(result, BoolOp) assert result.op == "or" assert len(result.operands) == 2 @@ -816,7 +815,7 @@ def test_mixed_transform_kwarg_and_top_level_equality(self): # DEV-1492: a transform kwarg `=` inside a call and a SQL equality # `=` at top level must be classified independently in the same # expression. Proves the scanner is selective, not global. - result = parse_filter_expr( + result = parse_expr( "ntile(revenue:sum, n=4) <= 1 and status = 'paid'" ) assert isinstance(result, BoolOp) @@ -837,7 +836,7 @@ def test_equality_inside_string_literal_untouched(self): # DEV-1492 literal-awareness: the `=` inside the string literal # `'a=b'` must not be touched by the scanner; only the outer SQL # equality is rewritten. - result = parse_filter_expr("status = 'a=b'") + result = parse_expr("status = 'a=b'") assert isinstance(result, Cmp) assert result.op == "==" assert result.left == Ref(name="status") @@ -850,7 +849,7 @@ def test_comparison_inside_scalar_call_preserved(self): # kwargs per architecture). Without the narrowing, the kwarg- # preservation rule would re-read `status='paid'` as a kwarg and # the binder would reject it. - result = parse_filter_expr("coalesce(status = 'paid', False)") + result = parse_expr("coalesce(status = 'paid', False)") assert isinstance(result, ScalarCall) assert result.name == "coalesce" assert len(result.args) == 2 @@ -866,7 +865,7 @@ def test_colon_agg_kwarg_preserved_in_filter(self): # is the same failure mode as transform kwargs — the colon-agg # callee `percentile` is not a SCALAR_FUNCTION, so the scanner # preserves `p=0.5` as a kwarg on the AggCall. - result = parse_filter_expr("revenue:percentile(p=0.5) > 100") + result = parse_expr("revenue:percentile(p=0.5) > 100") assert isinstance(result, Cmp) assert result.op == ">" assert isinstance(result.left, AggCall) @@ -882,7 +881,7 @@ def test_transform_kwarg_with_whitespace_around_equals_preserved(self): # DEV-1492 (Codex review): the scanner skips whitespace when # checking the `IDENT = value` shape, so a spaced kwarg still # parses as a kwarg, not a positional comparison. - result = parse_filter_expr("ntile(revenue:sum, n = 4) <= 1") + result = parse_expr("ntile(revenue:sum, n = 4) <= 1") assert isinstance(result, Cmp) assert isinstance(result.left, TransformCall) assert result.left.op == "ntile" @@ -895,7 +894,7 @@ def test_not_paren_is_grouping_not_call(self): # CALL). The `=` inside is therefore a SQL comparison and gets # rewritten to `==`. Python's `not(x)` is UnaryOp(Not, x), not a # function call. - result = parse_filter_expr("not(status = 'paid')") + result = parse_expr("not(status = 'paid')") assert isinstance(result, UnaryOp) assert result.op == "not" assert isinstance(result.operand, Cmp) @@ -909,7 +908,7 @@ def test_nested_scalar_in_transform_with_kwarg(self): # which must be preserved; inner scalar `coalesce(status = 'paid', # 0)` has a `=` which must be rewritten (SCALAR_FUNCTIONS # narrowing). Exercises the per-frame (kind, callee) stack. - result = parse_filter_expr( + result = parse_expr( "rank(coalesce(status = 'paid', 0), partition_by=region, direction='desc') <= 1" ) assert isinstance(result, Cmp) @@ -937,7 +936,7 @@ def test_string_literal_with_parens_commas_equals_does_not_corrupt_stack(self): # and `=` must not perturb the scanner's paren-kind stack or # previous-token tracking. Both the kwarg `n=4` and the top-level # `status = '),='` must be classified correctly. - result = parse_filter_expr( + result = parse_expr( "ntile(revenue:sum, n=4) <= 1 and status = '),='" ) assert isinstance(result, BoolOp) @@ -962,7 +961,7 @@ def test_colon_agg_with_ambiguous_name_preserves_kwarg_in_filter(self): # a `:`. Built-in `first`/`last` take the order column positionally # today, but custom aggregations overriding these names can declare # any kwarg shape; this pins the architectural rule. - result = parse_filter_expr("revenue:first(order=created_at) > 0") + result = parse_expr("revenue:first(order=created_at) > 0") assert isinstance(result, Cmp) assert result.op == ">" assert isinstance(result.left, AggCall) @@ -980,7 +979,7 @@ def test_transform_bare_ref_equality_predicate_preserved(self): # accept kwargs AFTER the value (i.e. after the first `,`). The colon- # agg form (`revenue:sum > 0`) already worked because the `:` token # breaks the LPAREN/IDENT pattern; the bare-ref form is the gap. - result = parse_filter_expr("consecutive_periods(status = 'paid') >= 2") + result = parse_expr("consecutive_periods(status = 'paid') >= 2") assert isinstance(result, Cmp) assert result.op == ">=" assert isinstance(result.left, TransformCall) @@ -998,7 +997,7 @@ def test_comparison_inside_ifnull_scalar_call_preserved(self): # DEV-1492 (Codex review): the SCALAR_FUNCTIONS narrowing must # apply to every scalar in the allowlist, not just `coalesce`. # `ifnull` (also a null-handling scalar) gets the same treatment. - result = parse_filter_expr("ifnull(status = 'paid', False)") + result = parse_expr("ifnull(status = 'paid', False)") assert isinstance(result, ScalarCall) assert result.name == "ifnull" assert len(result.args) == 2 diff --git a/tests/test_variables_planner.py b/tests/test_variables_planner.py index b6b435d8..6f3d7f33 100644 --- a/tests/test_variables_planner.py +++ b/tests/test_variables_planner.py @@ -175,26 +175,17 @@ def test_invalid_variable_name_raises_valueerror(self) -> None: with pytest.raises(ValueError, match="Invalid variable name"): apply_variables_to_query(query=q, variables={}) - def test_list_value_renders_a_mode_b_tuple(self) -> None: - """Was ``test_list_value_raises``. DEV-1730 made list values legal (the - ``IN``-pushdown primitive), so the rejection is superseded. - - Under Mode-B's ``python`` escaping the body carries a TRAILING COMMA — - ``('x',)`` is a Python tuple literal whereas ``('x')`` is a - parenthesised string, so the comma is what makes a single-element list - parse as a collection. The documented spelling puts the parentheses in - the filter (``region in ({regions})``); this pins the substituted body - itself. - """ + def test_list_value_renders_an_in_list(self) -> None: + """A list value fills the whole ``IN`` list; one element is still a list.""" q = SlayerQuery(source_model="orders", filters=["a in ({b})"]) out = apply_variables_to_query(query=q, variables={"b": [1, 2, 3]}) - assert out.filters == ["a in (1, 2, 3,)"], out.filters + assert out.filters == ["a in (1, 2, 3)"], out.filters single = apply_variables_to_query( query=SlayerQuery(source_model="orders", filters=["a in ({b})"]), variables={"b": ["x"]}, ) - assert single.filters == ["a in ('x',)"], single.filters + assert single.filters == ["a in ('x')"], single.filters def test_dict_value_raises(self) -> None: q = SlayerQuery(source_model="orders", filters=["a = {b}"]) @@ -414,11 +405,11 @@ def test_equal_template_time_dimensions_collapse_after_substitution(self) -> Non assert out.time_dimensions is not None assert [t.date_range for t in out.time_dimensions] == [["2024-01-01", None]] - def test_string_value_escaped_python_regime(self) -> None: + def test_string_value_sql_quoted(self) -> None: q = _q(measures=["sum(CASE WHEN status == '{name}' THEN 1 ELSE 0 END)"]) out = apply_variables_to_query(query=q, variables={"name": "O'Brien"}) assert out.measures is not None - assert out.measures[0].formula == "sum(CASE WHEN status == 'O\\'Brien' THEN 1 ELSE 0 END)" + assert out.measures[0].formula == "sum(CASE WHEN status == 'O''Brien' THEN 1 ELSE 0 END)" def test_list_value_renders_tuple_in_dimension(self) -> None: q = _q(dimensions=[{"expression": "region in ({regions})", "name": "ns"}]) @@ -426,7 +417,7 @@ def test_list_value_renders_tuple_in_dimension(self) -> None: assert out.dimensions is not None (d,) = out.dimensions assert isinstance(d, ComputedDimension) - assert d.expression == "region in ('North', 'South',)" + assert d.expression == "region in ('North', 'South')" @pytest.mark.parametrize("kw", [ {"measures": ["amount:sum * {? {k} ?}"]}, @@ -544,12 +535,12 @@ def test_saved_measure_formula_substituted(self) -> None: assert out.measures[0].name == "scaled" @pytest.mark.parametrize("backslash_escapes", [False, True]) - def test_saved_measure_uses_python_regime(self, backslash_escapes: bool) -> None: + def test_saved_measure_uses_sql_quoting(self, backslash_escapes: bool) -> None: out = substitute_model_sql_surfaces( model=_model_with_measure("sum(CASE WHEN status == '{n}' THEN 1 ELSE 0 END)"), variables={"n": "O'Brien"}, backslash_escapes=backslash_escapes, ) - assert out.measures[0].formula == "sum(CASE WHEN status == 'O\\'Brien' THEN 1 ELSE 0 END)" + assert out.measures[0].formula == "sum(CASE WHEN status == 'O''Brien' THEN 1 ELSE 0 END)" def test_saved_measure_optional_block_rejected(self) -> None: model = _model_with_measure("amount:sum * {? {k} ?}") From 3a271ba2bc66c5e5db500cea02bace00b86a92c1 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Wed, 21 Oct 2026 12:52:10 +0200 Subject: [PATCH 03/17] Move the Mode-B parser to slayer/core/syntax.py and the transform tables to core/enums.py Deterministic move (dr-refactor) of slayer/engine/syntax.py into core as the declared child core.syntax; ALL_TRANSFORMS / TIME_TRANSFORMS / TIMELESS_TRANSFORMS move from core/formula.py to core/enums.py. Approved harness edits H1-H7: model.c4 child + arrows, legacy baseline 8 -> 7, system P2 / P10, core and engine arc42 text, regenerated diagrams. Adds the pyparsing dependency. --- architecture/core.arc42.md | 14 +++++++---- architecture/engine.arc42.md | 23 ++++++++---------- architecture/index.yaml | 2 +- architecture/model.c4 | 8 +++---- architecture/system.arc42.md | 8 +++---- poetry.lock | 17 ++++++++++++- pyproject.toml | 1 + slayer/core/enums.py | 18 ++++++++++++++ slayer/core/formula.py | 24 +------------------ slayer/core/models.py | 17 +------------ slayer/core/query.py | 5 ++-- slayer/{engine => core}/syntax.py | 10 +------- slayer/engine/bind_inputs.py | 5 ++-- slayer/engine/binding.py | 18 ++------------ slayer/engine/elaborate_env.py | 3 +-- slayer/engine/model_reads.py | 2 +- slayer/engine/population.py | 2 +- slayer/engine/query_engine.py | 2 +- slayer/engine/schema_drift.py | 2 +- slayer/engine/unrelated.py | 2 +- slayer/ir/terms.py | 4 ++-- slayer/memories/resolver.py | 2 +- slayer/sql/sql_predicate.py | 4 ++-- slayer/usage_reporting/features/__init__.py | 3 +-- tests/facade/test_translator.py | 2 +- tests/test_agg_render_spec.py | 2 +- tests/test_binding.py | 2 +- tests/test_cube_converter.py | 2 +- tests/test_cube_views.py | 2 +- tests/test_dev1450fix_group2_correctness.py | 2 +- tests/test_dev1475_in_tuple_filters.py | 2 +- tests/test_dev1576_heals.py | 2 +- tests/test_dev1737_date_functions_parse.py | 2 +- tests/test_dev1737_date_functions_typing.py | 2 +- tests/test_dev1737_scalar_specs.py | 2 +- tests/test_dev1740_case_rewrite.py | 2 +- tests/test_dev1743_validators.py | 2 +- tests/test_dev1745_fragment_joins.py | 2 +- tests/test_dev1750_shifted_fragment_joins.py | 2 +- tests/test_dev1833_keyword_lexing.py | 2 +- .../test_dev1842_binder_measure_resolution.py | 2 +- tests/test_dev1847_gate.py | 2 +- tests/test_dev1859_plan_structure.py | 2 +- tests/test_dev1859_transform_row_leaf.py | 10 ++++---- tests/test_dev1901_param_binding.py | 2 +- tests/test_dev1903_first_last_dispatch.py | 4 ++-- tests/test_dev1903_transform_inputs.py | 2 +- tests/test_dev1946_transform_parameter.py | 2 +- tests/test_dev1952_derived_revisit.py | 2 +- tests/test_dev1953_partition_alias.py | 2 +- tests/test_dev1953_partition_membership.py | 2 +- tests/test_dev1954_canonical_paths.py | 2 +- tests/test_dev1958_row_leaf_ban.py | 4 ++-- tests/test_dev2046_boolean_valued.py | 2 +- tests/test_dev2046_typing.py | 2 +- tests/test_expression_aggregations.py | 2 +- tests/test_formula.py | 2 +- tests/test_functional_agg_text.py | 2 +- tests/test_functional_aggregations.py | 2 +- tests/test_functional_emission.py | 2 +- tests/test_functional_remedies.py | 2 +- tests/test_heal_date_parts.py | 2 +- tests/test_heal_functional_aggregations.py | 2 +- tests/test_memories_resolver_typed.py | 2 +- tests/test_mode_b_placeholder_parse.py | 2 +- tests/test_osi_converter.py | 2 +- tests/test_rank_direction.py | 2 +- tests/test_schema_drift_typed.py | 2 +- tests/test_sql_boolean_literal_filters.py | 2 +- tests/test_syntax.py | 4 ++-- tests/test_transforms_planner.py | 2 +- tests/test_usage_reporting_payload.py | 3 +-- 72 files changed, 139 insertions(+), 168 deletions(-) rename slayer/{engine => core}/syntax.py (99%) diff --git a/architecture/core.arc42.md b/architecture/core.arc42.md index 3e807d57..f47c6d1c 100644 --- a/architecture/core.arc42.md +++ b/architecture/core.arc42.md @@ -6,7 +6,7 @@ family, and the typed error/warning vocabulary. It is the bottom layer: it is meant to import no other SLayer node; the remaining crossings are the grandfathered child-level `#legacy` doors (`core.query`/`core.models` → -`engine.syntax`, `sql.*`, `storage.migrations`), slated to die (system +`sql.*`, `storage.migrations`), slated to die (system principle 2). ## 2. Building blocks @@ -20,6 +20,7 @@ flowchart TD subgraph core["Core domain models"] core__query["Query"] core__models["Models"] + core__syntax["Syntax"] end subgraph sql["SQL generation"] sql__dialects["Dialects"] @@ -27,7 +28,8 @@ flowchart TD sql__window_detect["Window detect"] end subgraph engine["Query engine"] - engine__syntax["Syntax"] + engine__binding["Binding"] + engine__bind_inputs["Bind inputs"] end ir["Intermediate representation"] subgraph storage["Storage backends"] @@ -41,7 +43,6 @@ flowchart TD usage_reporting__features["Features"] end surfaces("User-facing surfaces") - core__query -.-> engine__syntax core__models -.-> sql__dialects core__models -.-> sql__sql_predicate core__models -.-> sql__window_detect @@ -50,6 +51,9 @@ flowchart TD core__query -.-> storage__migrations core__models -.-> core__query core__query --> core__models + core__query --> core__syntax + engine__binding --> core__syntax + engine__bind_inputs --> core__syntax engine --> core importers --> core ir --> core @@ -61,14 +65,14 @@ flowchart TD surfaces --> core usage_reporting__features --> core classDef leaf fill:none; - class core__query,core__models,sql__dialects,sql__sql_predicate,sql__window_detect,engine__syntax,ir,storage__migrations,importers,search,memories,protocols,usage_reporting__features,surfaces leaf; + class core__query,core__models,core__syntax,sql__dialects,sql__sql_predicate,sql__window_detect,engine__binding,engine__bind_inputs,ir,storage__migrations,importers,search,memories,protocols,usage_reporting__features,surfaces leaf; ``` *Dashed arrows: legacy edges slated to die.* Identity: `keys.py`; scopes: `scope.py`; errors/warnings: `errors.py`, `warnings.py`; Pydantic domain -models alongside. +models alongside; the Mode B grammar: `syntax.py`. ## 3. Principles diff --git a/architecture/engine.arc42.md b/architecture/engine.arc42.md index 27402334..531d2ab8 100644 --- a/architecture/engine.arc42.md +++ b/architecture/engine.arc42.md @@ -18,6 +18,7 @@ flowchart TD subgraph core["Core domain models"] core__query["Query"] core__models["Models"] + core__syntax["Syntax"] end subgraph sql["SQL generation"] sql__render["Render"] @@ -26,7 +27,6 @@ flowchart TD sql__window_detect["Window detect"] end subgraph engine["Query engine"] - engine__syntax["Syntax"] engine__binding["Binding"] engine__bind_inputs["Bind inputs"] engine__elaborate_env["Checker"] @@ -38,7 +38,6 @@ flowchart TD subgraph storage["Storage backends"] storage__migrations["Migrations"] end - core__query -.-> engine__syntax core__models -.-> sql__dialects core__models -.-> sql__sql_predicate core__models -.-> sql__window_detect @@ -47,12 +46,13 @@ flowchart TD core__query -.-> storage__migrations core__models -.-> core__query core__query --> core__models + core__query --> core__syntax sql__render --> sql__dialects sql__sql_predicate --> sql__window_detect - engine__binding --> engine__syntax + engine__binding --> core__syntax engine__bind_inputs --> engine__binding engine__bind_inputs --> engine__elaborate_env - engine__bind_inputs --> engine__syntax + engine__bind_inputs --> core__syntax engine__compile --> engine__elaborate engine__compile --> engine__elaborate_env engine__elaborate --> engine__bind_inputs @@ -70,7 +70,7 @@ flowchart TD storage --> engine storage --> sql classDef leaf fill:none; - class core__query,core__models,sql__render,sql__dialects,sql__sql_predicate,sql__window_detect,engine__syntax,engine__binding,engine__bind_inputs,engine__elaborate_env,engine__elaborate,engine__compile,engine__plan,ir,storage__migrations leaf; + class core__query,core__models,core__syntax,sql__render,sql__dialects,sql__sql_predicate,sql__window_detect,engine__binding,engine__bind_inputs,engine__elaborate_env,engine__elaborate,engine__compile,engine__plan,ir,storage__migrations leaf; ``` *Dashed arrows: legacy edges slated to die.* @@ -83,7 +83,6 @@ only permitted arrows among them (DEV-1897 child-level model-truth): flowchart TD %% engine_focus: Engine seam subgraph engine["Query engine"] - engine__syntax["Syntax"] engine__binding["Binding"] engine__bind_inputs["Bind inputs"] engine__elaborate_env["Checker"] @@ -91,10 +90,8 @@ flowchart TD engine__compile["Compile"] engine__plan["Plan"] end - engine__binding --> engine__syntax engine__bind_inputs --> engine__binding engine__bind_inputs --> engine__elaborate_env - engine__bind_inputs --> engine__syntax engine__compile --> engine__elaborate engine__compile --> engine__elaborate_env engine__elaborate --> engine__bind_inputs @@ -102,12 +99,12 @@ flowchart TD engine__plan --> engine__compile engine__plan --> engine__elaborate classDef leaf fill:none; - class engine__syntax,engine__binding,engine__bind_inputs,engine__elaborate_env,engine__elaborate,engine__compile,engine__plan leaf; + class engine__binding,engine__bind_inputs,engine__elaborate_env,engine__elaborate,engine__compile,engine__plan leaf; ``` Stages: -`normalization.py` → `syntax.py` (parse) → `binding.py` + `bind_inputs.py` +`normalization.py` → `core/syntax.py` (parse) → `binding.py` + `bind_inputs.py` (bind) → `elaborate.py` / `elaborate_env.py` (type) → `compile/` (plan) → `slayer/ir/planned.py` (the typed hand-off to `sql`), fed by `bundle_builder.py` (builds the `ir` source bundle); `plan.py` composes @@ -155,11 +152,11 @@ elaborate→compile and `query_engine.py` orchestrates. unchanged; normalization applies to queries at execute time only. The one exception: a saved query stores each stage's inferred population as its `source_model`. [review] -9. **The term-interface boundary**: `syntax`, `binding`, `bind_inputs`, +9. **The term-interface boundary**: `binding`, `bind_inputs`, `elaborate`, `elaborate_env` (THE checker), `compile` and `plan` are declared children with only the modeled arrows among them — the compiler - consults the checker, never syntax or binding; every user-facing algebra - type error raises in the checker, as a `QueryTypeError`. + consults the checker, never `core.syntax` or binding; every user-facing + algebra type error raises in the checker, as a `QueryTypeError`. [enforced: arch_check:model-truth] [enforced: test:tests/test_dev1871_raise_parity.py] 10. **Dependencies are closures**: every predicate that classifies a column diff --git a/architecture/index.yaml b/architecture/index.yaml index abdbefc6..2650b5bc 100644 --- a/architecture/index.yaml +++ b/architecture/index.yaml @@ -6,7 +6,7 @@ python: # Count of `#legacy` arrows in the model; arch_check fails if the model's count # differs. Only ever lower this — every legacy arrow is a door slated to die. -legacy_arrows: {baseline: 8} +legacy_arrows: {baseline: 7} # Fail-closed deferral-site count (tests/test_law_guard_ratchet.py). Only ever lower. x-guards: {baseline: 0} diff --git a/architecture/model.c4 b/architecture/model.c4 index cada409e..96f9e179 100644 --- a/architecture/model.c4 +++ b/architecture/model.c4 @@ -23,6 +23,7 @@ model { } query = node 'Query' models = node 'Models' + syntax = node 'Syntax' } sql = node 'SQL generation' { metadata { @@ -40,7 +41,6 @@ model { package 'slayer.engine' arc42 'architecture/engine.arc42.md' } - syntax = node 'Syntax' binding = node 'Binding' bind_inputs = node 'Bind inputs' elaborate_env = node 'Checker' @@ -98,7 +98,6 @@ model { } } - core.query -> engine.syntax #legacy core.models -> sql.dialects #legacy core.models -> sql.sql_predicate #legacy core.models -> sql.window_detect #legacy @@ -107,12 +106,13 @@ model { core.query -> storage.migrations #legacy core.models -> core.query #legacy core.query -> core.models + core.query -> core.syntax sql.render -> sql.dialects sql.sql_predicate -> sql.window_detect - engine.binding -> engine.syntax + engine.binding -> core.syntax engine.bind_inputs -> engine.binding engine.bind_inputs -> engine.elaborate_env - engine.bind_inputs -> engine.syntax + engine.bind_inputs -> core.syntax engine.compile -> engine.elaborate engine.compile -> engine.elaborate_env engine.elaborate -> engine.bind_inputs diff --git a/architecture/system.arc42.md b/architecture/system.arc42.md index 50fa687c..eb83b460 100644 --- a/architecture/system.arc42.md +++ b/architecture/system.arc42.md @@ -28,7 +28,6 @@ flowchart TD protocols("BI wire protocols") usage_reporting["Usage reporting"] surfaces("User-facing surfaces") - core -.-> engine core -.-> sql core -.-> storage engine --> core @@ -91,7 +90,7 @@ All code, old and new, MUST obey these. [enforced: arch_check:model-truth] 2. **`core` imports no other SLayer node.** The remaining crossings are the grandfathered child-level `#legacy` doors (`core.query`/`core.models` → - `engine.syntax`, `sql.*`, `storage.migrations`), slated to die. + `sql.*`, `storage.migrations`), slated to die. [enforced: arch_check:model-truth] 3. **Every top-level `slayer.*` package/module belongs to exactly one node.** A new top-level package must be claimed in a node's model metadata in the same change. @@ -125,8 +124,9 @@ All code, old and new, MUST obey these. [enforced: test:tests/test_dev1743_resolution.py] [enforced: test:tests/test_dev1954_canonical_paths.py] 10. **Two expression layers**: Mode A free SQL (`Column.sql`, model `filters`) - vs Mode B Python-AST DSL (formulas, query fields, scalar allowlist only) — - one canonical scalar allowlist, extended never forked. [review] + vs Mode B, a SQL-compatible grammar (formulas, query fields) — one + canonical function library, extended never forked. [review] Every Mode B + construct means the same on every dialect. [target: DEV-2120] 11. **Versioned persistence**: models/queries/datasource configs carry `version`; migrations run automatically on load. [review] 12. **Pydantic v2 for all models; never dataclasses.** [review] diff --git a/poetry.lock b/poetry.lock index df678aac..4ab1a1c8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5741,6 +5741,21 @@ typing-extensions = {version = ">=4.9", markers = "python_version < \"3.13\" and docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx_rtd_theme"] test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"] +[[package]] +name = "pyparsing" +version = "3.3.3" +description = "pyparsing - Classes and methods to define and execute parsing grammars" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pyparsing-3.3.3-py3-none-any.whl", hash = "sha256:ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4"}, + {file = "pyparsing-3.3.3.tar.gz", hash = "sha256:928ae7e20211f3b6f3915a72f06a0cfd29ab9d24279dd6346b6b1a7146397d36"}, +] + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + [[package]] name = "pytest" version = "9.0.3" @@ -8282,4 +8297,4 @@ trino = ["trino"] [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "e318b76bd724a2ca9aa8658ba05c53396639fff8ceb2dc4523df6097f292af38" +content-hash = "bea59f700fb0ee40d6f20183dbe0341ac5238cb3fceaa945925dc8922864df8b" diff --git a/pyproject.toml b/pyproject.toml index 1834a4d4..6547e3a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ sqlglot = {version = ">=30.17", extras = ["c"]} sqlalchemy = ">=2.0,<2.1" greenlet = ">=1" pydantic = ">=2.0" +pyparsing = ">=3.1" python-dateutil = ">=2.8" pyyaml = ">=6.0" fastapi = ">=0.100" diff --git a/slayer/core/enums.py b/slayer/core/enums.py index 15c10c21..405fd609 100644 --- a/slayer/core/enums.py +++ b/slayer/core/enums.py @@ -272,6 +272,24 @@ def invert_cardinality( # Named once so classifier, planner and renderer agree. RANKED_AGGREGATIONS = ("first", "last") +# Transforms that require a time dimension for ORDER BY +TIME_TRANSFORMS = { + "cumsum", + "change", + "change_pct", + "time_shift", + "first", + "last", + "lag", + "lead", + "consecutive_periods", +} + +# Transforms that don't need time ordering +TIMELESS_TRANSFORMS = {"rank", "percent_rank", "dense_rank", "ntile"} + +ALL_TRANSFORMS = TIME_TRANSFORMS | TIMELESS_TRANSFORMS + # Default partition is the whole result set, not the group-by dimensions; # ``partition_by=`` opts into per-partition ranking. RANK_FAMILY_TRANSFORMS = {"rank", "percent_rank", "dense_rank", "ntile"} diff --git a/slayer/core/formula.py b/slayer/core/formula.py index 9ed75004..bd3d502f 100644 --- a/slayer/core/formula.py +++ b/slayer/core/formula.py @@ -26,35 +26,13 @@ from pydantic import BaseModel, Field from slayer.core.direction import rank_direction, with_direction_kwarg -from slayer.core.enums import ( - BUILTIN_AGGREGATIONS, - RANK_FAMILY_TRANSFORMS, - normalize_aggregation_name, -) +from slayer.core.enums import BUILTIN_AGGREGATIONS, RANK_FAMILY_TRANSFORMS, normalize_aggregation_name, ALL_TRANSFORMS from slayer.core.keys import SCALAR_FUNCTIONS from slayer.core.refs import ( AGG_REF_RE as _AGG_REF_RE, IDENT_OR_PATH_RE as _IDENT_OR_PATH_RE, ) -# Transforms that require a time dimension for ORDER BY -TIME_TRANSFORMS = { - "cumsum", - "change", - "change_pct", - "time_shift", - "first", - "last", - "lag", - "lead", - "consecutive_periods", -} - -# Transforms that don't need time ordering -TIMELESS_TRANSFORMS = {"rank", "percent_rank", "dense_rank", "ntile"} - -ALL_TRANSFORMS = TIME_TRANSFORMS | TIMELESS_TRANSFORMS - CallCategory = Literal["transform", "scalar", "unknown"] diff --git a/slayer/core/models.py b/slayer/core/models.py index a0befba7..48cb8119 100644 --- a/slayer/core/models.py +++ b/slayer/core/models.py @@ -15,24 +15,9 @@ ) from sqlalchemy.engine import URL as _SA_URL -from slayer.core.enums import ( - BUILTIN_AGGREGATION_FORMULAS, - BUILTIN_AGGREGATIONS, - DEFAULT_AGGREGATIONS_BY_TYPE, - GRANULARITY_NAMES, - DataType, - JoinCardinality, - JoinType, - ObjectKind, - PRIMARY_KEY_AGGREGATIONS, - RANKED_AGGREGATIONS, - RESERVED_DATE_SYNTAX_NAMES, - _coerce_legacy_datatype, - builtin_empty_value, -) +from slayer.core.enums import BUILTIN_AGGREGATION_FORMULAS, BUILTIN_AGGREGATIONS, DEFAULT_AGGREGATIONS_BY_TYPE, GRANULARITY_NAMES, DataType, JoinCardinality, JoinType, ObjectKind, PRIMARY_KEY_AGGREGATIONS, RANKED_AGGREGATIONS, RESERVED_DATE_SYNTAX_NAMES, _coerce_legacy_datatype, builtin_empty_value, ALL_TRANSFORMS from slayer.core.errors import GranularityDefinitionError, JoinKeyError from slayer.core.format import NumberFormat -from slayer.core.formula import ALL_TRANSFORMS from slayer.core.granularity import CustomGranularity, GranularitySpec, definition_error, granularity_key from slayer.core.keys import SCALAR_FUNCTIONS from slayer.core.refs import IDENTIFIER_RE diff --git a/slayer/core/query.py b/slayer/core/query.py index d94dd4d9..3eb804e4 100644 --- a/slayer/core/query.py +++ b/slayer/core/query.py @@ -23,8 +23,7 @@ ) from slayer.core.direction import normalize_direction -from slayer.core.enums import BUILTIN_AGGREGATIONS, GRANULARITY_NAMES, TimeGranularity, normalize_aggregation_name -from slayer.core.formula import ALL_TRANSFORMS +from slayer.core.enums import BUILTIN_AGGREGATIONS, GRANULARITY_NAMES, TimeGranularity, normalize_aggregation_name, ALL_TRANSFORMS from slayer.core.keys import SCALAR_FUNCTIONS from slayer.core.errors import DistinctDimensionValuesError, GranularityCallError, RefinementConflictError from slayer.core.granularity import GranularitySpec, granularity_key @@ -37,7 +36,7 @@ ) from slayer.core.refs import auto_name_from_expression from slayer.core.time_points import TIME_POINT_FORMS, is_time_point_shape -from slayer.engine.syntax import AggCall, DottedRef, Ref, parse_expr, walk_parsed_refs +from slayer.core.syntax import AggCall, DottedRef, Ref, parse_expr, walk_parsed_refs from slayer.sql.window_detect import WINDOW_IN_FILTER_ERROR, has_window_function from slayer.storage.migrations import CURRENT_VERSIONS, migrate as _migrate_schema diff --git a/slayer/engine/syntax.py b/slayer/core/syntax.py similarity index 99% rename from slayer/engine/syntax.py rename to slayer/core/syntax.py index f8aac171..174006a0 100644 --- a/slayer/engine/syntax.py +++ b/slayer/core/syntax.py @@ -19,16 +19,8 @@ from pydantic import BaseModel, ConfigDict from slayer.core.direction import normalize_direction -from slayer.core.enums import ( - AMBIGUOUS_WEEKDAY_NAMES, - BUILTIN_AGGREGATIONS, - DATE_PART_ALIASES, - GRANULARITY_NAMES, - ambiguous_weekday_message, - normalize_aggregation_name, -) +from slayer.core.enums import AMBIGUOUS_WEEKDAY_NAMES, BUILTIN_AGGREGATIONS, DATE_PART_ALIASES, GRANULARITY_NAMES, ambiguous_weekday_message, normalize_aggregation_name, ALL_TRANSFORMS from slayer.core.errors import GranularityCallError, IllegalWindowInFilterError, UnknownFunctionError -from slayer.core.formula import ALL_TRANSFORMS from slayer.core.keys import DATE_UNIT_ARGS, SCALAR_FUNCTIONS, check_scalar_arity from slayer.core.refs import split_agg_suffix diff --git a/slayer/engine/bind_inputs.py b/slayer/engine/bind_inputs.py index a10a94e7..423cb5b1 100644 --- a/slayer/engine/bind_inputs.py +++ b/slayer/engine/bind_inputs.py @@ -8,7 +8,7 @@ from datetime import date, datetime from typing import Callable, Dict, FrozenSet, List, Optional, Tuple, cast -from slayer.core.enums import BUILTIN_AGGREGATIONS, DataType, TimeGranularity, normalize_aggregation_name +from slayer.core.enums import BUILTIN_AGGREGATIONS, DataType, TimeGranularity, normalize_aggregation_name, TIME_TRANSFORMS from slayer.core.errors import ( AmbiguousJoinPathError, AmbiguousReferenceError, @@ -17,7 +17,6 @@ UnknownReferenceError, ) from slayer.core.format import NumberFormat -from slayer.core.formula import TIME_TRANSFORMS from slayer.core.time_spine import is_spine, is_spine_query, names_spine from slayer.core.join_walker import resolve_hop, resolved_terminal, walk_terminal from slayer.core.keys import ( @@ -98,7 +97,7 @@ scope_column_type, stage_measure_type, ) -from slayer.engine.syntax import ( +from slayer.core.syntax import ( AggCall, DottedRef, ParsedExpr, diff --git a/slayer/engine/binding.py b/slayer/engine/binding.py index 9b15d720..12ce36f1 100644 --- a/slayer/engine/binding.py +++ b/slayer/engine/binding.py @@ -26,22 +26,8 @@ UnresolvableDimensionJoinError, UnresolvedPlaceholderError, ) -from slayer.core.enums import ( - AMBIGUOUS_WEEKDAY_NAMES, - BUILTIN_AGGREGATION_PARAM_ORDER, - BUILTIN_AGGREGATIONS, - DATE_PART_ALIASES, - DEFAULT_AGGREGATIONS_BY_TYPE, - GRANULARITY_NAMES, - NUMERIC_ONLY_AGGREGATIONS, - PRIMARY_KEY_AGGREGATIONS, - RANKED_AGGREGATIONS, - DataType, - ambiguous_weekday_message, - normalize_aggregation_name, -) +from slayer.core.enums import AMBIGUOUS_WEEKDAY_NAMES, BUILTIN_AGGREGATION_PARAM_ORDER, BUILTIN_AGGREGATIONS, DATE_PART_ALIASES, DEFAULT_AGGREGATIONS_BY_TYPE, GRANULARITY_NAMES, NUMERIC_ONLY_AGGREGATIONS, PRIMARY_KEY_AGGREGATIONS, RANKED_AGGREGATIONS, DataType, ambiguous_weekday_message, normalize_aggregation_name, ALL_TRANSFORMS from slayer.core.direction import rank_direction, with_direction_kwarg -from slayer.core.formula import ALL_TRANSFORMS from slayer.core.enums import RANK_FAMILY_TRANSFORMS from slayer.core.granularity import CustomGranularity, Granularity, resolve_granularity from slayer.core.refs import EXPRESSION_SOURCE_KINDS, key_display @@ -68,7 +54,7 @@ from slayer.core.scope import ModelScope, StageSchema, resolve_generated_column from slayer.core.time_points import is_time_point from slayer.ir.source_bundle import ResolvedSourceBundle -from slayer.engine.syntax import ( +from slayer.core.syntax import ( AggCall, Arith, BoolOp, diff --git a/slayer/engine/elaborate_env.py b/slayer/engine/elaborate_env.py index 3f63a6f5..13987f61 100644 --- a/slayer/engine/elaborate_env.py +++ b/slayer/engine/elaborate_env.py @@ -15,7 +15,7 @@ from pydantic import BaseModel, ConfigDict -from slayer.core.enums import RANK_FAMILY_TRANSFORMS, DataType, TimeGranularity +from slayer.core.enums import RANK_FAMILY_TRANSFORMS, DataType, TimeGranularity, TIME_TRANSFORMS from slayer.core.errors import ( AssociationError, CanonicalAliasShadowsColumnError, @@ -42,7 +42,6 @@ UnrelatedDatasetError, UnsafeJoinInputError, ) -from slayer.core.formula import TIME_TRANSFORMS from slayer.core.granularity import CustomGranularity, Granularity, nests_into, unknown_granularity_message from slayer.core.window_duration import parse_window_duration from slayer.core.keys import ( diff --git a/slayer/engine/model_reads.py b/slayer/engine/model_reads.py index 18c1e54f..2b25222a 100644 --- a/slayer/engine/model_reads.py +++ b/slayer/engine/model_reads.py @@ -13,7 +13,7 @@ from slayer.core.query import ComputedDimension, ModelExtension, SlayerQuery, SourceSpec, render_probe_text from slayer.engine.param_binding import default_reference_sites from slayer.engine.reference_closure import parse_fragment_any_dialect -from slayer.engine.syntax import DottedRef, Ref, parse_expr, parse_filter_expr +from slayer.core.syntax import DottedRef, Ref, parse_expr, parse_filter_expr from slayer.ir.source_bundle import apply_extension_overlay, follow_sibling_chain, source_name_if_sibling from slayer.sql.column_expansion import is_trivial_base, reference_sites, root_scope_column_ids diff --git a/slayer/engine/population.py b/slayer/engine/population.py index b8c054ee..e25127d7 100644 --- a/slayer/engine/population.py +++ b/slayer/engine/population.py @@ -23,7 +23,7 @@ from slayer.core.query import ComputedDimension, SlayerQuery, render_probe_text from slayer.core.time_spine import TIME_SPINE_MODEL, mentions_spine, names_spine from slayer.engine.dimension_routing import resolve_route, safe_route_reachable -from slayer.engine.syntax import ( +from slayer.core.syntax import ( AggCall, DottedRef, Ref, diff --git a/slayer/engine/query_engine.py b/slayer/engine/query_engine.py index 3323eb22..f4911ba6 100644 --- a/slayer/engine/query_engine.py +++ b/slayer/engine/query_engine.py @@ -98,7 +98,7 @@ ItemPath, RootModelRecommendation, ) -from slayer.engine.syntax import split_entity_agg_ref +from slayer.core.syntax import split_entity_agg_ref from slayer.engine.cache import ( CacheConfig, QueryCache, diff --git a/slayer/engine/schema_drift.py b/slayer/engine/schema_drift.py index bc2c2727..58abb821 100644 --- a/slayer/engine/schema_drift.py +++ b/slayer/engine/schema_drift.py @@ -55,7 +55,7 @@ from slayer.core.join_walker import OrientedJoin, resolve_hop, walk from slayer.sql.column_expansion import resolve_ref_target from slayer.engine.dimension_routing import short_form_route_or_none -from slayer.engine.syntax import ( +from slayer.core.syntax import ( AggCall, DottedRef, ParsedExpr, diff --git a/slayer/engine/unrelated.py b/slayer/engine/unrelated.py index b8360ee2..71d559b8 100644 --- a/slayer/engine/unrelated.py +++ b/slayer/engine/unrelated.py @@ -21,7 +21,7 @@ from slayer.core.query import ComputedDimension, SlayerQuery, render_probe_text from slayer.core.time_spine import is_spine from slayer.engine.reference_closure import key_closure -from slayer.engine.syntax import DottedRef, parse_expr, parse_filter_expr +from slayer.core.syntax import DottedRef, parse_expr, parse_filter_expr from slayer.ir.source_bundle import ResolvedSourceBundle __all__ = ["ProductSides", "first_hop_names", "stamp_unrelated", "unrelated_targets"] diff --git a/slayer/ir/terms.py b/slayer/ir/terms.py index 65ef1d5c..33e1b297 100644 --- a/slayer/ir/terms.py +++ b/slayer/ir/terms.py @@ -13,8 +13,8 @@ from pydantic import BaseModel, model_validator -from slayer.core.formula import TIME_TRANSFORMS from slayer.core.keys import AggregateKey, Grain, TransformKey, ValueKey +import slayer.core.enums @runtime_checkable @@ -84,7 +84,7 @@ class Transform(_Term, frozen=True): @model_validator(mode="after") def _require_time_axis(self) -> "Transform": - if self.recipe.op in TIME_TRANSFORMS and self.recipe.time_key is None: + if self.recipe.op in slayer.core.enums.TIME_TRANSFORMS and self.recipe.time_key is None: raise ValueError( f"Transform {self.recipe.op!r} requires a time axis " f"(recipe.time_key is unset).", diff --git a/slayer/memories/resolver.py b/slayer/memories/resolver.py index 61670d76..a0486972 100644 --- a/slayer/memories/resolver.py +++ b/slayer/memories/resolver.py @@ -43,7 +43,7 @@ from slayer.core.models import SlayerModel from slayer.core.query import ComputedDimension, ModelExtension, SlayerQuery from slayer.engine.model_reads import FieldKind, query_field_texts -from slayer.engine.syntax import ( +from slayer.core.syntax import ( AggCall, DottedRef, ParsedExpr, diff --git a/slayer/sql/sql_predicate.py b/slayer/sql/sql_predicate.py index ccda2a82..afed9db0 100644 --- a/slayer/sql/sql_predicate.py +++ b/slayer/sql/sql_predicate.py @@ -22,9 +22,9 @@ from pydantic import BaseModel, Field -from slayer.core.formula import ALL_TRANSFORMS from slayer.core.refs import AGG_REF_RE from slayer.sql.window_detect import WINDOW_IN_FILTER_ERROR, has_window_function +import slayer.core.enums class ParsedFilter(BaseModel): @@ -36,7 +36,7 @@ class ParsedFilter(BaseModel): _STRING_LITERAL_RE = re.compile(r"'(?:[^'\\]|\\.)*'") _DSL_TRANSFORM_CALL_RE = re.compile( - r"\b(" + "|".join(sorted(ALL_TRANSFORMS, key=len, reverse=True)) + r")\s*\(", + r"\b(" + "|".join(sorted(slayer.core.enums.ALL_TRANSFORMS, key=len, reverse=True)) + r")\s*\(", re.IGNORECASE, ) diff --git a/slayer/usage_reporting/features/__init__.py b/slayer/usage_reporting/features/__init__.py index 9602294c..06fc8fc2 100644 --- a/slayer/usage_reporting/features/__init__.py +++ b/slayer/usage_reporting/features/__init__.py @@ -7,8 +7,7 @@ from pydantic import BaseModel, Field -from slayer.core.enums import BUILTIN_AGGREGATIONS, normalize_aggregation_name -from slayer.core.formula import ALL_TRANSFORMS +from slayer.core.enums import BUILTIN_AGGREGATIONS, normalize_aggregation_name, ALL_TRANSFORMS from slayer.core.keys import SCALAR_FUNCTIONS from slayer.core.models import DatasourceConfig from slayer.core.query import ComputedDimension, SlayerQuery diff --git a/tests/facade/test_translator.py b/tests/facade/test_translator.py index 117d1615..448c30ce 100644 --- a/tests/facade/test_translator.py +++ b/tests/facade/test_translator.py @@ -14,7 +14,7 @@ from slayer.core.enums import DataType, JoinType, TimeGranularity from slayer.core.models import Column, ModelJoin, ModelMeasure, SlayerModel from slayer.core.query import ModelExtension -from slayer.engine.syntax import Cmp, Ref, parse_expr +from slayer.core.syntax import Cmp, Ref, parse_expr from slayer.facade.catalog import FacadeCatalog, build_catalog from slayer.facade.rows import FacadeColumn, RowBatch from slayer.facade.translator import ( diff --git a/tests/test_agg_render_spec.py b/tests/test_agg_render_spec.py index 20b9cfaf..628a2890 100644 --- a/tests/test_agg_render_spec.py +++ b/tests/test_agg_render_spec.py @@ -46,7 +46,7 @@ from slayer.core.models import Aggregation, AggregationParam, Column, SlayerModel from slayer.core.scope import ModelScope from slayer.engine.binding import bind_expr -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr from slayer.ir.planned import ValueSlot from slayer.ir.source_bundle import ResolvedSourceBundle from slayer.sql.generator import ( # type: ignore[attr-defined] diff --git a/tests/test_binding.py b/tests/test_binding.py index e0851c92..8a3d360a 100644 --- a/tests/test_binding.py +++ b/tests/test_binding.py @@ -33,7 +33,7 @@ bind_filter, ) from slayer.ir.source_bundle import ResolvedSourceBundle -from slayer.engine.syntax import Ref, parse_expr +from slayer.core.syntax import Ref, parse_expr # --------------------------------------------------------------------------- diff --git a/tests/test_cube_converter.py b/tests/test_cube_converter.py index 25132dd3..c786a6a4 100644 --- a/tests/test_cube_converter.py +++ b/tests/test_cube_converter.py @@ -19,7 +19,7 @@ CubeSegment, ) from slayer.cube.report import CubeIssueCategory -from slayer.engine.syntax import AggCall, DottedRef, Ref, parse_expr +from slayer.core.syntax import AggCall, DottedRef, Ref, parse_expr from tests._cube_helpers import DS, column, convert, measure, meta diff --git a/tests/test_cube_views.py b/tests/test_cube_views.py index 65d0bdbd..80d0cf59 100644 --- a/tests/test_cube_views.py +++ b/tests/test_cube_views.py @@ -14,7 +14,7 @@ CubeViewCubeRef, ) from slayer.cube.report import CubeIssueCategory -from slayer.engine.syntax import AggCall, DottedRef, Ref, parse_expr +from slayer.core.syntax import AggCall, DottedRef, Ref, parse_expr from tests._cube_helpers import column, convert, measure, meta diff --git a/tests/test_dev1450fix_group2_correctness.py b/tests/test_dev1450fix_group2_correctness.py index 701d81c7..1c744926 100644 --- a/tests/test_dev1450fix_group2_correctness.py +++ b/tests/test_dev1450fix_group2_correctness.py @@ -30,7 +30,7 @@ from slayer.engine.schema_drift import _measure_formula_refs from slayer.storage.sqlite_conn import transaction from slayer.ir.source_bundle import ResolvedSourceBundle -from slayer.engine.syntax import parse_expr, parse_expr as _parse +from slayer.core.syntax import parse_expr, parse_expr as _parse from slayer.storage.yaml_storage import YAMLStorage diff --git a/tests/test_dev1475_in_tuple_filters.py b/tests/test_dev1475_in_tuple_filters.py index 86b27601..87dd0d17 100644 --- a/tests/test_dev1475_in_tuple_filters.py +++ b/tests/test_dev1475_in_tuple_filters.py @@ -39,7 +39,7 @@ from slayer.core.keys import InKey, LiteralKey from slayer.core.models import Column, ModelJoin, SlayerModel from slayer.core.query import SlayerQuery -from slayer.engine.syntax import ( +from slayer.core.syntax import ( Cmp, DottedRef, Literal, diff --git a/tests/test_dev1576_heals.py b/tests/test_dev1576_heals.py index 75c55769..9148f3d4 100644 --- a/tests/test_dev1576_heals.py +++ b/tests/test_dev1576_heals.py @@ -22,7 +22,7 @@ from slayer.core.models import Aggregation, Column, SlayerModel from slayer.core.scope import ModelScope from slayer.engine.binding import bind_expr -from slayer.engine.syntax import AggCall, Literal, Ref, ScalarCall, TransformCall, parse_expr +from slayer.core.syntax import AggCall, Literal, Ref, ScalarCall, TransformCall, parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle diff --git a/tests/test_dev1737_date_functions_parse.py b/tests/test_dev1737_date_functions_parse.py index 3d5cc0e6..80f50b2f 100644 --- a/tests/test_dev1737_date_functions_parse.py +++ b/tests/test_dev1737_date_functions_parse.py @@ -8,7 +8,7 @@ from slayer.core.keys import SCALAR_FUNCTION_ARITY, SCALAR_FUNCTIONS, check_scalar_arity from slayer.engine.binding import bind_expr, bind_filter -from slayer.engine.syntax import ( +from slayer.core.syntax import ( AggCall, Cmp, Literal, diff --git a/tests/test_dev1737_date_functions_typing.py b/tests/test_dev1737_date_functions_typing.py index 3d6785cd..47e62ff1 100644 --- a/tests/test_dev1737_date_functions_typing.py +++ b/tests/test_dev1737_date_functions_typing.py @@ -25,7 +25,7 @@ from slayer.engine.binding import bind_expr from slayer.engine.key_metadata import dimension_key_metadata, measure_key_type from slayer.engine.query_engine import SlayerQueryEngine -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle from slayer.storage.yaml_storage import YAMLStorage from tests._dev1737_fixtures import customers_model, orders_model diff --git a/tests/test_dev1737_scalar_specs.py b/tests/test_dev1737_scalar_specs.py index a1bdf1c6..79a59e1c 100644 --- a/tests/test_dev1737_scalar_specs.py +++ b/tests/test_dev1737_scalar_specs.py @@ -20,7 +20,7 @@ from slayer.core.models import Column from slayer.core.scope import ModelScope from slayer.engine.binding import bind_expr -from slayer.engine.syntax import ScalarCall, parse_expr +from slayer.core.syntax import ScalarCall, parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle DATE_NAMES = {"date_part", "date_diff", "date_add", "interval", "current_date", "now"} diff --git a/tests/test_dev1740_case_rewrite.py b/tests/test_dev1740_case_rewrite.py index 256838bb..9c691b8a 100644 --- a/tests/test_dev1740_case_rewrite.py +++ b/tests/test_dev1740_case_rewrite.py @@ -18,7 +18,7 @@ SCALAR_FUNCTIONS, check_scalar_arity, ) -from slayer.engine.syntax import ( +from slayer.core.syntax import ( BoolOp, Cmp, Literal, diff --git a/tests/test_dev1743_validators.py b/tests/test_dev1743_validators.py index cc17b4b0..4c0d5473 100644 --- a/tests/test_dev1743_validators.py +++ b/tests/test_dev1743_validators.py @@ -18,7 +18,7 @@ from slayer.core.enums import DataType from slayer.core.models import Column, ModelMeasure, SlayerModel from slayer.core.query import SlayerQuery -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr from slayer.storage.sqlite_storage import SQLiteStorage from slayer.storage.yaml_storage import YAMLStorage diff --git a/tests/test_dev1745_fragment_joins.py b/tests/test_dev1745_fragment_joins.py index 8ca5dbe9..1173dc83 100644 --- a/tests/test_dev1745_fragment_joins.py +++ b/tests/test_dev1745_fragment_joins.py @@ -35,7 +35,7 @@ from slayer.core.query import SlayerQuery from slayer.core.scope import ModelScope from slayer.engine.binding import bind_expr -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle from tests._dev1746_fixtures import cte_names_in_order, find_cte diff --git a/tests/test_dev1750_shifted_fragment_joins.py b/tests/test_dev1750_shifted_fragment_joins.py index 78422ad8..161f63c9 100644 --- a/tests/test_dev1750_shifted_fragment_joins.py +++ b/tests/test_dev1750_shifted_fragment_joins.py @@ -20,7 +20,7 @@ from slayer.core.keys import AggregateKey, ColumnKey from slayer.core.scope import ModelScope from slayer.engine.binding import bind_expr -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle from slayer.sql.scope_check import assert_scope_closed diff --git a/tests/test_dev1833_keyword_lexing.py b/tests/test_dev1833_keyword_lexing.py index 92ec1bf8..351de65d 100644 --- a/tests/test_dev1833_keyword_lexing.py +++ b/tests/test_dev1833_keyword_lexing.py @@ -14,7 +14,7 @@ import pytest from slayer.core.errors import IllegalWindowInFilterError -from slayer.engine.syntax import ( +from slayer.core.syntax import ( AggCall, Arith, BoolOp, diff --git a/tests/test_dev1842_binder_measure_resolution.py b/tests/test_dev1842_binder_measure_resolution.py index d4263dc3..fb71881d 100644 --- a/tests/test_dev1842_binder_measure_resolution.py +++ b/tests/test_dev1842_binder_measure_resolution.py @@ -34,7 +34,7 @@ from slayer.core.scope import StageColumn, StageSchema from slayer.engine.binding import bind_expr from slayer.ir.source_bundle import ResolvedSourceBundle -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr from tests._engine_helpers import _engine_generate diff --git a/tests/test_dev1847_gate.py b/tests/test_dev1847_gate.py index 0be8e46d..cb4cebfa 100644 --- a/tests/test_dev1847_gate.py +++ b/tests/test_dev1847_gate.py @@ -11,7 +11,7 @@ from slayer.core.keys import AggregateKey, TransformKey from slayer.core.scope import ModelScope from slayer.engine.binding import _source_is_reaggregation, bind_expr -from slayer.engine.syntax import AggCall, parse_expr +from slayer.core.syntax import AggCall, parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle from tests._dev1847_fixtures import ( diff --git a/tests/test_dev1859_plan_structure.py b/tests/test_dev1859_plan_structure.py index 18ecdf4c..e0132051 100644 --- a/tests/test_dev1859_plan_structure.py +++ b/tests/test_dev1859_plan_structure.py @@ -15,7 +15,7 @@ from slayer.core.keys import AggregateKey, ColumnKey, walk_value_keys from slayer.engine.plan import plan_query -from slayer.engine.syntax import AggCall, parse_expr +from slayer.core.syntax import AggCall, parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle from tests._dev1847_fixtures import ( diff --git a/tests/test_dev1859_transform_row_leaf.py b/tests/test_dev1859_transform_row_leaf.py index 34355852..858a76b6 100644 --- a/tests/test_dev1859_transform_row_leaf.py +++ b/tests/test_dev1859_transform_row_leaf.py @@ -16,7 +16,6 @@ import pytest -from slayer.core.formula import ALL_TRANSFORMS from slayer.engine.plan import plan_query from slayer.ir.source_bundle import ResolvedSourceBundle @@ -30,9 +29,10 @@ rows_by, ) from tests.test_law_guard_ratchet import DEFERRAL_CLASSIFIER +import slayer.core.enums SHIFT_OPS = frozenset({"time_shift", "change", "change_pct"}) -NON_SHIFT_OPS = sorted(ALL_TRANSFORMS - SHIFT_OPS - {"first", "last"}) +NON_SHIFT_OPS = sorted(slayer.core.enums.ALL_TRANSFORMS - SHIFT_OPS - {"first", "last"}) #: input-shape id -> the formula body around the unprojected row leaf. SHAPES = { @@ -81,9 +81,9 @@ async def exec_engine(request): class TestDerivedSet: def test_shift_family_exemption_is_exactly_three_ops(self): """A new transform op must default into the rejected set.""" - assert SHIFT_OPS <= ALL_TRANSFORMS - assert {"first", "last"} <= ALL_TRANSFORMS - assert set(NON_SHIFT_OPS) <= ALL_TRANSFORMS + assert SHIFT_OPS <= slayer.core.enums.ALL_TRANSFORMS + assert {"first", "last"} <= slayer.core.enums.ALL_TRANSFORMS + assert set(NON_SHIFT_OPS) <= slayer.core.enums.ALL_TRANSFORMS assert not set(NON_SHIFT_OPS) & SHIFT_OPS diff --git a/tests/test_dev1901_param_binding.py b/tests/test_dev1901_param_binding.py index 4a5dda74..71a0e26a 100644 --- a/tests/test_dev1901_param_binding.py +++ b/tests/test_dev1901_param_binding.py @@ -24,7 +24,7 @@ from slayer.core.scope import ModelScope, StageColumn, StageSchema from slayer.engine.binding import bind_expr from slayer.engine.compile.stages import _first_unattributable_arg_leaf -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle from tests._dev1901_fixtures import ( BAD_DEFAULTS, diff --git a/tests/test_dev1903_first_last_dispatch.py b/tests/test_dev1903_first_last_dispatch.py index 48117233..57508adf 100644 --- a/tests/test_dev1903_first_last_dispatch.py +++ b/tests/test_dev1903_first_last_dispatch.py @@ -16,10 +16,9 @@ from slayer.core.keys import AggregateKey, ColumnKey, TransformKey from slayer.core.models import ModelMeasure from slayer.core.query import SlayerQuery -from slayer.engine import syntax from slayer.engine.elaborate import elaborate_query from slayer.engine.plan import plan_query -from slayer.engine.syntax import AggCall, parse_expr +from slayer.core.syntax import AggCall, parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle from tests._dev1846_fixtures import ( @@ -30,6 +29,7 @@ sales_model, ) from tests._engine_helpers import seeded_exec_engine +from slayer.core import syntax SAVED = [ ModelMeasure(formula="revenue:sum", name="rev"), diff --git a/tests/test_dev1903_transform_inputs.py b/tests/test_dev1903_transform_inputs.py index 32093b15..f0130b31 100644 --- a/tests/test_dev1903_transform_inputs.py +++ b/tests/test_dev1903_transform_inputs.py @@ -14,7 +14,7 @@ from slayer.engine.binding import bind_expr from slayer.engine.elaborate_env import check_transform_inputs from slayer.engine.plan import plan_query -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle from tests._dev1846_fixtures import SlayerQuery, dev1846_models, month_td diff --git a/tests/test_dev1946_transform_parameter.py b/tests/test_dev1946_transform_parameter.py index e1188eae..0d628fdc 100644 --- a/tests/test_dev1946_transform_parameter.py +++ b/tests/test_dev1946_transform_parameter.py @@ -21,7 +21,7 @@ from slayer.core.scope import ModelScope from slayer.engine.binding import bind_expr from slayer.engine.plan import plan_query -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr from slayer.ir.planned import PlainProducerKernel from slayer.ir.source_bundle import ResolvedSourceBundle from slayer.sql.scope_check import assert_scope_closed diff --git a/tests/test_dev1952_derived_revisit.py b/tests/test_dev1952_derived_revisit.py index 87cdb73e..a5075188 100644 --- a/tests/test_dev1952_derived_revisit.py +++ b/tests/test_dev1952_derived_revisit.py @@ -44,7 +44,7 @@ from slayer.engine.binding import bind_expr from slayer.engine.compile.stages import _canonical_path from slayer.engine.query_engine import SlayerQueryEngine -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle from slayer.engine.param_binding import resolve_default_qualifier_path from slayer.sql.column_expansion import ( diff --git a/tests/test_dev1953_partition_alias.py b/tests/test_dev1953_partition_alias.py index 5fb3e3bd..d44376f7 100644 --- a/tests/test_dev1953_partition_alias.py +++ b/tests/test_dev1953_partition_alias.py @@ -15,7 +15,7 @@ from slayer.core.keys import AggregateKey, ColumnKey, Grain, TransformKey from slayer.core.scope import ModelScope from slayer.engine.binding import bind_expr -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle from tests._dev1847_fixtures import ( diff --git a/tests/test_dev1953_partition_membership.py b/tests/test_dev1953_partition_membership.py index c5531cfc..1768bec6 100644 --- a/tests/test_dev1953_partition_membership.py +++ b/tests/test_dev1953_partition_membership.py @@ -11,7 +11,7 @@ import pytest from slayer.engine.bind_inputs import bind_query_inputs -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle from tests._dev1840_fixtures import bundle as orders_bundle, q as orders_q diff --git a/tests/test_dev1954_canonical_paths.py b/tests/test_dev1954_canonical_paths.py index 5dc365d1..26ae21e8 100644 --- a/tests/test_dev1954_canonical_paths.py +++ b/tests/test_dev1954_canonical_paths.py @@ -25,7 +25,7 @@ from slayer.engine.binding import bind_expr, bind_time_dimension from slayer.engine.join_safety import attributable_from_root, reroot_from_root from slayer.engine.query_engine import SlayerQueryEngine -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle from slayer.sql import column_expansion diff --git a/tests/test_dev1958_row_leaf_ban.py b/tests/test_dev1958_row_leaf_ban.py index d1e5b06c..71e99cec 100644 --- a/tests/test_dev1958_row_leaf_ban.py +++ b/tests/test_dev1958_row_leaf_ban.py @@ -8,7 +8,6 @@ import pytest -from slayer.core.formula import ALL_TRANSFORMS from slayer.engine.plan import plan_query from slayer.ir.source_bundle import ResolvedSourceBundle @@ -21,8 +20,9 @@ month_td, ) from tests.test_law_guard_ratchet import DEFERRAL_CLASSIFIER +import slayer.core.enums -RULED_OPS = sorted(ALL_TRANSFORMS - {"first", "last"}) +RULED_OPS = sorted(slayer.core.enums.ALL_TRANSFORMS - {"first", "last"}) SHIFT_CASES = [ ("time_shift", "time_shift(weight, -1)", "weight"), ("change", "change(weight)", "weight"), diff --git a/tests/test_dev2046_boolean_valued.py b/tests/test_dev2046_boolean_valued.py index 67317f5e..7feb58e3 100644 --- a/tests/test_dev2046_boolean_valued.py +++ b/tests/test_dev2046_boolean_valued.py @@ -14,7 +14,7 @@ from slayer.core.scope import ModelScope, StageColumn, StageSchema from slayer.engine.binding import bind_expr from slayer.engine.key_metadata import stage_column_type -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle from tests._dev2046_fixtures import customers_model, orders_model diff --git a/tests/test_dev2046_typing.py b/tests/test_dev2046_typing.py index 3fa55ebc..2ed553b9 100644 --- a/tests/test_dev2046_typing.py +++ b/tests/test_dev2046_typing.py @@ -23,7 +23,7 @@ from slayer.engine.binding import bind_expr from slayer.engine.key_metadata import aggregated_type, measure_key_type, stage_measure_type from slayer.engine.response_meta import _infer_aggregated_format -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr from slayer.facade.catalog import build_local_view from slayer.ir.source_bundle import ResolvedSourceBundle diff --git a/tests/test_expression_aggregations.py b/tests/test_expression_aggregations.py index 9b3d5309..bf46ecdd 100644 --- a/tests/test_expression_aggregations.py +++ b/tests/test_expression_aggregations.py @@ -27,7 +27,7 @@ ) from slayer.core.query import OrderItem, SlayerQuery, TimeDimension from slayer.engine.query_engine import SlayerQueryEngine -from slayer.engine.syntax import AggCall, Ref, canonical_measure_text, parse_expr +from slayer.core.syntax import AggCall, Ref, canonical_measure_text, parse_expr from slayer.storage.yaml_storage import YAMLStorage from tests._engine_helpers import _norm diff --git a/tests/test_formula.py b/tests/test_formula.py index c2f31d8e..4af5d8e1 100644 --- a/tests/test_formula.py +++ b/tests/test_formula.py @@ -7,7 +7,7 @@ from slayer.core.models import Aggregation from slayer.core.query import _FUNCSTYLE_PENDING, OrderItem -from slayer.engine.syntax import AggCall, BoolOp, Cmp, Literal, Ref, ScalarCall, parse_expr +from slayer.core.syntax import AggCall, BoolOp, Cmp, Literal, Ref, ScalarCall, parse_expr class TestFilterInjection: diff --git a/tests/test_functional_agg_text.py b/tests/test_functional_agg_text.py index 0af2cc01..a76e7019 100644 --- a/tests/test_functional_agg_text.py +++ b/tests/test_functional_agg_text.py @@ -3,7 +3,7 @@ import pytest from slayer.core import refs -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr @pytest.mark.parametrize( diff --git a/tests/test_functional_aggregations.py b/tests/test_functional_aggregations.py index de063e7b..cbbb128e 100644 --- a/tests/test_functional_aggregations.py +++ b/tests/test_functional_aggregations.py @@ -27,7 +27,7 @@ from slayer.core.query import SlayerQuery from slayer.core.warnings import NormalizationWarning, SlayerNormalizationWarning from slayer.engine.query_engine import SlayerQueryEngine -from slayer.engine.syntax import ( +from slayer.core.syntax import ( AggCall, Arith, DottedRef, diff --git a/tests/test_functional_emission.py b/tests/test_functional_emission.py index 401b775b..6a2eec27 100644 --- a/tests/test_functional_emission.py +++ b/tests/test_functional_emission.py @@ -34,7 +34,7 @@ DbtProject, DbtSemanticModel, ) -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr from slayer.mcp.server import create_mcp_server from slayer.memories.help_seed import HELP_TOPICS from slayer.osi.converter import OsiToSlayerConverter diff --git a/tests/test_functional_remedies.py b/tests/test_functional_remedies.py index 1351d9e7..566bd43a 100644 --- a/tests/test_functional_remedies.py +++ b/tests/test_functional_remedies.py @@ -11,7 +11,7 @@ from slayer.dbt.converter import DbtToSlayerConverter from slayer.dbt.models import DbtProject from slayer.engine.column_dependency import _unproven_arity_message -from slayer.engine.syntax import split_entity_agg_ref +from slayer.core.syntax import split_entity_agg_ref from slayer.sql.generator import AggRenderSpec, SQLGenerator from tests._dev1832_fixtures import gen, orders_q from tests._engine_helpers import orders_agg_sql diff --git a/tests/test_heal_date_parts.py b/tests/test_heal_date_parts.py index 999ade0e..651ee2aa 100644 --- a/tests/test_heal_date_parts.py +++ b/tests/test_heal_date_parts.py @@ -11,7 +11,7 @@ from slayer.core.errors import GranularityDefinitionError, SlayerError from slayer.core.models import Aggregation, DatasourceConfig from slayer.engine.query_engine import SlayerQueryEngine -from slayer.engine.syntax import Arith, Cmp, Literal, Ref, parse_expr +from slayer.core.syntax import Arith, Cmp, Literal, Ref, parse_expr from tests._dev1737_fixtures import PARTS from tests._query_heal_fixtures import ( BACKENDS, diff --git a/tests/test_heal_functional_aggregations.py b/tests/test_heal_functional_aggregations.py index 422acd5a..1d535158 100644 --- a/tests/test_heal_functional_aggregations.py +++ b/tests/test_heal_functional_aggregations.py @@ -12,7 +12,7 @@ from slayer.core.models import Column from slayer.core.query import SlayerQuery from slayer.engine.query_engine import SlayerQueryEngine -from slayer.engine.syntax import AggCall, Cmp, Literal, Ref, parse_expr +from slayer.core.syntax import AggCall, Cmp, Literal, Ref, parse_expr from tests._engine_helpers import _engine_generate from tests._query_heal_fixtures import ( BACKENDS, diff --git a/tests/test_memories_resolver_typed.py b/tests/test_memories_resolver_typed.py index 4086d362..cef10c25 100644 --- a/tests/test_memories_resolver_typed.py +++ b/tests/test_memories_resolver_typed.py @@ -16,7 +16,7 @@ from __future__ import annotations -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr from slayer.memories.resolver import _formula_entity_tokens diff --git a/tests/test_mode_b_placeholder_parse.py b/tests/test_mode_b_placeholder_parse.py index 58956170..97809547 100644 --- a/tests/test_mode_b_placeholder_parse.py +++ b/tests/test_mode_b_placeholder_parse.py @@ -7,7 +7,7 @@ import pytest from slayer.core.errors import QueryTypeError, SlayerError, UnresolvedPlaceholderError -from slayer.engine.syntax import ( +from slayer.core.syntax import ( AggCall, Arith, Cmp, diff --git a/tests/test_osi_converter.py b/tests/test_osi_converter.py index 780e823d..eff5cb1d 100644 --- a/tests/test_osi_converter.py +++ b/tests/test_osi_converter.py @@ -13,7 +13,7 @@ import slayer.osi.converter as osi_converter_module from slayer.core.enums import DataType, JoinType -from slayer.engine.syntax import AggCall, Ref, parse_expr +from slayer.core.syntax import AggCall, Ref, parse_expr from slayer.osi.converter import OsiConversionError, OsiToSlayerConverter from slayer.osi.models import ( OSIDataset, diff --git a/tests/test_rank_direction.py b/tests/test_rank_direction.py index f6a2caca..49b6d8ee 100644 --- a/tests/test_rank_direction.py +++ b/tests/test_rank_direction.py @@ -11,7 +11,7 @@ from slayer.core.models import ModelMeasure from slayer.core.scope import ModelScope from slayer.engine.binding import bind_expr -from slayer.engine.syntax import Literal, TransformCall, parse_expr +from slayer.core.syntax import Literal, TransformCall, parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle from tests._dev1847_fixtures import ( diff --git a/tests/test_schema_drift_typed.py b/tests/test_schema_drift_typed.py index c88bcd05..4e285458 100644 --- a/tests/test_schema_drift_typed.py +++ b/tests/test_schema_drift_typed.py @@ -31,7 +31,7 @@ _filter_refs_dsl, _measure_formula_refs, ) -from slayer.engine.syntax import ( +from slayer.core.syntax import ( AggCall, DottedRef, Ref, diff --git a/tests/test_sql_boolean_literal_filters.py b/tests/test_sql_boolean_literal_filters.py index ae1974e8..ba5059ba 100644 --- a/tests/test_sql_boolean_literal_filters.py +++ b/tests/test_sql_boolean_literal_filters.py @@ -19,7 +19,7 @@ from slayer.core.models import Column, DatasourceConfig, SlayerModel from slayer.core.query import SlayerQuery from slayer.engine.query_engine import SlayerQueryEngine -from slayer.engine.syntax import BoolOp, Cmp, Literal, Ref, parse_expr +from slayer.core.syntax import BoolOp, Cmp, Literal, Ref, parse_expr from slayer.storage.yaml_storage import YAMLStorage diff --git a/tests/test_syntax.py b/tests/test_syntax.py index 5f597e8e..27850bf5 100644 --- a/tests/test_syntax.py +++ b/tests/test_syntax.py @@ -1,6 +1,6 @@ """Stage 7a.3 (DEV-1450) — Mode-B Python-AST parser tests. -The parser at ``slayer.engine.syntax.parse_expr`` consumes a Mode-B +The parser at ``slayer.core.syntax.parse_expr`` consumes a Mode-B expression string (the SLayer DSL) and emits a typed ``ParsedExpr`` AST the binder (stage 7a.5) compiles to ``BoundExpr``. Coverage: @@ -22,7 +22,7 @@ import pytest from slayer.core.errors import IllegalWindowInFilterError, UnknownFunctionError -from slayer.engine.syntax import ( +from slayer.core.syntax import ( AggCall, Arith, BoolOp, diff --git a/tests/test_transforms_planner.py b/tests/test_transforms_planner.py index 9984f45e..0bf77444 100644 --- a/tests/test_transforms_planner.py +++ b/tests/test_transforms_planner.py @@ -51,7 +51,7 @@ from slayer.engine.compile.projection import _iter_slot_deps from slayer.ir.source_bundle import ResolvedSourceBundle from slayer.engine.plan import plan_query -from slayer.engine.syntax import parse_expr +from slayer.core.syntax import parse_expr # --------------------------------------------------------------------------- diff --git a/tests/test_usage_reporting_payload.py b/tests/test_usage_reporting_payload.py index f9dac843..54f0dfcc 100644 --- a/tests/test_usage_reporting_payload.py +++ b/tests/test_usage_reporting_payload.py @@ -18,8 +18,7 @@ import slayer.cli from slayer import usage_reporting from slayer.api.server import create_app -from slayer.core.enums import BUILTIN_AGGREGATIONS -from slayer.core.formula import ALL_TRANSFORMS +from slayer.core.enums import BUILTIN_AGGREGATIONS, ALL_TRANSFORMS from slayer.flight.translator import TranslationError from slayer.mcp.server import create_mcp_server from slayer.storage.yaml_storage import YAMLStorage From 22b6b5463f82d508fe0fb328c3397da62a32efb7 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Wed, 21 Oct 2026 13:39:25 +0200 Subject: [PATCH 04/17] Mode-B grammar: pyparsing parser, SQL forms in every position, casts, FILTER, quoted identifiers Replaces the ast.parse pipeline and its text pre-passes with one pyparsing grammar (parse_filter_expr deleted). Adds the Cast and TemporalLiteral nodes, CastKey (database cast of the author's type, TRY_CAST lowered per dialect, Postgres 16+ via the server profile), FILTER (WHERE c) (inputs masked at binding; star counts and first/last carry the filter on the key), ILIKE, IS [NOT] DISTINCT FROM, the quoted-identifier hint, the round() ndigits check, hidden row-level ORDER BY expressions, and canonical names for unnamed computed dimensions. Tests respelled for the SQL string rule (backslash escapes and double-quoted strings no longer literals): test_dev1901_param_binding, test_dev1934_formula_agg, test_quoted_aggregation_placeholders. Totality registries extended with CastKey / ilike: test_dev1747_reroot_visitor, test_dev1842_keys_prepend, test_dev1825_substitute_keys, test_dev1827_value_key_traversal, test_dev2013_expression_attribution, test_dev1753_last_four_scalars, test_dev1744_value_expr. --- slayer/core/errors.py | 8 + slayer/core/keys.py | 75 +- slayer/core/query.py | 30 +- slayer/core/refs.py | 18 +- slayer/core/syntax.py | 2023 +++++++---------- slayer/engine/bind_inputs.py | 10 +- slayer/engine/binding.py | 148 +- slayer/engine/compile/projection.py | 3 + slayer/engine/compile/shift.py | 3 +- slayer/engine/compile/stages.py | 9 +- slayer/engine/elaborate_env.py | 15 +- slayer/engine/key_metadata.py | 5 + slayer/engine/model_reads.py | 12 +- slayer/engine/population.py | 5 +- slayer/engine/reference_closure.py | 5 +- slayer/engine/schema_drift.py | 3 +- slayer/engine/unrelated.py | 4 +- slayer/sql/client.py | 6 +- slayer/sql/dialects/_tier2.py | 6 + slayer/sql/dialects/base.py | 59 + slayer/sql/dialects/clickhouse.py | 12 +- slayer/sql/dialects/duckdb.py | 3 + slayer/sql/dialects/mysql.py | 19 + slayer/sql/dialects/postgres.py | 33 +- slayer/sql/dialects/snowflake.py | 1 + slayer/sql/dialects/sqlite.py | 50 +- slayer/sql/dialects/tsql.py | 3 + slayer/sql/generator.py | 24 +- slayer/sql/render/row_expr.py | 25 +- slayer/sql/render/value_expr.py | 11 + slayer/sql/scope.py | 5 +- slayer/sql/sql_expr.py | 7 + tests/test_dev1744_value_expr.py | 2 +- tests/test_dev1747_reroot_visitor.py | 6 +- tests/test_dev1753_last_four_scalars.py | 2 +- tests/test_dev1825_substitute_keys.py | 4 +- tests/test_dev1827_value_key_traversal.py | 7 +- tests/test_dev1842_keys_prepend.py | 4 +- tests/test_dev1901_param_binding.py | 8 +- tests/test_dev1934_formula_agg.py | 2 +- tests/test_dev2013_expression_attribution.py | 2 + tests/test_quoted_aggregation_placeholders.py | 8 +- 42 files changed, 1434 insertions(+), 1251 deletions(-) diff --git a/slayer/core/errors.py b/slayer/core/errors.py index 9b44c61e..b5c9f1fb 100644 --- a/slayer/core/errors.py +++ b/slayer/core/errors.py @@ -407,6 +407,14 @@ class UnrelatedDatasetError(QueryTypeError): """A row-level read of a dataset no join path reaches, outside an aggregation or a one-sided filter.""" +class CastTypeError(QueryTypeError): + """A CAST target type the query's dialect cannot express.""" + + +class AggregationFilterError(QueryTypeError): + """A ``FILTER (WHERE …)`` condition that is not a row-level predicate of its aggregation.""" + + class UnreachableDefinitionError(SlayerError, ValueError): """A stored ``Column.sql`` / ``Column.filter`` / model filter naming a dataset no join path reaches from its model.""" diff --git a/slayer/core/keys.py b/slayer/core/keys.py index fc81d9e8..c4dede58 100644 --- a/slayer/core/keys.py +++ b/slayer/core/keys.py @@ -88,6 +88,7 @@ def _spec(low: int, high: Optional[int], *, sql: bool = True) -> ScalarSpec: "replace": _spec(3, 3), "substr": _spec(2, 3), "substring": _spec(2, 3), "instr": _spec(2, 2), "concat": _spec(1, None), "like": _spec(2, 2, sql=False), # emits the LIKE operator + "ilike": _spec(2, 2, sql=False), # case-insensitive LIKE on every dialect "iif": _spec(3, 3, sql=False), # emits CASE WHEN "date_part": _spec(2, 2, sql=False), "date_diff": _spec(3, 3, sql=False), @@ -402,7 +403,7 @@ def __eq__(self, other: object) -> bool: AggregateSource = Union[ ColumnKey, ColumnSqlKey, StarKey, TimeTruncKey, "ArithmeticKey", "ScalarCallKey", "LiteralKey", "AggregateKey", "TransformKey", - "InKey", "TimePointCmpKey", + "InKey", "TimePointCmpKey", "CastKey", ] # Positional and kwarg arg values share one union: `last(created_at)` binds an # identifier column, `weighted_avg(weight=qty)` a column, @@ -411,6 +412,8 @@ def __eq__(self, other: object) -> bool: # all via `_bind_agg_arg`. _AggregateArgValue = Union[ ColumnKey, ColumnSqlKey, SqlFragmentKey, "AggregateKey", "TransformKey", + # A FILTER-masked row parameter. + "ScalarCallKey", Decimal, str, bool, None, ] _AggregateKwargValue = _AggregateArgValue @@ -447,6 +450,8 @@ class AggregateKey(_FrozenKey, frozen=True): locus: Literal["target", "host"] = "target" # None = grain inherited from context; Grain.EMPTY = explicitly scalar. partition_keys: Optional["Grain"] = None + # A ranked (first / last) aggregate's ``FILTER (WHERE …)``; other families mask their inputs at binding. + filter: Optional["ValueKey"] = None @field_validator("kwargs", mode="before") @classmethod @@ -465,6 +470,8 @@ def children(self) -> Tuple["ValueKey", ...]: ] if self.partition_keys is not None: embedded.extend(self.partition_keys) + if self.filter is not None: + embedded.append(self.filter) return tuple(embedded) def map_children( @@ -479,6 +486,7 @@ def map_children( None if self.partition_keys is None else Grain.of(m(p) for p in self.partition_keys) ), + "filter": None if self.filter is None else m(self.filter), } return self.model_copy(update=update) if m.changed else self @@ -491,6 +499,7 @@ def __hash__(self) -> int: _typed_kwargs(self.kwargs), self.locus, self.partition_keys, + self.filter, )) def __eq__(self, other: object) -> bool: @@ -503,6 +512,7 @@ def __eq__(self, other: object) -> bool: and _typed_kwargs(self.kwargs) == _typed_kwargs(other.kwargs) and self.locus == other.locus and self.partition_keys == other.partition_keys + and self.filter == other.filter ) @@ -677,6 +687,50 @@ def __eq__(self, other: object) -> bool: ) +# A cast target's SLayer type, by the type's leading word. +_CAST_TYPE_WORDS: Mapping[str, DataType] = MappingProxyType({ + **{w: DataType.INT for w in ("INT", "INTEGER", "BIGINT", "SMALLINT", "TINYINT", "MEDIUMINT", "HUGEINT", + "INT2", "INT4", "INT8", "INT64", "LONG")}, + **{w: DataType.DOUBLE for w in ("FLOAT", "FLOAT4", "FLOAT8", "FLOAT64", "DOUBLE", "REAL", "DECIMAL", + "NUMERIC", "NUMBER", "BIGNUMERIC", "DEC")}, + **{w: DataType.TEXT for w in ("TEXT", "VARCHAR", "CHAR", "CHARACTER", "NVARCHAR", "NCHAR", "STRING", + "CLOB", "NTEXT", "BPCHAR")}, + **{w: DataType.BOOLEAN for w in ("BOOLEAN", "BOOL")}, + "DATE": DataType.DATE, + **{w: DataType.TIMESTAMP for w in ("TIMESTAMP", "DATETIME", "DATETIME2", "TIMESTAMPTZ", "SMALLDATETIME")}, +}) + + +def cast_data_type(type_text: str) -> DataType: + """SLayer's type for a cast to ``type_text`` (UNKNOWN for any unmapped type).""" + word = type_text.split("(")[0].split()[0].upper() if type_text.strip() else "" + return _CAST_TYPE_WORDS.get(word, DataType.UNKNOWN) + + +class CastKey(_FrozenKey, frozen=True): + """Identity for ``CAST(source AS type_text)`` (``try_``: ``TRY_CAST``) — the database's + own cast of the author's type; ``data_type`` is SLayer's type for the target.""" + + source: "ValueKey" + type_text: str + data_type: DataType + try_: bool = False + + @property + def phase(self) -> Phase: + return self.source.phase + + def children(self) -> Tuple["ValueKey", ...]: + return (self.source,) + + def map_children( + self, fn: Callable[["ValueKey"], "ValueKey"], + ) -> "CastKey": + m = _ChildMapper(fn) + source = m(self.source) + return self.model_copy(update={"source": source}) if m.changed else self + + TimePointOp = Literal["=", "!=", "<", "<=", ">", ">=", "in", "not in"] MIRRORED_TIME_POINT_OP: Dict[TimePointOp, TimePointOp] = {"=": "=", "!=": "!=", "<": ">", "<=": ">=", ">": "<", ">=": "<="} @@ -765,6 +819,7 @@ def map_children( TimePointCmpKey, InKey, SqlFragmentKey, + CastKey, ] @@ -871,6 +926,7 @@ def __sub__(self, other: object) -> "Grain": ScalarCallKey.model_rebuild() TimePointCmpKey.model_rebuild() InKey.model_rebuild() +CastKey.model_rebuild() TimeTruncKey.model_rebuild() # AggregateKey.source forward-references the expression composites. AggregateKey.model_rebuild() @@ -879,6 +935,8 @@ def __sub__(self, other: object) -> "Grain": VALUE_KEY_TYPES: Tuple[type, ...] = get_args(ValueKey) +# Key fields spelled only when set, so a key without one keeps its historical spelling. +OPTIONAL_KEY_FIELDS = frozenset({"filter"}) class KindPolicy(BaseModel): @@ -908,6 +966,7 @@ class KindPolicy(BaseModel): TimePointCmpKey: KindPolicy(), InKey: KindPolicy(slot_composite=True, materialised_order=True), SqlFragmentKey: KindPolicy(), + CastKey: KindPolicy(slot_composite=True, materialised_order=True), } SLOT_COMPOSITE_KINDS: Tuple[type, ...] = tuple(k for k, p in KIND_POLICY.items() if p.slot_composite) @@ -1238,6 +1297,8 @@ def temporal_type(key: object, *, column_type: ColumnTypeFn) -> Optional[DataTyp return temporal_type(key.source, column_type=column_type) if from_source else None if isinstance(key, ScalarCallKey): return _scalar_temporal_type(key, column_type=column_type) + if isinstance(key, CastKey): + return key.data_type if key.data_type in TEMPORAL_TYPES else None return None @@ -1282,8 +1343,10 @@ def boolean_valued(key: object, *, column_type: ColumnTypeFn) -> bool: return key.agg.lower() in _BOOLEAN_PRESERVING_AGGS and boolean_valued(key.source, column_type=column_type) if isinstance(key, TransformKey): return key.op in AXIS_COLLAPSING_TRANSFORMS and boolean_valued(key.input, column_type=column_type) + if isinstance(key, CastKey): + return key.data_type is DataType.BOOLEAN if isinstance(key, ScalarCallKey): - if key.name == "like": + if key.name in ("like", "ilike"): return True if key.name not in _BOOLEAN_BRANCH_SCALARS: return False @@ -1340,6 +1403,8 @@ def numeric_valued(key: object, *, column_type: ColumnTypeFn) -> bool: return agg in _BOOLEAN_PRESERVING_AGGS and numeric_valued(key.source, column_type=column_type) if isinstance(key, TransformKey): return key.op in _NUMBER_RESULT_TRANSFORMS or numeric_valued(key.input, column_type=column_type) + if isinstance(key, CastKey): + return key.data_type in _NUMERIC_TYPES if isinstance(key, ScalarCallKey): if key.name in _NUMBER_RESULT_SCALARS: return True @@ -1418,6 +1483,8 @@ def aggregation_source_type(source: object, *, column_type: ColumnTypeFn) -> Opt return DataType.INT if isinstance(source, (ColumnKey, ColumnSqlKey)): return column_type(source) + if isinstance(source, CastKey) and source.data_type is not DataType.UNKNOWN: + return source.data_type return None @@ -1493,8 +1560,10 @@ def is_local_combined_regroup_ref( ) +# Null-safe comparisons: never UNKNOWN. +NULL_SAFE_COMPARISON_OPS = frozenset({"is distinct from", "is not distinct from"}) PREDICATE_COMPARISON_OPS = frozenset( - {"==", "=", "!=", "<>", "<", "<=", ">", ">=", "is", "is not"} + {"==", "=", "!=", "<>", "<", "<=", ">", ">=", "is", "is not", *NULL_SAFE_COMPARISON_OPS} ) BOOL_CONNECTIVE_OPS = frozenset({"and", "or", "not"}) diff --git a/slayer/core/query.py b/slayer/core/query.py index 3eb804e4..4dbab984 100644 --- a/slayer/core/query.py +++ b/slayer/core/query.py @@ -36,7 +36,7 @@ ) from slayer.core.refs import auto_name_from_expression from slayer.core.time_points import TIME_POINT_FORMS, is_time_point_shape -from slayer.core.syntax import AggCall, DottedRef, Ref, parse_expr, walk_parsed_refs +from slayer.core.syntax import AggCall, DottedRef, Ref, canonical_measure_text, parse_expr, walk_parsed_refs from slayer.sql.window_detect import WINDOW_IN_FILTER_ERROR, has_window_function from slayer.storage.migrations import CURRENT_VERSIONS, migrate as _migrate_schema @@ -671,11 +671,21 @@ def substituted(self, expression: str) -> "ComputedDimension": @model_validator(mode="after") def _fill_name(self) -> "ComputedDimension": if self.name is None: - self.name = auto_name_from_expression(self.expression) + self.name = computed_dimension_name(self.expression) _validate_model_name(self.name, "Computed dimension") return self +def computed_dimension_name(expression: str) -> str: + """An unnamed computed dimension's name: its expression's canonical text, sanitised, so every + spelling of one expression names it alike.""" + try: + text = canonical_measure_text(parse_expr(expression)) + except Exception: # noqa: BLE001 — an unparseable expression is reported at binding + text = expression + return auto_name_from_expression(text) + + def _coerce_column_ref(v: Any) -> Any: """Allow plain string where a ColumnRef is expected: "x" → {"name": "x"}.""" if isinstance(v, str): @@ -696,24 +706,18 @@ def _coerce_column_ref(v: Any) -> Any: def _order_formula_candidate(v: str) -> str | None: - """``v`` verbatim if it carries a measure expression (legacy colon spelling, - call-style text, or an expression containing an aggregation), else - ``None``; shared by ``_capture_raw_formula`` and ``_coerce_order_column`` - so they can't drift. The author's spelling is preserved — resolution - happens at binding via the native parser. Bare-alias - arithmetic (``rev / cnt``) is deliberately NOT a candidate — it falls - through to ColumnRef validation and fails fast.""" + """``v`` verbatim if it is an expression rather than a column reference, else ``None``; + shared by ``_capture_raw_formula`` and ``_coerce_order_column`` so they can't drift. + Binding judges whether the expression is legal there.""" if ":" in v or _FUNCSTYLE_CALL_PATTERN.match(v): return v if _is_valid_column_ref_name(v): return None try: - parsed = parse_expr(v) + parse_expr(v) except Exception: return None - if any(isinstance(node, AggCall) for node in walk_parsed_refs(parsed)): - return v - return None + return v def _is_valid_column_ref_name(name: str) -> bool: diff --git a/slayer/core/refs.py b/slayer/core/refs.py index 9f2ab0e8..e9926670 100644 --- a/slayer/core/refs.py +++ b/slayer/core/refs.py @@ -11,8 +11,10 @@ from slayer.core.granularity import CustomGranularity from slayer.core.keys import ( MIRRORED_TIME_POINT_OP, + OPTIONAL_KEY_FIELDS, AggregateKey, ArithmeticKey, + CastKey, ColumnKey, ColumnSqlKey, InKey, @@ -65,7 +67,7 @@ def auto_name_from_expression(expression: str) -> str: # The row-level ``ValueKey`` kinds an ``AggregateKey.source`` may take when it # is a same-model scalar EXPRESSION rather than a column / star. -EXPRESSION_SOURCE_KINDS = (ArithmeticKey, ScalarCallKey, LiteralKey, TimeTruncKey, InKey, TimePointCmpKey) +EXPRESSION_SOURCE_KINDS = (ArithmeticKey, ScalarCallKey, LiteralKey, TimeTruncKey, InKey, TimePointCmpKey, CastKey) # The pinned legacy key spelling: the historical Pydantic @@ -94,6 +96,7 @@ def auto_name_from_expression(expression: str) -> str: ("partition_keys", lambda k: ( None if k.partition_keys is None else k.partition_keys.keys )), + ("filter", attrgetter("filter")), )), TransformKey: ("TransformKey", ( ("op", attrgetter("op")), ("input", attrgetter("input")), @@ -120,6 +123,10 @@ def auto_name_from_expression(expression: str) -> str: SqlFragmentKey: ("SqlFragmentKey", ( ("template", attrgetter("template")), ("refs", attrgetter("refs")), )), + CastKey: ("CastKey", ( + ("source", attrgetter("source")), ("type_text", attrgetter("type_text")), + ("try_", attrgetter("try_")), + )), } @@ -146,6 +153,7 @@ def _legacy_key_fields(key: Any) -> tuple[str, list[tuple[str, str]]]: spelled_cls, fields = _LEGACY_KEY_SPELLINGS[type(key)] return spelled_cls, [ (name, _legacy_value_spelling(get(key))) for name, get in fields + if not (name in OPTIONAL_KEY_FIELDS and get(key) is None) ] @@ -231,7 +239,8 @@ def _aggregate_display(key: AggregateKey) -> str: args.extend(f"{k}={_arg_display(v)}" for k, v in key.kwargs) if key.partition_keys is not None: args.append(f"partition_by={_grain_display_list(key.partition_keys)}") - return _call_display(key.agg, args) + text = _call_display(key.agg, args) + return text if key.filter is None else f"{text} FILTER (WHERE {key_display(key.filter)})" def _transform_display(key: TransformKey) -> str: @@ -285,6 +294,8 @@ def key_display(key: ValueKey) -> str: return _arithmetic_display(key) if isinstance(key, ScalarCallKey): return _call_display(key.name, [_arg_display(a) for a in key.args]) + if isinstance(key, CastKey): + return f"{'TRY_CAST' if key.try_ else 'CAST'}({key_display(key.source)} AS {key.type_text})" if isinstance(key, TimePointCmpKey): return _time_point_cmp_display(key) if isinstance(key, InKey): @@ -345,6 +356,9 @@ def agg_kwarg_canonical_str(value: Any) -> str: return _transform_key_canonical_str(value) if isinstance(value, SqlFragmentKey): return sql_fragment_digest(value) + if isinstance(value, ScalarCallKey): + # A FILTER-masked row parameter. + return auto_name_from_expression(key_display(value)) raise TypeError( f"AggregateKey kwarg value of type {type(value).__name__!r} " f"is not supported: {value!r}", diff --git a/slayer/core/syntax.py b/slayer/core/syntax.py index 174006a0..9c18d14c 100644 --- a/slayer/core/syntax.py +++ b/slayer/core/syntax.py @@ -1,25 +1,36 @@ -"""Mode-B Python-AST parser. - -``parse_expr(text) -> ParsedExpr`` lowers a Mode-B DSL string -(``ModelMeasure.formula``, ``SlayerQuery.measures`` / ``.filters``) to a typed -tree — pure syntax, no scope resolution or named-measure expansion (binder's -job). Grammar: bare/dotted refs; functional or legacy colon aggregations, which -collapse to one ``AggCall``; transform calls; a closed scalar -allowlist; arithmetic / comparison / boolean / unary; grouping. Rejects -non-allowlisted calls, raw ``OVER(...)``, and chained comparisons. +"""The Mode-B grammar. + +``parse_expr(text) -> ParsedExpr`` parses a Mode-B expression (``ModelMeasure.formula``, +query measures / dimensions / filters / order) with one SQL-compatible pyparsing grammar — +pure syntax, no scope resolution or named-measure expansion (binder's job). Every spelling +of a construct collapses to one node: CASE → ``iif``, ``EXTRACT`` → ``date_part``, +``count(distinct x)`` → ``count_distinct``, ``BETWEEN`` → its conjunction, ``x::t`` → +``Cast``, interval literals → ``date_add``. """ from __future__ import annotations -import ast import re +import threading +import unicodedata from decimal import Decimal +from functools import lru_cache from typing import Any, Dict, Iterator, List, Optional, Tuple, Union +import pyparsing as pp from pydantic import BaseModel, ConfigDict from slayer.core.direction import normalize_direction -from slayer.core.enums import AMBIGUOUS_WEEKDAY_NAMES, BUILTIN_AGGREGATIONS, DATE_PART_ALIASES, GRANULARITY_NAMES, ambiguous_weekday_message, normalize_aggregation_name, ALL_TRANSFORMS +from slayer.core.enums import ( + ALL_TRANSFORMS, + AMBIGUOUS_WEEKDAY_NAMES, + BUILTIN_AGGREGATIONS, + DATE_PART_ALIASES, + GRANULARITY_NAMES, + TimeGranularity, + ambiguous_weekday_message, + normalize_aggregation_name, +) from slayer.core.errors import GranularityCallError, IllegalWindowInFilterError, UnknownFunctionError from slayer.core.keys import DATE_UNIT_ARGS, SCALAR_FUNCTIONS, check_scalar_arity from slayer.core.refs import split_agg_suffix @@ -36,10 +47,26 @@ class _BaseNode(BaseModel): class Ref(_BaseNode): name: str + # Written double-quoted; not part of identity. + quoted: bool = False + + def __eq__(self, other: object) -> bool: + return isinstance(other, Ref) and self.name == other.name + + def __hash__(self) -> int: + return hash(("Ref", self.name)) class DottedRef(_BaseNode): parts: Tuple[str, ...] + # Some segment was written double-quoted; not part of identity. + quoted: bool = False + + def __eq__(self, other: object) -> bool: + return isinstance(other, DottedRef) and self.parts == other.parts + + def __hash__(self) -> int: + return hash(("DottedRef", self.parts)) class StarSource(_BaseNode): @@ -50,30 +77,32 @@ class Literal(_BaseNode): value: Decimal | str | bool | None = None +class TemporalLiteral(_BaseNode): + """``DATE '…'`` / ``TIMESTAMP '…'``; the text is validated at binding.""" + + kind: str + text: str + + class Placeholder(_BaseNode): - """An unsubstituted ``{name}`` variable placeholder; admitted wherever a ``Literal`` is.""" + """An unsubstituted ``{name}`` variable in a literal slot.""" name: str class TupleLit(_BaseNode): - """Literal-only tuple/list RHS for ``IN`` / ``NOT IN``; non-literal - elements and empty tuples are rejected at parse time.""" + """Literal-only list RHS for ``IN`` / ``NOT IN``.""" elements: Tuple[Literal, ...] class AggCall(_BaseNode): - # source may also be an aggregation-free scalar expression (``sum(a - b)``), - # or — for a re-aggregation — a nested AggCall or a grained - # TransformCall, alone or composed. - source: ( - Ref | DottedRef | StarSource | Literal | Placeholder | ScalarCall | Arith | UnaryOp - | AggCall | TransformCall | Cmp | BoolOp - ) + source: Any agg: str args: Tuple[Any, ...] = () kwargs: Tuple[Tuple[str, Any], ...] = () + # ``FILTER (WHERE …)`` predicate. + filter: Any = None class TransformCall(_BaseNode): @@ -88,6 +117,14 @@ class ScalarCall(_BaseNode): args: Tuple[Any, ...] = () +class Cast(_BaseNode): + """``CAST(x AS t)`` / ``x::t`` / ``TRY_CAST(x AS t)``; ``type_text`` is the normalised author type.""" + + source: Any + type_text: str + try_: bool = False + + class Arith(_BaseNode): op: str left: Any @@ -111,1195 +148,884 @@ class BoolOp(_BaseNode): ParsedExpr = Union[ - Ref, DottedRef, StarSource, Literal, Placeholder, TupleLit, - AggCall, TransformCall, ScalarCall, + Ref, DottedRef, StarSource, Literal, TemporalLiteral, Placeholder, TupleLit, + AggCall, TransformCall, ScalarCall, Cast, Arith, UnaryOp, Cmp, BoolOp, ] -# ``AggCall.source`` forward-references ScalarCall / Arith / UnaryOp (defined -# below it) for expression aggregation sources. -AggCall.model_rebuild() - # --------------------------------------------------------------------------- -# Constants +# Lexical helpers # --------------------------------------------------------------------------- -# The ``__slayer_`` prefix is reserved from user input (P3). Matched only at an -# identifier boundary so a legal embedded name (``foo__slayer_bar``) stays -# referenceable; ``\w`` is Unicode-aware, covering ``é__slayer_bar`` too. +# The ``__slayer_`` prefix is reserved from user input (P3). _RESERVED_EXPR_PREFIX = "__slayer_" -_RESERVED_EXPR_PREFIX_RE = re.compile(r"(? str: - """``col LIKE 'p%'`` → ``like(col, 'p%')``; ``col NOT LIKE 'p%'`` → - ``not like(col, 'p%')``. Matches starting inside string literals are - left untouched.""" - spans = [(m.start(), m.end()) for m in _PY_STRING_LITERAL_RE.finditer(text)] - - def _sub(m: "re.Match[str]") -> str: - # Skip a match inside a string literal, or one whose LHS is only part of - # a larger reference the ``\b``/``\w`` LHS token can't lex whole — a - # ``.``-rooted dotted path (``℘.name``) or a Unicode-fused prefix - # (``℘name``, decomposed ``éname``); that name is not a LIKE operand. - if any(s <= m.start() < e for s, e in spans) or _continues_ref( - text=text, pos=m.start() - 1 - ): - return m.group(0) - lhs, neg, pat = m.group(1), m.group(2), m.group(3) - call = f"like({lhs}, {pat})" - return f"not {call}" if neg else call - - return _SQL_LIKE_RE.sub(_sub, text) - - -# SQL ``CASE WHEN … END`` → nested ``iif(cond, then, otherwise)``. Lowered to the -# host language (a plain call) so binding + per-dialect emission reuse the whole -# scalar-call machinery. Only the WHEN-condition segments are operator-normalised -# (so ``CASE WHEN a = 5 …`` works in measures too); THEN/ELSE values are sliced -# verbatim and only recursed for nested CASE, so a value like ``'a AND b'`` is -# never rewritten. String literals are single tokens, so keywords inside them are -# invisible to the keyword scan. Identifiers lex complete (Unicode-aware start, -# dotted paths as ONE token), so a name containing or qualified by a keyword -# (``customers.case``, ``écase``) can never equal one. -_CASE_TOKEN_RE = re.compile( - r"(?P'(?:\\.|[^'\\])*'|\"(?:\\.|[^\"\\])*\")" - r"|(?P[^\W\d]\w*(?:\s*\.\s*[^\W\d]\w*)*)" - r"|(?P[(\[])" - r"|(?P[)\]])", - re.DOTALL, -) -_CASE_STOPS_OPERAND = frozenset({"WHEN", "THEN", "ELSE", "END"}) -_CASE_KEYWORDS = _CASE_STOPS_OPERAND | {"CASE"} +class _Problem(ValueError): + """A grammar-level rejection; ``parse_expr`` prefixes the expression text.""" + def __init__(self, detail: str) -> None: + super().__init__(detail) + self.detail = detail -def _is_ident_adjacent(text: str, pos: int) -> bool: - """Whether ``text[pos]`` is identifier material the ``\\w`` token class - misses (combining marks, ``Other_ID_Start`` symbols like ``℘``).""" - return 0 <= pos < len(text) and ("a" + text[pos]).isidentifier() +class _Region(BaseModel): + kind: str # "code" | "string" | "ident" + start: int + end: int -def _continues_ref(text: str, pos: int) -> bool: - """Whether ``text[pos]`` extends a reference token past a keyword span — a - ``.`` join separator, or identifier material the ``\\w`` class misses. A - keyword touching such a char is a name component (``a.NULL``, ``℘case``), - not a keyword.""" - return _is_ident_adjacent(text=text, pos=pos) or ( - 0 <= pos < len(text) and text[pos] == "." - ) +def _regions(text: str) -> List[_Region]: + """Split ``text`` into code, ``'…'`` string and ``"…"`` identifier spans (doubled quotes escape).""" + out: List[_Region] = [] + i = start = 0 + n = len(text) + while i < n: + quote = text[i] + if quote not in ("'", '"'): + i += 1 + continue + if i > start: + out.append(_Region(kind="code", start=start, end=i)) + j = i + 1 + while j < n: + if text[j] == quote: + if j + 1 < n and text[j + 1] == quote: + j += 2 + continue + break + j += 1 + end = min(j + 1, n) + out.append(_Region(kind="string" if quote == "'" else "ident", start=i, end=end)) + i = start = end + if start < n: + out.append(_Region(kind="code", start=start, end=n)) + return out + + +def _mask_placeholders(text: str) -> str: + """``{name}`` outside string literals → sentinel-bracketed name of the same length.""" + if _PH_OPEN in text or _PH_CLOSE in text: + raise _Problem("unsupported character in expression.") + parts: List[str] = [] + for region in _regions(text): + chunk = text[region.start:region.end] + if region.kind == "code": + chunk = _CODE_PLACEHOLDER_RE.sub( + lambda m: f"{_PH_OPEN}{m.group(1)}{_PH_CLOSE}" if m.group(1) else m.group(0), chunk, + ) + if "{" in chunk or "}" in chunk: + raise _Problem( + "unsupported AST node Set: a brace expression is not a variable placeholder; " + "write a variable as {name}." + ) + elif region.kind == "ident" and re.search(r"\{" + _VAR_NAME + r"\}", chunk): + raise _Problem(f"{_LITERAL_VALUES}, never an identifier.") + parts.append(chunk) + return "".join(parts) -def _sub_keyword_isolated(pattern: "re.Pattern[str]", repl: str, text: str) -> str: - """``pattern.sub(repl, text)`` but leaving a keyword match that is really a - reference component untouched — a dotted leaf/root (``a.NULL`` → keeps - ``NULL``) or one fused to ``Other_ID_Start`` / combining marks the ``\\b`` - boundary splits (``℘NULL`` → ``℘None``, ``℘AND`` → ``℘and``). Both would - silently rebind the reference (mirrors ``_case_keyword``'s guard).""" - def _guard(m: "re.Match[str]") -> str: - if _continues_ref(text=text, pos=m.start() - 1) or _continues_ref( - text=text, pos=m.end() - ): - return m.group(0) - return repl - return pattern.sub(_guard, text) +class _State: + """Per-parse scratch: the expression text and the farthest failure offset.""" -def _case_keyword(text: str, tok: Tuple[Optional[str], str, int, int]) -> Optional[str]: - """The CASE-grammar keyword a token spells, or None for an identifier. + def __init__(self) -> None: + self.text = "" + self.farthest = 0 - ASCII-exact (``caſe`` uppercases to ``CASE`` only via Unicode folding) and - rejected when adjacent raw text continues an identifier around the token. - """ - kind, val, start, end = tok - if kind != "id" or not val.isascii(): - return None - up = val.upper() - if up not in _CASE_KEYWORDS: - return None - if _is_ident_adjacent(text, start - 1) or _is_ident_adjacent(text, end): - return None - return up +_STATE = _State() +_PARSE_LOCK = threading.Lock() -def _case_has_when( - text: str, toks: List[Tuple[Optional[str], str, int, int]], i: int, -) -> bool: - """Whether the CASE at ``toks[i]`` opens a conditional: a depth-0 ``WHEN`` - follows before any other structural keyword, an unmatched ``)``, or end. - Accepted mislex edge: a BARE keyword-named simple-CASE operand - (``CASE case WHEN 1 …``) is genuinely ambiguous and errors loudly — - parenthesize the operand (``CASE (case) WHEN 1 …``) to disambiguate. - """ - depth = 0 - for j in range(i + 1, len(toks)): - kind = toks[j][0] - if kind == "lp": - depth += 1 - elif kind == "rp": - if depth == 0: - return False - depth -= 1 - elif kind == "id" and depth == 0: - kw = _case_keyword(text, toks[j]) - if kw is not None: - return kw == "WHEN" - return False - - -def _rewrite_case_when(text: str) -> str: - if "case" not in text.lower(): - return text - toks = [ - (m.lastgroup, m.group(), m.start(), m.end()) - for m in _CASE_TOKEN_RE.finditer(text) - ] - if not any(_case_keyword(text, t) == "CASE" for t in toks): - return text - result, _ = _rw_value(text, toks, 0, frozenset(), 0) - return result +def _record_failure(_s: str, loc: int, _expr: Any, _err: Any) -> None: + if loc > _STATE.farthest: + _STATE.farthest = loc -def _rw_value(text, toks, i, stop_kws, start_char): # NOSONAR(S3776) — one cohesive token-scan over a value/condition span: paren-depth tracking, the depth-0 stop-keyword break, and nested-CASE recursion are each one decision, and splitting them scatters the slice-and-recurse contract that preserves the original text. - """Rewrite one value/condition span → ``(rewritten, next_index)``. +def _t(element: pp.ParserElement) -> pp.ParserElement: + """A terminal whose failures feed the farthest-failure error position.""" + element.set_fail_action(_record_failure) + return element - ``start_char`` is the span start (end of the keyword just consumed): bare - values (``1``) have no token, so non-CASE content is recovered by slicing the - original text (preserving colons/dots/spacing). Stops at a depth-0 stop - keyword or an unmatched ``)``. - """ - parts: List[str] = [] - depth = 0 - last = start_char - while i < len(toks): - kind, _, start, _ = toks[i] - if kind == "lp": - depth += 1 - elif kind == "rp": - if depth == 0: - break - depth -= 1 - elif kind == "id": - kw = _case_keyword(text, toks[i]) - if depth == 0 and kw in stop_kws: - break - if kw == "CASE" and _case_has_when(text, toks, i): - parts.append(text[last:start]) - nested, i = _rw_case(text, toks, i) - parts.append(nested) - last = toks[i - 1][3] - continue - i += 1 - stop_char = toks[i][2] if i < len(toks) else len(text) - parts.append(text[last:stop_char]) - return "".join(parts).strip(), i - - -def _rw_case(text, toks, i): - """Rewrite a ``CASE … END`` starting at ``toks[i]`` (the CASE keyword).""" - i += 1 # consume CASE - operand, i = _rw_value(text, toks, i, _CASE_STOPS_OPERAND, toks[i - 1][3]) - is_simple = bool(operand) - branches: List[Tuple[str, str]] = [] - while i < len(toks) and _case_keyword(text, toks[i]) == "WHEN": - i += 1 - cond, i = _rw_value(text, toks, i, frozenset({"THEN"}), toks[i - 1][3]) - if not (i < len(toks) and _case_keyword(text, toks[i]) == "THEN"): - raise ValueError( - f"Malformed CASE expression in {text!r}: a WHEN branch is " - f"missing its THEN." - ) - i += 1 - then_val, i = _rw_value( - text, toks, i, frozenset({"WHEN", "ELSE", "END"}), toks[i - 1][3], - ) - branches.append((cond, then_val)) - else_val = "None" - if i < len(toks) and _case_keyword(text, toks[i]) == "ELSE": - i += 1 - else_val, i = _rw_value(text, toks, i, frozenset({"END"}), toks[i - 1][3]) - else_val = else_val or "None" - if not (i < len(toks) and _case_keyword(text, toks[i]) == "END"): - raise ValueError( - f"Malformed CASE expression in {text!r}: missing END." - ) - i += 1 # consume END - if not branches: - raise ValueError( - f"Malformed CASE expression in {text!r}: needs at least one " - f"WHEN … THEN branch." + +def _fail(detail: str) -> pp.ParserElement: + """Matches nothing and raises ``detail`` — the committed branch of a construct.""" + def _raise(_s: str, _loc: int, _toks: pp.ParseResults) -> None: + raise _Problem(detail) + return pp.Empty().set_parse_action(_raise) + + +def _need(element: pp.ParserElement, detail: str) -> pp.ParserElement: + return element | _fail(detail) + + +_RUN_CHAR = r"[^\x00-\x20\x7f!-/:-@\[-^`{-~\s]" +_RUN = r"(?!\d)" + _RUN_CHAR + r"+" + + +class _Word: + """A lexed identifier: the raw spelling, its NFKC name, and whether it was double-quoted.""" + + __slots__ = ("raw", "name", "quoted") + + def __init__(self, raw: str, name: str, quoted: bool) -> None: + self.raw = raw + self.name = name + self.quoted = quoted + + +def _check_reserved(name: str) -> None: + if name.startswith(_RESERVED_EXPR_PREFIX): + raise _Problem( + f"the identifier {name!r} uses the reserved {_RESERVED_EXPR_PREFIX!r} prefix, which is " + f"reserved for SLayer-internal names." ) - result = else_val - for cond, then_val in reversed(branches): - if is_simple: - cond_expr = f"{operand} == {cond}" - else: - cond_expr = _normalize_sql_filter_operators(cond) - result = f"iif({cond_expr}, {then_val}, {result})" - return result, i - - -# Tokens of the call-shape rewriters (``extract``, ``count(distinct …)``): literals -# whole, identifiers Unicode-aware, so keyword text inside either never matches. -_CALL_TOKEN_RE = re.compile( - r"(?P'(?:\\.|[^'\\])*'|\"(?:\\.|[^\"\\])*\")" - r"|(?P[^\W\d]\w*)" - r"|(?P\()|(?P\))|(?P,)" - r"|(?P==|!=|<=|>=|=)" - r"|(?P\S)", - re.DOTALL, -) -_Token = Tuple[str, str, int, int] -_DISTINCT_ONLY = "DISTINCT is supported only as count(distinct x), i.e. count_distinct(x)" -def _call_tokens(text: str) -> List[_Token]: - return [(m.lastgroup or "", m.group(), m.start(), m.end()) for m in _CALL_TOKEN_RE.finditer(text)] +def _bare_word(s: str, loc: int, toks: pp.ParseResults) -> _Word: + raw = toks[0] + name = unicodedata.normalize("NFKC", raw) + if not name.isidentifier(): + raise pp.ParseException(s, loc, f"{raw!r} is not an identifier") + _check_reserved(name) + return _Word(raw=raw, name=name, quoted=False) -def _is_word(text: str, toks: List[_Token], i: int, word: str) -> bool: - """Whether ``toks[i]`` is the bare ASCII keyword ``word`` (any case), not part of a reference.""" - kind, val, start, end = toks[i] - return ( - kind == "id" and val.isascii() and val.lower() == word - and not _is_ident_adjacent(text, start - 1) and not _continues_ref(text=text, pos=end) - and not (i > 0 and toks[i - 1][1] == ".") - ) +def _quoted_word(toks: pp.ParseResults) -> _Word: + raw = toks[0] + name = raw[1:-1].replace('""', '"') + if not name: + raise _Problem('an empty quoted identifier "" names nothing.') + _check_reserved(name) + return _Word(raw=raw, name=name, quoted=True) -def _matching_rp(toks: List[_Token], lp: int) -> Optional[int]: - depth = 0 - for j in range(lp, len(toks)): - if toks[j][0] == "lp": - depth += 1 - elif toks[j][0] == "rp": - depth -= 1 - if depth == 0: - return j - return None +def _kw(*words: str) -> pp.ParserElement: + """An ASCII-exact, case-insensitive keyword (a Unicode case-fold spoof is never one).""" + wanted = frozenset(words) + return _t(pp.Regex(_RUN).add_condition( + lambda toks: toks[0].isascii() and toks[0].lower() in wanted, + ).set_name("/".join(w.upper() for w in words))) -def _top_level(toks: List[_Token], lp: int, rp: int) -> Iterator[int]: - """Indices of the tokens directly inside the parens at ``lp`` / ``rp``.""" - depth = 0 - for j in range(lp + 1, rp): - kind = toks[j][0] - if kind == "lp": - depth += 1 - elif kind == "rp": - depth -= 1 - elif depth == 0: - yield j +def _sym(pattern: str, name: str) -> pp.ParserElement: + return _t(pp.Regex(pattern).set_name(name)) -def _extract_call(text: str, toks: List[_Token], i: int) -> Optional[Tuple[int, int]]: - """``(rp, from)`` token indices when ``toks[i]`` opens ``extract(… from …)``.""" - if not (_is_word(text, toks, i, "extract") and i + 1 < len(toks) and toks[i + 1][0] == "lp"): - return None - rp = _matching_rp(toks, i + 1) - if rp is None: - return None - frm = next((j for j in _top_level(toks, i + 1, rp) if _is_word(text, toks, j, "from")), None) - return None if frm is None else (rp, frm) - - -def _rewrite_extract(text: str, *, original: str) -> str: - """``extract( from x)`` → ``date_part('', x)``.""" - if "extract" not in text.lower(): - return text - toks = _call_tokens(text) - out: List[str] = [] - last = i = 0 - while i < len(toks): - call = _extract_call(text, toks, i) - if call is None: - i += 1 - continue - rp, frm = call - part = toks[i + 2:frm] - operand = text[toks[frm][3]:toks[rp][2]].strip() - if len(part) != 1 or part[0][0] not in ("id", "str") or not operand: - raise ValueError( - f"Invalid Mode-B expression {original!r}: write extract( from ), " - f"e.g. extract(day from created_at)." - ) - word = part[0][1] if part[0][0] == "str" else f"'{part[0][1]}'" - out += [text[last:toks[i][2]], f"date_part({word}, {_rewrite_extract(operand, original=original)})"] - last, i = toks[rp][3], rp + 1 - out.append(text[last:]) - return "".join(out) +LPAR, RPAR = pp.Suppress(_sym(r"\(", "'('")), pp.Suppress(_sym(r"\)", "')'")) +LBRACK, RBRACK = pp.Suppress(_sym(r"\[", "'['")), pp.Suppress(_sym(r"\]", "']'")) +COMMA = pp.Suppress(_sym(",", "','")) +DOT = pp.Suppress(_sym(r"\.", "'.'")) +STAR = _sym(r"\*(?!\*)", "'*'") +COLON = pp.Suppress(_sym(r":(?!:)", "':'")) +DCOLON = pp.Suppress(_sym("::", "'::'")) +EQ = pp.Suppress(_sym(r"=(?!=)", "'='")) +_BARE = _t(pp.Regex(_RUN).set_name("identifier")).set_parse_action(_bare_word) +_QUOTED = _t(pp.Regex(r'"(?:[^"]|"")*"').set_name("quoted identifier")).set_parse_action(_quoted_word) +_PLACEHOLDER = _t(pp.Regex(_PH_OPEN + "(" + _VAR_NAME + ")" + _PH_CLOSE).set_name("variable")) -def _single_distinct_operand(toks: List[_Token], i: int, rp: int) -> bool: - """Whether ``distinct`` at ``toks[i]`` is followed by one non-star operand and only keyword arguments.""" - commas = [j for j in _top_level(toks, i - 1, rp) if toks[j][0] == "comma"] - operand = toks[i + 1:commas[0] if commas else rp] - if not operand or [t[1] for t in operand] == ["*"]: - return False - return all(toks[c + 1][0] == "id" and toks[c + 2][1] == "=" for c in commas) +def _number(toks: pp.ParseResults) -> Literal: + text = toks[0] + return Literal(value=Decimal(int(text)) if text.isdigit() else Decimal(str(float(text)))) -def _in_call_top_level(toks: List[_Token], i: int) -> bool: - """Whether ``toks[i]`` sits directly inside a call's parentheses.""" - depth = 0 - for j in range(i - 1, -1, -1): - kind = toks[j][0] - if kind == "rp": - depth += 1 - elif kind == "lp": - if depth == 0: - return j > 0 and toks[j - 1][0] == "id" - depth -= 1 - return False - - -def _is_later_distinct_arg(text: str, toks: List[_Token], i: int) -> bool: - """Whether ``toks[i]`` is a ``distinct`` keyword opening a non-first call argument or a keyword value.""" - opens_arg = 0 < i < len(toks) - 1 and (toks[i - 1][0] == "comma" or toks[i - 1][1] == "=") - if not (opens_arg and _is_word(text, toks, i, "distinct")): - return False - kind, val = toks[i + 1][:2] - return (kind in ("id", "str", "lp") or val.isdigit() or val == "*") and _in_call_top_level(toks, i) - - -def _rewrite_count_distinct(text: str, *, original: str) -> str: - """``count(distinct x, …)`` → ``count_distinct(x, …)``; any other ``distinct`` keyword in a call raises.""" - if "distinct" not in text.lower(): - return text - toks = _call_tokens(text) - edits: List[Tuple[int, int, str]] = [] - for i in range(2, len(toks)): - if _is_later_distinct_arg(text, toks, i): - raise ValueError(f"Invalid Mode-B expression {original!r}: {_DISTINCT_ONLY}.") - if not (_is_word(text, toks, i, "distinct") and toks[i - 1][0] == "lp" and toks[i - 2][0] == "id"): - continue - rp = _matching_rp(toks, i - 1) - if rp is None: - continue - if not (_is_word(text, toks, i - 2, "count") and _single_distinct_operand(toks, i, rp)): - raise ValueError(f"Invalid Mode-B expression {original!r}: {_DISTINCT_ONLY}.") - edits += [(toks[i - 2][2], toks[i - 2][3], "count_distinct"), (toks[i][2], toks[i][3], "")] - for start, end, repl in reversed(edits): - text = text[:start] + repl + text[end:] - return text +def _string(toks: pp.ParseResults) -> Literal: + return Literal(value=toks[0][1:-1].replace("''", "'")) -_COLON_AGG_RE = re.compile( - r"(\*|[a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*(?:\.\*)?)" # source: * / ident / dotted - r":" - r"([a-zA-Z_]\w*)" - # No (args) consumption — Python's AST handles that. -) -_FILTER_KEYWORDS = frozenset({"and", "or", "not", "in", "is"}) -_SCAN_TOKEN_RE = re.compile( - r"(?P\s+)" - r"|(?P'(?:\\.|[^'\\])*'|\"(?:\\.|[^\"\\])*\")" - r"|(?P[A-Za-z_]\w*)" - r"|(?P\d+(?:\.\d*)?|\.\d+)" - r"|(?P==|<=|>=|!=)" - r"|(?P=)" - r"|(?P\()" - r"|(?P\))" - r"|(?P\[)" - r"|(?P\])" - r"|(?P,)" - r"|(?P.)", - re.DOTALL, -) +_NUMBER = _t(pp.Regex(r"(?:\d+\.\d*|\.\d+|\d+)(?:[eE][+-]?\d+)?").set_name("number")).set_parse_action(_number) +_STRING = _t(pp.Regex(r"'(?:[^']|'')*'").set_name("string")).set_parse_action(_string) +_CONSTANT_WORDS = {"True": True, "False": False, "None": None} +_SQL_CONSTANT_WORDS = {"true": True, "false": False, "null": None} + + +def _constant_value(raw: str) -> Tuple[bool, Any]: + if raw in _CONSTANT_WORDS: + return True, _CONSTANT_WORDS[raw] + if raw.isascii() and raw.lower() in _SQL_CONSTANT_WORDS: + return True, _SQL_CONSTANT_WORDS[raw.lower()] + return False, None + + +def _constant(s: str, loc: int, toks: pp.ParseResults) -> Literal: + known, value = _constant_value(toks[0]) + if not known: + raise pp.ParseException(s, loc, "not a constant") + return Literal(value=value) + + +_CONSTANT = _t(pp.Regex(_RUN).set_name("constant")).set_parse_action(_constant) + + +def _placeholder_node(s: str, loc: int, toks: pp.ParseResults) -> Placeholder: + end = loc + len(toks[0]) + rest = s[end:].lstrip() + if rest.startswith(("(", ".")) or (rest.startswith(":") and not rest.startswith("::")): + raise _Problem(f"{_LITERAL_VALUES}; a variable cannot name a function, column or path.") + return Placeholder(name=toks[0][1:-1]) + +def _reject_placeholder(_s: str, _loc: int, _toks: pp.ParseResults) -> None: + raise _Problem(f"{_LITERAL_VALUES}; a variable cannot name a column or path segment.") -_BIN_OP_MAP: Dict[type, str] = { - ast.Add: "+", ast.Sub: "-", ast.Mult: "*", ast.Div: "/", - ast.Mod: "%", ast.Pow: "**", ast.FloorDiv: "//", -} -_CMP_OP_MAP: Dict[type, str] = { - ast.Eq: "==", ast.NotEq: "!=", - ast.Lt: "<", ast.LtE: "<=", - ast.Gt: ">", ast.GtE: ">=", - # SQL ``IS [NOT] NULL`` lowers to ``is [not] None``; rendered back as - # ``IS [NOT] NULL`` by the SQL generator. - ast.Is: "is", ast.IsNot: "is not", - # ``IN`` / ``NOT IN`` with a literal-tuple RHS (shape enforced in - # the ``ast.Compare`` branch). - ast.In: "in", ast.NotIn: "not in", -} + +def _reject_placeholder_kwarg(_s: str, _loc: int, _toks: pp.ParseResults) -> None: + raise _Problem(f"{_LITERAL_VALUES}; a variable cannot name a keyword argument.") # --------------------------------------------------------------------------- -# Public entry point +# Grammar # --------------------------------------------------------------------------- -def parse_expr(text: str) -> ParsedExpr: - """Parse a Mode-B expression string into a ``ParsedExpr``. +pp.ParserElement.enable_packrat() - ``__`` in identifiers is legal; only the ``__slayer_`` prefix is - reserved. Raises ``ValueError`` (empty/syntax/unsupported node/chained - comparison/reserved prefix), ``UnknownFunctionError``, or - ``IllegalWindowInFilterError`` (raw ``OVER(...)``). - """ - if not text or not text.strip(): - raise ValueError("Empty Mode-B expression.") +expr = pp.Forward().set_name("expression") +_concat_level = pp.Forward() - _reject_reserved_expr_token(text) +_SEGMENT = _BARE | _QUOTED +_PATH = pp.Group(_SEGMENT + pp.ZeroOrMore(DOT + (_SEGMENT | _PLACEHOLDER.copy().set_parse_action(_reject_placeholder)))) - # Scan for raw ``OVER(`` after blanking string literals so a quoted value - # (``status == 'OVER('``) isn't mistaken for window usage. Skip an ``OVER`` - # that only continues a reference — a Unicode-fused (``℘OVER(``) or dotted - # (``a.OVER(``) name the leading ``\b`` splits. - blanked = _PY_STRING_LITERAL_RE.sub("", text) - if any( - not _continues_ref(text=blanked, pos=m.start() - 1) - for m in _OVER_RE.finditer(blanked) - ): - raise IllegalWindowInFilterError( - filter_expr=text, - source="raw OVER(...) is not allowed in Mode-B DSL", - suggestion=( - "use a transform instead (rank, percent_rank, dense_rank, " - "ntile, cumsum, lag, lead, time_shift, …)." - ), - ) - sql_spelled = _rewrite_count_distinct(_rewrite_extract(text, original=text), original=text) - preprocessed, agg_map = _preprocess_colons(_rewrite_case_when(sql_spelled)) - # AFTER the legacy colon pass — ``customers.*:count`` must become a - # placeholder first, so the only ``*`` left in call-first-arg position is - # the functional spelling (``count(*)`` / ``count(customers.*)``). - preprocessed = _preprocess_star_args(preprocessed) +def _path_node(words: List[_Word]) -> Ref | DottedRef: + quoted = any(w.quoted for w in words) + if len(words) == 1: + return Ref(name=words[0].name, quoted=quoted) + return DottedRef(parts=tuple(w.name for w in words), quoted=quoted) - try: - py_ast = ast.parse(preprocessed, mode="eval").body - except SyntaxError as e: - raise ValueError( - f"Invalid Mode-B expression {text!r}: {e}" - ) - parsed = _convert(py_ast, agg_map=agg_map, original=text) - if any(_is_interval(n) for n in _walk_parsed(parsed)): - raise ValueError( - f"Invalid Mode-B expression {text!r}: interval(n, unit) is only valid as " - f"`x + interval(n, unit)`, `interval(n, unit) + x` or `x - interval(n, unit)`; " - f"otherwise use date_add(x, n, unit)." - ) - return parsed +_PATH_STAR = pp.Group(_PATH + DOT + pp.Suppress(STAR)) -def parse_filter_expr(text: str) -> ParsedExpr: - """Parse a Mode-B *filter* string, accepting SQL operator spellings. +class _Kw: + __slots__ = ("name", "value") - Filters historically accepted SQL-style operators (``=``, ``<>``, ``NULL``, - and the keyword forms ``AND`` / ``OR`` / ``NOT`` / ``IS`` / ``IN``) - alongside the Python spellings. This wrapper normalizes those to their - Python equivalents (string-literal-aware, so quoted contents are - untouched) and then delegates to :func:`parse_expr`. Measures / order use - ``parse_expr`` directly — only filters get the SQL-operator leniency. - """ - return parse_expr(_normalize_sql_filter_operators(text)) + def __init__(self, name: str, value: Any) -> None: + self.name = name + self.value = value + + +class _Distinct: + __slots__ = ("value",) + def __init__(self, value: Any) -> None: + self.value = value -# SQL keyword rewrites as explicit ASCII character classes — ``re.IGNORECASE`` -# folds Unicode spoofs (``ıs`` → ``is``) into keywords. -_SQL_NULL_RE = re.compile(r"\b[nN][uU][lL][lL]\b") -_SQL_KEYWORD_RES: Tuple[Tuple[re.Pattern, str], ...] = tuple( - (re.compile(r"\b" + "".join(f"[{c}{c.upper()}]" for c in kw) + r"\b"), kw) - for kw in ("is", "not", "and", "or", "in") + +_ARG_END = pp.FollowedBy(_sym(r"[,)]", "',' or ')'")) +_STAR_ARG = (pp.Suppress(STAR) + _ARG_END).set_parse_action(lambda: StarSource()) +_PATH_STAR_ARG = (_PATH_STAR + _ARG_END).set_parse_action( + lambda toks: DottedRef(parts=(*(w.name for w in toks[0][0]), "*"), quoted=any(w.quoted for w in toks[0][0])), +) +_SEQ_ITEMS = pp.Group(pp.Optional(pp.DelimitedList(expr, allow_trailing_delim=True))) +_LIST_VALUE = (LBRACK + _SEQ_ITEMS + RBRACK).set_parse_action(lambda toks: tuple(toks[0])) +_TUPLE_VALUE = (LPAR + pp.Group(expr + pp.OneOrMore(COMMA + expr) + pp.Optional(COMMA)) + RPAR).set_parse_action( + lambda toks: tuple(toks[0]), +) +_DISTINCT_VALUE = _kw("distinct") + _fail(f"{_DISTINCT_ONLY}.") +_KW_VALUE = _DISTINCT_VALUE | _LIST_VALUE | _TUPLE_VALUE | _STAR_ARG | _PATH_STAR_ARG | expr +_KWARG = (_BARE + EQ + _KW_VALUE).set_parse_action(lambda toks: _Kw(name=toks[0].name, value=toks[1])) +_PH_KWARG = (_PLACEHOLDER + EQ).set_parse_action(_reject_placeholder_kwarg) +_VALUE_ARG = _STAR_ARG | _PATH_STAR_ARG | expr +_DISTINCT_ARG = (pp.Suppress(_kw("distinct")) + _need(_VALUE_ARG, f"{_DISTINCT_ONLY}.")).set_parse_action( + lambda toks: _Distinct(value=toks[0]), ) +_UNPACKING = _sym(r"\*\*", "'**'") + _fail("dictionary unpacking (**kwargs) is not supported in calls.") +_ANY_ARG = _UNPACKING | _PH_KWARG | _KWARG | _DISTINCT_ARG | _VALUE_ARG +_POSITIONAL_ARG = _UNPACKING | _PH_KWARG | _DISTINCT_ARG | _VALUE_ARG -def _normalize_sql_filter_operators(text: str) -> str: - """Rewrite SQL operator spellings to Python ones outside string literals. +def _arg_list(first: pp.ParserElement, rest: pp.ParserElement) -> pp.ParserElement: + return pp.Group(pp.Optional(first + pp.ZeroOrMore(COMMA + rest) + pp.Optional(COMMA))) - ``NULL`` → ``None``; ``IS`` / ``NOT`` / ``AND`` / ``OR`` / ``IN`` → - lowercase; standalone ``=`` → ``==``; ``<>`` → ``!=``; ``col [NOT] LIKE - 'p%'`` → ``[not ]like(col, 'p%')``. - """ - # LIKE first, on the whole string (its pattern is a quoted literal). The - # rest run per non-literal part (escape-aware split) so a keyword inside a - # quoted value isn't rewritten. - text = _rewrite_sql_like(text) - parts = _PY_STRING_LITERAL_RE.split(text) - literals = _PY_STRING_LITERAL_RE.findall(text) - result: List[str] = [] - for i, part in enumerate(parts): - part = _sub_keyword_isolated(_SQL_NULL_RE, "None", part) - for kw_re, kw in _SQL_KEYWORD_RES: - part = _sub_keyword_isolated(kw_re, kw, part) - part = part.replace("<>", "!=") - # SQL ``||`` → Python ``|`` (BitOr), desugared to ``concat`` in - # ``_convert``; same precedence relative to comparisons. - part = part.replace("||", "|") - result.append(part) - if i < len(literals): - result.append(literals[i]) - # ``=`` → ``==`` runs last, on the rejoined string, via a paren-aware scanner - # that leaves kwarg ``=`` inside non-scalar calls alone. - return _rewrite_comparison_equals("".join(result)) - - -def _classify_paren( - hist: List[Tuple[str, str]], -) -> Tuple[bool, Optional[str]]: - """Classify an open ``(`` as a CALL or GROUPING paren. - - A call paren follows a bare identifier or a ``)`` / ``]``; lowercase keywords - (``and`` / ``not`` / ``in`` …) do not open one. After a legacy colon - (``revenue:first(...)``) the callee is dropped to ``None`` so - :func:`_is_kwarg_equals` treats it as an aggregation, not a transform. - """ - prev_kind = hist[-1][0] if hist else None - prev_text = hist[-1][1] if hist else "" - is_call = prev_kind == "NAME" or prev_text in (")", "]") - callee = hist[-1][1] if (is_call and prev_kind == "NAME") else None - if callee is not None and len(hist) >= 2 and hist[-2] == ("OTHER", ":"): - callee = None - return is_call, callee - - -def _is_kwarg_equals( - stack: List[Tuple[bool, Optional[str]]], - hist: List[Tuple[str, str]], -) -> bool: - """Whether a lone ``=`` is a Python keyword-argument separator. - - Scalars never take kwargs. Transforms take the value first, so a kwarg only - follows a ``,`` (keeping ``consecutive_periods(status = 'paid')`` a - predicate). Aggregations/unknowns may take a kwarg first, so ``=`` follows - ``(`` or ``,``. - """ - top = stack[-1] if stack else None - if top is None or not top[0]: - return False - callee = top[1] - if callee is not None and callee.lower() in SCALAR_FUNCTIONS: - return False - prev_kind = hist[-1][0] if hist else None - if prev_kind != "NAME": - return False - prev_prev_kind = hist[-2][0] if len(hist) >= 2 else None - if callee is not None and callee in ALL_TRANSFORMS: - return prev_prev_kind == "COMMA" - return prev_prev_kind in ("LPAREN", "COMMA") - - -def _push_hist(hist: List[Tuple[str, str]], kind: str, text: str) -> None: - """Append ``(kind, text)`` and trim ``hist`` to the last 2 entries.""" - hist.append((kind, text)) - if len(hist) > 2: - del hist[0] - - -def _handle_pass_through( - m: "re.Match[str]", - out: List[str], - stack: List[Tuple[bool, Optional[str]]], - hist: List[Tuple[str, str]], -) -> None: - """Whitespace and string literals: emit verbatim, don't touch hist.""" - out.append(m.group(0)) +def _callee_is(kind: str): + def _check(toks: pp.ParseResults) -> bool: + name = toks[0][-1].name + if kind == "scalar": + return name.lower() in SCALAR_FUNCTIONS or name.lower() in DATE_PART_ALIASES + return normalize_aggregation_name(name) in BUILTIN_AGGREGATIONS or name in ALL_TRANSFORMS + return _check -def _handle_ident( - m: "re.Match[str]", - out: List[str], - stack: List[Tuple[bool, Optional[str]]], - hist: List[Tuple[str, str]], -) -> None: - ident = m.group(0) - out.append(ident) - _push_hist(hist, "KW" if ident in _FILTER_KEYWORDS else "NAME", ident) +_SCALAR_CALL = _PATH.copy().add_condition(_callee_is("scalar")) + LPAR + _arg_list(_POSITIONAL_ARG, _POSITIONAL_ARG) + RPAR +_VALUE_FIRST_CALL = _PATH.copy().add_condition(_callee_is("value")) + LPAR + _arg_list(_POSITIONAL_ARG, _ANY_ARG) + RPAR +_OTHER_CALL = _PATH + LPAR + _arg_list(_ANY_ARG, _ANY_ARG) + RPAR +_CALL = _SCALAR_CALL | _VALUE_FIRST_CALL | _OTHER_CALL -def _handle_other( - m: "re.Match[str]", - out: List[str], - stack: List[Tuple[bool, Optional[str]]], - hist: List[Tuple[str, str]], -) -> None: - """Numbers, compound ops (``==``/``<=``/``>=``/``!=``), brackets, and - any single char not otherwise classified (``:``, ``.``, ``+``, ``-``, - ``*``, ``/``, ``%``, ``<``, ``>``, ``!``, ``|``, ``{``, ``}``, ...).""" - text = m.group(0) - out.append(text) - _push_hist(hist, "OTHER", text) - - -def _handle_comma( - m: "re.Match[str]", - out: List[str], - stack: List[Tuple[bool, Optional[str]]], - hist: List[Tuple[str, str]], -) -> None: - out.append(",") - _push_hist(hist, "COMMA", ",") +def _split_args(items: List[Any], *, call: str) -> Tuple[Tuple[Any, ...], Tuple[Tuple[str, Any], ...], bool]: + positional: List[Any] = [] + kwargs: List[Tuple[str, Any]] = [] + distinct = False + for i, item in enumerate(items): + if isinstance(item, _Kw): + if any(k == item.name for k, _ in kwargs): + raise _Problem(f"{call}() got keyword argument {item.name!r} more than once.") + kwargs.append((item.name, item.value)) + continue + if kwargs: + raise _Problem(f"{call}(): a positional argument follows a keyword argument.") + if isinstance(item, _Distinct): + if i != 0: + raise _Problem(f"{_DISTINCT_ONLY}.") + distinct = True + item = item.value + if isinstance(item, tuple): + raise _Problem(f"{call}(): a list is only valid as a keyword-argument value.") + positional.append(item) + return tuple(positional), tuple(kwargs), distinct + + +class _PendingCall: + """A parsed call, dispatched once its postfix is known (a trailing ``OVER (`` rejects first).""" + + __slots__ = ("words", "items") + + def __init__(self, words: List[_Word], items: List[Any]) -> None: + self.words = words + self.items = items + + +def _resolve(node: Any) -> Any: + return _dispatch_pending(node) if isinstance(node, _PendingCall) else node + + +def _dispatch_pending(call: _PendingCall) -> Any: + words = call.words + name = words[-1].name + args, kwargs, distinct = _split_args(call.items, call=name) + if distinct: + if name.lower() != "count" or len(words) > 1 or len(args) != 1 or isinstance(args[0], StarSource): + raise _Problem(f"{_DISTINCT_ONLY}.") + name = "count_distinct" + if len(words) > 1: + return _method_call(path=tuple(w.name for w in words[:-1]), name=name, args=args, kwargs=kwargs) + return _dispatch_call(name, args=args, kwargs=kwargs, original=_STATE.text) + + +_CALL.set_parse_action(lambda toks: _PendingCall(words=list(toks[0]), items=list(toks[1]))) + +_COLON_SOURCE = (pp.Suppress(STAR).set_parse_action(lambda: StarSource()) + | _PATH_STAR.copy().set_parse_action( + lambda toks: DottedRef(parts=(*(w.name for w in toks[0][0]), "*"), + quoted=any(w.quoted for w in toks[0][0]))) + | _PATH.copy().set_parse_action(lambda toks: _path_node(list(toks[0])))) +_COLON_AGG = _COLON_SOURCE + COLON + _BARE + pp.Optional(LPAR + _arg_list(_ANY_ARG, _ANY_ARG) + RPAR) + + +def _colon_node(toks: pp.ParseResults) -> AggCall: + source, agg = toks[0], toks[1].name + args, kwargs, distinct = _split_args(list(toks[2]) if len(toks) > 2 else [], call=agg) + if distinct: + raise _Problem(f"{_DISTINCT_ONLY}.") + _reject_bare_star_args(args, kwargs, func_name=agg, original=_STATE.text) + return AggCall(source=source, agg=agg, args=args, kwargs=kwargs) + + +_COLON_AGG.set_parse_action(_colon_node) +_REF = _PATH.copy().set_parse_action(lambda toks: _path_node(list(toks[0]))) + +# -- CASE ---------------------------------------------------------------------- + +_CASE_WORDS = ("case", "when", "then", "else", "end") +WHEN, THEN, ELSE, END = (pp.Suppress(_kw(w)) for w in _CASE_WORDS[1:]) +_MISSING_THEN = "Malformed CASE expression: a WHEN branch is missing its THEN." +_WHEN_CLAUSE = pp.Group(WHEN + _need(expr, "Malformed CASE expression: a WHEN needs a condition.") + + _need(THEN, _MISSING_THEN) + + _need(expr, "Malformed CASE expression: a THEN needs a value.")) +_CASE_TAIL = (pp.Group(pp.OneOrMore(_WHEN_CLAUSE)) + + pp.Group(pp.Optional(ELSE + _need(expr, "Malformed CASE expression: an ELSE needs a value."))) + + _need(END, "Malformed CASE expression: missing END.")) +_SEARCHED_CASE = pp.Suppress(_kw("case")) + pp.FollowedBy(WHEN) + pp.Group(pp.Empty()) + _CASE_TAIL +# A bare keyword-named simple-CASE operand (``CASE case WHEN …``) is ambiguous: no CASE. +_SIMPLE_CASE = (pp.Suppress(_kw("case")) + ~(_kw(*_CASE_WORDS) + WHEN) + + pp.Group(expr) + pp.FollowedBy(WHEN) + _CASE_TAIL) + + +def _case_node(toks: pp.ParseResults) -> ScalarCall: + operand = toks[0][0] if len(toks[0]) else None + branches = [(b[0], b[1]) for b in toks[1]] + result = toks[2][0] if len(toks[2]) else Literal(value=None) + for cond, value in reversed(branches): + test = Cmp(op="==", left=operand, right=cond) if operand is not None else cond + result = ScalarCall(name="iif", args=(test, value, result)) + return result -def _handle_lparen( - m: "re.Match[str]", - out: List[str], - stack: List[Tuple[bool, Optional[str]]], - hist: List[Tuple[str, str]], -) -> None: - stack.append(_classify_paren(hist)) - out.append("(") - _push_hist(hist, "LPAREN", "(") +_CASE = (_SEARCHED_CASE | _SIMPLE_CASE).set_parse_action(_case_node) -def _handle_rparen( - m: "re.Match[str]", - out: List[str], - stack: List[Tuple[bool, Optional[str]]], - hist: List[Tuple[str, str]], -) -> None: - if stack: - stack.pop() - out.append(")") - _push_hist(hist, "OTHER", ")") +# -- CAST / EXTRACT / POSITION / SUBSTRING / TRIM ------------------------------ +_TYPE_EXTRA_WORDS = ("precision", "varying", "with", "without", "time", "zone", "local", "unsigned", "signed") +_TYPE_WORD = _t(pp.Regex(r"[A-Za-z_][A-Za-z0-9_]*").set_name("type name")) +_TYPE_SPEC = (pp.Group(_TYPE_WORD + pp.ZeroOrMore(_kw(*_TYPE_EXTRA_WORDS))) + + pp.Group(pp.Optional(LPAR + pp.DelimitedList(_sym(r"\d+", "type parameter")) + RPAR))) -def _handle_op_eq( - m: "re.Match[str]", - out: List[str], - stack: List[Tuple[bool, Optional[str]]], - hist: List[Tuple[str, str]], -) -> None: - out.append("=" if _is_kwarg_equals(stack, hist) else "==") - _push_hist(hist, "OTHER", "=") - - -_HANDLERS: Dict[str, Any] = { - "ws": _handle_pass_through, - "string": _handle_pass_through, - "ident": _handle_ident, - "number": _handle_other, - "op_eq2": _handle_other, - "op_eq": _handle_op_eq, - "lparen": _handle_lparen, - "rparen": _handle_rparen, - "lbrack": _handle_other, - "rbrack": _handle_other, - "comma": _handle_comma, - "other": _handle_other, -} - - -def _rewrite_comparison_equals(text: str) -> str: - """Rewrite SQL ``=`` to Python ``==`` except where it is a kwarg separator. - - A lone ``=`` is kept (kwarg) inside a non-scalar CALL paren when it follows - ``ident`` after ``(`` or ``,`` — Python's kwarg grammar (see - :func:`_is_kwarg_equals`); inside scalars a ``=`` is always the user's - comparison. Compound ops and string literals pass through untouched. - Token dispatch is keyed by the scan regex's ``lastgroup``. - """ - out: List[str] = [] - # Each frame: (is_call, callee) — callee is the ident before ``(``, else None. - stack: List[Tuple[bool, Optional[str]]] = [] - # Last 2 significant tokens (NAME / KW / LPAREN / COMMA / OTHER; no strings). - hist: List[Tuple[str, str]] = [] - for m in _SCAN_TOKEN_RE.finditer(text): - _HANDLERS[m.lastgroup](m, out, stack, hist) - return "".join(out) +def _type_text(toks: pp.ParseResults) -> str: + words, params = toks + text = " ".join(w.upper() for w in words) + return f"{text}({','.join(params)})" if len(params) else text -# --------------------------------------------------------------------------- -# Reference walk (best-effort textual extraction) -# --------------------------------------------------------------------------- +_TYPE_SPEC.set_parse_action(_type_text) +_CAST = ((pp.Suppress(_kw("cast")) | _kw("try_cast")) + LPAR + + _need(expr, "CAST needs a value: CAST(x AS ).") + + _need(pp.Suppress(_kw("as")), "CAST needs AS : CAST(x AS ).") + + _need(_TYPE_SPEC, "CAST needs a type after AS.") + + _need(RPAR, "CAST(x AS ) is missing its ')'.")) -def walk_parsed_refs( - parsed: ParsedExpr, -) -> Iterator[Ref | DottedRef | AggCall]: - """Yield the reference-bearing leaves (``Ref`` / ``DottedRef`` / ``AggCall``) - of a tree — scope-free name extraction for schema-drift / memory tagging. - Descent (matches the binder's walk): ``AggCall`` yielded whole (args/kwargs - opaque, so ``weighted_avg(weight=quantity)`` surfaces ``price`` not - ``quantity``); ``TransformCall`` descends ``input`` only; ``ScalarCall`` / - arithmetic / comparison / boolean descend operands; literals yield nothing. - """ - if isinstance(parsed, (Ref, DottedRef, AggCall)): - yield parsed - return - if isinstance(parsed, TransformCall): - yield from walk_parsed_refs(parsed.input) - return - if isinstance(parsed, ScalarCall): - for a in parsed.args: - yield from walk_parsed_refs(a) - return - if isinstance(parsed, Arith): - yield from walk_parsed_refs(parsed.left) - yield from walk_parsed_refs(parsed.right) - return - if isinstance(parsed, Cmp): - yield from walk_parsed_refs(parsed.left) - yield from walk_parsed_refs(parsed.right) - return - if isinstance(parsed, UnaryOp): - yield from walk_parsed_refs(parsed.operand) - return - if isinstance(parsed, BoolOp): - for op in parsed.operands: - yield from walk_parsed_refs(op) - # Literal / Placeholder / StarSource / TupleLit → no references (TupleLit - # holds only Literals by construction). +def _cast_node(toks: pp.ParseResults) -> Cast: + try_ = len(toks) == 3 + return Cast(source=toks[-2], type_text=toks[-1], try_=try_) -# --------------------------------------------------------------------------- -# Internals -# --------------------------------------------------------------------------- +_CAST.set_parse_action(_cast_node) +_EXTRACT = (pp.Suppress(_kw("extract")) + LPAR + (_STRING | _BARE) + pp.Suppress(_kw("from")) + + expr + RPAR) -def _reject_reserved_expr_token(text: str) -> None: - """Reject the reserved ``__slayer_`` token in RAW Mode-B input (P3). - Runs BEFORE ``_preprocess_colons`` mints its own ``__slayer_agg_N__`` - placeholders, so a legacy colon-spelling ``revenue:sum`` is unaffected while a - literal ``__slayer_agg_0__`` spoof is rejected [C3]. String literals are - blanked first (Python syntax, so escapes count) so quoted data is never - mistaken for an identifier. Matched at an identifier boundary so a legal - embedded name like ``foo__slayer_bar`` stays referenceable. - """ - if _RESERVED_EXPR_PREFIX_RE.search(_PY_STRING_LITERAL_RE.sub("", text)): - raise ValueError( - f"Mode-B expression {text!r} uses the reserved " - f"{_RESERVED_EXPR_PREFIX!r} identifier prefix, which is reserved " - f"for SLayer-internal placeholders." - ) +def _extract_node(toks: pp.ParseResults) -> ScalarCall: + part, value = toks + word = part.value if isinstance(part, Literal) else part.name + return ScalarCall(name="date_part", args=_date_args( + name="date_part", args=(Literal(value=word), value), original=_STATE.text, + )) -def _preprocess_colons( - text: str, -) -> Tuple[str, Dict[int, Tuple[Ref | DottedRef | StarSource, str]]]: - """Replace the legacy colon spelling ``:`` with placeholder identifiers. +_EXTRACT.set_parse_action(_extract_node) +_POSITION = (pp.Suppress(_kw("position")) + LPAR + _concat_level + pp.Suppress(_kw("in")) + expr + RPAR).set_parse_action( + lambda toks: ScalarCall(name="instr", args=(toks[1], toks[0])), +) +_SUBSTRING = (pp.Suppress(_kw("substring")) + LPAR + expr + pp.Suppress(_kw("from")) + expr + + pp.Optional(pp.Suppress(_kw("for")) + expr) + RPAR).set_parse_action( + lambda toks: ScalarCall(name="substr", args=tuple(toks)), +) +_TRIM_FORM = (_kw("trim") + LPAR + _kw("both", "leading", "trailing")) + _fail( + "TRIM(BOTH | LEADING | TRAILING …) is not supported; use trim(x), ltrim(x) or rtrim(x).", +) - Captures source kind + agg name. Any trailing ``(args)`` is left in - place so Python's AST parses it naturally as a Call. String literal - spans are skipped — the literal text is user data, not DSL syntax. - """ - agg_map: Dict[int, Tuple[Ref | DottedRef | StarSource, str]] = {} - counter = [0] - literal_spans = [ - # Escape-aware, so escaped quotes don't leak colon rewrites into the string body. - (m.start(), m.end()) for m in _PY_STRING_LITERAL_RE.finditer(text) - ] - - def _in_literal(pos: int) -> bool: - return any(s <= pos < e for s, e in literal_spans) - - def _replace(match: re.Match) -> str: - if _in_literal(match.start()): - return match.group(0) - source_str = match.group(1) - agg_name = match.group(2) - source: Ref | DottedRef | StarSource - if source_str == "*": - source = StarSource() - elif "." in source_str: - source = DottedRef(parts=tuple(source_str.split("."))) - else: - source = Ref(name=source_str) - idx = counter[0] - counter[0] += 1 - agg_map[idx] = (source, agg_name) - return f"{_PLACEHOLDER_PREFIX}{idx}__" - - return _COLON_AGG_RE.sub(_replace, text), agg_map - - -def _preprocess_star_args(text: str) -> str: # NOSONAR(S3776) — one token-scan with two star shapes (bare ``(*``, dotted ``path.*``); the prev/next-significant-token checks ARE the grammar being recognised, and splitting them hides the call-first-argument contract. - """Replace a call's first-argument ``*`` / ``path.*`` with the reserved - star token (``count(*)`` → ``count(__slayer_star__)``) so Python's AST can - parse it. Token-aware, string-literal-safe; multiplication is untouched - (a multiplying ``*`` never directly follows a call's ``(`` or a ``.``). - """ - if "*" not in text: - return text - toks = [(m.lastgroup, m.group(0)) for m in _SCAN_TOKEN_RE.finditer(text)] - n = len(toks) - - def _prev(i: int) -> int: - j = i - 1 - while j >= 0 and toks[j][0] == "ws": - j -= 1 - return j - - def _next(i: int) -> int: - j = i + 1 - while j < n and toks[j][0] == "ws": - j += 1 - return j - - def _is_call_lparen(i: int) -> bool: - p = _prev(i) - return p >= 0 and toks[p][0] == "ident" - - def _dotted_star_opens_call(dot_idx: int) -> bool: - # Walk ``ident(.ident)*`` back from the ``.`` before the star; the - # path must open a call's first argument (directly after ``ident(``). - j = dot_idx - while True: - q = _prev(j) - if q < 0 or toks[q][0] != "ident": - return False - r = _prev(q) - if r >= 0 and toks[r] == ("other", "."): - j = r - continue - return r >= 0 and toks[r][0] == "lparen" and _is_call_lparen(r) - - out: List[str] = [] - for i, (kind, val) in enumerate(toks): - if kind == "other" and val == "*": - nx = _next(i) - closes_arg = nx < n and toks[nx][1] in (",", ")") - p = _prev(i) - if closes_arg and p >= 0: - if toks[p][0] == "lparen" and _is_call_lparen(p): - out.append(_STAR_ARG_TOKEN) - continue - if toks[p] == ("other", ".") and _dotted_star_opens_call(p): - out.append(_STAR_ARG_TOKEN) - continue - out.append(val) - return "".join(out) - - -def _convert(node: ast.AST, *, agg_map: Dict, original: str) -> ParsedExpr: # NOSONAR(S3776) — one-pass dispatch over ast node kinds (Constant/Name/Compare/BinOp/UnaryOp/BoolOp/Call/Attribute…) producing typed ParsedExpr; the branches are flat and short, and splitting hides the exhaustive ast-kind coverage one read scans for. Surfaces ParsedExpr.kind contract directly. - if isinstance(node, ast.Constant): - return _convert_constant(node, original=original) - - if isinstance(node, ast.Name): - m = _PLACEHOLDER_RE.match(node.id) - if m: - idx = int(m.group(1)) - source, agg = agg_map[idx] - return AggCall(source=source, agg=agg) - if node.id == _STAR_ARG_TOKEN: - return StarSource() - # SQL-cased boolean / null literals: Python's ast only treats True/False/None - # as constants, so `true`/`FALSE`/`NULL`/... arrive as names. All are - # reserved words in every target dialect, never columns. - if node.id.lower() in ("true", "false"): - return Literal(value=node.id.lower() == "true") - if node.id.lower() == "null": - return Literal(value=None) - return Ref(name=node.id) - - if isinstance(node, ast.Attribute): - parts = _flatten_attribute(node, original=original) - # ``count(customers.*)`` — the star pre-pass turned the trailing ``*`` - # into the star token; restore it so the DottedRef matches the legacy colon form. - parts = ["*" if p == _STAR_ARG_TOKEN else p for p in parts] - return DottedRef(parts=tuple(parts)) - - if isinstance(node, ast.Call): - return _convert_call(node, agg_map=agg_map, original=original) - - if isinstance(node, ast.BinOp): - op_type = type(node.op) - if op_type is ast.BitOr: - # SQL ``||`` (normalized to ``|``) desugars to the ``concat`` scalar. - return ScalarCall( - name="concat", - args=( - _convert(node.left, agg_map=agg_map, original=original), - _convert(node.right, agg_map=agg_map, original=original), - ), - ) - if op_type not in _BIN_OP_MAP: - raise ValueError( - f"Invalid Mode-B expression {original!r}: unsupported " - f"binary operator {op_type.__name__}." - ) - left = _convert(node.left, agg_map=agg_map, original=original) - right = _convert(node.right, agg_map=agg_map, original=original) - return _interval_as_date_add(op=_BIN_OP_MAP[op_type], left=left, right=right) or Arith( - op=_BIN_OP_MAP[op_type], left=left, right=right, - ) +# -- typed literals, intervals, clock keywords ---------------------------------- - if isinstance(node, ast.UnaryOp): - op_type = type(node.op) - if op_type is ast.USub: - return UnaryOp( - op="-", - operand=_convert(node.operand, agg_map=agg_map, original=original), - ) - if op_type is ast.UAdd: - # `+x` is a no-op; collapse to the operand directly. - return _convert(node.operand, agg_map=agg_map, original=original) - if op_type is ast.Not: - return UnaryOp( - op="not", - operand=_convert(node.operand, agg_map=agg_map, original=original), - ) - raise ValueError( - f"Invalid Mode-B expression {original!r}: unsupported unary " - f"operator {op_type.__name__}." - ) +_TEMPORAL = (_kw("date", "timestamp") + _STRING).set_parse_action( + lambda toks: TemporalLiteral(kind=toks[0].lower(), text=toks[1].value), +) +_TIME_LITERAL = (_kw("time") + _STRING) + _fail( + "TIME '…' literals are not supported (there is no time-of-day type); compare with a TIMESTAMP '…' literal.", +) +_UNIT_WORDS = frozenset(g.value for g in TimeGranularity) - if isinstance(node, ast.Compare): - if len(node.ops) != 1 or len(node.comparators) != 1: - raise ValueError( - f"Invalid Mode-B expression {original!r}: chained " - f"comparisons are not supported. Each Cmp must be a " - f"single comparison; split (e.g.) `1 < x < 10` into " - f"`1 < x and x < 10`." - ) - op_type = type(node.ops[0]) - if op_type not in _CMP_OP_MAP: - raise ValueError( - f"Invalid Mode-B expression {original!r}: unsupported " - f"comparison operator {op_type.__name__}." - ) - # ``IN`` / ``NOT IN`` carry a literal-only tuple RHS, or a single string - # (a time-point membership); empty and non-literal RHS are rejected. - if op_type in (ast.In, ast.NotIn): - rhs_node = node.comparators[0] - if isinstance(rhs_node, ast.Constant) and isinstance(rhs_node.value, str): - return Cmp( - op=_CMP_OP_MAP[op_type], - left=_convert(node.left, agg_map=agg_map, original=original), - right=Literal(value=rhs_node.value), - ) - placeholder = _placeholder_of(rhs_node) - if placeholder is not None: - return Cmp( - op=_CMP_OP_MAP[op_type], - left=_convert(node.left, agg_map=agg_map, original=original), - right=placeholder, - ) - if not isinstance(rhs_node, (ast.Tuple, ast.List)): - raise ValueError( - f"Invalid Mode-B expression {original!r}: the right-" - f"hand side of ``in`` / ``not in`` must be a tuple/" - f"list literal (e.g. ``status in ('a', 'b')``) or a " - f"time-point string (``created_at in '2025-Q1'``); got " - f"{type(rhs_node).__name__}." - ) - if not rhs_node.elts: - raise ValueError( - f"Invalid Mode-B expression {original!r}: empty " - f"tuple is not allowed on the right-hand side of " - f"``in`` / ``not in`` (dialect-dependent SQL); use " - f"a non-empty literal tuple." - ) - elements: List[Literal] = [] - for elt in rhs_node.elts: - converted = _convert_in_rhs_element( - elt, agg_map=agg_map, original=original, - ) - elements.append(converted) - return Cmp( - op=_CMP_OP_MAP[op_type], - left=_convert(node.left, agg_map=agg_map, original=original), - right=TupleLit(elements=tuple(elements)), - ) - return Cmp( - op=_CMP_OP_MAP[op_type], - left=_convert(node.left, agg_map=agg_map, original=original), - right=_convert(node.comparators[0], agg_map=agg_map, original=original), + +def _unit_of(word: str) -> Optional[str]: + low = word.lower() + if low in _UNIT_WORDS: + return low + if low.endswith("s") and low[:-1] in _UNIT_WORDS: + return low[:-1] + return None + + +_UNIT = _t(pp.Regex(_RUN).add_condition(lambda toks: toks[0].isascii() and _unit_of(toks[0]) is not None)) +_INTERVAL = (pp.Suppress(_kw("interval")) + (_STRING | _NUMBER) + pp.Optional(_UNIT)) + + +def _interval_count(text: str) -> Literal: + if not re.fullmatch(r"[+-]?\d+", text.strip()): + raise _Problem(f"an interval count must be a whole number; got {text.strip()!r}.") + return Literal(value=Decimal(int(text))) + + +def _interval_node(toks: pp.ParseResults) -> ScalarCall: + quantity = toks[0] + unit_word = toks[1] if len(toks) > 1 else None + if isinstance(quantity.value, str): + pieces = quantity.value.split() + else: + pieces = [str(quantity.value)] + if unit_word is not None: + pieces.append(unit_word) + if len(pieces) > 2 and len(pieces) % 2 == 0 and unit_word is None: + chain = " ".join(f"+ INTERVAL '{n} {u}'" for n, u in zip(pieces[::2], pieces[1::2])) + raise _Problem(f"an interval literal names one unit; write a chain: x {chain}.") + if len(pieces) != 2: + raise _Problem( + "write an interval literal as INTERVAL ' ', INTERVAL '' or INTERVAL ." ) + count, word = pieces + unit = _unit_of(word) + if unit is None: + accepted = ", ".join(g.value for g in TimeGranularity) + raise _Problem(f"unknown interval unit {word!r}; accepted: {accepted}.") + return ScalarCall(name="interval", args=(_interval_count(count), Literal(value=unit))) + + +_INTERVAL.set_parse_action(_interval_node) +_CLOCK = (_kw("current_date", "current_timestamp") + ~pp.FollowedBy(LPAR)).set_parse_action( + lambda toks: ScalarCall(name="current_date" if toks[0].lower() == "current_date" else "now"), +) + +# -- primary, postfix ---------------------------------------------------------- + +_PAREN = LPAR + expr + RPAR +_PLACEHOLDER_ATOM = _PLACEHOLDER.copy().set_parse_action(_placeholder_node) +_ATOM = (_CASE | _CAST | _EXTRACT | _POSITION | _SUBSTRING | _TRIM_FORM | _TEMPORAL | _TIME_LITERAL + | _INTERVAL | _CLOCK | _CONSTANT | _PLACEHOLDER_ATOM | _NUMBER | _STRING | _PAREN + | _CALL | _COLON_AGG | _REF) + +_FILTER_CLAUSE = (pp.Suppress(_kw("filter")) + LPAR + + _need(pp.Suppress(_kw("where")), "FILTER needs (WHERE ).") + + _need(expr, "FILTER (WHERE …) needs a condition.") + + _need(RPAR, "FILTER (WHERE ) is missing its ')'.")) + + +def _reject_over(_s: str, _loc: int, _toks: pp.ParseResults) -> None: + raise IllegalWindowInFilterError( + filter_expr=_STATE.text, + source="raw OVER(...) is not allowed in Mode-B DSL", + suggestion="use a transform instead (rank, percent_rank, dense_rank, ntile, cumsum, lag, lead, time_shift, …).", + ) + + +_OVER = (_kw("over") + pp.FollowedBy(LPAR)).set_parse_action(_reject_over) +_PRIMARY = _ATOM + pp.Group(pp.ZeroOrMore(pp.Group(_FILTER_CLAUSE) | _OVER)) + + +def _primary_node(toks: pp.ParseResults) -> Any: + node = _resolve(toks[0]) + for clause in toks[1]: + if not isinstance(node, AggCall): + raise _Problem("FILTER (WHERE …) applies only to an aggregation, e.g. sum(x) FILTER (WHERE c).") + if node.filter is not None: + raise _Problem("an aggregation takes one FILTER (WHERE …) clause.") + node = node.model_copy(update={"filter": clause[0]}) + return node + + +_PRIMARY.set_parse_action(_primary_node) +_CAST_LEVEL = (_PRIMARY + pp.ZeroOrMore(DCOLON + _need(_TYPE_SPEC, "'::' needs a type."))).set_parse_action( + lambda toks: _fold_casts(list(toks)), +) + + +def _fold_casts(items: List[Any]) -> Any: + node = items[0] + for type_text in items[1:]: + node = Cast(source=node, type_text=type_text) + return node + + +# -- operator ladder ----------------------------------------------------------- + +_MINUS, _PLUS = _sym("-", "'-'"), _sym(r"\+", "'+'") +_power = pp.Forward() +_power_rhs = pp.Forward() +_power_rhs <<= ((_MINUS | _PLUS) + _power_rhs) | _power +_power <<= _CAST_LEVEL + pp.Optional(pp.Suppress(_sym(r"\*\*", "'**'")) + _power_rhs) + + +def _signed(items: List[Any]) -> Any: + node = items[-1] + for sign in reversed(items[:-1]): + if sign == "-": + node = UnaryOp(op="-", operand=node) + return node + + +_power_rhs.set_parse_action(lambda toks: _signed(list(toks))) +_power.set_parse_action(lambda toks: ScalarCall(name="pow", args=(toks[0], toks[1])) if len(toks) == 2 else toks[0]) +_UNARY = (pp.ZeroOrMore(_MINUS | _PLUS) + _power).set_parse_action(lambda toks: _signed(list(toks))) + +_FLOORDIV = _sym("//", "'//'") + _fail("'//' is not an operator; use div(a, b) or floor(a / b).") +_MUL_LEVEL = _UNARY + pp.ZeroOrMore((_FLOORDIV | _sym(r"\*(?!\*)|/(?!/)|%", "'*', '/' or '%'")) + _UNARY) +_ADD_LEVEL = _MUL_LEVEL + pp.ZeroOrMore(_sym("[+-]", "'+' or '-'") + _MUL_LEVEL) + + +def _fold_arith(items: List[Any]) -> Any: + node = items[0] + for op, right in zip(items[1::2], items[2::2]): + node = _interval_as_date_add(op=op, left=node, right=right) or Arith(op=op, left=node, right=right) + return node - if isinstance(node, ast.BoolOp): - op_str = "and" if isinstance(node.op, ast.And) else "or" - operands = tuple( - _convert(v, agg_map=agg_map, original=original) for v in node.values + +_MUL_LEVEL.set_parse_action(lambda toks: _fold_arith(list(toks))) +_ADD_LEVEL.set_parse_action(lambda toks: _fold_arith(list(toks))) +_PIPE = _sym(r"\|(?!\|)", "'|'") + _fail("'|' is not an operator; use || to concatenate strings.") +_concat_level <<= _ADD_LEVEL + pp.ZeroOrMore(pp.Suppress(_PIPE | _sym(r"\|\|", "'||'")) + _ADD_LEVEL) + + +def _fold_concat(items: List[Any]) -> Any: + node = items[0] + for right in items[1:]: + node = ScalarCall(name="concat", args=(node, right)) + return node + + +_concat_level.set_parse_action(lambda toks: _fold_concat(list(toks))) + +# -- comparison level ---------------------------------------------------------- + +_CMP_OPS = {"=": "==", "==": "==", "<>": "!=", "!=": "!=", "<": "<", "<=": "<=", ">": ">", ">=": ">="} +_IS = pp.Suppress(_kw("is")) +_NOT_FLAG = pp.Optional(_kw("not")).set_parse_action(lambda toks: bool(len(toks))) + + +class _Suffix: + """A comparison-level suffix: builds the predicate node from its left operand.""" + + __slots__ = ("build",) + + def __init__(self, build) -> None: + self.build = build + + +def _suffix(element: pp.ParserElement, build) -> pp.ParserElement: + return element.set_parse_action(lambda toks: _Suffix(lambda left: build(left, toks))) + + +def _negated(node: Any, negate: bool) -> Any: + return UnaryOp(op="not", operand=node) if negate else node + + +def _truth_test(left: Any, toks: pp.ParseResults) -> Any: + negate, word = toks[0], toks[1].lower() + known = Cmp(op="is not", left=left, right=Literal(value=None)) + test = BoolOp(op="and", operands=(known, left if word == "true" else UnaryOp(op="not", operand=left))) + return _negated(test, negate) + + +def _in_rhs_elements(items: List[Any]) -> TupleLit: + if not items: + raise _Problem( + "an empty list is not allowed on the right-hand side of in / not in (dialect-dependent SQL); " + "use a non-empty literal list." ) - return BoolOp(op=op_str, operands=operands) + elements: List[Literal] = [] + for item in items: + if (isinstance(item, UnaryOp) and item.op == "-" and isinstance(item.operand, Literal) + and isinstance(item.operand.value, Decimal)): + item = Literal(value=-item.operand.value) + if not isinstance(item, Literal): + raise _Problem( + "every element on the right-hand side of in / not in must be a literal (string, number, or " + f"boolean); got {type(item).__name__}." + ) + elements.append(item) + return TupleLit(elements=tuple(elements)) + + +_IN_LIST = ((LPAR + _PLACEHOLDER_ATOM + RPAR) + | _PLACEHOLDER_ATOM + | (LPAR + _SEQ_ITEMS + RPAR).set_parse_action(lambda toks: _in_rhs_elements(list(toks[0]))) + | (LBRACK + _SEQ_ITEMS + RBRACK).set_parse_action(lambda toks: _in_rhs_elements(list(toks[0]))) + | _STRING + | _fail("the right-hand side of in / not in must be a literal list, e.g. status in ('a', 'b').")) + +_CMP_SUFFIX = ( + _suffix(_sym(r"==|!=|<>|<=|>=|=|<|>", "comparison") + _concat_level, + lambda left, t: Cmp(op=_CMP_OPS[t[0]], left=left, right=t[1])) + | _suffix(_IS + _NOT_FLAG + pp.Suppress(_kw("distinct")) + pp.Suppress(_kw("from")) + _concat_level, + lambda left, t: Cmp(op="is not distinct from" if t[0] else "is distinct from", left=left, right=t[1])) + | _suffix(_IS + _NOT_FLAG + pp.Suppress(_kw("null") | pp.Regex(r"None\b")), + lambda left, t: Cmp(op="is not" if t[0] else "is", left=left, right=Literal(value=None))) + | _suffix(_IS + _NOT_FLAG + (_kw("true", "false")), _truth_test) + | _suffix(_IS + _NOT_FLAG + _concat_level, + lambda left, t: Cmp(op="is not" if t[0] else "is", left=left, right=t[1])) + | _suffix(_NOT_FLAG + pp.Suppress(_kw("in")) + _IN_LIST, + lambda left, t: Cmp(op="not in" if t[0] else "in", left=left, right=t[1])) + | _suffix(_NOT_FLAG + pp.Suppress(_kw("between")) + _concat_level + pp.Suppress(_kw("and")) + _concat_level, + lambda left, t: _negated(BoolOp(op="and", operands=( + Cmp(op=">=", left=left, right=t[1]), Cmp(op="<=", left=left, right=t[2]), + )), t[0])) + | _suffix(_NOT_FLAG + _kw("like", "ilike") + _concat_level, + lambda left, t: _negated(ScalarCall(name=t[1].lower(), args=(left, t[2])), t[0])) + | (_NOT_FLAG + _kw("similar") + _kw("to")) + _fail("SIMILAR TO is not supported; use LIKE or ILIKE.") + | (_kw("regexp", "rlike") + _fail("the REGEXP / RLIKE operators are not supported; use LIKE or ILIKE.")) +) - placeholder = _placeholder_of(node) - if placeholder is not None: - return placeholder - if isinstance(node, ast.IfExp): - raise ValueError( - f"Invalid Mode-B expression {original!r}: the Python conditional " - f"'x if cond else y' is not supported. Use SQL " - f"'CASE WHEN cond THEN x ELSE y END' or iif(cond, x, y)." +def _comparison_node(toks: pp.ParseResults) -> Any: + left, suffixes = toks[0], list(toks[1:]) + if len(suffixes) > 1: + raise _Problem( + "chained comparisons are not supported. Each comparison must be a single comparison; split " + "(e.g.) `1 < x < 10` into `1 < x and x < 10`." ) + return suffixes[0].build(left) if suffixes else left - raise ValueError( - f"Invalid Mode-B expression {original!r}: unsupported AST node " - f"{type(node).__name__}." - ) +_COMPARISON = (_concat_level + pp.ZeroOrMore(_CMP_SUFFIX)).set_parse_action(_comparison_node) +_NOT_LEVEL = (pp.ZeroOrMore(_kw("not")) + _COMPARISON).set_parse_action( + lambda toks: _negate_n(len(toks) - 1, toks[-1]), +) -_PLACEHOLDER_NAME_RE = re.compile(r"[a-zA-Z_]\w*", re.ASCII) +def _negate_n(count: int, node: Any) -> Any: + for _ in range(count): + node = UnaryOp(op="not", operand=node) + return node -def _placeholder_of(node: ast.AST) -> Optional[Placeholder]: - """``Placeholder`` for a set literal holding exactly one bare variable name, else ``None``.""" - if not (isinstance(node, ast.Set) and len(node.elts) == 1 and isinstance(node.elts[0], ast.Name)): - return None - name = node.elts[0].id - if name == _STAR_ARG_TOKEN or _PLACEHOLDER_RE.match(name) or not _PLACEHOLDER_NAME_RE.fullmatch(name): - return None - return Placeholder(name=name) +def _bool_level(operand: pp.ParserElement, word: str) -> pp.ParserElement: + level = operand + pp.ZeroOrMore(pp.Suppress(_kw(word)) + operand) + return level.set_parse_action(lambda toks: BoolOp(op=word, operands=tuple(toks)) if len(toks) > 1 else toks[0]) -def placeholder_names(parsed: Any) -> List[str]: - """Names of every ``Placeholder`` in a parsed tree, in walk order.""" - return [n.name for n in _walk_parsed(parsed) if isinstance(n, Placeholder)] + +_TERNARY = _kw("if") + _fail( + "the Python conditional 'x if cond else y' is not supported; use CASE WHEN cond THEN x ELSE y END or iif(cond, x, y).", +) +expr <<= _bool_level(_bool_level(_NOT_LEVEL, "and"), "or") + pp.Optional(_TERNARY) +_EXPRESSION = expr + pp.StringEnd().set_fail_action(_record_failure) + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- -def _convert_in_rhs_element( - node: ast.AST, *, agg_map: Dict, original: str, -) -> Literal: - """Convert one element of an ``IN`` / ``NOT IN`` literal-tuple RHS. +@lru_cache(maxsize=8192) +def parse_expr(text: str) -> ParsedExpr: + """Parse a Mode-B expression string into a ``ParsedExpr``. - The Python parser emits a negative numeric literal as - ``UnaryOp(USub, Constant(int|float))`` rather than a bare - ``Constant`` with a negative value, so a strict ``isinstance(_, - Literal)`` check against ``_convert``'s output would reject - ``amount in (-1, -2)``. This helper collapses the sign onto the - inner numeric before the literal check (Codex review). Boolean and - string literals are unaffected. + Raises ``ValueError`` (empty / syntax / unsupported form / reserved prefix), + ``UnknownFunctionError``, or ``IllegalWindowInFilterError`` (raw ``OVER(...)``). """ - if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)): - inner = node.operand - if ( - isinstance(inner, ast.Constant) - and isinstance(inner.value, (int, float)) - and not isinstance(inner.value, bool) - ): - signed = -inner.value if isinstance(node.op, ast.USub) else inner.value - if isinstance(signed, int): - return Literal(value=Decimal(signed)) - return Literal(value=Decimal(str(signed))) - converted = _convert(node, agg_map=agg_map, original=original) - if not isinstance(converted, Literal): + if not text or not text.strip(): + raise ValueError("Empty Mode-B expression.") + with _PARSE_LOCK: + _STATE.text, _STATE.farthest = text, 0 + try: + masked = _mask_placeholders(text) + parsed = _EXPRESSION.parse_string(masked, parse_all=True)[0] + except _Problem as problem: + raise ValueError(f"Invalid Mode-B expression {text!r}: {problem.detail}") from None + except pp.ParseBaseException as e: + raise ValueError(f"Invalid Mode-B expression {text!r}: {_syntax_message(text, e)}") from None + if any(_is_interval(n) for n in _walk_parsed(parsed)): raise ValueError( - f"Invalid Mode-B expression {original!r}: every element on " - f"the right-hand side of ``in`` / ``not in`` must be a " - f"literal (string, number, or boolean); got " - f"{type(converted).__name__}." + f"Invalid Mode-B expression {text!r}: interval(n, unit) is only valid as " + f"`x + interval(n, unit)`, `interval(n, unit) + x` or `x - interval(n, unit)`; " + f"otherwise use date_add(x, n, unit)." ) - return converted - - -def _convert_constant(node: ast.Constant, *, original: str) -> Literal: - val = node.value - if isinstance(val, bool): - return Literal(value=val) - if val is None: - return Literal(value=None) - if isinstance(val, int): - return Literal(value=Decimal(val)) - if isinstance(val, float): - return Literal(value=Decimal(str(val))) - if isinstance(val, str): - return Literal(value=val) - raise ValueError( - f"Invalid Mode-B expression {original!r}: unsupported literal " - f"type {type(val).__name__}." - ) + return parsed -def _flatten_attribute( - node: ast.Attribute, *, original: str, -) -> List[str]: - parts: List[str] = [node.attr] - cur: ast.AST = node.value - while isinstance(cur, ast.Attribute): - parts.append(cur.attr) - cur = cur.value - if isinstance(cur, ast.Name): - parts.append(cur.id) - else: - raise ValueError( - f"Invalid Mode-B expression {original!r}: unsupported " - f"attribute base {type(cur).__name__}." - ) - return list(reversed(parts)) +def _syntax_message(text: str, error: pp.ParseBaseException) -> str: + pos = max(_STATE.farthest, error.loc) + if pos >= len(text.rstrip()): + return f"unexpected end of expression at column {pos + 1}." + return f"syntax error at column {pos + 1}, near {text[pos:pos + 12]!r}." + + +# --------------------------------------------------------------------------- +# Variables +# --------------------------------------------------------------------------- + + +def placeholder_names(parsed: Any) -> List[str]: + """Names of every ``Placeholder`` in a parsed tree, in walk order.""" + return [n.name for n in _walk_parsed(parsed) if isinstance(n, Placeholder)] -def _convert_kwarg_value(node: ast.AST, *, agg_map: Dict, original: str): - """Convert a call keyword-argument value. +# --------------------------------------------------------------------------- +# Reference walk +# --------------------------------------------------------------------------- + + +def walk_parsed_refs( + parsed: ParsedExpr, +) -> Iterator[Ref | DottedRef | AggCall]: + """Yield the reference-bearing leaves (``Ref`` / ``DottedRef`` / ``AggCall``) + of a tree — scope-free name extraction for schema-drift / memory tagging. - List / tuple values (e.g. ``partition_by=[region, channel]`` for the - rank family) convert to a tuple of converted elements so the parser - accepts the documented multi-column transform-kwarg grammar instead of - raising on the bare ``ast.List`` node; scalar values convert normally. + ``AggCall`` is yielded whole (args/kwargs/filter opaque); ``TransformCall`` + descends ``input`` only; every other composite descends its operands. """ - if isinstance(node, (ast.List, ast.Tuple)): - return tuple( - _convert(e, agg_map=agg_map, original=original) for e in node.elts - ) - return _convert(node, agg_map=agg_map, original=original) + if isinstance(parsed, (Ref, DottedRef, AggCall)): + yield parsed + return + if isinstance(parsed, TransformCall): + yield from walk_parsed_refs(parsed.input) + return + if isinstance(parsed, ScalarCall): + for a in parsed.args: + yield from walk_parsed_refs(a) + return + if isinstance(parsed, Cast): + yield from walk_parsed_refs(parsed.source) + return + if isinstance(parsed, (Arith, Cmp)): + yield from walk_parsed_refs(parsed.left) + yield from walk_parsed_refs(parsed.right) + return + if isinstance(parsed, UnaryOp): + yield from walk_parsed_refs(parsed.operand) + return + if isinstance(parsed, BoolOp): + for op in parsed.operands: + yield from walk_parsed_refs(op) + + +# --------------------------------------------------------------------------- +# Call dispatch +# --------------------------------------------------------------------------- # The node kinds an ``AggCall.source`` may take: a column, star, or an aggregation-free # scalar expression, predicates included (a bare TupleLit is not a value). -_AGG_SOURCE_KINDS = (Ref, DottedRef, StarSource, Literal, Placeholder, ScalarCall, Arith, UnaryOp, Cmp, BoolOp) +_AGG_SOURCE_KINDS = ( + Ref, DottedRef, StarSource, Literal, TemporalLiteral, Placeholder, ScalarCall, Cast, Arith, UnaryOp, Cmp, BoolOp, +) def _contains_agg_or_transform(node: Any) -> bool: """Whether a parsed subtree contains any AggCall / TransformCall.""" if isinstance(node, (AggCall, TransformCall)): return True + return any(_contains_agg_or_transform(c) for c in _operands(node)) + + +def _operands(node: Any) -> Tuple[Any, ...]: + """The value operands of a row-level composite (empty for leaves and calls with their own scope).""" if isinstance(node, ScalarCall): - return any(_contains_agg_or_transform(a) for a in node.args) + return node.args + if isinstance(node, Cast): + return (node.source,) if isinstance(node, (Arith, Cmp)): - return _contains_agg_or_transform(node.left) or _contains_agg_or_transform( - node.right - ) + return (node.left, node.right) if isinstance(node, UnaryOp): - return _contains_agg_or_transform(node.operand) + return (node.operand,) if isinstance(node, BoolOp): - return any(_contains_agg_or_transform(o) for o in node.operands) - return False + return node.operands + return () def _source_leaves(node: Any): """Composition leaves of an agg source, treating an inner AggCall / - TransformCall as opaque (not descended — its own source was already - validated when it was converted).""" - if isinstance(node, ScalarCall): - for a in node.args: - yield from _source_leaves(a) - elif isinstance(node, (Arith, Cmp)): - yield from _source_leaves(node.left) - yield from _source_leaves(node.right) - elif isinstance(node, UnaryOp): - yield from _source_leaves(node.operand) - elif isinstance(node, BoolOp): - for o in node.operands: - yield from _source_leaves(o) - else: + TransformCall as opaque.""" + children = _operands(node) + if not children: yield node + return + for child in children: + yield from _source_leaves(child) def _validated_agg_source(source: Any, *, func_name: str, original: str) -> Any: - """Validate a functional aggregation's first argument as its source. - - A source resolving to attached values — nested AggCalls and/or grained - transforms, alone or composed through arithmetic / scalar calls — is a - (second-order) re-aggregation; a source mixing row-level - references with attached values is a row-grain aggregation; both - are accepted.""" + """Validate a functional aggregation's first argument as its source; an attached + (re-aggregation) or mixed source is accepted.""" leaves = list(_source_leaves(source)) if any(isinstance(leaf, (AggCall, TransformCall)) for leaf in leaves): return source @@ -1330,72 +1056,18 @@ def _reject_bare_star_args( ) -def _reject_repeated_keywords(node: ast.Call, *, call: str, original: str) -> None: - seen: set = set() - for kw in node.keywords: - if kw.arg in seen: - raise ValueError( - f"Invalid Mode-B expression {original!r}: {call}() got keyword " - f"argument {kw.arg!r} more than once." - ) - seen.add(kw.arg) - - -def _call_arguments( - node: ast.Call, *, call: str, agg_map: Dict, original: str, -) -> Tuple[Tuple[Any, ...], Tuple[Tuple[str, Any], ...]]: - args = tuple( - _convert(a, agg_map=agg_map, original=original) for a in node.args - ) - # Reject ``**kwargs`` dictionary unpacking (``kw.arg is None``) rather - # than silently dropping it (CR) — a dropped ``**`` would change call - # semantics without warning. - if any(kw.arg is None for kw in node.keywords): - raise ValueError( - f"Invalid Mode-B expression {original!r}: dictionary unpacking " - f"(**kwargs) is not supported in calls." - ) - _reject_repeated_keywords(node, call=call, original=original) - kwargs = tuple( - (kw.arg, _convert_kwarg_value(kw.value, agg_map=agg_map, original=original)) - for kw in node.keywords - if kw.arg is not None # guarded above; narrows kw.arg to str - ) - return args, kwargs - - -def _convert_call( - node: ast.Call, *, agg_map: Dict, original: str, -) -> ParsedExpr: - if isinstance(node.func, ast.Attribute): - return _convert_method_call(node, func=node.func, agg_map=agg_map, original=original) - if not isinstance(node.func, ast.Name): - raise ValueError( - f"Invalid Mode-B expression {original!r}: function calls " - f"with non-name callee are not supported." - ) - func_name = node.func.id - m = _PLACEHOLDER_RE.match(func_name) - args, kwargs = _call_arguments( - node, call=agg_map[int(m.group(1))][1] if m else func_name, agg_map=agg_map, original=original, - ) - # Legacy colon-spelling placeholder? - if m: - source, agg = agg_map[int(m.group(1))] - return AggCall(source=source, agg=agg, args=args, kwargs=kwargs) - return _dispatch_call(func_name, args=args, kwargs=kwargs, original=original) - - def _onto_path(node: Any, *, path: Tuple[str, ...]) -> Any: """``node`` with every column leaf (and ``*``) read under the join ``path``.""" if isinstance(node, Ref): - return DottedRef(parts=(*path, node.name)) + return DottedRef(parts=(*path, node.name), quoted=node.quoted) if isinstance(node, DottedRef): - return DottedRef(parts=(*path, *node.parts)) + return DottedRef(parts=(*path, *node.parts), quoted=node.quoted) if isinstance(node, StarSource): return DottedRef(parts=(*path, "*")) if isinstance(node, ScalarCall): return node.model_copy(update={"args": tuple(_onto_path(a, path=path) for a in node.args)}) + if isinstance(node, Cast): + return node.model_copy(update={"source": _onto_path(node.source, path=path)}) if isinstance(node, (Arith, Cmp)): return node.model_copy(update={"left": _onto_path(node.left, path=path), "right": _onto_path(node.right, path=path)}) if isinstance(node, UnaryOp): @@ -1428,11 +1100,9 @@ def _method_call_refusal(name: str, *, args: Tuple[Any, ...], kwargs: Tuple[Tupl return None -def _convert_method_call(node: ast.Call, *, func: ast.Attribute, agg_map: Dict, original: str) -> ParsedExpr: +def _method_call(*, path: Tuple[str, ...], name: str, args: Tuple[Any, ...], kwargs: Tuple[Tuple[str, Any], ...]) -> ParsedExpr: """``.(, )`` as its functional twin ``(., …)``.""" - *path_parts, name = _flatten_attribute(func, original=original) - path = tuple(path_parts) - args, kwargs = _call_arguments(node, call=name, agg_map=agg_map, original=original) + original = _STATE.text twin_args = (_onto_path(args[0], path=path), *args[1:]) if args else () reason = _method_call_refusal(name, args=args, kwargs=kwargs) if reason is not None: @@ -1443,26 +1113,33 @@ def _convert_method_call(node: ast.Call, *, func: ast.Attribute, agg_map: Dict, return _dispatch_call(name, args=twin_args, kwargs=kwargs, original=original) -def _dispatch_call( # NOSONAR(S3776) — the one call-dispatch ladder (builtin functional aggregation → transform → scalar → date-part alias → unknown-name AggCall deferral); each rung IS the documented dispatch order and splitting them hides it. first/last carry no grain decision here — they dispatch at bind. +_SCALAR_MIN_MAX = {"min": "least", "max": "greatest"} + + +def _dispatch_call( # NOSONAR(S3776) — the one call-dispatch ladder (builtin functional aggregation → transform → scalar → date-part alias → unknown-name AggCall deferral); each rung IS the documented dispatch order and splitting them hides it. func_name: str, *, args: Tuple[Any, ...], kwargs: Tuple[Tuple[str, Any], ...], original: str, ) -> ParsedExpr: - # Functional builtin aggregation? Matched via alias/case healing, exactly - # like legacy colon names heal at binding; ``agg`` stores the RAW token so both - # spellings collapse to the identical AggCall. ``first``/``last`` dispatch at - # bind by the operand's type. - if normalize_aggregation_name(func_name) in BUILTIN_AGGREGATIONS: + # ``agg`` stores the RAW token so colon and functional spellings collapse to one + # AggCall; ``first``/``last`` dispatch at bind by the operand's type. + canonical_agg = normalize_aggregation_name(func_name) + if canonical_agg in BUILTIN_AGGREGATIONS: if not args: raise ValueError( f"Invalid Mode-B expression {original!r}: aggregation " f"{func_name!r} requires a value argument, e.g. " f"{func_name}(column)." ) + if canonical_agg in _SCALAR_MIN_MAX and len(args) == 2 and not kwargs: + scalar = _SCALAR_MIN_MAX[canonical_agg] + raise ValueError( + f"Invalid Mode-B expression {original!r}: {func_name}() with two values is the " + f"aggregation, not a scalar; use {scalar}{_canonical_call_params(args, ())}." + ) source = _validated_agg_source( args[0], func_name=func_name, original=original, ) return AggCall(source=source, agg=func_name, args=args[1:], kwargs=kwargs) - # Transform? if func_name in ALL_TRANSFORMS: if not args: raise ValueError( @@ -1471,15 +1148,9 @@ def _dispatch_call( # NOSONAR(S3776) — the one call-dispatch ladder (builtin f"(the value to transform)." ) _reject_bare_star_args(args, kwargs, func_name=func_name, original=original) - return TransformCall( - op=func_name, - input=args[0], - args=args[1:], - kwargs=kwargs, - ) + return TransformCall(op=func_name, input=args[0], args=args[1:], kwargs=kwargs) - # Scalar function? Case-insensitive match; normalised to lower case into - # ``ScalarCall`` so ``COALESCE`` / ``coalesce`` intern to one key. + # Normalised to lower case so ``COALESCE`` / ``coalesce`` intern to one key. if func_name.lower() in SCALAR_FUNCTIONS: if kwargs: raise ValueError( @@ -1501,9 +1172,7 @@ def _dispatch_call( # NOSONAR(S3776) — the one call-dispatch ladder (builtin _reject_bare_star_args(args, kwargs, func_name=func_name, original=original) return ScalarCall(name="date_part", args=(Literal(value=part.value), args[0])) - # Unknown name with an aggregatable first arg → AggCall candidate, validated - # at binding. A custom aggregation over an - # attached source is a re-aggregation, validated like any other. + # Unknown name with an aggregatable first arg → AggCall candidate, validated at binding. if args and isinstance(args[0], _AGG_SOURCE_KINDS) and not _contains_agg_or_transform(args[0]): return AggCall(source=args[0], agg=func_name, args=args[1:], kwargs=kwargs) if args and any(isinstance(leaf, AggCall) for leaf in _source_leaves(args[0])): @@ -1595,7 +1264,7 @@ def _interval_as_date_add(*, op: str, left: Any, right: Any) -> Optional[ScalarC def _walk_parsed(node: Any) -> Iterator[Any]: - """Every node of a parsed tree, including call args, kwargs and agg sources.""" + """Every node of a parsed tree, including call args, kwargs, agg sources and filters.""" if isinstance(node, tuple): for item in node: yield from _walk_parsed(item) @@ -1635,6 +1304,10 @@ def _canonical_kwarg_text(value: Any) -> str: return canonical_measure_text(value) +def _identifier_text(name: str) -> str: + return name if name.isidentifier() else '"' + name.replace('"', '""') + '"' + + def canonical_measure_text(parsed: Any) -> str: # NOSONAR(S3776) — flat per-node-kind rendering table; each branch is one spelling rule. """Render a ``ParsedExpr`` back to canonical legacy colon-spelling text. @@ -1643,35 +1316,32 @@ def canonical_measure_text(parsed: Any) -> str: # NOSONAR(S3776) — flat per-n verbatim round-trip: grouping parens are dropped and spacing normalised. """ if isinstance(parsed, Ref): - return parsed.name + return _identifier_text(parsed.name) if isinstance(parsed, DottedRef): - return ".".join(parsed.parts) + return ".".join(p if p == "*" else _identifier_text(p) for p in parsed.parts) if isinstance(parsed, StarSource): return "*" if isinstance(parsed, Literal): if isinstance(parsed.value, str): - return f"'{parsed.value}'" + return "'" + parsed.value.replace("'", "''") + "'" return str(parsed.value) + if isinstance(parsed, TemporalLiteral): + return f"{parsed.kind.upper()} '{parsed.text}'" if isinstance(parsed, Placeholder): return f"{{{parsed.name}}}" if isinstance(parsed, TupleLit): return f"({', '.join(canonical_measure_text(e) for e in parsed.elements)})" if isinstance(parsed, AggCall): - source = canonical_measure_text(parsed.source) - if isinstance(parsed.source, (Ref, DottedRef, StarSource)): - return f"{source}:{parsed.agg}{_canonical_call_params(parsed.args, parsed.kwargs)}" - # Expression source (``sum(amount - cost)``): render functionally so the - # text can't collide with a distinct parse tree like ``amount - cost:sum``. - parts = [source] - parts += [canonical_measure_text(a) for a in parsed.args] - parts += [f"{k}={_canonical_kwarg_text(v)}" for k, v in parsed.kwargs] - return f"{parsed.agg}({', '.join(parts)})" + return _canonical_agg_text(parsed) if isinstance(parsed, TransformCall): inner = canonical_measure_text(parsed.input) params = _canonical_call_params(parsed.args, parsed.kwargs, bare_direction=True) return f"{parsed.op}({inner}{', ' + params[1:-1] if params else ''})" if isinstance(parsed, ScalarCall): return f"{parsed.name}({', '.join(canonical_measure_text(a) for a in parsed.args)})" + if isinstance(parsed, Cast): + fn = "TRY_CAST" if parsed.try_ else "CAST" + return f"{fn}({canonical_measure_text(parsed.source)} AS {parsed.type_text})" if isinstance(parsed, (Arith, Cmp)): left = canonical_measure_text(parsed.left) right = canonical_measure_text(parsed.right) @@ -1686,6 +1356,22 @@ def canonical_measure_text(parsed: Any) -> str: # NOSONAR(S3776) — flat per-n return str(parsed) +def _canonical_agg_text(parsed: AggCall) -> str: + source = canonical_measure_text(parsed.source) + if isinstance(parsed.source, (Ref, DottedRef, StarSource)): + text = f"{source}:{parsed.agg}{_canonical_call_params(parsed.args, parsed.kwargs)}" + else: + # Expression source (``sum(amount - cost)``): render functionally so the + # text can't collide with a distinct parse tree like ``amount - cost:sum``. + parts = [source] + parts += [canonical_measure_text(a) for a in parsed.args] + parts += [f"{k}={_canonical_kwarg_text(v)}" for k, v in parsed.kwargs] + text = f"{parsed.agg}({', '.join(parts)})" + if parsed.filter is not None: + text += f" FILTER (WHERE {canonical_measure_text(parsed.filter)})" + return text + + def _functional_suffix_text(raw: str, *, agg: str) -> str: """Rebuild the ``agg`` / ``agg(rest-args)`` suffix of a functional entity ref, preserving any extra-arg text verbatim (``last(balance, updated_at)`` @@ -1728,7 +1414,7 @@ def split_entity_agg_ref(raw: str) -> Tuple[str, Optional[str]]: if suffix is not None or "(" not in raw: return prefix, suffix parsed = parse_expr(raw) - if not isinstance(parsed, AggCall) or not isinstance( + if not isinstance(parsed, AggCall) or parsed.filter is not None or not isinstance( parsed.source, (Ref, DottedRef, StarSource) ): raise ValueError( @@ -1743,3 +1429,10 @@ def split_entity_agg_ref(raw: str) -> Tuple[str, Optional[str]]: else: prefix = ".".join(source.parts) return prefix, _functional_suffix_text(raw, agg=parsed.agg) + + +__all__ = [ + "AggCall", "Arith", "BoolOp", "Cast", "Cmp", "DottedRef", "Literal", "ParsedExpr", "Placeholder", "Ref", + "ScalarCall", "StarSource", "TemporalLiteral", "TransformCall", "TupleLit", "UnaryOp", + "canonical_measure_text", "parse_expr", "placeholder_names", "split_entity_agg_ref", "walk_parsed_refs", +] diff --git a/slayer/engine/bind_inputs.py b/slayer/engine/bind_inputs.py index 423cb5b1..13de93f7 100644 --- a/slayer/engine/bind_inputs.py +++ b/slayer/engine/bind_inputs.py @@ -48,6 +48,7 @@ SlayerQuery, TimeDimension, call_callee, + computed_dimension_name, functional_call_column, granularity_call_parts, ) @@ -105,7 +106,6 @@ TransformCall, canonical_measure_text, parse_expr, - parse_filter_expr, ) from slayer.ir.bound import ( GranularityConflict, @@ -202,7 +202,7 @@ def _reject_measure_refs_in_filters( if not isinstance(f, str): continue try: - parsed = parse_filter_expr(f) + parsed = parse_expr(f) except Exception: # noqa: BLE001 — binder reports parse errors properly continue check_raw_rows_filter_measure_ref( @@ -375,7 +375,7 @@ def bind_query_inputs( # NOSONAR(S3776) — one cohesive bind pass. The stages if not isinstance(f, str): continue bf = bind_filter( - parsed=parse_filter_expr(f), + parsed=parse_expr(f), scope=scope, bundle=bundle, alias_map=filter_alias_map, @@ -950,7 +950,7 @@ def _declared_computed_dimension( bound=bound, declared_name=name, public_name=name, - name_is_explicit=name != auto_name_from_expression(d.template), + name_is_explicit=name != computed_dimension_name(d.template), type=dim_type, is_dimension=True, ) @@ -1218,7 +1218,7 @@ def _resolve_granularity_calls(query: SlayerQuery, *, bundle: ResolvedSourceBund td = ( _datasource_time_dimension(item.expression, bundle=bundle) if isinstance(item, ComputedDimension) - and item.name == auto_name_from_expression(item.template) + and item.name == computed_dimension_name(item.template) else None ) if td is None: diff --git a/slayer/engine/binding.py b/slayer/engine/binding.py index 12ce36f1..60b26cf9 100644 --- a/slayer/engine/binding.py +++ b/slayer/engine/binding.py @@ -5,6 +5,7 @@ import difflib import os +from datetime import datetime from decimal import Decimal from typing import Dict, Final, List, Mapping, Optional, Tuple, TypeGuard @@ -12,7 +13,9 @@ from slayer.core.errors import ( AggregationArgumentError, + AggregationFilterError, AggregationNotAllowedError, + CastTypeError, CircularJoinPathError, GranularityCallError, IllegalScopeReferenceError, @@ -20,6 +23,7 @@ MeasureCycleError, MeasureRecursionLimitError, PartitionKeyError, + TimeLiteralError, TransformArgumentError, UnknownFunctionError, UnknownReferenceError, @@ -31,7 +35,7 @@ from slayer.core.enums import RANK_FAMILY_TRANSFORMS from slayer.core.granularity import CustomGranularity, Granularity, resolve_granularity from slayer.core.refs import EXPRESSION_SOURCE_KINDS, key_display -from slayer.core.keys import DATE_ADD_COUNT_ARG, DATE_OPERAND_ARGS, MIRRORED_TIME_POINT_OP, SCALAR_FUNCTIONS, check_scalar_arity, type_date_values, AggregateKey, AggregateSource, ArithmeticKey, ColumnKey, ColumnSqlKey, Grain, InKey, LiteralKey, ScalarCallKey, SqlFragmentKey, StarKey, TimePointCmpKey, TimePointOp, TimeTruncKey, TransformKey, ValueKey, boolean_valued, column_leaf, column_path, is_attached_source, normalize_scalar, prepend_value_key, temporal_type, walk_value_keys +from slayer.core.keys import source_anchor_path, DATE_ADD_COUNT_ARG, DATE_OPERAND_ARGS, MIRRORED_TIME_POINT_OP, SCALAR_FUNCTIONS, cast_data_type, check_scalar_arity, type_date_values, AggregateKey, AggregateSource, ArithmeticKey, CastKey, ColumnKey, ColumnSqlKey, Grain, InKey, LiteralKey, ScalarCallKey, SqlFragmentKey, StarKey, TimePointCmpKey, TimePointOp, TimeTruncKey, TransformKey, ValueKey, boolean_valued, column_leaf, column_path, is_attached_source, normalize_scalar, prepend_value_key, temporal_type, walk_value_keys from slayer.core.join_walker import ( aggregation_owner, canonical_path, @@ -52,12 +56,13 @@ from slayer.engine.param_binding import bind_aggregation_params from slayer.core.query import TimeDimension from slayer.core.scope import ModelScope, StageSchema, resolve_generated_column -from slayer.core.time_points import is_time_point +from slayer.core.time_points import is_time_point, parse_temporal_value from slayer.ir.source_bundle import ResolvedSourceBundle from slayer.core.syntax import ( AggCall, Arith, BoolOp, + Cast, Cmp, DottedRef, Literal, @@ -65,6 +70,7 @@ Ref, ScalarCall, StarSource, + TemporalLiteral, TransformCall, TupleLit, UnaryOp, @@ -72,7 +78,7 @@ parse_expr, placeholder_names, ) -from slayer.sql.sql_expr import has_window_function +from slayer.sql.sql_expr import build_cast_type, has_window_function from slayer.sql.sql_template import SqlTemplateError, aggregation_reads from slayer.ir.bound import BoundExpr, BoundFilter, BoundTimeDimension @@ -339,17 +345,28 @@ def _bind( if isinstance(parsed, Literal): return LiteralKey(value=normalize_scalar(parsed.value)) - if isinstance(parsed, Ref): - return _resolve_ref( - parsed.name, scope=scope, bundle=bundle, alias_map=alias_map, - measure_ctx=measure_ctx, - ) + if isinstance(parsed, TemporalLiteral): + return LiteralKey(value=_temporal_literal_value(parsed)) - if isinstance(parsed, DottedRef): - return _resolve_dotted( - parsed.parts, scope=scope, bundle=bundle, alias_map=alias_map, - measure_ctx=measure_ctx, - ) + if isinstance(parsed, (Ref, DottedRef)): + try: + if isinstance(parsed, Ref): + return _resolve_ref( + parsed.name, scope=scope, bundle=bundle, alias_map=alias_map, + measure_ctx=measure_ctx, + ) + return _resolve_dotted( + parsed.parts, scope=scope, bundle=bundle, alias_map=alias_map, + measure_ctx=measure_ctx, + ) + except UnknownReferenceError as e: + raise _with_quote_hint(e) if parsed.quoted else e + + if isinstance(parsed, Cast): + return _bind_cast(parsed, source=_bind( + parsed.source, scope=scope, bundle=bundle, in_filter=in_filter, alias_map=alias_map, + measure_ctx=measure_ctx, dim_alias_map=dim_alias_map, + ), bundle=bundle) if isinstance(parsed, StarSource): return StarKey() @@ -425,6 +442,40 @@ def _bind( ) +def _with_quote_hint(error: UnknownReferenceError) -> UnknownReferenceError: + """``error`` for a double-quoted name, adding that strings take single quotes.""" + hint = f"{error.name!r} was written in double quotes, which name a column; strings use single quotes ('…')." + return UnknownReferenceError( + name=error.name, scope_kind=error.scope_kind, scope_summary=error.scope_summary, + suggestion=f"{hint} {error.suggestion}" if error.suggestion else hint, + ) + + +def _temporal_literal_value(parsed: TemporalLiteral): + """A ``DATE '…'`` / ``TIMESTAMP '…'`` literal's value, checked like an ISO literal in a date position.""" + value = parse_temporal_value(parsed.text) + if parsed.kind == "date" and isinstance(value, datetime): + value = None + if value is None: + raise TimeLiteralError( + summary=f"{parsed.kind.upper()} '{parsed.text}' is not a valid {parsed.kind} literal.", + suggestion="write an ISO time point: DATE 'YYYY-MM-DD' or TIMESTAMP 'YYYY-MM-DD HH:MM[:SS]'.", + ) + if parsed.kind == "timestamp" and not isinstance(value, datetime): + value = datetime(value.year, value.month, value.day) + return value + + +def _bind_cast(parsed: Cast, *, source: ValueKey, bundle: ResolvedSourceBundle) -> CastKey: + try: + build_cast_type(parsed.type_text, dialect=bundle.dialect) + except ValueError as e: + raise CastTypeError(summary=str(e), suggestion="cast to a type this database has.") from None + return CastKey( + source=source, type_text=parsed.type_text, data_type=cast_data_type(parsed.type_text), try_=parsed.try_, + ) + + def _reject_placeholders(parsed: ParsedExpr, *, text: Optional[str] = None) -> None: """Raise ``UnresolvedPlaceholderError`` for the first unsubstituted ``{name}`` in ``parsed``.""" names = placeholder_names(parsed) @@ -1086,6 +1137,11 @@ def _bind_agg_call( ) -> AggregateKey | TransformKey | TimeTruncKey: """Bind an ``AggCall`` (a granularity callee is a row-level bucket); ``first`` / ``last`` dispatch by the bound operand's type — attached → the series transform, row grain → the ranked aggregation.""" + if parsed.filter is not None: + return _bind_filtered_agg( + parsed, scope=scope, bundle=bundle, alias_map=alias_map, measure_ctx=measure_ctx, + dim_alias_map=dim_alias_map, + ) granularity = _callee_granularity(parsed.agg, bundle=bundle) if granularity is not None: return _bind_granularity_call(parsed, scope=scope, bundle=bundle, granularity=granularity) @@ -1103,6 +1159,56 @@ def _bind_agg_call( return _bind_agg(parsed, scope=scope, bundle=bundle, dim_alias_map=dim_alias_map) +_FILTER_CLAUSE = "FILTER (WHERE …)" + + +def _filter_error(reason: str) -> AggregationFilterError: + return AggregationFilterError( + summary=f"{_FILTER_CLAUSE} {reason}", + location=_FILTER_CLAUSE, + suggestion="filter on columns of the aggregated rows, e.g. sum(amount) FILTER (WHERE status = 'paid').", + ) + + +def _bind_filtered_agg( + parsed: AggCall, *, + scope: ModelScope | StageSchema, + bundle: ResolvedSourceBundle, + alias_map: Optional[Dict[str, "ValueKey"]] = None, + measure_ctx: Optional[MeasureResolutionCtx] = None, + dim_alias_map: Optional[Dict[str, "ValueKey"]] = None, +) -> AggregateKey: + """``agg(…) FILTER (WHERE c)``: the aggregation over the rows where ``c`` holds. Every row-level + input is masked by ``c``; a star source and a ranked pick carry ``c`` on the key instead.""" + condition = _bind(parsed.filter, scope=scope, bundle=bundle, in_filter=False, dim_alias_map=dim_alias_map) + if any(isinstance(k, (AggregateKey, TransformKey)) for k in walk_value_keys(condition)): + raise _filter_error("needs a row-level condition; it cannot read an aggregate, a transform or a measure.") + key = _bind_agg_call( + parsed.model_copy(update={"filter": None}), scope=scope, bundle=bundle, alias_map=alias_map, + measure_ctx=measure_ctx, dim_alias_map=dim_alias_map, + ) + if not isinstance(key, AggregateKey) or is_attached_source(key.source): + raise _filter_error("applies only to an aggregation over rows.") + if isinstance(key.source, StarKey) or key.agg in RANKED_AGGREGATIONS: + return key.model_copy(update={"filter": condition}) + masked = key.model_copy(update={ + "source": _masked(key.source, condition=condition), + "kwargs": tuple((k, _masked(v, condition=condition) if isinstance(v, _ROW_INPUT_KINDS) else v) + for k, v in key.kwargs), + }) + if source_anchor_path(masked.source) != source_anchor_path(key.source): + raise _filter_error("must read the aggregated rows' own model or models joined to it one-to-one.") + return masked + + +# Row-valued aggregation parameters a FILTER masks (literals and attached values are not rows). +_ROW_INPUT_KINDS = (ColumnKey, ColumnSqlKey, SqlFragmentKey) + + +def _masked(value: ValueKey, *, condition: ValueKey) -> ScalarCallKey: + return ScalarCallKey(name="iif", args=(condition, value, LiteralKey(value=None))) + + def _bind_granularity_call( parsed: AggCall, *, scope: ModelScope | StageSchema, bundle: ResolvedSourceBundle, granularity: Granularity, @@ -1691,6 +1797,12 @@ def _bind_scalar( f"(value, pattern); got {len(parsed.args)}." ) raise ValueError(arity_error) + if parsed.name == "round" and len(parsed.args) == 2 and not _is_integer_literal(parsed.args[1]): + raise ValueError( + "'round' ndigits (2nd argument) must be an integer literal; got " + f"{canonical_measure_text(parsed.args[1])!r}." + ) + def bind(a: ParsedExpr, *, in_temporal_slot: bool) -> ValueKey: key = _bind( a, scope=scope, bundle=bundle, in_filter=in_filter, alias_map=alias_map, @@ -1706,6 +1818,16 @@ def bind(a: ParsedExpr, *, in_temporal_slot: bool) -> ValueKey: return ScalarCallKey(name=parsed.name, args=args) +def _is_integer_literal(parsed: ParsedExpr) -> bool: + """An integer literal, signed or not (``2``, ``-2``).""" + if isinstance(parsed, UnaryOp) and parsed.op == "-": + parsed = parsed.operand + return ( + isinstance(parsed, Literal) and isinstance(parsed.value, Decimal) + and parsed.value == parsed.value.to_integral_value() + ) + + def _check_literal_count(count: ValueKey) -> None: if not isinstance(count, LiteralKey): return diff --git a/slayer/engine/compile/projection.py b/slayer/engine/compile/projection.py index 91c655da..e5ef3f78 100644 --- a/slayer/engine/compile/projection.py +++ b/slayer/engine/compile/projection.py @@ -25,6 +25,7 @@ Phase, REGROUP_LEAF_PREFIX, SLOT_COMPOSITE_KINDS, + CastKey, ScalarCallKey, StarKey, TimeTruncKey, @@ -529,6 +530,8 @@ def _canonical_name(key: ValueKey) -> str: # NOSONAR(S3776) — sequential isin return f"_arith_{key.op}" if isinstance(key, ScalarCallKey): return f"_scalar_{key.name}" + if isinstance(key, CastKey): + return f"_cast_{_canonical_name(key.source)}" if isinstance(key, LiteralKey): return f"_lit_{key.value}" if isinstance(key, StarKey): diff --git a/slayer/engine/compile/shift.py b/slayer/engine/compile/shift.py index cc2d12bd..65b9f13f 100644 --- a/slayer/engine/compile/shift.py +++ b/slayer/engine/compile/shift.py @@ -10,6 +10,7 @@ ColumnKey, ColumnSqlKey, REGROUP_LEAF_PREFIX, + CastKey, ScalarCallKey, TimeTruncKey, TransformKey, @@ -57,7 +58,7 @@ def _series_mode(inner: ValueKey, *, to_original: Dict[ValueKey, ValueKey]) -> b transform or cross-model aggregate leaf.""" if isinstance(inner, TransformKey) or is_boolean_shaped(inner): return True - if isinstance(inner, (ArithmeticKey, ScalarCallKey)): + if isinstance(inner, (ArithmeticKey, ScalarCallKey, CastKey)): has_transform, has_cross_model = _series_flags(inner, to_original=to_original) return has_transform or has_cross_model return False diff --git a/slayer/engine/compile/stages.py b/slayer/engine/compile/stages.py index e0a41e8d..bed6daed 100644 --- a/slayer/engine/compile/stages.py +++ b/slayer/engine/compile/stages.py @@ -30,7 +30,7 @@ from slayer.core.enums import DataType, JoinType, RANKED_AGGREGATIONS from slayer.core.errors import AmbiguousJoinPathError -from slayer.core.keys import SLOT_COMPOSITE_KINDS, AggregateKey, Grain, ArithmeticKey, ColumnKey, ColumnSqlKey, InKey, LiteralKey, Phase, PREDICATE_COMPARISON_OPS, StarKey, TimeTruncKey, TransformKey, ValueKey, column_leaf, effective_root_grain, constituent_grain, attached_parameter_grain, substitute_value_keys, substitute_consumer_keys, walk_value_keys, REGROUP_LEAF_PREFIX, is_cross_model_agg, is_kernel_requiring, split_top_level_and, window_kwarg_of, is_row_attach_root, attached_inputs, operand_aggregates, operand_constituents, parameter_row_leaves, source_anchor_path, source_row_leaves, shift_offset_of, VALUE_KEY_TYPES +from slayer.core.keys import SLOT_COMPOSITE_KINDS, AggregateKey, Grain, ArithmeticKey, ColumnKey, ColumnSqlKey, InKey, LiteralKey, Phase, PREDICATE_COMPARISON_OPS, StarKey, TimeTruncKey, TransformKey, ValueKey, column_leaf, effective_root_grain, constituent_grain, attached_parameter_grain, substitute_value_keys, substitute_consumer_keys, walk_value_keys, REGROUP_LEAF_PREFIX, is_cross_model_agg, is_kernel_requiring, split_top_level_and, window_kwarg_of, is_row_attach_root, attached_inputs, NULL_SAFE_COMPARISON_OPS, OPTIONAL_KEY_FIELDS, operand_aggregates, operand_constituents, parameter_row_leaves, source_anchor_path, source_row_leaves, shift_offset_of, VALUE_KEY_TYPES from slayer.core.models import Column, SlayerModel, aggregation_definition, empty_value from slayer.core.refs import key_display from slayer.engine.reference_closure import ( @@ -249,7 +249,10 @@ def _windowed_slot_id_set( def _stable_text(obj: Any) -> str: """A process-independent spelling of a key (sets sorted, never hash order).""" if isinstance(obj, BaseModel): - fields = ",".join(_stable_text(getattr(obj, f)) for f in type(obj).model_fields) + fields = ",".join( + _stable_text(getattr(obj, f)) for f in type(obj).model_fields + if not (f in OPTIONAL_KEY_FIELDS and getattr(obj, f) is None) + ) return f"{type(obj).__name__}({fields})" if isinstance(obj, (set, frozenset)): return "{" + ",".join(sorted(_stable_text(x) for x in obj)) + "}" @@ -1652,6 +1655,8 @@ def _pred_value(cj: ValueKey, *, h: Tuple[str, ...], srcs: _NullSources) -> str: return _is_value( a=cj.operands[0], b=cj.operands[1], negated=op == "is not", h=h, srcs=srcs, ) + if op in NULL_SAFE_COMPARISON_OPS: + return _D if op in PREDICATE_COMPARISON_OPS: return _unknown_if_null(cj.operands, h=h, srcs=srcs) return _D diff --git a/slayer/engine/elaborate_env.py b/slayer/engine/elaborate_env.py index 13987f61..64cdfd6f 100644 --- a/slayer/engine/elaborate_env.py +++ b/slayer/engine/elaborate_env.py @@ -64,6 +64,7 @@ Grain, InKey, LiteralKey, + CastKey, ScalarCallKey, TimePointCmpKey, TimeTruncKey, @@ -1023,6 +1024,8 @@ def _operand_display(k: object) -> str: return f"'{k}'" if isinstance(k, ScalarCallKey): return f"{k.name}({', '.join(_operand_display(a) for a in k.args)})" + if isinstance(k, CastKey): + return f"{'TRY_CAST' if k.try_ else 'CAST'}({_operand_display(k.source)} AS {k.type_text})" if isinstance(k, ArithmeticKey): if len(k.operands) == 1: return f"{k.op}{_operand_display(k.operands[0])}" @@ -1036,7 +1039,7 @@ def _operand_display(k: object) -> str: bool: DataType.BOOLEAN, str: DataType.TEXT, }) _SCALAR_RESULT_TYPES: Mapping[str, DataType] = MappingProxyType({ - **{name: DataType.TEXT for name in _TEXT_SCALARS}, "like": DataType.BOOLEAN, + **{name: DataType.TEXT for name in _TEXT_SCALARS}, "like": DataType.BOOLEAN, "ilike": DataType.BOOLEAN, }) @@ -1049,6 +1052,8 @@ def _leaf_count_type(key: object, *, column_type: ColumnTypeFn) -> Optional[Data return DataType.BOOLEAN if isinstance(key, ScalarCallKey): return _SCALAR_RESULT_TYPES.get(key.name) + if isinstance(key, CastKey) and key.data_type is not DataType.UNKNOWN: + return key.data_type return None @@ -1245,9 +1250,9 @@ def _time_search_children(key: ValueKey) -> List[ValueKey]: return [ key.source, *[a for a in key.args if isinstance( - a, (AggregateKey, TransformKey, ArithmeticKey, ScalarCallKey))], + a, (AggregateKey, TransformKey, ArithmeticKey, ScalarCallKey, CastKey))], *[v for _, v in key.kwargs if isinstance( - v, (AggregateKey, TransformKey, ArithmeticKey, ScalarCallKey))], + v, (AggregateKey, TransformKey, ArithmeticKey, ScalarCallKey, CastKey))], ] if isinstance(key, TransformKey): return [key.input] @@ -1257,9 +1262,11 @@ def _time_search_children(key: ValueKey) -> List[ValueKey]: return [ a for a in key.args if isinstance( - a, (AggregateKey, TransformKey, ArithmeticKey, ScalarCallKey, TimePointCmpKey, InKey), + a, (AggregateKey, TransformKey, ArithmeticKey, ScalarCallKey, TimePointCmpKey, InKey, CastKey), ) ] + if isinstance(key, CastKey): + return [key.source] if isinstance(key, TimePointCmpKey): return [key.operand] if isinstance(key, InKey): diff --git a/slayer/engine/key_metadata.py b/slayer/engine/key_metadata.py index 788c3872..fa31e9d1 100644 --- a/slayer/engine/key_metadata.py +++ b/slayer/engine/key_metadata.py @@ -21,6 +21,7 @@ ColumnTypeFn, ColumnSqlKey, LiteralKey, + CastKey, ScalarCallKey, StarKey, TimeTruncKey, @@ -180,6 +181,8 @@ def measure_key_type( *, model: SlayerModel, key: ValueKey, bundle=None, ) -> Optional[DataType]: """``type`` for a measure slot, from its bound key alone.""" + if isinstance(key, CastKey): + return key.data_type date_type = date_scalar_type(key, column_type=_model_types(model=model, bundle=bundle)) if date_type is not None: return date_type @@ -249,6 +252,8 @@ def dimension_key_metadata( pre-existing: joined refs surface format / description through ``response_meta``, which resolves them against the owning model. """ + if isinstance(key, CastKey): + return key.data_type, None, None if not isinstance(key, (ColumnKey, ColumnSqlKey, TimeTruncKey)): date_type = date_scalar_type(key, column_type=_model_types(model=model, bundle=bundle)) if date_type is not None: diff --git a/slayer/engine/model_reads.py b/slayer/engine/model_reads.py index 2b25222a..d1d2d05c 100644 --- a/slayer/engine/model_reads.py +++ b/slayer/engine/model_reads.py @@ -13,7 +13,7 @@ from slayer.core.query import ComputedDimension, ModelExtension, SlayerQuery, SourceSpec, render_probe_text from slayer.engine.param_binding import default_reference_sites from slayer.engine.reference_closure import parse_fragment_any_dialect -from slayer.core.syntax import DottedRef, Ref, parse_expr, parse_filter_expr +from slayer.core.syntax import DottedRef, Ref, parse_expr from slayer.ir.source_bundle import apply_extension_overlay, follow_sibling_chain, source_name_if_sibling from slayer.sql.column_expansion import is_trivial_base, reference_sites, root_scope_column_ids @@ -98,15 +98,15 @@ def query_reference_parts(query: SlayerQuery) -> list[tuple[str, ...]]: if kind == "ref": out.append(tuple(text.split("."))) elif kind == "formula": - out.extend(_expr_parts(text, parse=parse_expr)) + out.extend(_expr_parts(text)) else: - out.extend(_expr_parts(render_probe_text(text), parse=parse_filter_expr)) + out.extend(_expr_parts(render_probe_text(text))) return out -def _expr_parts(text: str, *, parse) -> list[tuple[str, ...]]: +def _expr_parts(text: str) -> list[tuple[str, ...]]: try: - parsed = parse(text) + parsed = parse_expr(text) except Exception: # noqa: BLE001 — unparseable text reads nothing; the binder reports it return [] return list(_node_parts(parsed)) @@ -192,7 +192,7 @@ def measure(self, *, host: SlayerModel, name: str) -> None: measure = host.get_measure(name) if measure is None or not measure.formula or not self._first_visit(host, f"measure:{name}"): return - for parts in _expr_parts(measure.formula, parse=parse_expr): + for parts in _expr_parts(measure.formula): self.ref(parts, host=host) def _first_visit(self, host: SlayerModel, key: str) -> bool: diff --git a/slayer/engine/population.py b/slayer/engine/population.py index e25127d7..22f8d4da 100644 --- a/slayer/engine/population.py +++ b/slayer/engine/population.py @@ -28,7 +28,6 @@ DottedRef, Ref, parse_expr, - parse_filter_expr, walk_parsed_refs, ) from slayer.memories.resolver import _all_models_in_datasource @@ -121,14 +120,14 @@ def _parsed_filter_refs(filter_str: str) -> tuple[list[str], bool]: def _all_filter_refs(filter_str: str) -> tuple[list[str], bool]: """``(row-valued refs, ok)`` of one masked filter via the execution filter parser. - Uses ``parse_filter_expr`` (the same parser execution binds with), so SQL operator + Uses ``parse_expr`` (the same parser execution binds with), so SQL operator spellings and aggregations in either spelling (``sum(x)``) are all handled uniformly and string literals never surface as references. ``ok`` is False when the filter is aggregate-bearing or unparseable — either way it contributes nothing to determination. """ try: - parsed = parse_filter_expr(render_probe_text(filter_str)) + parsed = parse_expr(render_probe_text(filter_str)) except Exception: return [], False nodes = list(walk_parsed_refs(parsed)) diff --git a/slayer/engine/reference_closure.py b/slayer/engine/reference_closure.py index f6faf218..8e3ab9ad 100644 --- a/slayer/engine/reference_closure.py +++ b/slayer/engine/reference_closure.py @@ -24,6 +24,7 @@ from slayer.core.keys import ( AggregateKey, ArithmeticKey, + CastKey, ColumnKey, ColumnSqlKey, InKey, @@ -236,7 +237,7 @@ def _child_keys(node, *, descend_aggregates: bool = True) -> List: if isinstance(node, AggregateKey): if not descend_aggregates: return [] - return [node.source, *node.args, *(v for _name, v in node.kwargs)] + return [node.source, *node.args, *(v for _name, v in node.kwargs), *([node.filter] if node.filter else [])] if isinstance(node, TransformKey): return [node.input, *sorted(node.partition_keys, key=repr), node.time_key] if isinstance(node, ArithmeticKey): @@ -249,6 +250,8 @@ def _child_keys(node, *, descend_aggregates: bool = True) -> List: return [node.operand] if isinstance(node, SqlFragmentKey): return list(node.refs) + if isinstance(node, CastKey): + return [node.source] raise UnhandledValueKindError(node) diff --git a/slayer/engine/schema_drift.py b/slayer/engine/schema_drift.py index 58abb821..d2368027 100644 --- a/slayer/engine/schema_drift.py +++ b/slayer/engine/schema_drift.py @@ -62,7 +62,6 @@ Ref, StarSource, parse_expr, - parse_filter_expr, walk_parsed_refs, ) from slayer.sql import engine_factory, sqlite_introspect @@ -575,7 +574,7 @@ def _filter_refs_dsl(filter_str: str) -> list[str]: """Column/measure references in a DSL (Mode B) filter, in expression order (deduplicated); ``[]`` on parse failure.""" try: - parsed = parse_filter_expr(filter_str) + parsed = parse_expr(filter_str) except Exception: return [] return list(dict.fromkeys(_walk_ref_names(parsed))) diff --git a/slayer/engine/unrelated.py b/slayer/engine/unrelated.py index 71d559b8..4dc89d28 100644 --- a/slayer/engine/unrelated.py +++ b/slayer/engine/unrelated.py @@ -21,7 +21,7 @@ from slayer.core.query import ComputedDimension, SlayerQuery, render_probe_text from slayer.core.time_spine import is_spine from slayer.engine.reference_closure import key_closure -from slayer.core.syntax import DottedRef, parse_expr, parse_filter_expr +from slayer.core.syntax import DottedRef, parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle __all__ = ["ProductSides", "first_hop_names", "stamp_unrelated", "unrelated_targets"] @@ -43,7 +43,7 @@ def _dotted_heads(node: Any) -> Iterator[str]: def _parsed_heads(text: str, *, as_filter: bool) -> Iterator[str]: try: - parsed = parse_filter_expr(render_probe_text(text)) if as_filter else parse_expr(text) + parsed = parse_expr(render_probe_text(text) if as_filter else text) except Exception: # noqa: BLE001 — the binder reports a malformed expression return yield from _dotted_heads(parsed) diff --git a/slayer/sql/client.py b/slayer/sql/client.py index a9d73c8c..48a37951 100644 --- a/slayer/sql/client.py +++ b/slayer/sql/client.py @@ -389,7 +389,7 @@ def _statement_timeout( conn, *, engine: Any, dialect: SqlDialect, datasource_name: str, timeout_seconds: int, ) -> Iterator[list[StatementTimeoutSkippedWarning]]: """Put the dialect's timeout on ``conn`` for the block; yields the skips to report.""" - if not dialect.timeout_permitted(_server_profile(conn, engine=engine, dialect=dialect)): + if dialect.timeout_reads_profile and not dialect.timeout_permitted(_server_profile(conn, engine=engine, dialect=dialect)): yield [_skip(datasource_name, timeout_seconds, "readonly_user")] return dbapi_connection = conn.connection.dbapi_connection @@ -415,7 +415,9 @@ async def _statement_timeout_async( conn, *, engine: Any, dialect: SqlDialect, datasource_name: str, timeout_seconds: int, ) -> AsyncIterator[list[StatementTimeoutSkippedWarning]]: """Async sibling of ``_statement_timeout``.""" - if not dialect.timeout_permitted(await _server_profile_async(conn, engine=engine, dialect=dialect)): + if dialect.timeout_reads_profile and not dialect.timeout_permitted( + await _server_profile_async(conn, engine=engine, dialect=dialect), + ): yield [_skip(datasource_name, timeout_seconds, "readonly_user")] return dbapi_connection = (await conn.get_raw_connection()).dbapi_connection diff --git a/slayer/sql/dialects/_tier2.py b/slayer/sql/dialects/_tier2.py index 19cc0ef4..50d9f96a 100644 --- a/slayer/sql/dialects/_tier2.py +++ b/slayer/sql/dialects/_tier2.py @@ -12,6 +12,8 @@ from __future__ import annotations +from typing import ClassVar + from sqlglot import exp from slayer.sql.dialects.base import SqlDialect @@ -20,6 +22,8 @@ class RedshiftDialect(SqlDialect): sqlglot_name: str = "redshift" + ilike_native: ClassVar[bool] = True + vendor_cast_types: ClassVar[frozenset[exp.DataType.Type]] = frozenset({exp.DataType.Type.MONEY}) ds_type_aliases: frozenset[str] = frozenset({"redshift"}) explain_prefix: str | None = "EXPLAIN" explain_postfix: str = "" @@ -44,6 +48,7 @@ class PrestoDialect(PrestoFamilyDialect): class DatabricksDialect(SqlDialect): sqlglot_name: str = "databricks" + ilike_native: ClassVar[bool] = True ds_type_aliases: frozenset[str] = frozenset({"databricks"}) explain_prefix: str | None = "EXPLAIN EXTENDED" explain_postfix: str = "" @@ -55,6 +60,7 @@ class DatabricksDialect(SqlDialect): class SparkDialect(SqlDialect): sqlglot_name: str = "spark" + ilike_native: ClassVar[bool] = True ds_type_aliases: frozenset[str] = frozenset({"spark"}) explain_prefix: str | None = "EXPLAIN EXTENDED" explain_postfix: str = "" diff --git a/slayer/sql/dialects/base.py b/slayer/sql/dialects/base.py index bdb0837d..ef09d61d 100644 --- a/slayer/sql/dialects/base.py +++ b/slayer/sql/dialects/base.py @@ -15,6 +15,7 @@ from sqlglot import exp from sqlglot.expressions.core import Expression from sqlglot.dialects.dialect import Dialect as _SqlglotDialect +from sqlglot.errors import SqlglotError from slayer.core.enums import SUB_DAY_GRANULARITIES, DataType, DatePart, TimeGranularity from slayer.core.errors import IdentifierCollisionError, IdentifierLengthError @@ -104,6 +105,39 @@ def iso_text(value: date) -> str: } +#: The validity grammar where a database cannot check a cast itself: a signed decimal with an optional +#: fraction and exponent, surrounding spaces ignored (POSIX classes, so no backslash meets a string literal). +NUMBER_TEXT_PATTERN = r"^[[:space:]]*[+-]?([0-9]+[.]?[0-9]*|[.][0-9]+)([eE][+-]?[0-9]+)?[[:space:]]*$" +TRUE_WORDS = ("true", "t", "yes", "1") +FALSE_WORDS = ("false", "f", "no", "0") + + +def as_text(value: Expression) -> Expression: + """``CAST(value AS TEXT)`` — the input a validity check reads.""" + return exp.Cast(this=value.copy(), to=exp.DataType.build("TEXT")) + + +def cast_target(type_text: str, *, dialect: "SqlDialect") -> exp.DataType: + """The author's cast type as ``dialect``'s ``DataType``; ``ValueError`` when it cannot express it.""" + sqlglot_name = dialect.sqlglot_name + built: Optional[exp.DataType] = None + for reader in (sqlglot_name, "postgres"): + try: + built = exp.DataType.build(type_text, dialect=reader) + break + except (SqlglotError, ValueError): + continue + if built is None or built.this == exp.DataType.Type.USERDEFINED or not dialect.cast_type_supported(built): + raise ValueError(f"CAST to {type_text} cannot be expressed on {sqlglot_name}.") + return built + + +#: Types only some databases have; a dialect without one rejects a CAST to it at plan time. +VENDOR_ONLY_CAST_TYPES: frozenset[exp.DataType.Type] = frozenset({ + exp.DataType.Type.MONEY, exp.DataType.Type.SMALLMONEY, +}) + + def temporal_type_of(value: date) -> DataType: return DataType.TIMESTAMP if isinstance(value, datetime) else DataType.DATE @@ -287,6 +321,12 @@ class SqlDialect(BaseModel): # Correlated subqueries need a server setting this user may be unable to set. correlated_subqueries_gated: bool = False + # Whether setting the statement timeout depends on the server profile (probed only then). + timeout_reads_profile: bool = False + + # The server's profile, when SQL generation needs it (Postgres TRY_CAST); unknown otherwise. + server_profile: ServerProfile = ServerProfile() + # An ``IN`` subquery and its joins run once on the initiator (``GLOBAL``): exact over shards. global_in_subqueries: bool = False @@ -329,6 +369,25 @@ def declared_cast_type(self, dt: Optional[DataType]) -> Optional[DataType]: """The CAST target for a value of type ``dt`` (``None`` skips the cast).""" return dt + # Vendor-only types sqlglot parses for every dialect; a dialect lists the ones it has. + vendor_cast_types: ClassVar[frozenset[exp.DataType.Type]] = frozenset() + + def cast_type_supported(self, data_type: exp.DataType) -> bool: + """Whether this dialect can express a CAST to ``data_type``.""" + return data_type.this not in VENDOR_ONLY_CAST_TYPES or data_type.this in self.vendor_cast_types + + def build_try_cast(self, value: Expression, to: exp.DataType) -> Expression: + """``TRY_CAST``: NULL for an input the cast cannot convert; the input is read as text first.""" + return exp.TryCast(this=as_text(value), to=to) + + # A native case-insensitive ``ILIKE``; elsewhere it is ``LOWER(x) LIKE LOWER(p)``. + ilike_native: ClassVar[bool] = False + + def build_ilike(self, value: Expression, pattern: Expression) -> Expression: + if self.ilike_native: + return exp.ILike(this=value, expression=pattern) + return exp.Like(this=exp.Lower(this=value), expression=exp.Lower(this=pattern)) + def build_null_safe_eq( self, left: Expression, right: Expression, ) -> Expression: diff --git a/slayer/sql/dialects/clickhouse.py b/slayer/sql/dialects/clickhouse.py index 8e9308eb..fccfa8b1 100644 --- a/slayer/sql/dialects/clickhouse.py +++ b/slayer/sql/dialects/clickhouse.py @@ -10,13 +10,13 @@ import re from collections.abc import Sequence from datetime import date -from typing import Any +from typing import Any, ClassVar from sqlglot import exp from sqlglot.expressions.core import Expression from slayer.core.enums import SUB_DAY_GRANULARITIES, DataType, DatePart, TimeGranularity -from slayer.sql.dialects.base import ServerProfile, SqlDialect, iso_text +from slayer.sql.dialects.base import ServerProfile, SqlDialect, as_text, iso_text _CH_CORRELATED_SETTING = "allow_experimental_correlated_subqueries" _CORRELATED_MIN_VERSION = (25, 4) @@ -106,6 +106,7 @@ def _settings_holder(ast: Expression) -> Expression: class ClickhouseDialect(SqlDialect): sqlglot_name: str = "clickhouse" + ilike_native: ClassVar[bool] = True ds_type_aliases: frozenset[str] = frozenset({"clickhouse"}) explain_prefix: str | None = "EXPLAIN" explain_postfix: str = "" @@ -114,6 +115,7 @@ class ClickhouseDialect(SqlDialect): max_identifier_bytes: int | None = None # unbounded approx_count_distinct_native: bool = True correlated_subqueries_gated: bool = True + timeout_reads_profile: bool = True global_in_subqueries: bool = True url_scheme: str | None = "clickhouse+http" install_extra: str | None = "clickhouse" @@ -132,6 +134,12 @@ def restore_connection_timeout(self, dbapi_connection: Any, prior: object) -> No else: settings[_TIMEOUT_SETTING] = prior + def build_try_cast(self, value: Expression, to: exp.DataType) -> Expression: + spelled = to.sql(dialect=self.sqlglot_name) + if spelled.startswith("Nullable(") and spelled.endswith(")"): + spelled = spelled[len("Nullable("):-1] + return exp.Anonymous(this="accurateCastOrNull", expressions=[as_text(value), exp.Literal.string(spelled)]) + def server_profile_sql(self) -> str | None: return "SELECT version(), getSetting('readonly')" diff --git a/slayer/sql/dialects/duckdb.py b/slayer/sql/dialects/duckdb.py index eaf061ce..5c310f2c 100644 --- a/slayer/sql/dialects/duckdb.py +++ b/slayer/sql/dialects/duckdb.py @@ -7,6 +7,8 @@ from __future__ import annotations +from typing import ClassVar + from sqlglot import exp from slayer.sql.dialects.base import SqlDialect @@ -14,6 +16,7 @@ class DuckdbDialect(SqlDialect): sqlglot_name: str = "duckdb" + ilike_native: ClassVar[bool] = True ds_type_aliases: frozenset[str] = frozenset({"duckdb"}) explain_prefix: str | None = "EXPLAIN ANALYZE" explain_postfix: str = "" diff --git a/slayer/sql/dialects/mysql.py b/slayer/sql/dialects/mysql.py index 8fd89cf8..ebc9eee3 100644 --- a/slayer/sql/dialects/mysql.py +++ b/slayer/sql/dialects/mysql.py @@ -16,7 +16,11 @@ from slayer.core.enums import DataType, DatePart, TimeGranularity from slayer.sql.dialects.base import ( + FALSE_WORDS, + NUMBER_TEXT_PATTERN, + TRUE_WORDS, SqlDialect, + as_text, StatAgg1Name, StatAgg2Name, _build_covar_decomposition, @@ -38,6 +42,21 @@ class MysqlDialect(SqlDialect): sqlglot_name: str = "mysql" ds_type_aliases: frozenset[str] = frozenset({"mysql"}) + + def build_try_cast(self, value: Expression, to: exp.DataType) -> Expression: + """Numeric and boolean targets are guarded by the validity grammar; MySQL's date casts already yield NULL.""" + text = as_text(value) + if to.is_type(*exp.DataType.NUMERIC_TYPES): + guard = exp.RegexpLike(this=text, expression=exp.Literal.string(NUMBER_TEXT_PATTERN)) + return exp.Case(ifs=[exp.If(this=guard, true=exp.Cast(this=value.copy(), to=to))]) + if to.is_type(exp.DataType.Type.BOOLEAN): + word = exp.Lower(this=exp.Trim(this=text)) + return exp.Case(ifs=[ + exp.If(this=exp.In(this=word, expressions=[exp.Literal.string(w) for w in TRUE_WORDS]), true=exp.true()), + exp.If(this=exp.In(this=word.copy(), expressions=[exp.Literal.string(w) for w in FALSE_WORDS]), + true=exp.false()), + ]) + return exp.Cast(this=value, to=to) explain_prefix: str | None = "EXPLAIN FORMAT=JSON" explain_postfix: str = "" log10_native: bool = True diff --git a/slayer/sql/dialects/postgres.py b/slayer/sql/dialects/postgres.py index 6844a98d..d80819b0 100644 --- a/slayer/sql/dialects/postgres.py +++ b/slayer/sql/dialects/postgres.py @@ -2,10 +2,16 @@ from __future__ import annotations +from collections.abc import Sequence +from typing import Any, ClassVar from sqlglot import exp +from sqlglot.expressions.core import Expression -from slayer.sql.dialects.base import SqlDialect +from slayer.sql.dialects.base import ServerProfile, SqlDialect, as_text + +#: The first Postgres major version with ``pg_input_is_valid``. +_PG_INPUT_IS_VALID_SINCE = (16, 0) def _cast_round_arg_to_numeric(node: exp.Expression) -> exp.Expression: @@ -37,6 +43,8 @@ def _cast_round_arg_to_numeric(node: exp.Expression) -> exp.Expression: class PostgresDialect(SqlDialect): sqlglot_name: str = "postgres" ds_type_aliases: frozenset[str] = frozenset({"postgres", "postgresql"}) + vendor_cast_types: ClassVar[frozenset[exp.DataType.Type]] = frozenset({exp.DataType.Type.MONEY}) + ilike_native: ClassVar[bool] = True explain_prefix: str | None = "EXPLAIN ANALYZE" explain_postfix: str = "" log10_native: bool = True @@ -48,6 +56,29 @@ class PostgresDialect(SqlDialect): async_driver: str | None = "asyncpg" install_extra: str | None = "postgres" + def server_profile_sql(self) -> str | None: + return "SELECT current_setting('server_version_num')" + + def parse_server_profile( + self, *, base_row: Sequence[Any] | None, correlated_row: Sequence[Any] | None = None, + ) -> ServerProfile: + number = base_row[0] if base_row else None + try: + version_num = int(number) if number is not None else None + except (TypeError, ValueError): + version_num = None + return ServerProfile(version=None if version_num is None else (version_num // 10000, version_num % 10000)) + + def build_try_cast(self, value: Expression, to: exp.DataType) -> Expression: + """Postgres 16+ checks the input with ``pg_input_is_valid``; an older or unknown server gets a plain cast.""" + version = self.server_profile.version + if version is None or version < _PG_INPUT_IS_VALID_SINCE: + return exp.Cast(this=value, to=to) + check = exp.Anonymous(this="pg_input_is_valid", expressions=[ + as_text(value), exp.Literal.string(to.sql(dialect=self.sqlglot_name)), + ]) + return exp.Case(ifs=[exp.If(this=check, true=exp.Cast(this=value.copy(), to=to))]) + def build_integer_sequence(self, *, size: int) -> exp.Select: return exp.select(exp.column("i")).from_(exp.Table( this=exp.Anonymous(this="GENERATE_SERIES", expressions=[ diff --git a/slayer/sql/dialects/snowflake.py b/slayer/sql/dialects/snowflake.py index 77fd8e17..6cb0fa82 100644 --- a/slayer/sql/dialects/snowflake.py +++ b/slayer/sql/dialects/snowflake.py @@ -100,6 +100,7 @@ class SnowflakeDialect(SqlDialect): """Snowflake dialect.""" sqlglot_name: str = "snowflake" + ilike_native: ClassVar[bool] = True ds_type_aliases: frozenset[str] = frozenset({"snowflake"}) explain_prefix: str | None = "EXPLAIN USING JSON" explain_postfix: str = "" diff --git a/slayer/sql/dialects/sqlite.py b/slayer/sql/dialects/sqlite.py index 444532b0..dfb45fb2 100644 --- a/slayer/sql/dialects/sqlite.py +++ b/slayer/sql/dialects/sqlite.py @@ -10,10 +10,21 @@ from sqlglot import exp from sqlglot.expressions.core import Expression -from typing import Optional +from typing import ClassVar, Optional from slayer.core.enums import DataType, DatePart, TimeGranularity -from slayer.sql.dialects.base import COMPARISON_NODES, SqlDialect, TemporalComparisonOp, iso_text +from slayer.core.keys import cast_data_type +from slayer.core.time_points import parse_temporal_value +from slayer.sql.dialects.base import ( + COMPARISON_NODES, + FALSE_WORDS, + TRUE_WORDS, + VENDOR_ONLY_CAST_TYPES, + SqlDialect, + TemporalComparisonOp, + as_text, + iso_text, +) def rewrite_sqlite_json_extract(node: Expression) -> Expression: @@ -345,6 +356,33 @@ def _slayer_date_add(ts, n, unit): return moved.isoformat(sep=" ") +_NUMBER_TEXT_RE = re.compile(r"\s*[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?\s*") + + +def _slayer_try_cast(text, target): + """``TRY_CAST(text AS target)``: the converted value, or NULL where the cast cannot convert ``text``.""" + if text is None: + return None + kind = cast_data_type(target) + if kind in (DataType.INT, DataType.DOUBLE): + if not _NUMBER_TEXT_RE.fullmatch(text): + return None + number = float(text) + return int(number) if kind is DataType.INT else number + if kind is DataType.BOOLEAN: + word = text.strip().lower() + return 1 if word in TRUE_WORDS else 0 if word in FALSE_WORDS else None + if kind in (DataType.DATE, DataType.TIMESTAMP): + moment = parse_temporal_value(text.strip()) + if moment is None: + return None + if kind is DataType.DATE: + return (moment.date() if isinstance(moment, datetime) else moment).isoformat() + stamp = moment if isinstance(moment, datetime) else datetime(moment.year, moment.month, moment.day) + return stamp.isoformat(sep=" ") + return text + + def register_sqlite_udfs(dbapi_connection) -> None: """Register all SLayer SQLite UDFs on a fresh DBAPI connection (idempotent).""" dbapi_connection.create_function("ln", 1, _ln) @@ -357,6 +395,7 @@ def register_sqlite_udfs(dbapi_connection) -> None: dbapi_connection.create_function("pow", 2, _pow) dbapi_connection.create_function("power", 2, _pow) dbapi_connection.create_function("slayer_date_add", 3, _slayer_date_add) + dbapi_connection.create_function("slayer_try_cast", 2, _slayer_try_cast) dbapi_connection.create_aggregate("median", 1, _MedianAgg) dbapi_connection.create_aggregate("percentile_cont", 2, _PercentileContAgg) @@ -376,6 +415,8 @@ def register_sqlite_udfs(dbapi_connection) -> None: class SqliteDialect(SqlDialect): sqlglot_name: str = "sqlite" + # Any type name is a column affinity on SQLite. + vendor_cast_types: ClassVar[frozenset[exp.DataType.Type]] = VENDOR_ONLY_CAST_TYPES ds_type_aliases: frozenset[str] = frozenset({"sqlite"}) explain_prefix: str | None = "EXPLAIN QUERY PLAN" explain_postfix: str = "" @@ -392,6 +433,11 @@ def declared_cast_type(self, dt: Optional[DataType]) -> Optional[DataType]: return None return dt + def build_try_cast(self, value: Expression, to: exp.DataType) -> Expression: + return exp.Anonymous(this="slayer_try_cast", expressions=[ + as_text(value), exp.Literal.string(to.sql(dialect="postgres")), + ]) + def build_null_safe_eq( self, left: Expression, right: Expression, ) -> Expression: diff --git a/slayer/sql/dialects/tsql.py b/slayer/sql/dialects/tsql.py index da39729f..05a66f35 100644 --- a/slayer/sql/dialects/tsql.py +++ b/slayer/sql/dialects/tsql.py @@ -96,6 +96,9 @@ def _bit_value(predicate: Expr) -> Expression: class TsqlDialect(DottedAliasManglingMixin, SqlDialect): sqlglot_name: str = "tsql" ds_type_aliases: frozenset[str] = frozenset({"mssql", "sqlserver", "tsql"}) + vendor_cast_types: ClassVar[frozenset[exp.DataType.Type]] = frozenset({ + exp.DataType.Type.MONEY, exp.DataType.Type.SMALLMONEY, + }) explain_prefix: str | None = "SET SHOWPLAN_ALL ON;" explain_postfix: str = "; SET SHOWPLAN_ALL OFF" log10_native: bool = True diff --git a/slayer/sql/generator.py b/slayer/sql/generator.py index cd994188..17d738bd 100644 --- a/slayer/sql/generator.py +++ b/slayer/sql/generator.py @@ -40,7 +40,7 @@ from slayer.core.enums import RANK_FAMILY_TRANSFORMS from slayer.core.granularity import CustomGranularity, Granularity, granularity_parts from slayer.core.time_spine import TIME_SPINE_COLUMN, is_spine -from slayer.core.keys import BOOL_CONNECTIVE_OPS, KIND_POLICY, REGROUP_LEAF_PREFIX, SLOT_COMPOSITE_KINDS, TEMPORAL_TYPES, VALUE_KEY_TYPES, AggregateKey, ArithmeticKey, ColumnKey, ColumnSqlKey, ColumnTypeFn, InKey, LiteralKey, Phase, ScalarCallKey, SqlFragmentKey, StarKey, TimePointCmpKey, TimeTruncKey, TransformKey, aggregation_source_type, column_path, date_add_type, is_boolean_shaped, parameter_row_leaves, shift_offset_of, source_anchor_path, substitute_value_keys, temporal_type, walk_value_keys +from slayer.core.keys import BOOL_CONNECTIVE_OPS, KIND_POLICY, REGROUP_LEAF_PREFIX, SLOT_COMPOSITE_KINDS, TEMPORAL_TYPES, VALUE_KEY_TYPES, AggregateKey, ArithmeticKey, CastKey, ColumnKey, ColumnSqlKey, ColumnTypeFn, InKey, LiteralKey, Phase, ScalarCallKey, SqlFragmentKey, StarKey, TimePointCmpKey, TimeTruncKey, TransformKey, aggregation_source_type, column_path, date_add_type, is_boolean_shaped, parameter_row_leaves, shift_offset_of, source_anchor_path, substitute_value_keys, temporal_type, walk_value_keys from slayer.core.join_walker import ( aggregation_owner, follow, @@ -599,7 +599,7 @@ def _conjunction(parts: Sequence[Expression]) -> Optional[Expression]: # pyrigh return out # Parameter values rendered through a scope; scalars render as literals, markers never. -_ROW_PARAM_KINDS = (ColumnKey, ColumnSqlKey, SqlFragmentKey) +_ROW_PARAM_KINDS = (ColumnKey, ColumnSqlKey, SqlFragmentKey, ScalarCallKey) # Shift units whose whole-unit offsets map each bucket start onto another, making the outer re-trunc a per-row no-op. _BUCKET_ALIGNED_SHIFT_UNITS: dict[str, frozenset[str]] = { @@ -669,7 +669,7 @@ def _is_host_grain(key) -> bool: # Real ValueKey args (a ScalarCallKey / iif may also carry raw scalar literals). _COMPOUND_VALUE_KEYS = ( ColumnKey, ColumnSqlKey, TimeTruncKey, StarKey, - AggregateKey, TransformKey, ArithmeticKey, ScalarCallKey, InKey, + AggregateKey, TransformKey, ArithmeticKey, ScalarCallKey, InKey, CastKey, ) @@ -706,6 +706,8 @@ def _walk_cp_predicate(*, op: str, key, connective_operand: bool) -> None: _walk_cp_predicate(op=op, key=a, connective_operand=False) elif isinstance(key, InKey): _walk_cp_predicate(op=op, key=key.column, connective_operand=False) + elif isinstance(key, CastKey): + _walk_cp_predicate(op=op, key=key.source, connective_operand=False) def _effective_src_filters(*, lowered_filters, plan) -> list: @@ -1990,6 +1992,8 @@ def _walk(key, fn) -> None: elif isinstance(key, ScalarCallKey): for a in key.args: _walk(a, fn) + elif isinstance(key, CastKey): + _walk(key.source, fn) def _for_each_local_agg(fn) -> None: for sid in base_render_order: @@ -5525,6 +5529,8 @@ def _walk(k) -> None: _walk(a) elif isinstance(k, InKey): _walk(k.column) + elif isinstance(k, CastKey): + _walk(k.source) _walk(key) return out @@ -6250,6 +6256,18 @@ def _host_base_order_ref( # NOSONAR(S3776) — per-key-kind resolution of ONE h if expr is not None: return expr + # A hidden row-level expression in an ungrouped query renders in place over the host's + # rows; its joins were collected with the order slots. + if ( + source_model is not None and bundle is not None and slot.phase == Phase.ROW + and isinstance(key, SLOT_COMPOSITE_KINDS) + ): + frame = self._scope_frame( + model=source_model, relation=source_relation, bundle=bundle, + allocator=self._gen_allocator or self._new_allocator(), + ) + return render_value_key(key=key, ctx=RenderContext(scope=frame, dialect=self._dialect)) + raise NotImplementedError( f"ORDER BY references a hidden slot (id={slot.id!r}, key=" f"{type(slot.key).__name__}) that was not resolved at plan " diff --git a/slayer/sql/render/row_expr.py b/slayer/sql/render/row_expr.py index f15beaf4..c328119c 100644 --- a/slayer/sql/render/row_expr.py +++ b/slayer/sql/render/row_expr.py @@ -26,7 +26,9 @@ CLOCK_FUNCTIONS, DATE_ADD_COUNT_ARG, DATE_OPERAND_ARGS, + NULL_SAFE_COMPARISON_OPS, ArithmeticKey, + CastKey, ColumnKey, ColumnSqlKey, ColumnTypeFn, @@ -41,7 +43,7 @@ temporal_type, unit_word, ) -from slayer.sql.dialects.base import SqlDialect, TemporalComparisonOp, temporal_type_of +from slayer.sql.dialects.base import SqlDialect, TemporalComparisonOp, cast_target, operand_copy, temporal_type_of from slayer.sql.render.parse import rewrite_log_aliases as rewrite_log_alias _BINARY_OPS: Dict[str, Any] = { @@ -79,7 +81,7 @@ def _literal(value: Any, *, dialect: Optional[SqlDialect] = None) -> Expression: exp.And: 2, exp.Not: 3, exp.EQ: 4, exp.NEQ: 4, exp.LT: 4, exp.LTE: 4, exp.GT: 4, exp.GTE: 4, - exp.Is: 4, exp.In: 4, exp.Like: 4, exp.Between: 4, + exp.Is: 4, exp.In: 4, exp.Like: 4, exp.ILike: 4, exp.Between: 4, exp.Add: 5, exp.Sub: 5, exp.Mul: 6, exp.Div: 6, exp.Mod: 6, exp.Neg: 7, @@ -241,6 +243,19 @@ def temporal_comparison(key: ArithmeticKey) -> Optional[Tuple[TemporalComparison return None +def render_null_safe(*, op: str, operands: List[Expression], dialect: SqlDialect) -> Expression: + """``a IS [NOT] DISTINCT FROM b`` through the dialect's null-safe equality.""" + left, right = (operand_copy(o) for o in operands) + same = dialect.build_null_safe_eq(left, right) + return same if op == "is not distinct from" else exp.Not(this=exp.Paren(this=same)) + + +def render_cast(*, key: CastKey, value: Expression, dialect: SqlDialect) -> Expression: + """The database's own cast of the author's type; ``TRY_CAST`` through the dialect's lowering.""" + to = cast_target(key.type_text, dialect=dialect) + return dialect.build_try_cast(value, to) if key.try_ else exp.Cast(this=value, to=to) + + def render_temporal_comparison( *, op: TemporalComparisonOp, operand: Expression, value: date, dialect: SqlDialect, ) -> Expression: @@ -261,6 +276,8 @@ def render_scalar_call( raise RuntimeError(f"Date function {name!r} renders only from its key (render_date_call).") if name == "like": return exp.Like(this=args[0], expression=args[1]) + if name == "ilike": + return dialect.build_ilike(args[0], args[1]) if name == "mod": # ``%`` is an operator, not a Func; the ``%`` composer parenthesises both sides correctly. return render_arithmetic(op="%", operands=args) @@ -394,6 +411,8 @@ def _part(a: Any) -> Expression: if comparison is not None: op, operand, value = comparison return render_temporal_comparison(op=op, operand=_part(operand), value=value, dialect=dialect) + if key.op in NULL_SAFE_COMPARISON_OPS: + return render_null_safe(op=key.op, operands=[_part(o) for o in key.operands], dialect=dialect) return render_arithmetic( op=key.op.lower(), operands=numeric_parts(key=key, parts=[_part(o) for o in key.operands], column_type=column_type), @@ -410,6 +429,8 @@ def _part(a: Any) -> Expression: if isinstance(key, InKey): (column,) = numeric_parts(key=key, parts=[_part(key.column)], column_type=column_type) return render_in_predicate(key=key, column=column, dialect=dialect) + if isinstance(key, CastKey): + return render_cast(key=key, value=_part(key.source), dialect=dialect) raise NotImplementedError( f"Row-level expression cannot contain {type(key).__name__}.", ) diff --git a/slayer/sql/render/value_expr.py b/slayer/sql/render/value_expr.py index 1898c250..955e8bac 100644 --- a/slayer/sql/render/value_expr.py +++ b/slayer/sql/render/value_expr.py @@ -13,8 +13,10 @@ from slayer.core.enums import DataType from slayer.core.errors import RenderContextMissingFacilityError from slayer.core.keys import ( + NULL_SAFE_COMPARISON_OPS, AggregateKey, ArithmeticKey, + CastKey, ColumnKey, ColumnSqlKey, ColumnTypeFn, @@ -53,7 +55,9 @@ iif_case_chain, numeric_parts, render_arithmetic, + render_cast, render_in_predicate, + render_null_safe, render_date_call, render_temporal_comparison, render_scalar_call, @@ -385,6 +389,10 @@ def render_value_key( # NOSONAR(S3776) — sequential dispatch over the closed if ctx.filters is not None and ctx.filters.paren_comparison_operands: rendered = _paren_if_binary(rendered) return render_temporal_comparison(op=op, operand=rendered, value=value, dialect=ctx.dialect) + if key.op in NULL_SAFE_COMPARISON_OPS: + return render_null_safe( + op=key.op, operands=[render_value_key(key=o, ctx=ctx) for o in key.operands], dialect=ctx.dialect, + ) op = key.op.lower() operands = numeric_parts( key=key, parts=[render_value_key(key=o, ctx=ctx) for o in key.operands], @@ -423,6 +431,9 @@ def render_value_key( # NOSONAR(S3776) — sequential dispatch over the closed ) return render_in_predicate(key=key, column=column, dialect=ctx.dialect) + if isinstance(key, CastKey): + return render_cast(key=key, value=render_value_key(key=key.source, ctx=ctx), dialect=ctx.dialect) + if isinstance(key, AggregateKey): return _render_aggregate(key, ctx) diff --git a/slayer/sql/scope.py b/slayer/sql/scope.py index 01e1fa5a..f1c1fd7a 100644 --- a/slayer/sql/scope.py +++ b/slayer/sql/scope.py @@ -33,6 +33,7 @@ from slayer.core.keys import ( REGROUP_LEAF_PREFIX, ArithmeticKey, + CastKey, ColumnKey, ColumnSqlKey, InKey, @@ -71,7 +72,7 @@ # A ref that can enter a scope: structural column refs, derived columns, bound # parameter expressions, free Mode-A / predicate text, and row-level expression # composites — an aggregate's same-model expression source anchors through the same door. -Ref = Union[ColumnKey, ColumnSqlKey, SqlFragmentKey, ArithmeticKey, ScalarCallKey, LiteralKey, TimeTruncKey, InKey, str] +Ref = Union[ColumnKey, ColumnSqlKey, SqlFragmentKey, ArithmeticKey, ScalarCallKey, LiteralKey, TimeTruncKey, InKey, CastKey, str] class _OrderedPathSet: @@ -457,7 +458,7 @@ def _anchor(self, ref: Ref) -> exp.Expression: # NOSONAR(S3776) — flat dispat # its joins) here, substituted into the parsed template. template = sql_template(text=ref.template, dialect=self.dialect.sqlglot_name) return template.render({f"r{i}": self._anchor(r) for i, r in enumerate(ref.refs)}) - if isinstance(ref, (ArithmeticKey, ScalarCallKey, LiteralKey, TimeTruncKey, InKey)): + if isinstance(ref, (ArithmeticKey, ScalarCallKey, LiteralKey, TimeTruncKey, InKey, CastKey)): # An aggregate's row-level expression source — column # leaves anchor recursively through this scope, so join # registration and derived expansion apply per leaf. diff --git a/slayer/sql/sql_expr.py b/slayer/sql/sql_expr.py index c63fcf3d..d31445fe 100644 --- a/slayer/sql/sql_expr.py +++ b/slayer/sql/sql_expr.py @@ -37,6 +37,7 @@ from slayer.core.errors import IllegalWindowInFilterError from slayer.sql.dialects import get_dialect +from slayer.sql.dialects.base import cast_target from slayer.sql.dialects.sqlite import rewrite_sqlite_json_extract from slayer.sql.render.parse import rewrite_log_aliases from slayer.sql.window_detect import has_window_function as _has_window_function @@ -139,3 +140,9 @@ def assert_no_window_in_filter(text: str, *, source: str) -> None: "into an earlier multi-stage source_query." ), ) + + + +def build_cast_type(type_text: str, *, dialect: str) -> exp.DataType: + """The author's cast type as ``dialect``'s ``DataType``; ``ValueError`` when it cannot express it.""" + return cast_target(type_text, dialect=get_dialect(dialect)) diff --git a/tests/test_dev1744_value_expr.py b/tests/test_dev1744_value_expr.py index d94f1b48..845894cb 100644 --- a/tests/test_dev1744_value_expr.py +++ b/tests/test_dev1744_value_expr.py @@ -1871,7 +1871,7 @@ def test_binder_only_names_are_like_iif_and_date_functions(self) -> None: """Names whose Mode-B call is not the same SQL function: the LIKE and CASE rewrite targets, and the dialect-rendered date functions.""" assert SCALAR_FUNCTIONS - SCALAR_PASSTHROUGH == { - "like", "iif", "date_part", "date_diff", "date_add", "interval", + "like", "ilike", "iif", "date_part", "date_diff", "date_add", "interval", "current_date", "now", } diff --git a/tests/test_dev1747_reroot_visitor.py b/tests/test_dev1747_reroot_visitor.py index fdfcd712..ac3c182e 100644 --- a/tests/test_dev1747_reroot_visitor.py +++ b/tests/test_dev1747_reroot_visitor.py @@ -21,10 +21,11 @@ import pytest -from slayer.core.enums import TimeGranularity +from slayer.core.enums import DataType, TimeGranularity from slayer.core.keys import ( AggregateKey, ArithmeticKey, + CastKey, ColumnKey, ColumnSqlKey, InKey, @@ -347,6 +348,9 @@ def test_every_union_member_is_handled(self) -> None: SqlFragmentKey: SqlFragmentKey( template="{r0} * 2", refs=(ColumnKey(path=("customers",), leaf="spend"),), ), + CastKey: CastKey( + source=ColumnKey(path=("customers",), leaf="spend"), type_text="INT", data_type=DataType.INT, + ), } members = set(get_args(ValueKey)) missing = members - set(samples) diff --git a/tests/test_dev1753_last_four_scalars.py b/tests/test_dev1753_last_four_scalars.py index c123c7bf..0cbfc342 100644 --- a/tests/test_dev1753_last_four_scalars.py +++ b/tests/test_dev1753_last_four_scalars.py @@ -75,7 +75,7 @@ def test_no_parser_only_names_remain(self) -> None: def test_binder_only_names_are_like_iif_and_date_functions(self) -> None: # Rewrite targets and dialect-rendered date functions are not SQL pass-throughs. assert SCALAR_FUNCTIONS - SCALAR_PASSTHROUGH == { - "like", "iif", "date_part", "date_diff", "date_add", "interval", + "like", "ilike", "iif", "date_part", "date_diff", "date_add", "interval", "current_date", "now", } diff --git a/tests/test_dev1825_substitute_keys.py b/tests/test_dev1825_substitute_keys.py index 235c357d..fec7ede7 100644 --- a/tests/test_dev1825_substitute_keys.py +++ b/tests/test_dev1825_substitute_keys.py @@ -13,10 +13,11 @@ import pytest -from slayer.core.enums import TimeGranularity +from slayer.core.enums import DataType, TimeGranularity from slayer.core.keys import ( AggregateKey, ArithmeticKey, + CastKey, ColumnKey, ColumnSqlKey, InKey, @@ -118,6 +119,7 @@ def test_every_union_member_has_a_sample(self) -> None: TimePointCmpKey: TimePointCmpKey(op="=", operand=CITY, point="2025-Q1"), InKey: InKey(column=CITY, values=(LiteralKey(value="gold"),)), SqlFragmentKey: SqlFragmentKey(template="{r0} * 2", refs=(CITY,)), + CastKey: CastKey(source=CITY, type_text="TEXT", data_type=DataType.TEXT), } members = set(get_args(ValueKey)) missing = members - set(samples) diff --git a/tests/test_dev1827_value_key_traversal.py b/tests/test_dev1827_value_key_traversal.py index 959e4747..4fd13f66 100644 --- a/tests/test_dev1827_value_key_traversal.py +++ b/tests/test_dev1827_value_key_traversal.py @@ -16,7 +16,7 @@ from slayer.core.enums import DataType, TimeGranularity from slayer.core.keys import Grain -from slayer.core.keys import KIND_POLICY, VALUE_KEY_TYPES, AggregateKey, ArithmeticKey, ColumnKey, ColumnSqlKey, InKey, KindPolicy, LiteralKey, Phase, ScalarCallKey, SqlFragmentKey, StarKey, TimePointCmpKey, TimeTruncKey, TransformKey, ValueKey, _FrozenKey, reroot_value_key, substitute_value_keys, walk_value_keys +from slayer.core.keys import KIND_POLICY, VALUE_KEY_TYPES, AggregateKey, ArithmeticKey, CastKey, ColumnKey, ColumnSqlKey, InKey, KindPolicy, LiteralKey, Phase, ScalarCallKey, SqlFragmentKey, StarKey, TimePointCmpKey, TimeTruncKey, TransformKey, ValueKey, _FrozenKey, reroot_value_key, substitute_value_keys, walk_value_keys from slayer.core.models import Column, ModelJoin, ModelMeasure, SlayerModel from slayer.core.query import ColumnRef, SlayerQuery, TimeDimension from slayer.engine.reference_closure import UnhandledValueKindError, aggregate_input_closure @@ -83,6 +83,7 @@ TimePointCmpKey: TP, InKey: IK, SqlFragmentKey: SqlFragmentKey(template="{r0} * {r1}", refs=(CITY, JOINED)), + CastKey: CastKey(source=JOINED, type_text="INT", data_type=DataType.INT), } LEAF_KINDS = (ColumnKey, ColumnSqlKey, StarKey, LiteralKey) @@ -291,12 +292,12 @@ def test_slottable_membership(self) -> None: def test_slot_composite_membership(self) -> None: assert {k for k, p in KIND_POLICY.items() if p.slot_composite} == { - ArithmeticKey, ScalarCallKey, InKey, + ArithmeticKey, ScalarCallKey, InKey, CastKey, } def test_materialised_order_membership(self) -> None: assert {k for k, p in KIND_POLICY.items() if p.materialised_order} == { - AggregateKey, ArithmeticKey, ScalarCallKey, TransformKey, InKey, + AggregateKey, ArithmeticKey, ScalarCallKey, TransformKey, InKey, CastKey, } def test_kind_policy_is_frozen(self) -> None: diff --git a/tests/test_dev1842_keys_prepend.py b/tests/test_dev1842_keys_prepend.py index 817c62bf..972e2f4a 100644 --- a/tests/test_dev1842_keys_prepend.py +++ b/tests/test_dev1842_keys_prepend.py @@ -20,11 +20,12 @@ import pytest -from slayer.core.enums import TimeGranularity +from slayer.core.enums import DataType, TimeGranularity from slayer.core import keys as K from slayer.core.keys import ( AggregateKey, ArithmeticKey, + CastKey, ColumnKey, ColumnSqlKey, InKey, @@ -79,6 +80,7 @@ def _samples() -> dict: SqlFragmentKey: SqlFragmentKey( template="{r0} * 2", refs=(ColumnKey(path=(), leaf="spend"),), ), + CastKey: CastKey(source=ColumnKey(path=(), leaf="spend"), type_text="INT", data_type=DataType.INT), } diff --git a/tests/test_dev1901_param_binding.py b/tests/test_dev1901_param_binding.py index 71a0e26a..ee4aa903 100644 --- a/tests/test_dev1901_param_binding.py +++ b/tests/test_dev1901_param_binding.py @@ -225,14 +225,14 @@ def test_default(self, agg: str) -> None: @pytest.mark.parametrize("text", sorted(BAD_DEFAULTS.values())) def test_string_argument(self, text: str) -> None: - quoted = text.replace("'", "\\'") + quoted = text.replace("'", "''") with pytest.raises(UnanalyzableAggregationParameterError) as ei: _key(f"amount:wsum(weight='{quoted}')") self._assert_names(ei.value, agg="wsum", text=text) @pytest.mark.parametrize("text", ROW_TEXTS) def test_row_expression_binds(self, text: str) -> None: - quoted = text.replace("'", "\\'") + quoted = text.replace("'", "''") assert _kw(f"amount:wsum(weight='{quoted}')")["weight"] is not None @pytest.mark.parametrize("dialect,text", [ @@ -241,7 +241,7 @@ def test_row_expression_binds(self, text: str) -> None: ]) def test_dialect_row_expression_binds(self, dialect: str, text: str) -> None: models = dev1901_models() - quoted = text.replace("'", "\\'") + quoted = text.replace("'", "''") key = bind_expr( parse_expr(f"amount:wsum(weight='{quoted}')"), scope=ModelScope(source_model=models[0]), @@ -317,7 +317,7 @@ def _first_token_revisit_models() -> List[SlayerModel]: class TestFragmentTextSurvives: def test_a_literal_spelling_the_placeholder_sentinel_is_kept(self) -> None: frag = _fragment(_kw( - "amount:wsum(weight='CASE WHEN status = \\'__slayer_r0__\\' THEN amount ELSE 0 END')")["weight"]) + "amount:wsum(weight='CASE WHEN status = ''__slayer_r0__'' THEN amount ELSE 0 END')")["weight"]) assert "'__slayer_r0__'" in frag.template, frag.template assert set(frag.refs) == {AMOUNT, ColumnKey(path=(), leaf="status")} diff --git a/tests/test_dev1934_formula_agg.py b/tests/test_dev1934_formula_agg.py index bb0dcdeb..7a6b267b 100644 --- a/tests/test_dev1934_formula_agg.py +++ b/tests/test_dev1934_formula_agg.py @@ -306,7 +306,7 @@ async def _pct_sql(p: str | None = None, *, default: str | None = None) -> str: """``price:percentile`` with ``p`` passed as a query-time string and/or a model default.""" aggs = (Aggregation(name="percentile", params=[AggregationParam(name="p", sql=default)]),) \ if default is not None else () - arg = "" if p is None else "(p='" + p.replace("'", "\\'") + "')" + arg = "" if p is None else "(p='" + p.replace("'", "''") + "')" return await _sql(f"price:percentile{arg}", aggs=aggs) diff --git a/tests/test_dev2013_expression_attribution.py b/tests/test_dev2013_expression_attribution.py index ce156944..34d21e79 100644 --- a/tests/test_dev2013_expression_attribution.py +++ b/tests/test_dev2013_expression_attribution.py @@ -20,6 +20,7 @@ KIND_POLICY, AggregateKey, ArithmeticKey, + CastKey, ColumnKey, ColumnSqlKey, Grain, @@ -652,6 +653,7 @@ def test_unanalysable_definition_inside_composite(self): ["city", " in ", "'Alpha'", "'Beta'"]), (SqlFragmentKey(template="LENGTH({r0}) + {r1}", refs=(CITY, ColumnKey(leaf="quantity"))), ["LENGTH(city) + quantity"]), + (CastKey(source=CITY, type_text="INT", data_type=DataType.INT), ["CAST(city AS INT)"]), ] _REPR_MARKERS = ("Key(", "path=", "leaf=", "operands=", "op=", "refs=", "template=") diff --git a/tests/test_quoted_aggregation_placeholders.py b/tests/test_quoted_aggregation_placeholders.py index d3c15097..2ea96d27 100644 --- a/tests/test_quoted_aggregation_placeholders.py +++ b/tests/test_quoted_aggregation_placeholders.py @@ -57,6 +57,10 @@ def _q(formula: str) -> SlayerQuery: return SlayerQuery(source_model="orders", measures=[ModelMeasure(formula=formula, name="m")]) +def _mode_b_string(text: str) -> str: + return "'" + text.replace("'", "''") + "'" + + async def _dry(formula: str, *, agg: Aggregation, dialect: str = "postgres", validate: bool = False) -> str: return await _engine_generate(query=_q(formula), model=_orders(agg), dialect=dialect, validate=validate) @@ -204,7 +208,7 @@ class TestQuotedBindingValue: async def test_string_kwarg_stays_inside_the_literal(self, dialect: str) -> None: # The kwarg is the dialect's own spelling of a string literal valued a'b\c. kwarg = exp.Literal.string(_TRICKY).sql(dialect=dialect) - sql = await _dry(f"lbl(amount, n={kwarg!r})", agg=_agg(_LABEL, name="lbl", n="'x'"), dialect=dialect) + sql = await _dry(f"lbl(amount, n={_mode_b_string(kwarg)})", agg=_agg(_LABEL, name="lbl", n="'x'"), dialect=dialect) lit = _then_literal(sql, dialect=dialect) assert isinstance(lit, exp.Literal) assert lit.is_string @@ -269,7 +273,7 @@ async def test_interval_value_cannot_hold_a_quote_or_backslash(self, dialect: st kwarg = exp.Literal.string(value).sql(dialect=dialect) agg = _agg(formula, name="days", n="2") with pytest.raises(SqlTemplateError) as ei: - await _dry(f"days(amount, n={kwarg!r})", agg=agg, dialect=dialect) + await _dry(f"days(amount, n={_mode_b_string(kwarg)})", agg=agg, dialect=dialect) _names_placeholder(err=ei.value, name="n", formula=formula) From 1737b01dbffb1742d9b21734bbb6ad3e0074f9a5 Mon Sep 17 00:00:00 2001 From: Egor Kraev Date: Wed, 21 Oct 2026 14:15:59 +0200 Subject: [PATCH 05/17] NULL ordering, variables as literals, stored-text migration, one importer parser - OrderItem.nulls feeds OrderEntry.nulls; dialects emit a list of order terms (build_order_terms) and T-SQL builds its NULL key over the resolved projection expression. - {var} placeholders bind as literals (bind_variables): strings, bare number/boolean slots, whole IN lists; structure-changing values rejected; date_range bounds keep their own literal substitution. ComputedDimension .substituted removed. - Stored-only migration SlayerModel 15->16, SlayerQuery 6->7, Memory 4->5 re-spells Python string literals as SQL literals and | as ||; stored queries are migrated before the legacy time-literal repair. - Importers use parse_expr and one namespace-aware dependency check; slayer/core/formula.py (parse_formula, FieldSpec) deleted. - Postgres TRY_CAST reads the server profile only when a plan casts with it; the statement timeout probes the profile only where it depends on it. - Literal dimensions render without a GROUP BY entry. Test changes (mechanical): version pins bumped to 16/7/5 in test_stored_upgrade_rank_orders, test_stored_upgrade_backing_sql, test_dev1934_save_time_formula, test_model_access_caches, test_model_access_tags, test_rank_direction_migration, test_memory_string_ids, test_memories_storage, test_sqlite_memories_pk_migration; type narrowing in test_syntax, test_dev1737_scalar_specs, test_dev1838_kernels, test_dev1576_heals, test_rank_direction and the dev2101 suites; composite asserts split and pytest.raises blocks reduced to the call under test in the dev2101 suites, test_dev1576_heals, test_dev1737_scalar_specs, test_osi_converter and test_quoted_aggregation_placeholders. --- .basedpyright/baseline.json | 320 ------- slayer/core/formula.py | 902 ------------------ slayer/core/query.py | 13 +- slayer/core/syntax.py | 378 ++++++-- slayer/cube/converter.py | 27 +- slayer/dbt/converter.py | 46 +- slayer/engine/bind_inputs.py | 7 +- slayer/engine/binding.py | 26 +- slayer/engine/compile/stages.py | 2 + slayer/engine/query_engine.py | 20 +- slayer/ir/bound.py | 3 +- slayer/ir/planned.py | 11 + slayer/ir/variables.py | 55 +- slayer/osi/converter.py | 4 +- slayer/sql/dialects/_tier2.py | 3 +- slayer/sql/dialects/base.py | 19 +- slayer/sql/dialects/postgres.py | 4 +- slayer/sql/dialects/sqlite.py | 3 +- slayer/sql/dialects/tsql.py | 30 +- slayer/sql/generator.py | 88 +- slayer/sql/render/order_terms.py | 13 +- slayer/sql/render/ranked.py | 3 +- slayer/storage/base.py | 10 +- slayer/storage/migrations.py | 7 +- slayer/storage/rank_direction_migration.py | 66 +- slayer/storage/sql_string_migration.py | 76 ++ tests/integration/test_dev2101_postgres.py | 6 +- tests/test_dev1576_heals.py | 16 +- tests/test_dev1737_scalar_specs.py | 9 +- tests/test_dev1838_kernels.py | 2 +- tests/test_dev1934_save_time_formula.py | 2 +- tests/test_dev2101_agg_filter.py | 3 +- tests/test_dev2101_casts.py | 15 +- tests/test_dev2101_date_literals.py | 12 +- tests/test_dev2101_grammar_parse.py | 27 +- tests/test_dev2101_importer_dependencies.py | 3 +- tests/test_dev2101_null_ordering.py | 12 +- tests/test_dev2101_sql_forms.py | 12 +- tests/test_dev2101_stored_upgrade.py | 17 +- tests/test_dev2101_variables.py | 5 +- tests/test_memories_storage.py | 2 +- tests/test_memory_string_ids.py | 6 +- tests/test_model_access_caches.py | 2 +- tests/test_model_access_tags.py | 10 +- tests/test_osi_converter.py | 4 +- tests/test_quoted_aggregation_placeholders.py | 3 +- tests/test_rank_direction.py | 4 +- tests/test_rank_direction_migration.py | 6 +- tests/test_sqlite_memories_pk_migration.py | 2 +- tests/test_stored_upgrade_backing_sql.py | 2 +- tests/test_stored_upgrade_rank_orders.py | 6 +- tests/test_syntax.py | 2 +- 52 files changed, 795 insertions(+), 1531 deletions(-) delete mode 100644 slayer/core/formula.py create mode 100644 slayer/storage/sql_string_migration.py diff --git a/.basedpyright/baseline.json b/.basedpyright/baseline.json index 0955fbba..bc65ca83 100644 --- a/.basedpyright/baseline.json +++ b/.basedpyright/baseline.json @@ -18,96 +18,6 @@ } } ], - "./slayer/core/formula.py": [ - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 21, - "endColumn": 23, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 20, - "endColumn": 64, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 21, - "endColumn": 66, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 70, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 20, - "endColumn": 64, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 13, - "endColumn": 24, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 13, - "endColumn": 19, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 13, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 66, - "lineCount": 1 - } - }, - { - "code": "reportOperatorIssue", - "range": { - "startColumn": 15, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportOptionalOperand", - "range": { - "startColumn": 16, - "endColumn": 34, - "lineCount": 1 - } - } - ], "./slayer/demo/jaffle_shop.py": [ { "code": "reportOptionalSubscript", @@ -188,16 +98,6 @@ } } ], - "./slayer/engine/syntax.py": [ - { - "code": "reportArgumentType", - "range": { - "startColumn": 8, - "endColumn": 30, - "lineCount": 1 - } - } - ], "./slayer/facade/catalog_sql.py": [ { "code": "reportPrivateImportUsage", @@ -1815,14 +1715,6 @@ "lineCount": 1 } }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 26, - "endColumn": 40, - "lineCount": 1 - } - }, { "code": "reportArgumentType", "range": { @@ -10578,32 +10470,6 @@ } } ], - "./tests/test_dev1576_heals.py": [ - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 22, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 22, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 22, - "endColumn": 38, - "lineCount": 1 - } - } - ], "./tests/test_dev1645_invalid_postgres_sql.py": [ { "code": "reportArgumentType", @@ -15239,78 +15105,6 @@ "lineCount": 1 } }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, { "code": "reportArgumentType", "range": { @@ -22221,14 +22015,6 @@ "lineCount": 1 } }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 28, - "lineCount": 1 - } - }, { "code": "reportArgumentType", "range": { @@ -22317,22 +22103,6 @@ "lineCount": 1 } }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 24, - "endColumn": 35, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 18, - "endColumn": 23, - "lineCount": 1 - } - }, { "code": "reportCallIssue", "range": { @@ -34058,96 +33828,6 @@ } } ], - "./tests/test_syntax.py": [ - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - } - ], "./tests/test_time_dimensions_filters.py": [ { "code": "reportArgumentType", diff --git a/slayer/core/formula.py b/slayer/core/formula.py deleted file mode 100644 index bd3d502f..00000000 --- a/slayer/core/formula.py +++ /dev/null @@ -1,902 +0,0 @@ -"""Formula parser for SLayer measure formulas. - -Parses formula strings into structured FieldSpec objects using Python's ast module. - -A formula can be: -- An aggregated measure ref: "sum(revenue)" → AggregatedMeasureRef -- Star count: "count(*)" → AggregatedMeasureRef("*", "count") -- With agg args: "weighted_avg(price, weight=quantity)" → AggregatedMeasureRef with kwargs -- Arithmetic: "sum(revenue) / count(*)" → ArithmeticField -- A transform: "cumsum(sum(revenue))" → TransformField wrapping AggregatedMeasureRef -- Nested transforms: "change(cumsum(sum(revenue)))" → TransformField wrapping TransformField -- Arithmetic on transforms: "cumsum(sum(revenue)) / count(*)" → MixedArithmeticField - -Functional aggregations are rewritten internally to the legacy colon spelling -(``revenue:sum``), which this parser also accepts as input. -""" - -import ast -import difflib -import io -import re -import tokenize -from typing import Any, Literal -from collections.abc import Mapping - -from pydantic import BaseModel, Field - -from slayer.core.direction import rank_direction, with_direction_kwarg -from slayer.core.enums import BUILTIN_AGGREGATIONS, RANK_FAMILY_TRANSFORMS, normalize_aggregation_name, ALL_TRANSFORMS -from slayer.core.keys import SCALAR_FUNCTIONS -from slayer.core.refs import ( - AGG_REF_RE as _AGG_REF_RE, - IDENT_OR_PATH_RE as _IDENT_OR_PATH_RE, -) - -CallCategory = Literal["transform", "scalar", "unknown"] - - -def _classify_call_name(name: str) -> CallCategory: - """Categorize a Mode B function-call identifier.""" - if name in ALL_TRANSFORMS: - return "transform" - if name.lower() in SCALAR_FUNCTIONS: - return "scalar" - return "unknown" - - -def _validate_scalar_call(node: ast.Call, original: str) -> None: - """Arity check for top-level scalar calls. ``round`` / ``abs`` have specific - shapes worth catching early; other scalars in ``SCALAR_FUNCTIONS`` have - variable arity (``coalesce``, ``greatest``, …) and are validated by sqlglot - at SQL-emission time.""" - name = node.func.id.lower() - if node.keywords: - raise ValueError( - f"'{name}' does not accept keyword arguments. Formula: {original!r}" - ) - if name == "abs": - if len(node.args) != 1: - raise ValueError( - f"'abs' requires exactly one argument. Formula: {original!r}" - ) - elif name == "round": - if len(node.args) not in (1, 2): - raise ValueError( - f"'round' accepts 1 or 2 arguments (expression[, ndigits]). " - f"Formula: {original!r}" - ) - if len(node.args) == 2 and not _is_integer_literal(node.args[1]): - raise ValueError( - f"'round' ndigits (2nd argument) must be an integer literal. " - f"Formula: {original!r}" - ) - - -def _is_integer_literal(node: ast.AST) -> bool: - """True for an integer constant or a negated integer constant (``-2``). - - Booleans are rejected — ``True``/``False`` are ``int`` subclasses but are - not meaningful ndigits values. - """ - if isinstance(node, ast.Constant): - return isinstance(node.value, int) and not isinstance(node.value, bool) - if ( - isinstance(node, ast.UnaryOp) - and isinstance(node.op, ast.USub) - and isinstance(node.operand, ast.Constant) - ): - v = node.operand.value - return isinstance(v, int) and not isinstance(v, bool) - return False - - -class AggregatedMeasureRef(BaseModel): - """A measure reference with explicit aggregation. - - Examples: - "sum(revenue)" → AggregatedMeasureRef("revenue", "sum") - "count(*)" → AggregatedMeasureRef("*", "count") - "sum(customers.revenue)" → AggregatedMeasureRef("customers.revenue", "sum") - "weighted_avg(price, weight=quantity)" → AggregatedMeasureRef("price", "weighted_avg", - agg_kwargs={"weight": "quantity"}) - "last(revenue, ordered_at)" → AggregatedMeasureRef("revenue", "last", - agg_args=["ordered_at"]) - """ - measure_name: str = Field(description="Measure name, e.g. 'revenue', 'customers.revenue', '*'") - aggregation_name: str = Field(description="Aggregation name, e.g. 'sum', 'weighted_avg'") - agg_args: list[str] = Field(default_factory=list, description="Positional aggregation args") - agg_kwargs: dict[str, str] = Field(default_factory=dict, description="Keyword aggregation args") - - -class ArithmeticField(BaseModel): - """An arithmetic expression over measures only (no transform calls inside).""" - sql: str = Field(description="Preprocessed formula with placeholders for aggregated refs") - measure_names: list[str] = Field(description="Placeholder IDs or bare measure names") - agg_refs: dict[str, AggregatedMeasureRef] = Field(default_factory=dict) - is_predicate: bool = Field( - default=False, - description="True when the top-level AST node is a comparison or boolean op " - "(so the field renders as a boolean expression). Drives boolean-aware SQL " - "generation for transforms like consecutive_periods.", - ) - - -class TransformField(BaseModel): - """A transform function call, possibly wrapping another transform or arithmetic.""" - transform: str = Field(description="Transform name: cumsum, lag, lead, change, change_pct, rank, percent_rank, dense_rank, ntile, time_shift, first, last, consecutive_periods") - inner: "FieldSpec" = Field(description="The measure or expression being transformed") - args: list[Any] = Field(default_factory=list, description="Extra transform args (offset, granularity, etc.)") - kwargs: dict[str, Any] = Field( - default_factory=dict, - description=( - "Keyword args from the call site, e.g. partition_by=[...] for the " - "rank family, direction='desc' for rank / dense_rank or n=4 for ntile. " - "Validated per-transform at parse time." - ), - ) - - -class MixedArithmeticField(BaseModel): - """Arithmetic that contains transform sub-expressions. - - E.g., "cumsum(sum(revenue)) / count(*)" — the cumsum needs to be computed first - as a CTE step, then the arithmetic references its result. - """ - sql: str = Field(description="Preprocessed formula with placeholders") - measure_names: list[str] = Field(description="Placeholder IDs or bare measure names") - sub_transforms: list[tuple] = Field(description="List of (placeholder_name, TransformField)") - agg_refs: dict[str, AggregatedMeasureRef] = Field(default_factory=dict) - is_predicate: bool = Field( - default=False, - description="True when the top-level AST node is a comparison or boolean op " - "(so the field renders as a boolean expression). Drives boolean-aware SQL " - "generation for transforms like consecutive_periods.", - ) - - -# The parsed result of a single field -FieldSpec = AggregatedMeasureRef | ArithmeticField | TransformField | MixedArithmeticField - -# Rebuild TransformField which uses a forward reference to FieldSpec -TransformField.model_rebuild() - - -# --------------------------------------------------------------------------- -# Function-style aggregation rewrite -# --------------------------------------------------------------------------- - -# Aggregation names that are also transform names — ambiguous, need special handling -_AMBIGUOUS_AGG_TRANSFORMS = BUILTIN_AGGREGATIONS & ALL_TRANSFORMS # {"first", "last"} - -_STRING_LITERAL_RE = re.compile(r"'(?:[^'\\]|\\.)*'") - - -def _find_balanced_close(s: str, start: int) -> int: - """Find the index of the balanced closing paren starting after the open paren at `start`.""" - depth = 1 - i = start + 1 - in_string = False - string_char = "" - while i < len(s): - ch = s[i] - if in_string: - if ch == "\\" and i + 1 < len(s): - i += 2 # skip escaped character - continue - if ch == string_char: - in_string = False - elif ch in ("'", '"'): - in_string = True - string_char = ch - elif ch == "(": - depth += 1 - elif ch == ")": - depth -= 1 - if depth == 0: - return i - i += 1 - return -1 # unbalanced - - -def _rewrite_funcstyle_aggregations( - formula: str, - extra_agg_names: frozenset[str] | None = None, -) -> str: - """Rewrite function-style aggregation calls to the legacy colon spelling (internal). - - E.g., ``sum(revenue)`` → ``revenue:sum``, ``count(customers.*)`` → ``customers.*:count``. - - For aggregation names that are also transform names (``first``, ``last``), - the rewrite only fires when the first argument holds no colon, so a - transform over an already-rewritten aggregate (``last(revenue:sum)``) is left alone. - - Args: - formula: The formula string to rewrite. - extra_agg_names: Additional (custom) aggregation names to recognise. - """ - agg_names = BUILTIN_AGGREGATIONS | (extra_agg_names or frozenset()) - - # Build word-boundary regex for known aggregation names. - # Sort by length descending so longer names match first (e.g., count_distinct before count). - # Negative lookbehind for ':' avoids matching a legacy colon suffix (e.g., revenue:last(...)). - sorted_names = sorted(agg_names, key=len, reverse=True) - pattern = re.compile( - r'(?.*``. - if first_arg != "*" and not _IDENT_OR_PATH_RE.fullmatch(first_arg.removesuffix(".*")): - search_start = close_paren + 1 - continue - - remaining_args = [p.strip() for p in parts[1:]] - if remaining_args: - replacement = f"{first_arg}:{agg_name}({', '.join(remaining_args)})" - else: - replacement = f"{first_arg}:{agg_name}" - - formula = formula[:match.start()] + replacement + formula[close_paren + 1:] - rewritten = True - break # restart scanning from the beginning after a rewrite - - if not rewritten: - break - - return formula - - -def _split_args(s: str) -> list[str]: - """Split a comma-separated argument string respecting parentheses and quotes.""" - parts = [] - depth = 0 - current: list[str] = [] - in_string = False - string_char = "" - i = 0 - while i < len(s): - ch = s[i] - if in_string: - if ch == "\\" and i + 1 < len(s): - current.append(ch) - current.append(s[i + 1]) - i += 2 - continue - current.append(ch) - if ch == string_char: - in_string = False - elif ch in ("'", '"'): - current.append(ch) - in_string = True - string_char = ch - elif ch == "(": - depth += 1 - current.append(ch) - elif ch == ")": - depth -= 1 - current.append(ch) - elif ch == "," and depth == 0: - parts.append("".join(current)) - current = [] - else: - current.append(ch) - i += 1 - if current: - parts.append("".join(current)) - return parts - - -# --------------------------------------------------------------------------- -# Legacy colon-spelling preprocessing (regex source-of-truth lives in slayer.core.refs) -# --------------------------------------------------------------------------- - - -def _split_agg_arglist(args_str: str | None) -> tuple[list[str], dict[str, str]]: - """Parse a legacy colon-spelling ``(...)`` arglist (with parens, or ``None``) into - (positional, keyword): ``(weight=quantity)`` → ``([], {"weight": "quantity"})``.""" - agg_args: list[str] = [] - agg_kwargs: dict[str, str] = {} - if not args_str: - return agg_args, agg_kwargs - inner = args_str[1:-1].strip() - if not inner: - return agg_args, agg_kwargs - for part in inner.split(","): - part = part.strip() - if "=" in part: - key, val = part.split("=", 1) - agg_kwargs[key.strip()] = val.strip() - else: - agg_args.append(part) - return agg_args, agg_kwargs - - -def _preprocess_agg_refs( - formula: str, - custom_agg_names: frozenset[str] = frozenset(), -) -> tuple[str, dict[str, AggregatedMeasureRef]]: - """Replace legacy colon-spelling aggregated refs with placeholder identifiers. - - Returns (preprocessed_formula, {placeholder: AggregatedMeasureRef}). - An aggregation name exactly matching one of ``custom_agg_names`` (the reachable - custom aggregations) skips alias/casing healing, so the custom aggregation wins. - """ - refs: dict[str, AggregatedMeasureRef] = {} - counter = [0] - - def _replace(match: re.Match) -> str: - measure_name = match.group(1) - # Heal aggregation aliases/casing; unknown names pass through to the enrichment error. - raw_agg = match.group(2) - agg_name = ( - raw_agg if raw_agg in custom_agg_names - else normalize_aggregation_name(raw_agg) - ) - agg_args, agg_kwargs = _split_agg_arglist(match.group(3)) - - placeholder = f"__agg{counter[0]}__" - counter[0] += 1 - refs[placeholder] = AggregatedMeasureRef( - measure_name=measure_name, - aggregation_name=agg_name, - agg_args=agg_args, - agg_kwargs=agg_kwargs, - ) - return placeholder - - processed = _AGG_REF_RE.sub(_replace, formula) - return processed, refs - - -def _expand_named_measures( - formula: str, - named_measures: Mapping[str, str], - _visited: frozenset[str] = frozenset(), -) -> str: - """Inline-expand bare references to ``ModelMeasure`` saved formulas. - - For each NAME token whose value is a key in ``named_measures``, replaces it - with ``()``. The expansion is a textual - substitution that runs *before* the rest of the formula pipeline (function- - style rewrite, legacy colon-spelling preprocessing, AST parsing), so the expanded - string is parsed as if the user had written the inlined formula directly. - - Substitution is **skipped** when the NAME token is: - - - Preceded by ``.`` — right-hand side of a cross-model reference - (e.g. ``customers.aov``); cross-model resolution is handled separately. - - Followed by ``.`` — left-hand side of a cross-model reference - (e.g. ``customers.aov`` where ``customers`` happens to be a saved name). - - Preceded or followed by ``:`` — part of a legacy colon-spelling aggregation - (``revenue:sum``). - - Followed by ``(`` — function call (transform like ``cumsum(...)``). - - Followed by ``=`` — keyword argument name in an aggregation call - (e.g. ``weight`` in ``weighted_avg(price, weight=quantity)``). - - Cycles (``a → b → a``) raise ``ValueError`` with the chain in the message. - - Saved-measure names are pre-validated to be Python identifiers - (``slayer/core/models.py:_NAME_PATTERN``), so plain Python tokenization is - sufficient. If tokenization fails for any reason, returns the formula - unchanged so the downstream parser can produce a clear error. - """ - if not named_measures: - return formula - - try: - tokens = list(tokenize.generate_tokens(io.StringIO(formula).readline)) - except (tokenize.TokenError, SyntaxError): # IndentationError is a SyntaxError - return formula - - def _significant(idx: int, step: int) -> tokenize.TokenInfo | None: - skip_types = {tokenize.NEWLINE, tokenize.NL, tokenize.ENCODING, - tokenize.ENDMARKER, tokenize.COMMENT, tokenize.INDENT, - tokenize.DEDENT} - j = idx + step - while 0 <= j < len(tokens): - t = tokens[j] - if t.type in skip_types: - j += step - continue - return t - return None - - replacements: list[tuple] = [] # (start_pos, end_pos, replacement_text) - for i, tok in enumerate(tokens): - if tok.type != tokenize.NAME or tok.string not in named_measures: - continue - prev_tok = _significant(i, -1) - next_tok = _significant(i, 1) - if prev_tok is not None and prev_tok.type == tokenize.OP and prev_tok.string in (".", ":"): - continue - if next_tok is not None and next_tok.type == tokenize.OP and next_tok.string in (".", ":", "(", "="): - continue - - if tok.string in _visited: - chain = " → ".join([*_visited, tok.string]) - raise ValueError( - f"Saved measure '{tok.string}' has a cyclic reference: {chain}" - ) - inner_expanded = _expand_named_measures( - named_measures[tok.string], - named_measures, - _visited | {tok.string}, - ) - replacements.append((tok.start, tok.end, f"({inner_expanded})")) - - if not replacements: - return formula - - # Position-based substitution assumes a single-line formula. Multi-line - # formulas are not part of the supported grammar; if encountered, fall back - # to no expansion and let the downstream parser produce a sensible error. - if any(start[0] != end[0] or start[0] != 1 for start, end, _ in replacements): - return formula - - result = formula - for (_, start_col), (_, end_col), repl in sorted( - replacements, key=lambda r: -r[0][1] - ): - result = result[:start_col] + repl + result[end_col:] - return result - - -def parse_formula( - formula: str, - extra_agg_names: frozenset[str] | None = None, - named_measures: Mapping[str, str] | None = None, -) -> FieldSpec: - """Parse a formula string into a FieldSpec. - - Examples: - "sum(revenue)" → AggregatedMeasureRef("revenue", "sum") - "count(*)" → AggregatedMeasureRef("*", "count") - "sum(revenue) / count(*)" → ArithmeticField(...) - "cumsum(sum(revenue))" → TransformField("cumsum", AggregatedMeasureRef(...)) - "weighted_avg(price, weight=qty)" → AggregatedMeasureRef(..., agg_kwargs={"weight": "qty"}) - "last(revenue, ordered_at)" → AggregatedMeasureRef(..., agg_args=["ordered_at"]) - - Bare measure names (e.g., "revenue") are valid only when ``named_measures`` - is supplied and contains the name — they are inline-expanded to the saved - formula. Otherwise bare names raise. - - Args: - formula: The formula string to parse. - extra_agg_names: Additional aggregation names for function-style rewriting. - named_measures: Mapping of saved-measure name → its formula. Bare - references to these names are inline-expanded (recursively, with - cycle detection) before the rest of parsing. - """ - if named_measures: - formula = _expand_named_measures(formula, named_measures) - # Functional aggregations → legacy colon spelling → ast-parseable placeholders. - formula = _rewrite_funcstyle_aggregations(formula, extra_agg_names) - processed, agg_refs = _preprocess_agg_refs( - formula=formula, custom_agg_names=extra_agg_names or frozenset() - ) - - try: - tree = ast.parse(processed, mode="eval") - except SyntaxError as e: - raise ValueError(f"Invalid formula syntax: {formula!r} — {e}") - - return _parse_node( - tree.body, - original=formula, - agg_refs=agg_refs, - known_measures=frozenset(named_measures or ()), - ) - - -def _bare_name_message(*, name: str, known_measures: frozenset[str]) -> str: - """Explain a bare name that expanded to no saved measure. - - A bare name is only ever a saved measure, and by this point expansion has - already failed, so the name is either misspelled or a column. Suggesting a - real measure first stops the reader from wrapping a measure in an - aggregation it would reject anyway. - """ - suggestion = difflib.get_close_matches( - word=name, possibilities=sorted(known_measures), n=1 - ) - hint = f" Did you mean '{suggestion[0]}'?" if suggestion else "" - return ( - f"'{name}' is not a saved measure.{hint} Reference a saved measure by its " - f"bare name, or aggregate a column (e.g., 'sum({name})'). " - f"For COUNT(*), use 'count(*)'." - ) - - -def _parse_node( - node: ast.AST, - original: str, - agg_refs: dict[str, AggregatedMeasureRef] | None = None, - known_measures: frozenset[str] = frozenset(), -) -> FieldSpec: - """Recursively parse an AST node into a FieldSpec.""" - if agg_refs is None: - agg_refs = {} - - # Simple name → aggregation placeholder or error for bare names - if isinstance(node, ast.Name): - if node.id in agg_refs: - return agg_refs[node.id] - raise ValueError( - _bare_name_message(name=node.id, known_measures=known_measures) - ) - - # Dotted name → cross-model measure must include aggregation - if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name): - name = f"{node.value.id}.{node.attr}" - raise ValueError( - f"Cross-model measure '{name}' must include an aggregation " - f"(e.g., 'sum({name})')." - ) - - # Function call → transform - if isinstance(node, ast.Call): - if not isinstance(node.func, ast.Name): - raise ValueError(f"Unsupported function call in formula: {original!r}") - - func_name = node.func.id - category = _classify_call_name(func_name) - if category == "scalar": - # Allowlisted scalar call. Route through - # _parse_mixed_arithmetic so inner aggregated refs and nested - # transforms are registered/extracted. - _validate_scalar_call(node, original) - return _parse_mixed_arithmetic(node, original, agg_refs, known_measures) - if category != "transform": - raise ValueError( - f"Unknown function '{func_name}' in formula {original!r}. " - f"Supported scalar functions: " - f"{', '.join(sorted(SCALAR_FUNCTIONS))}. " - f"Transforms: {', '.join(sorted(ALL_TRANSFORMS))}." - ) - - if not node.args: - raise ValueError(f"Transform '{func_name}' requires at least one argument (the measure)") - - # First arg is the measure/expression being transformed - inner = _parse_node(node.args[0], original, agg_refs, known_measures) - - # Remaining positional args are transform parameters (offset, granularity, etc.) - # The rank family is keyword-only after the measure; reject extra positionals - # so calls like `rank(sum(revenue), 'desc')` or `ntile(sum(revenue), 4, n=2)` fail - # fast instead of silently dropping the extra arg downstream. - if func_name in RANK_FAMILY_TRANSFORMS and len(node.args) > 1: - raise ValueError( - f"Transform '{func_name}' does not accept positional arguments " - f"beyond the measure; use keyword args (e.g. partition_by=, direction=, n=). " - f"Formula: {original!r}" - ) - extra_args = [] - for arg in node.args[1:]: - extra_args.append(_parse_literal(node=arg, original=original)) - - # Keyword args, validated per-transform. - kwargs = _parse_transform_kwargs( - transform=func_name, keywords=node.keywords, original=original - ) - - return TransformField(transform=func_name, inner=inner, args=extra_args, kwargs=kwargs) - - # Binary/unary/comparison/boolean operation → check if it contains transform calls - if isinstance(node, (ast.BinOp, ast.UnaryOp, ast.Compare, ast.BoolOp)): - if _contains_call(node): - return _parse_mixed_arithmetic(node, original, agg_refs, known_measures) - measure_names = _collect_names(node) - # Reject bare names (aggregations were preprocessed into placeholders) - for mname in measure_names: - if mname not in agg_refs: - if "." in mname: - raise ValueError( - f"Cross-model measure '{mname}' must include an aggregation " - f"(e.g., 'sum({mname})')." - ) - raise ValueError( - _bare_name_message( - name=mname, known_measures=known_measures - ) - ) - field_agg_refs = {n: agg_refs[n] for n in measure_names if n in agg_refs} - return ArithmeticField( - sql=ast.unparse(node), - measure_names=measure_names, - agg_refs=field_agg_refs, - is_predicate=isinstance(node, (ast.Compare, ast.BoolOp)), - ) - - # Constant (bare number) - if isinstance(node, ast.Constant): - return ArithmeticField(sql=ast.unparse(node), measure_names=[]) - - raise ValueError(f"Unsupported formula syntax: {original!r}") - - -def _contains_call(node: ast.AST) -> bool: - """Check if an AST subtree contains any function Call nodes.""" - for child in ast.walk(node): - if isinstance(child, ast.Call): - return True - return False - - -def _replace_calls_in_arith( - node: ast.AST, - *, - sub_transforms: list[tuple], - measure_names: list[str], - counter: list[int], - agg_refs: dict[str, AggregatedMeasureRef], - original: str, - known_measures: frozenset[str] = frozenset(), -) -> ast.AST: - """Walk the AST, replacing transform Call nodes with Name placeholders.""" - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id in ALL_TRANSFORMS: - placeholder = f"_t{counter[0]}" - counter[0] += 1 - transform = _parse_node(node, original, agg_refs, known_measures) - sub_transforms.append((placeholder, transform)) - return ast.Name(id=placeholder, ctx=ast.Load()) - - kwargs = { - "sub_transforms": sub_transforms, - "measure_names": measure_names, - "counter": counter, - "agg_refs": agg_refs, - "original": original, - "known_measures": known_measures, - } - - if isinstance(node, ast.Name): - if node.id not in [p for p, _ in sub_transforms]: - measure_names.append(node.id) - return node - - if isinstance(node, ast.BinOp): - node.left = _replace_calls_in_arith(node.left, **kwargs) - node.right = _replace_calls_in_arith(node.right, **kwargs) - return node - - if isinstance(node, ast.UnaryOp): - node.operand = _replace_calls_in_arith(node.operand, **kwargs) - return node - - if isinstance(node, ast.Compare): - node.left = _replace_calls_in_arith(node.left, **kwargs) - node.comparators = [_replace_calls_in_arith(c, **kwargs) for c in node.comparators] - return node - - if isinstance(node, ast.BoolOp): - node.values = [_replace_calls_in_arith(v, **kwargs) for v in node.values] - return node - - if isinstance(node, ast.Call): - # Non-transform scalar call wrapping aggregated refs (e.g. nullif, - # coalesce, abs). Validate the name against the canonical Mode B - # allowlist — previously any name passed through, which was a - # consistency bug (top-level had a strict allowlist while - # inside-arithmetic accepted anything). - if isinstance(node.func, ast.Name): - name = node.func.id - if name not in ALL_TRANSFORMS and name.lower() not in SCALAR_FUNCTIONS: - raise ValueError( - f"Unknown function {name!r} in formula {kwargs['original']!r}. " - f"Supported scalar functions: " - f"{', '.join(sorted(SCALAR_FUNCTIONS))}. " - f"Transforms: {', '.join(sorted(ALL_TRANSFORMS))}." - ) - node.args = [_replace_calls_in_arith(a, **kwargs) for a in node.args] - for kw in node.keywords: - kw.value = _replace_calls_in_arith(kw.value, **kwargs) - return node - - return node - - -def _parse_mixed_arithmetic( - node: ast.AST, - original: str, - agg_refs: dict[str, AggregatedMeasureRef] | None = None, - known_measures: frozenset[str] = frozenset(), -) -> MixedArithmeticField: - """Parse arithmetic that contains transform calls. - - Extracts transform calls, replaces them with placeholder names, - and returns a MixedArithmeticField. - """ - if agg_refs is None: - agg_refs = {} - - sub_transforms: list[tuple] = [] - measure_names: list[str] = [] - counter = [0] - - modified = _replace_calls_in_arith( - node, - sub_transforms=sub_transforms, - measure_names=measure_names, - counter=counter, - agg_refs=agg_refs, - original=original, - known_measures=known_measures, - ) - modified_sql = ast.unparse(modified) - - field_agg_refs = {n: agg_refs[n] for n in measure_names if n in agg_refs} - is_predicate = isinstance(node, (ast.Compare, ast.BoolOp)) - - return MixedArithmeticField( - sql=modified_sql, - measure_names=measure_names, - sub_transforms=sub_transforms, - agg_refs=field_agg_refs, - is_predicate=is_predicate, - ) - - -def _parse_literal(node: ast.AST, original: str) -> Any: - """Parse a literal value from an AST node (number, string, negative number).""" - if isinstance(node, ast.Constant): - return node.value - if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub) and isinstance(node.operand, ast.Constant): - return -node.operand.value - raise ValueError(f"Expected a literal value (number or string) in formula: {original!r}") - - -# Per-transform kwarg whitelist. Empty set means the transform takes no kwargs. -_ALLOWED_TRANSFORM_KWARGS: dict[str, frozenset] = { - "rank": frozenset({"partition_by"}), - "percent_rank": frozenset({"partition_by"}), - "dense_rank": frozenset({"partition_by"}), - "ntile": frozenset({"partition_by", "n"}), -} - - -def _parse_dotted_name(node: ast.AST, original: str) -> str: - """Render a Name / Attribute node back into a dotted-path string. - - ``region`` → ``"region"``; ``customers.region`` → ``"customers.region"``. - """ - if isinstance(node, ast.Name): - return node.id - if isinstance(node, ast.Attribute): - return f"{_parse_dotted_name(node=node.value, original=original)}.{node.attr}" - raise ValueError( - f"Expected a column name or dotted path in formula {original!r}, " - f"got {ast.dump(node)}" - ) - - -def _parse_transform_kwargs( # NOSONAR S3776 — straight-line whitelist + per-kwarg validation; splitting into helpers would force threading transform/original through every call just to preserve the error-message context - transform: str, keywords: list[ast.keyword], original: str -) -> dict[str, Any]: - """Parse and validate a transform's keyword arguments. - - Each transform has a fixed kwarg whitelist (see ``_ALLOWED_TRANSFORM_KWARGS``). - Unknown kwargs raise with the accepted set in the message. ``ntile`` requires - ``n`` (positive integer); ``partition_by`` accepts a column name, dotted - path, or list of those. - """ - allowed = _ALLOWED_TRANSFORM_KWARGS.get(transform, frozenset()) - parsed: dict[str, Any] = {} - direction_kw: ast.keyword | None = None - - for kw in keywords: - if kw.arg is None: - raise ValueError( - f"Transform '{transform}' does not accept **kwargs in formula {original!r}" - ) - if kw.arg == "direction" and transform in RANK_FAMILY_TRANSFORMS: - direction_kw = kw - continue - if kw.arg not in allowed: - if not allowed: - raise ValueError( - f"Transform '{transform}' does not accept keyword arguments; " - f"got '{kw.arg}=' in formula {original!r}" - ) - advertised = with_direction_kwarg(op=transform, accepted=allowed) - raise ValueError( - f"Transform '{transform}' does not accept keyword '{kw.arg}'. " - f"Accepted kwargs: {', '.join(sorted(advertised))}. " - f"Formula: {original!r}" - ) - - if kw.arg == "partition_by": - value = kw.value - if isinstance(value, ast.List): - cols = [ - _parse_dotted_name(node=elt, original=original) for elt in value.elts - ] - else: - cols = [_parse_dotted_name(node=value, original=original)] - if not cols: - raise ValueError( - f"Transform '{transform}': partition_by must reference at " - f"least one column in formula {original!r}" - ) - parsed["partition_by"] = cols - elif kw.arg == "n": - n_val = _parse_literal(node=kw.value, original=original) - if not isinstance(n_val, int) or isinstance(n_val, bool) or n_val <= 0: - raise ValueError( - f"Transform '{transform}': n must be a positive integer, " - f"got {n_val!r} in formula {original!r}" - ) - parsed["n"] = n_val - else: # pragma: no cover — guarded by the whitelist check above - parsed[kw.arg] = _parse_literal(node=kw.value, original=original) - - literal = direction_kw.value if direction_kw is not None else None - direction = rank_direction( - op=transform, given=direction_kw is not None, - value=literal.value if isinstance(literal, ast.Constant) else None, - ) - if direction is not None: - parsed["direction"] = direction - - if transform == "ntile" and "n" not in parsed: - raise ValueError( - f"Transform 'ntile' requires keyword argument 'n' (positive integer) " - f"in formula {original!r}" - ) - - return parsed - - -def _collect_names(node: ast.AST) -> list[str]: - """Collect all Name and dotted Attribute references from an AST subtree.""" - names = [] - # Collect dotted names (model.measure) first to avoid also collecting the bare Name part - dotted = set() - for child in ast.walk(node): - if isinstance(child, ast.Attribute) and isinstance(child.value, ast.Name): - dotted_name = f"{child.value.id}.{child.attr}" - names.append(dotted_name) - dotted.add(id(child.value)) # Mark the Name node as consumed - for child in ast.walk(node): - if isinstance(child, ast.Name) and id(child) not in dotted: - names.append(child.id) - return names diff --git a/slayer/core/query.py b/slayer/core/query.py index 4dbab984..1ba948ed 100644 --- a/slayer/core/query.py +++ b/slayer/core/query.py @@ -36,7 +36,7 @@ ) from slayer.core.refs import auto_name_from_expression from slayer.core.time_points import TIME_POINT_FORMS, is_time_point_shape -from slayer.core.syntax import AggCall, DottedRef, Ref, canonical_measure_text, parse_expr, walk_parsed_refs +from slayer.core.syntax import AggCall, DottedRef, Ref, canonical_measure_text, parse_expr from slayer.sql.window_detect import WINDOW_IN_FILTER_ERROR, has_window_function from slayer.storage.migrations import CURRENT_VERSIONS, migrate as _migrate_schema @@ -662,12 +662,6 @@ def template(self) -> str: """The expression as written, before ``{var}`` substitution.""" return self._template or self.expression - def substituted(self, expression: str) -> "ComputedDimension": - """A copy carrying the substituted ``expression`` and remembering its template.""" - out = self.model_copy(update={"expression": expression}) - out._template = self.template - return out - @model_validator(mode="after") def _fill_name(self) -> "ComputedDimension": if self.name is None: @@ -862,6 +856,7 @@ class OrderItem(BaseModel): column: Annotated[ColumnRef, BeforeValidator(_coerce_order_column)] direction: str = "asc" + nulls: Literal["first", "last"] | None = None raw_formula: str | None = Field( default=None, description="Internal — captured automatically from expression strings; do not set.", @@ -1071,7 +1066,9 @@ def _strip_order(order: list[OrderItem] | None, model_name: str, pattern: re.Pat if stripped is item.column and raw_formula == item.raw_formula: new_order.append(item) else: - new_order.append(OrderItem(column=stripped, direction=item.direction, raw_formula=raw_formula)) + new_order.append(OrderItem( + column=stripped, direction=item.direction, nulls=item.nulls, raw_formula=raw_formula, + )) return None if all(n is o for n, o in zip(new_order, order)) else new_order diff --git a/slayer/core/syntax.py b/slayer/core/syntax.py index 9c18d14c..bfe08a1b 100644 --- a/slayer/core/syntax.py +++ b/slayer/core/syntax.py @@ -10,12 +10,13 @@ from __future__ import annotations +import difflib import re import threading import unicodedata from decimal import Decimal from functools import lru_cache -from typing import Any, Dict, Iterator, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional, Tuple, Union, cast import pyparsing as pp from pydantic import BaseModel, ConfigDict @@ -166,6 +167,9 @@ class BoolOp(_BaseNode): _VAR_NAME = r"[A-Za-z_][A-Za-z0-9_]*" _CODE_PLACEHOLDER_RE = re.compile(r"\{\{|\}\}|\{(" + _VAR_NAME + r")\}") _LITERAL_VALUES = "variables supply literal values" +_OPTIONAL_BLOCKS = ( + "Optional blocks '{? ... ?}' are supported only on raw-SQL model surfaces, not in Mode-B expressions." +) _DISTINCT_ONLY = "DISTINCT is supported only as count(distinct x), i.e. count_distinct(x)" @@ -218,6 +222,8 @@ def _mask_placeholders(text: str) -> str: parts: List[str] = [] for region in _regions(text): chunk = text[region.start:region.end] + if region.kind == "code" and ("{?" in chunk or "?}" in chunk): + raise _Problem(_OPTIONAL_BLOCKS) if region.kind == "code": chunk = _CODE_PLACEHOLDER_RE.sub( lambda m: f"{_PH_OPEN}{m.group(1)}{_PH_CLOSE}" if m.group(1) else m.group(0), chunk, @@ -250,6 +256,18 @@ def _record_failure(_s: str, loc: int, _expr: Any, _err: Any) -> None: _STATE.farthest = loc +def _on(element: pp.ParserElement, fn: Callable[[List[Any]], Any]) -> pp.ParserElement: + """``element`` producing ``fn(tokens)``, groups as plain lists.""" + def action(toks: pp.ParseResults) -> Any: + return fn(toks.as_list()) + return element.set_parse_action(action) + + +def _first(toks: pp.ParseResults) -> str: + """The first token of a terminal match, as text.""" + return str(toks[0]) + + def _t(element: pp.ParserElement) -> pp.ParserElement: """A terminal whose failures feed the farthest-failure error position.""" element.set_fail_action(_record_failure) @@ -291,7 +309,7 @@ def _check_reserved(name: str) -> None: def _bare_word(s: str, loc: int, toks: pp.ParseResults) -> _Word: - raw = toks[0] + raw = _first(toks) name = unicodedata.normalize("NFKC", raw) if not name.isidentifier(): raise pp.ParseException(s, loc, f"{raw!r} is not an identifier") @@ -299,7 +317,7 @@ def _bare_word(s: str, loc: int, toks: pp.ParseResults) -> _Word: return _Word(raw=raw, name=name, quoted=False) -def _quoted_word(toks: pp.ParseResults) -> _Word: +def _quoted_word(toks: List[Any]) -> _Word: raw = toks[0] name = raw[1:-1].replace('""', '"') if not name: @@ -311,9 +329,10 @@ def _quoted_word(toks: pp.ParseResults) -> _Word: def _kw(*words: str) -> pp.ParserElement: """An ASCII-exact, case-insensitive keyword (a Unicode case-fold spoof is never one).""" wanted = frozenset(words) - return _t(pp.Regex(_RUN).add_condition( - lambda toks: toks[0].isascii() and toks[0].lower() in wanted, - ).set_name("/".join(w.upper() for w in words))) + def is_keyword(toks: pp.ParseResults) -> bool: + raw = _first(toks) + return raw.isascii() and raw.lower() in wanted + return _t(pp.Regex(_RUN).add_condition(is_keyword).set_name("/".join(w.upper() for w in words))) def _sym(pattern: str, name: str) -> pp.ParserElement: @@ -330,21 +349,21 @@ def _sym(pattern: str, name: str) -> pp.ParserElement: EQ = pp.Suppress(_sym(r"=(?!=)", "'='")) _BARE = _t(pp.Regex(_RUN).set_name("identifier")).set_parse_action(_bare_word) -_QUOTED = _t(pp.Regex(r'"(?:[^"]|"")*"').set_name("quoted identifier")).set_parse_action(_quoted_word) +_QUOTED = _on(_t(pp.Regex(r'"(?:[^"]|"")*"').set_name("quoted identifier")), _quoted_word) _PLACEHOLDER = _t(pp.Regex(_PH_OPEN + "(" + _VAR_NAME + ")" + _PH_CLOSE).set_name("variable")) -def _number(toks: pp.ParseResults) -> Literal: +def _number(toks: List[Any]) -> Literal: text = toks[0] return Literal(value=Decimal(int(text)) if text.isdigit() else Decimal(str(float(text)))) -def _string(toks: pp.ParseResults) -> Literal: +def _string(toks: List[Any]) -> Literal: return Literal(value=toks[0][1:-1].replace("''", "'")) -_NUMBER = _t(pp.Regex(r"(?:\d+\.\d*|\.\d+|\d+)(?:[eE][+-]?\d+)?").set_name("number")).set_parse_action(_number) -_STRING = _t(pp.Regex(r"'(?:[^']|'')*'").set_name("string")).set_parse_action(_string) +_NUMBER = _on(_t(pp.Regex(r"(?:\d+\.\d*|\.\d+|\d+)(?:[eE][+-]?\d+)?").set_name("number")), _number) +_STRING = _on(_t(pp.Regex(r"'(?:[^']|'')*'").set_name("string")), _string) _CONSTANT_WORDS = {"True": True, "False": False, "None": None} _SQL_CONSTANT_WORDS = {"true": True, "false": False, "null": None} @@ -358,7 +377,7 @@ def _constant_value(raw: str) -> Tuple[bool, Any]: def _constant(s: str, loc: int, toks: pp.ParseResults) -> Literal: - known, value = _constant_value(toks[0]) + known, value = _constant_value(_first(toks)) if not known: raise pp.ParseException(s, loc, "not a constant") return Literal(value=value) @@ -368,11 +387,12 @@ def _constant(s: str, loc: int, toks: pp.ParseResults) -> Literal: def _placeholder_node(s: str, loc: int, toks: pp.ParseResults) -> Placeholder: - end = loc + len(toks[0]) + raw = _first(toks) + end = loc + len(raw) rest = s[end:].lstrip() if rest.startswith(("(", ".")) or (rest.startswith(":") and not rest.startswith("::")): raise _Problem(f"{_LITERAL_VALUES}; a variable cannot name a function, column or path.") - return Placeholder(name=toks[0][1:-1]) + return Placeholder(name=raw[1:-1]) def _reject_placeholder(_s: str, _loc: int, _toks: pp.ParseResults) -> None: @@ -390,7 +410,8 @@ def _reject_placeholder_kwarg(_s: str, _loc: int, _toks: pp.ParseResults) -> Non pp.ParserElement.enable_packrat() -expr = pp.Forward().set_name("expression") +expr = pp.Forward() +expr.set_name("expression") _concat_level = pp.Forward() _SEGMENT = _BARE | _QUOTED @@ -423,23 +444,17 @@ def __init__(self, value: Any) -> None: _ARG_END = pp.FollowedBy(_sym(r"[,)]", "',' or ')'")) -_STAR_ARG = (pp.Suppress(STAR) + _ARG_END).set_parse_action(lambda: StarSource()) -_PATH_STAR_ARG = (_PATH_STAR + _ARG_END).set_parse_action( - lambda toks: DottedRef(parts=(*(w.name for w in toks[0][0]), "*"), quoted=any(w.quoted for w in toks[0][0])), -) +_STAR_ARG = _on((pp.Suppress(STAR) + _ARG_END), lambda _toks: StarSource()) +_PATH_STAR_ARG = _on((_PATH_STAR + _ARG_END), lambda toks: DottedRef(parts=(*(w.name for w in toks[0][0]), "*"), quoted=any(w.quoted for w in toks[0][0]))) _SEQ_ITEMS = pp.Group(pp.Optional(pp.DelimitedList(expr, allow_trailing_delim=True))) -_LIST_VALUE = (LBRACK + _SEQ_ITEMS + RBRACK).set_parse_action(lambda toks: tuple(toks[0])) -_TUPLE_VALUE = (LPAR + pp.Group(expr + pp.OneOrMore(COMMA + expr) + pp.Optional(COMMA)) + RPAR).set_parse_action( - lambda toks: tuple(toks[0]), -) +_LIST_VALUE = _on((LBRACK + _SEQ_ITEMS + RBRACK), lambda toks: tuple(toks[0])) +_TUPLE_VALUE = _on((LPAR + pp.Group(expr + pp.OneOrMore(COMMA + expr) + pp.Optional(COMMA)) + RPAR), lambda toks: tuple(toks[0])) _DISTINCT_VALUE = _kw("distinct") + _fail(f"{_DISTINCT_ONLY}.") _KW_VALUE = _DISTINCT_VALUE | _LIST_VALUE | _TUPLE_VALUE | _STAR_ARG | _PATH_STAR_ARG | expr -_KWARG = (_BARE + EQ + _KW_VALUE).set_parse_action(lambda toks: _Kw(name=toks[0].name, value=toks[1])) +_KWARG = _on((_BARE + EQ + _KW_VALUE), lambda toks: _Kw(name=toks[0].name, value=toks[1])) _PH_KWARG = (_PLACEHOLDER + EQ).set_parse_action(_reject_placeholder_kwarg) _VALUE_ARG = _STAR_ARG | _PATH_STAR_ARG | expr -_DISTINCT_ARG = (pp.Suppress(_kw("distinct")) + _need(_VALUE_ARG, f"{_DISTINCT_ONLY}.")).set_parse_action( - lambda toks: _Distinct(value=toks[0]), -) +_DISTINCT_ARG = _on((pp.Suppress(_kw("distinct")) + _need(_VALUE_ARG, f"{_DISTINCT_ONLY}.")), lambda toks: _Distinct(value=toks[0])) _UNPACKING = _sym(r"\*\*", "'**'") + _fail("dictionary unpacking (**kwargs) is not supported in calls.") _ANY_ARG = _UNPACKING | _PH_KWARG | _KWARG | _DISTINCT_ARG | _VALUE_ARG _POSITIONAL_ARG = _UNPACKING | _PH_KWARG | _DISTINCT_ARG | _VALUE_ARG @@ -451,7 +466,7 @@ def _arg_list(first: pp.ParserElement, rest: pp.ParserElement) -> pp.ParserEleme def _callee_is(kind: str): def _check(toks: pp.ParseResults) -> bool: - name = toks[0][-1].name + name = toks.as_list()[0][-1].name if kind == "scalar": return name.lower() in SCALAR_FUNCTIONS or name.lower() in DATE_PART_ALIASES return normalize_aggregation_name(name) in BUILTIN_AGGREGATIONS or name in ALL_TRANSFORMS @@ -514,17 +529,16 @@ def _dispatch_pending(call: _PendingCall) -> Any: return _dispatch_call(name, args=args, kwargs=kwargs, original=_STATE.text) -_CALL.set_parse_action(lambda toks: _PendingCall(words=list(toks[0]), items=list(toks[1]))) +_on(_CALL, lambda toks: _PendingCall(words=list(toks[0]), items=list(toks[1]))) -_COLON_SOURCE = (pp.Suppress(STAR).set_parse_action(lambda: StarSource()) - | _PATH_STAR.copy().set_parse_action( - lambda toks: DottedRef(parts=(*(w.name for w in toks[0][0]), "*"), +_COLON_SOURCE = (_on(pp.Suppress(STAR), lambda _toks: StarSource()) + | _on(_PATH_STAR.copy(), lambda toks: DottedRef(parts=(*(w.name for w in toks[0][0]), "*"), quoted=any(w.quoted for w in toks[0][0]))) - | _PATH.copy().set_parse_action(lambda toks: _path_node(list(toks[0])))) + | _on(_PATH.copy(), lambda toks: _path_node(list(toks[0])))) _COLON_AGG = _COLON_SOURCE + COLON + _BARE + pp.Optional(LPAR + _arg_list(_ANY_ARG, _ANY_ARG) + RPAR) -def _colon_node(toks: pp.ParseResults) -> AggCall: +def _colon_node(toks: List[Any]) -> AggCall: source, agg = toks[0], toks[1].name args, kwargs, distinct = _split_args(list(toks[2]) if len(toks) > 2 else [], call=agg) if distinct: @@ -533,8 +547,8 @@ def _colon_node(toks: pp.ParseResults) -> AggCall: return AggCall(source=source, agg=agg, args=args, kwargs=kwargs) -_COLON_AGG.set_parse_action(_colon_node) -_REF = _PATH.copy().set_parse_action(lambda toks: _path_node(list(toks[0]))) +_on(_COLON_AGG, _colon_node) +_REF = _on(_PATH.copy(), lambda toks: _path_node(list(toks[0]))) # -- CASE ---------------------------------------------------------------------- @@ -553,7 +567,7 @@ def _colon_node(toks: pp.ParseResults) -> AggCall: + pp.Group(expr) + pp.FollowedBy(WHEN) + _CASE_TAIL) -def _case_node(toks: pp.ParseResults) -> ScalarCall: +def _case_node(toks: List[Any]) -> Any: operand = toks[0][0] if len(toks[0]) else None branches = [(b[0], b[1]) for b in toks[1]] result = toks[2][0] if len(toks[2]) else Literal(value=None) @@ -563,7 +577,7 @@ def _case_node(toks: pp.ParseResults) -> ScalarCall: return result -_CASE = (_SEARCHED_CASE | _SIMPLE_CASE).set_parse_action(_case_node) +_CASE = _on(_SEARCHED_CASE | _SIMPLE_CASE, _case_node) # -- CAST / EXTRACT / POSITION / SUBSTRING / TRIM ------------------------------ @@ -573,13 +587,13 @@ def _case_node(toks: pp.ParseResults) -> ScalarCall: + pp.Group(pp.Optional(LPAR + pp.DelimitedList(_sym(r"\d+", "type parameter")) + RPAR))) -def _type_text(toks: pp.ParseResults) -> str: +def _type_text(toks: List[Any]) -> str: words, params = toks text = " ".join(w.upper() for w in words) return f"{text}({','.join(params)})" if len(params) else text -_TYPE_SPEC.set_parse_action(_type_text) +_on(_TYPE_SPEC, _type_text) _CAST = ((pp.Suppress(_kw("cast")) | _kw("try_cast")) + LPAR + _need(expr, "CAST needs a value: CAST(x AS ).") + _need(pp.Suppress(_kw("as")), "CAST needs AS : CAST(x AS ).") @@ -587,18 +601,18 @@ def _type_text(toks: pp.ParseResults) -> str: + _need(RPAR, "CAST(x AS ) is missing its ')'.")) -def _cast_node(toks: pp.ParseResults) -> Cast: +def _cast_node(toks: List[Any]) -> Cast: try_ = len(toks) == 3 return Cast(source=toks[-2], type_text=toks[-1], try_=try_) -_CAST.set_parse_action(_cast_node) +_on(_CAST, _cast_node) _EXTRACT = (pp.Suppress(_kw("extract")) + LPAR + (_STRING | _BARE) + pp.Suppress(_kw("from")) + expr + RPAR) -def _extract_node(toks: pp.ParseResults) -> ScalarCall: +def _extract_node(toks: List[Any]) -> ScalarCall: part, value = toks word = part.value if isinstance(part, Literal) else part.name return ScalarCall(name="date_part", args=_date_args( @@ -606,23 +620,17 @@ def _extract_node(toks: pp.ParseResults) -> ScalarCall: )) -_EXTRACT.set_parse_action(_extract_node) -_POSITION = (pp.Suppress(_kw("position")) + LPAR + _concat_level + pp.Suppress(_kw("in")) + expr + RPAR).set_parse_action( - lambda toks: ScalarCall(name="instr", args=(toks[1], toks[0])), -) -_SUBSTRING = (pp.Suppress(_kw("substring")) + LPAR + expr + pp.Suppress(_kw("from")) + expr - + pp.Optional(pp.Suppress(_kw("for")) + expr) + RPAR).set_parse_action( - lambda toks: ScalarCall(name="substr", args=tuple(toks)), -) +_on(_EXTRACT, _extract_node) +_POSITION = _on((pp.Suppress(_kw("position")) + LPAR + _concat_level + pp.Suppress(_kw("in")) + expr + RPAR), lambda toks: ScalarCall(name="instr", args=(toks[1], toks[0]))) +_SUBSTRING = _on((pp.Suppress(_kw("substring")) + LPAR + expr + pp.Suppress(_kw("from")) + expr + + pp.Optional(pp.Suppress(_kw("for")) + expr) + RPAR), lambda toks: ScalarCall(name="substr", args=tuple(toks))) _TRIM_FORM = (_kw("trim") + LPAR + _kw("both", "leading", "trailing")) + _fail( "TRIM(BOTH | LEADING | TRAILING …) is not supported; use trim(x), ltrim(x) or rtrim(x).", ) # -- typed literals, intervals, clock keywords ---------------------------------- -_TEMPORAL = (_kw("date", "timestamp") + _STRING).set_parse_action( - lambda toks: TemporalLiteral(kind=toks[0].lower(), text=toks[1].value), -) +_TEMPORAL = _on((_kw("date", "timestamp") + _STRING), lambda toks: TemporalLiteral(kind=toks[0].lower(), text=toks[1].value)) _TIME_LITERAL = (_kw("time") + _STRING) + _fail( "TIME '…' literals are not supported (there is no time-of-day type); compare with a TIMESTAMP '…' literal.", ) @@ -638,7 +646,12 @@ def _unit_of(word: str) -> Optional[str]: return None -_UNIT = _t(pp.Regex(_RUN).add_condition(lambda toks: toks[0].isascii() and _unit_of(toks[0]) is not None)) +def _is_unit(toks: pp.ParseResults) -> bool: + raw = _first(toks) + return raw.isascii() and _unit_of(raw) is not None + + +_UNIT = _t(pp.Regex(_RUN).add_condition(_is_unit)) _INTERVAL = (pp.Suppress(_kw("interval")) + (_STRING | _NUMBER) + pp.Optional(_UNIT)) @@ -648,7 +661,7 @@ def _interval_count(text: str) -> Literal: return Literal(value=Decimal(int(text))) -def _interval_node(toks: pp.ParseResults) -> ScalarCall: +def _interval_node(toks: List[Any]) -> ScalarCall: quantity = toks[0] unit_word = toks[1] if len(toks) > 1 else None if isinstance(quantity.value, str): @@ -672,10 +685,8 @@ def _interval_node(toks: pp.ParseResults) -> ScalarCall: return ScalarCall(name="interval", args=(_interval_count(count), Literal(value=unit))) -_INTERVAL.set_parse_action(_interval_node) -_CLOCK = (_kw("current_date", "current_timestamp") + ~pp.FollowedBy(LPAR)).set_parse_action( - lambda toks: ScalarCall(name="current_date" if toks[0].lower() == "current_date" else "now"), -) +_on(_INTERVAL, _interval_node) +_CLOCK = _on((_kw("current_date", "current_timestamp") + ~pp.FollowedBy(LPAR)), lambda toks: ScalarCall(name="current_date" if toks[0].lower() == "current_date" else "now")) # -- primary, postfix ---------------------------------------------------------- @@ -703,7 +714,7 @@ def _reject_over(_s: str, _loc: int, _toks: pp.ParseResults) -> None: _PRIMARY = _ATOM + pp.Group(pp.ZeroOrMore(pp.Group(_FILTER_CLAUSE) | _OVER)) -def _primary_node(toks: pp.ParseResults) -> Any: +def _primary_node(toks: List[Any]) -> Any: node = _resolve(toks[0]) for clause in toks[1]: if not isinstance(node, AggCall): @@ -714,10 +725,8 @@ def _primary_node(toks: pp.ParseResults) -> Any: return node -_PRIMARY.set_parse_action(_primary_node) -_CAST_LEVEL = (_PRIMARY + pp.ZeroOrMore(DCOLON + _need(_TYPE_SPEC, "'::' needs a type."))).set_parse_action( - lambda toks: _fold_casts(list(toks)), -) +_on(_PRIMARY, _primary_node) +_CAST_LEVEL = _on((_PRIMARY + pp.ZeroOrMore(DCOLON + _need(_TYPE_SPEC, "'::' needs a type."))), lambda toks: _fold_casts(list(toks))) def _fold_casts(items: List[Any]) -> Any: @@ -744,9 +753,9 @@ def _signed(items: List[Any]) -> Any: return node -_power_rhs.set_parse_action(lambda toks: _signed(list(toks))) -_power.set_parse_action(lambda toks: ScalarCall(name="pow", args=(toks[0], toks[1])) if len(toks) == 2 else toks[0]) -_UNARY = (pp.ZeroOrMore(_MINUS | _PLUS) + _power).set_parse_action(lambda toks: _signed(list(toks))) +_on(_power_rhs, lambda toks: _signed(list(toks))) +_on(_power, lambda toks: ScalarCall(name="pow", args=(toks[0], toks[1])) if len(toks) == 2 else toks[0]) +_UNARY = _on((pp.ZeroOrMore(_MINUS | _PLUS) + _power), lambda toks: _signed(list(toks))) _FLOORDIV = _sym("//", "'//'") + _fail("'//' is not an operator; use div(a, b) or floor(a / b).") _MUL_LEVEL = _UNARY + pp.ZeroOrMore((_FLOORDIV | _sym(r"\*(?!\*)|/(?!/)|%", "'*', '/' or '%'")) + _UNARY) @@ -760,8 +769,8 @@ def _fold_arith(items: List[Any]) -> Any: return node -_MUL_LEVEL.set_parse_action(lambda toks: _fold_arith(list(toks))) -_ADD_LEVEL.set_parse_action(lambda toks: _fold_arith(list(toks))) +_on(_MUL_LEVEL, lambda toks: _fold_arith(list(toks))) +_on(_ADD_LEVEL, lambda toks: _fold_arith(list(toks))) _PIPE = _sym(r"\|(?!\|)", "'|'") + _fail("'|' is not an operator; use || to concatenate strings.") _concat_level <<= _ADD_LEVEL + pp.ZeroOrMore(pp.Suppress(_PIPE | _sym(r"\|\|", "'||'")) + _ADD_LEVEL) @@ -773,13 +782,13 @@ def _fold_concat(items: List[Any]) -> Any: return node -_concat_level.set_parse_action(lambda toks: _fold_concat(list(toks))) +_on(_concat_level, lambda toks: _fold_concat(list(toks))) # -- comparison level ---------------------------------------------------------- _CMP_OPS = {"=": "==", "==": "==", "<>": "!=", "!=": "!=", "<": "<", "<=": "<=", ">": ">", ">=": ">="} _IS = pp.Suppress(_kw("is")) -_NOT_FLAG = pp.Optional(_kw("not")).set_parse_action(lambda toks: bool(len(toks))) +_NOT_FLAG = _on(pp.Optional(_kw("not")), lambda toks: bool(len(toks))) class _Suffix: @@ -792,14 +801,14 @@ def __init__(self, build) -> None: def _suffix(element: pp.ParserElement, build) -> pp.ParserElement: - return element.set_parse_action(lambda toks: _Suffix(lambda left: build(left, toks))) + return _on(element, lambda toks: _Suffix(lambda left: build(left, toks))) def _negated(node: Any, negate: bool) -> Any: return UnaryOp(op="not", operand=node) if negate else node -def _truth_test(left: Any, toks: pp.ParseResults) -> Any: +def _truth_test(left: Any, toks: List[Any]) -> Any: negate, word = toks[0], toks[1].lower() known = Cmp(op="is not", left=left, right=Literal(value=None)) test = BoolOp(op="and", operands=(known, left if word == "true" else UnaryOp(op="not", operand=left))) @@ -828,8 +837,8 @@ def _in_rhs_elements(items: List[Any]) -> TupleLit: _IN_LIST = ((LPAR + _PLACEHOLDER_ATOM + RPAR) | _PLACEHOLDER_ATOM - | (LPAR + _SEQ_ITEMS + RPAR).set_parse_action(lambda toks: _in_rhs_elements(list(toks[0]))) - | (LBRACK + _SEQ_ITEMS + RBRACK).set_parse_action(lambda toks: _in_rhs_elements(list(toks[0]))) + | _on((LPAR + _SEQ_ITEMS + RPAR), lambda toks: _in_rhs_elements(list(toks[0]))) + | _on((LBRACK + _SEQ_ITEMS + RBRACK), lambda toks: _in_rhs_elements(list(toks[0]))) | _STRING | _fail("the right-hand side of in / not in must be a literal list, e.g. status in ('a', 'b').")) @@ -856,7 +865,7 @@ def _in_rhs_elements(items: List[Any]) -> TupleLit: ) -def _comparison_node(toks: pp.ParseResults) -> Any: +def _comparison_node(toks: List[Any]) -> Any: left, suffixes = toks[0], list(toks[1:]) if len(suffixes) > 1: raise _Problem( @@ -866,10 +875,8 @@ def _comparison_node(toks: pp.ParseResults) -> Any: return suffixes[0].build(left) if suffixes else left -_COMPARISON = (_concat_level + pp.ZeroOrMore(_CMP_SUFFIX)).set_parse_action(_comparison_node) -_NOT_LEVEL = (pp.ZeroOrMore(_kw("not")) + _COMPARISON).set_parse_action( - lambda toks: _negate_n(len(toks) - 1, toks[-1]), -) +_COMPARISON = _on(_concat_level + pp.ZeroOrMore(_CMP_SUFFIX), _comparison_node) +_NOT_LEVEL = _on((pp.ZeroOrMore(_kw("not")) + _COMPARISON), lambda toks: _negate_n(len(toks) - 1, toks[-1])) def _negate_n(count: int, node: Any) -> Any: @@ -880,14 +887,14 @@ def _negate_n(count: int, node: Any) -> Any: def _bool_level(operand: pp.ParserElement, word: str) -> pp.ParserElement: level = operand + pp.ZeroOrMore(pp.Suppress(_kw(word)) + operand) - return level.set_parse_action(lambda toks: BoolOp(op=word, operands=tuple(toks)) if len(toks) > 1 else toks[0]) + return _on(level, lambda toks: BoolOp(op=word, operands=tuple(toks)) if len(toks) > 1 else toks[0]) _TERNARY = _kw("if") + _fail( "the Python conditional 'x if cond else y' is not supported; use CASE WHEN cond THEN x ELSE y END or iif(cond, x, y).", ) expr <<= _bool_level(_bool_level(_NOT_LEVEL, "and"), "or") + pp.Optional(_TERNARY) -_EXPRESSION = expr + pp.StringEnd().set_fail_action(_record_failure) +_EXPRESSION = (expr + pp.StringEnd().set_fail_action(_record_failure)).parse_with_tabs() # --------------------------------------------------------------------------- @@ -908,7 +915,7 @@ def parse_expr(text: str) -> ParsedExpr: _STATE.text, _STATE.farthest = text, 0 try: masked = _mask_placeholders(text) - parsed = _EXPRESSION.parse_string(masked, parse_all=True)[0] + parsed = cast(ParsedExpr, _EXPRESSION.parse_string(masked, parse_all=True)[0]) except _Problem as problem: raise ValueError(f"Invalid Mode-B expression {text!r}: {problem.detail}") from None except pp.ParseBaseException as e: @@ -939,6 +946,207 @@ def placeholder_names(parsed: Any) -> List[str]: return [n.name for n in _walk_parsed(parsed) if isinstance(n, Placeholder)] +_ANY_BRACE_RE = re.compile(r"\{\{|\}\}|\{(" + _VAR_NAME + r")\}|\{([^}]*)\}") +_NUMBER_TEXT_RE = re.compile(r"[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?") + + +def bind_variables(text: str, variables: Mapping[str, Any]) -> str: + """``text`` with every ``{name}`` bound as a literal value: inside a string literal its text, + in a bare literal slot a number or boolean, as a whole ``IN`` list a list. The template is + parsed first, so no value can change the expression's structure.""" + if not _ANY_BRACE_RE.search(text): + return text + regions = [r for r in _regions(text) if r.kind != "ident"] + for region in regions: + chunk = text[region.start:region.end] + if region.kind == "code" and ("{?" in chunk or "?}" in chunk): + raise ValueError(f"{_OPTIONAL_BLOCKS} Got: {text!r}.") + for m in _ANY_BRACE_RE.finditer(chunk): + _check_variable(m, text=text, variables=variables) + # Strings bound and ``{{`` / ``}}`` outside strings read as braces: the grammar judges the variable slots. + parse_expr(_spliced(text, variables=variables, regions=regions, kinds=("string",), code_braces=True)) + return _spliced(text, variables=variables, regions=regions, kinds=("string", "code"), code_braces=False) + + +def _spliced( + text: str, *, variables: Mapping[str, Any], regions: List[_Region], kinds: Tuple[str, ...], code_braces: bool, +) -> str: + out: List[str] = [] + cursor = 0 + for region in regions: + out.append(text[cursor:region.start]) + chunk = text[region.start:region.end] + if region.kind in kinds: + in_string = region.kind == "string" + chunk = _ANY_BRACE_RE.sub( + lambda m: _bound_text(m, text=text, region_start=region.start, in_string=in_string, variables=variables), + chunk, + ) + elif code_braces: + chunk = chunk.replace("{{", "{").replace("}}", "}") + out.append(chunk) + cursor = region.end + out.append(text[cursor:]) + return "".join(out) + + +def _check_variable(match: "re.Match[str]", *, text: str, variables: Mapping[str, Any]) -> None: + if match.group(0) in ("{{", "}}"): + return + name = match.group(1) + if name is None: + raise ValueError( + f"Invalid variable name '{match.group(2)}' in filter: {text!r}. " + f"Variable names must contain only letters, digits, and underscores." + ) + if name not in variables: + raise ValueError( + f"Undefined variable '{name}' in filter: {text!r}. Available variables: {sorted(variables)}" + ) + + +def _bound_text( + match: "re.Match[str]", *, text: str, region_start: int, in_string: bool, variables: Mapping[str, Any], +) -> str: + whole = match.group(0) + if whole in ("{{", "}}"): + return whole[0] + name = match.group(1) + value = variables[name] + if in_string: + return _value_text(name, value).replace("'", "''") + start = region_start + match.start() + if _opens_in_list(text, start=start, end=region_start + match.end()): + values = value if isinstance(value, (list, tuple)) else [value] + if not values: + raise ValueError( + f"Variable '{name}' cannot be an empty list; 'IN ()' is invalid SQL. " + f"For 'no filter' semantics, use a sentinel default." + ) + return ", ".join(_literal_text(name, v) for v in values) + return _bare_literal_text(name, value) + + +def _opens_in_list(text: str, *, start: int, end: int) -> bool: + """Whether the placeholder at ``[start, end)`` is the whole list of an ``IN``.""" + before = text[:start].rstrip() + if before.endswith("(") and text[end:].lstrip().startswith(")"): + before = before[:-1].rstrip() + word = re.search(r"([A-Za-z]+)$", before) + return word is not None and word.group(1).lower() == "in" + + +def _value_text(name: str, value: Any) -> str: + if isinstance(value, bool): + return str(value) + if isinstance(value, str): + return value + if isinstance(value, (int, float)): + return _number_text(name, value) + raise ValueError(f"Variable '{name}' must be a string, number, or list/tuple, got {type(value).__name__}") + + +def _number_text(name: str, value: int | float) -> str: + if isinstance(value, float) and value != value or value in (float("inf"), float("-inf")): + raise ValueError(f"Variable '{name}' must be finite, got {value!r}") + return str(value) + + +def _literal_text(name: str, value: Any) -> str: + """One value as a literal: a string quoted, a number or boolean as itself.""" + if isinstance(value, str): + return "'" + value.replace("'", "''") + "'" + if isinstance(value, (list, tuple)): + raise ValueError(f"{_LITERAL_VALUES}: variable '{name}' holds a list, which fills only a whole IN list.") + return _value_text(name, value) + + +def _bare_literal_text(name: str, value: Any) -> str: + if isinstance(value, (list, tuple)): + raise ValueError( + f"{_LITERAL_VALUES}: variable '{name}' holds a list, which fills only a whole IN list, e.g. x in ({{{name}}})." + ) + if isinstance(value, str): + stripped = value.strip() + if _NUMBER_TEXT_RE.fullmatch(stripped): + return stripped + raise ValueError( + f"{_LITERAL_VALUES}: variable '{name}' holds the text {value!r} in a number or boolean slot; " + f"put the placeholder inside quotes ('{{{name}}}') to use it as a string." + ) + return _value_text(name, value) + + +# --------------------------------------------------------------------------- +# Saved-measure dependencies +# --------------------------------------------------------------------------- + + +def measure_dependency_problems( + formulas: List[Tuple[Optional[str], str]], *, aggregations: frozenset[str] = frozenset(), +) -> List[Optional[str]]: + """Per ``(name, formula)``: why the saved measure cannot resolve among its peers, else ``None``. + + Outside an aggregation a bare name must be a peer measure; an aggregation must be built-in + or one of ``aggregations``; a measure on a cycle, or reading one that cannot resolve, cannot + resolve either.""" + names = {name for name, _ in formulas if name} + own: Dict[int, Optional[str]] = {} + deps: Dict[int, set] = {} + for i, (_name, formula) in enumerate(formulas): + try: + refs, problem = _measure_refs(parse_expr(formula), names=names, aggregations=aggregations) + except (ValueError, UnknownFunctionError, IllegalWindowInFilterError, GranularityCallError) as e: + refs, problem = set(), str(e) + own[i], deps[i] = problem, refs + index = {name: i for i, (name, _) in enumerate(formulas) if name} + out: List[Optional[str]] = [] + for i, (name, _) in enumerate(formulas): + out.append(_resolution_problem(i, own=own, deps=deps, index=index, path=(name or "",))) + return out + + +def _resolution_problem( + i: int, *, own: Dict[int, Optional[str]], deps: Dict[int, set], index: Dict[str, int], path: Tuple[str, ...], +) -> Optional[str]: + if own[i] is not None: + return own[i] + for dep in sorted(deps[i]): + if dep in path: + return f"saved measure {dep!r} has a cyclic reference: {' → '.join((*path, dep))}" + problem = _resolution_problem(index[dep], own=own, deps=deps, index=index, path=(*path, dep)) + if problem is not None: + return problem if dep == path[-1] else f"{dep!r} cannot resolve: {problem}" + return None + + +def _measure_refs(node: Any, *, names: set, aggregations: frozenset[str]) -> Tuple[set, Optional[str]]: + """The peer measures ``node`` reads outside aggregations, and the first problem found.""" + if isinstance(node, Ref): + if node.name in names: + return {node.name}, None + match = difflib.get_close_matches(word=node.name, possibilities=sorted(names), n=1) + hint = f" Did you mean '{match[0]}'?" if match else "" + return set(), ( + f"'{node.name}' is not a saved measure.{hint} Reference a saved measure by its bare name, or " + f"aggregate a column (e.g., 'sum({node.name})'). For COUNT(*), use 'count(*)'." + ) + if isinstance(node, DottedRef): + dotted = ".".join(node.parts) + return set(), f"Cross-model measure '{dotted}' must include an aggregation (e.g., 'sum({dotted})')." + if isinstance(node, AggCall): + known = normalize_aggregation_name(node.agg) in BUILTIN_AGGREGATIONS or node.agg in aggregations + return set(), None if known else f"unknown aggregation {node.agg!r}" + children = (node.input,) if isinstance(node, TransformCall) else _operands(node) + refs: set = set() + for child in children: + found, problem = _measure_refs(child, names=names, aggregations=aggregations) + if problem is not None: + return refs, problem + refs |= found + return refs, None + + # --------------------------------------------------------------------------- # Reference walk # --------------------------------------------------------------------------- @@ -1434,5 +1642,5 @@ def split_entity_agg_ref(raw: str) -> Tuple[str, Optional[str]]: __all__ = [ "AggCall", "Arith", "BoolOp", "Cast", "Cmp", "DottedRef", "Literal", "ParsedExpr", "Placeholder", "Ref", "ScalarCall", "StarSource", "TemporalLiteral", "TransformCall", "TupleLit", "UnaryOp", - "canonical_measure_text", "parse_expr", "placeholder_names", "split_entity_agg_ref", "walk_parsed_refs", + "bind_variables", "canonical_measure_text", "measure_dependency_problems", "parse_expr", "placeholder_names", "split_entity_agg_ref", "walk_parsed_refs", ] diff --git a/slayer/cube/converter.py b/slayer/cube/converter.py index 5fadce04..362139b9 100644 --- a/slayer/cube/converter.py +++ b/slayer/cube/converter.py @@ -19,7 +19,7 @@ invert_cardinality, ) from slayer.core.format import NumberFormat, NumberFormatType -from slayer.core.formula import parse_formula +from slayer.core.syntax import measure_dependency_problems from slayer.core.models import Column, ModelJoin, ModelMeasure, SlayerModel from slayer.core.query import render_probe_text from slayer.core.refs import IDENTIFIER_RE, functional_agg_text @@ -57,7 +57,6 @@ } _DURATION_UNITS = {"day": "d", "month": "m", "week": "w", "year": "y", "hour": "h", "minute": "min", "second": "s"} -_STAR_COUNT = "count(*)" _CUBE_INFRA_FIELDS = ("refresh_key", "calendar", "hierarchies", "access_policy", "sql_alias") @@ -842,27 +841,15 @@ def _drop_measures_over(self, *, cube_name, measures, info, dropped, report) -> return good_measures def _drop_unparseable_measures(self, cube_name, good_measures, report) -> list[ModelMeasure]: - # Fixpoint: dropping a measure can break a calc measure naming it. Only - # measures are bare-referenceable; a bare column needs an aggregation. - while True: - known = {m.name for m in good_measures if m.name} - broken = [m for m in good_measures if not self._formula_parses(m.formula, known)] - if not broken: - return good_measures - for m in broken: + # Only measures are bare-referenceable; a bare column needs an aggregation. + problems = measure_dependency_problems([(m.name, m.formula) for m in good_measures]) + for m, problem in zip(good_measures, problems): + if problem is not None: report.add(CubeConversionIssue( category=CubeIssueCategory.COMPLEX_MEASURE, severity="warning", cube=cube_name, member=m.name, - message=f"Measure formula does not parse; dropped: {m.formula!r}")) - good_measures = [m for m in good_measures if m not in broken] - - def _formula_parses(self, formula: str, known_names: set[str]) -> bool: - nm = dict.fromkeys(known_names, _STAR_COUNT) - try: - parse_formula(formula, named_measures=nm or None) - return True - except Exception: # noqa: BLE001 - return False + message=f"Measure formula does not resolve ({problem}); dropped: {m.formula!r}")) + return [m for m, problem in zip(good_measures, problems) if problem is None] def _dedisambiguate_namespace(self, columns, measures, report, *, cube) -> None: # The allocator already guarantees uniqueness; this is a defensive check. diff --git a/slayer/dbt/converter.py b/slayer/dbt/converter.py index ace0a2ab..553e2f5c 100644 --- a/slayer/dbt/converter.py +++ b/slayer/dbt/converter.py @@ -24,7 +24,7 @@ from slayer.core.enums import DataType from slayer.core.format import NumberFormat, NumberFormatType -from slayer.core.formula import parse_formula +from slayer.core.syntax import measure_dependency_problems from slayer.core.models import ( Column, ModelJoin, @@ -261,32 +261,24 @@ def _prune_dangling_measures(self) -> None: self._prune_model_measures(model) def _prune_model_measures(self, model: SlayerModel) -> None: - agg_names = frozenset(a.name for a in model.aggregations) - changed = True - while changed: - changed = False - named = {m.name: m.formula for m in model.measures if m.name} - survivors: list[ModelMeasure] = [] - for m in model.measures: - others = {k: v for k, v in named.items() if k != m.name} - try: - parse_formula(m.formula, extra_agg_names=agg_names, named_measures=others) - survivors.append(m) - except (ValueError, RecursionError) as exc: - self._unconverted.append(ConversionWarning( - model_name=model.name, - metric_name=m.name, - category="dangling_reference", - severity="dropped", - message=( - f"Measure '{m.name}' references a name that does not resolve " - f"on model '{model.name}' and was dropped: {exc}" - ), - suggestion="Ensure every referenced metric/measure converts successfully.", - )) - changed = True - if changed: - model.measures = survivors + problems = measure_dependency_problems( + [(m.name, m.formula) for m in model.measures], + aggregations=frozenset(a.name for a in model.aggregations), + ) + for m, problem in zip(model.measures, problems): + if problem is not None: + self._unconverted.append(ConversionWarning( + model_name=model.name, + metric_name=m.name, + category="dangling_reference", + severity="dropped", + message=( + f"Measure '{m.name}' references a name that does not resolve " + f"on model '{model.name}' and was dropped: {problem}" + ), + suggestion="Ensure every referenced metric/measure converts successfully.", + )) + model.measures = [m for m, problem in zip(model.measures, problems) if problem is None] def _convert_regular_models(self, existing_names: set) -> list[SlayerModel]: """Convert orphan dbt models (not wrapped by semantic_models) to hidden SLayer models.""" diff --git a/slayer/engine/bind_inputs.py b/slayer/engine/bind_inputs.py index 13de93f7..d66054b5 100644 --- a/slayer/engine/bind_inputs.py +++ b/slayer/engine/bind_inputs.py @@ -285,6 +285,7 @@ def _map_bound_keys( routed_dotted=spec.bound.routed_dotted, ), direction=spec.direction, + nulls=spec.nulls, ) for spec in order_specs ] @@ -430,6 +431,7 @@ def bind_query_inputs( # NOSONAR(S3776) — one cohesive bind pass. The stages granularity=_td_granularity(matching_td, bundle=bundle), ).bound, direction=o.direction, + nulls=o.nulls or "default", )) continue # A placeholder ColumnRef means the item is an EXPRESSION: bind raw_formula, skip alias lookups. @@ -442,6 +444,7 @@ def bind_query_inputs( # NOSONAR(S3776) — one cohesive bind pass. The stages dimension_alias_map=dim_alias_map, ), direction=o.direction, + nulls=o.nulls or "default", )) continue # An ORDER BY over a partition_by / window= aggregate must bind raw_formula (the alias shortcut would drop the partition/window). @@ -460,7 +463,7 @@ def bind_query_inputs( # NOSONAR(S3776) — one cohesive bind pass. The stages for k in walk_value_keys(_part_bound.value_key) ): order_specs.append(OrderSpec( - bound=_part_bound, direction=o.direction, + bound=_part_bound, direction=o.direction, nulls=o.nulls or "default", )) continue # A FOREIGN-qualified order ref must not resolve to a same-named local column via the bare-leaf shortcut. @@ -493,7 +496,7 @@ def bind_query_inputs( # NOSONAR(S3776) — one cohesive bind pass. The stages scope=scope, bundle=bundle, ) - order_specs.append(OrderSpec(bound=bo, direction=o.direction)) + order_specs.append(OrderSpec(bound=bo, direction=o.direction, nulls=o.nulls or "default")) column_type = scope_column_type(scope=scope, bundle=bundle) check_date_operands( diff --git a/slayer/engine/binding.py b/slayer/engine/binding.py index 60b26cf9..356e3ba1 100644 --- a/slayer/engine/binding.py +++ b/slayer/engine/binding.py @@ -35,7 +35,7 @@ from slayer.core.enums import RANK_FAMILY_TRANSFORMS from slayer.core.granularity import CustomGranularity, Granularity, resolve_granularity from slayer.core.refs import EXPRESSION_SOURCE_KINDS, key_display -from slayer.core.keys import source_anchor_path, DATE_ADD_COUNT_ARG, DATE_OPERAND_ARGS, MIRRORED_TIME_POINT_OP, SCALAR_FUNCTIONS, cast_data_type, check_scalar_arity, type_date_values, AggregateKey, AggregateSource, ArithmeticKey, CastKey, ColumnKey, ColumnSqlKey, Grain, InKey, LiteralKey, ScalarCallKey, SqlFragmentKey, StarKey, TimePointCmpKey, TimePointOp, TimeTruncKey, TransformKey, ValueKey, boolean_valued, column_leaf, column_path, is_attached_source, normalize_scalar, prepend_value_key, temporal_type, walk_value_keys +from slayer.core.keys import source_anchor_path, window_kwarg_of, DATE_ADD_COUNT_ARG, DATE_OPERAND_ARGS, MIRRORED_TIME_POINT_OP, SCALAR_FUNCTIONS, cast_data_type, check_scalar_arity, type_date_values, AggregateKey, AggregateSource, ArithmeticKey, CastKey, ColumnKey, ColumnSqlKey, Grain, InKey, LiteralKey, ScalarCallKey, SqlFragmentKey, StarKey, TimePointCmpKey, TimePointOp, TimeTruncKey, TransformKey, ValueKey, boolean_valued, column_leaf, column_path, is_attached_source, normalize_scalar, prepend_value_key, temporal_type, walk_value_keys from slayer.core.join_walker import ( aggregation_owner, canonical_path, @@ -1021,7 +1021,7 @@ def _bind_expression_agg_source( agg: str, scope: ModelScope | StageSchema, bundle: ResolvedSourceBundle, -) -> ArithmeticKey | ScalarCallKey | LiteralKey | TimeTruncKey | InKey | TimePointCmpKey: +) -> ArithmeticKey | ScalarCallKey | LiteralKey | TimeTruncKey | InKey | TimePointCmpKey | CastKey: """Bind a scalar-expression aggregate source. Dotted joined-model leaves and operands carrying ``Column.filter`` are both @@ -1189,16 +1189,26 @@ def _bind_filtered_agg( ) if not isinstance(key, AggregateKey) or is_attached_source(key.source): raise _filter_error("applies only to an aggregation over rows.") - if isinstance(key.source, StarKey) or key.agg in RANKED_AGGREGATIONS: - return key.model_copy(update={"filter": condition}) + if key.agg in RANKED_AGGREGATIONS: + return _carrying_filter(key, condition=condition) + value = LiteralKey(value=normalize_scalar(1)) if isinstance(key.source, StarKey) else key.source masked = key.model_copy(update={ - "source": _masked(key.source, condition=condition), + "source": _masked(value, condition=condition), "kwargs": tuple((k, _masked(v, condition=condition) if isinstance(v, _ROW_INPUT_KINDS) else v) for k, v in key.kwargs), }) - if source_anchor_path(masked.source) != source_anchor_path(key.source): - raise _filter_error("must read the aggregated rows' own model or models joined to it one-to-one.") - return masked + if source_anchor_path(masked.source) == source_anchor_path(key.source): + return masked + if isinstance(key.source, StarKey) and not source_anchor_path(key.source): + # The condition reads a joined model: the star keeps the count's home. + return _carrying_filter(key, condition=condition) + raise _filter_error("must read the aggregated rows' own model or models joined to it one-to-one.") + + +def _carrying_filter(key: AggregateKey, *, condition: ValueKey) -> AggregateKey: + if window_kwarg_of(key) is not None: + raise _filter_error("cannot combine with window= here; filter the query's rows instead.") + return key.model_copy(update={"filter": condition}) # Row-valued aggregation parameters a FILTER masks (literals and attached values are not rows). diff --git a/slayer/engine/compile/stages.py b/slayer/engine/compile/stages.py index bed6daed..8f0f2f67 100644 --- a/slayer/engine/compile/stages.py +++ b/slayer/engine/compile/stages.py @@ -3486,6 +3486,7 @@ def _rewrite_regrouped_prebound( value_key=substitute_value_keys(sp.bound.value_key, mapping), ), direction=sp.direction, + nulls=sp.nulls, ) for sp in prebound.order_specs ], @@ -4191,6 +4192,7 @@ def _emit_planned(routed: _Routed) -> PlannedQuery: # NOSONAR(S3776) — projec slot_id=sid, direction=direction, phase=order_slot.key.phase, + nulls=spec.nulls, )) transform_layers = _emit_transform_layers(slots=projection.registry.slots) diff --git a/slayer/engine/query_engine.py b/slayer/engine/query_engine.py index f4911ba6..c4c162c2 100644 --- a/slayer/engine/query_engine.py +++ b/slayer/engine/query_engine.py @@ -116,6 +116,7 @@ is_spliced, plan_has_semi_join_filters, plans_read_clock, + plans_try_cast, ) from slayer.engine.schema_drift import ( AppliedEntry, @@ -749,6 +750,20 @@ def _apply_policy( ), ) + async def _render_dialect( + self, *, dialect: str, datasource: DatasourceConfig, planned_list, + ) -> "str | SqlDialect": + """``dialect``, carrying the server profile when rendering a ``TRY_CAST`` reads it; an unreadable + profile renders as an unknown server.""" + sql_dialect = get_dialect(dialect) + if not sql_dialect.try_cast_reads_profile or not plans_try_cast(planned_list): + return dialect + try: + profile = await self._client_for(datasource).server_profile() + except Exception: # noqa: BLE001 — an unknown server gets the plain cast + profile = ServerProfile() + return sql_dialect.model_copy(update={"server_profile": profile}) + async def _require_correlated_subqueries( self, *, dialect: str, datasource: DatasourceConfig, planned_list, ) -> None: @@ -1171,6 +1186,7 @@ async def _plan_and_render( # NOSONAR S3776 — linear pipeline (localize→inf # Plan the DAG (root last) and render the whole chain to one SQL string. planned_list = plan_stages(queries=[*normed_named.values(), query], bundle=bundle) + render_dialect = await self._render_dialect(dialect=dialect, datasource=datasource, planned_list=planned_list) kept: Set[str] = set() sql: Optional[str] = None statement: Optional[Any] = None @@ -1178,12 +1194,12 @@ async def _plan_and_render( # NOSONAR S3776 — linear pipeline (localize→inf with collect_stale_spellings() as render_stale_spellings: if as_statement: statement = _build_planned_stages_ast( - planned_list, bundle=bundle, dialect=dialect, kept_stages=kept, + planned_list, bundle=bundle, dialect=render_dialect, kept_stages=kept, ) reads |= read_models(statement) else: sql = generate_planned_stages( - planned_queries=planned_list, bundle=bundle, dialect=dialect, + planned_queries=planned_list, bundle=bundle, dialect=render_dialect, # Plan-derived canonical projection keys drive the write-side length # fit; the read side decodes against the same set. projection_aliases=projection_result_keys(root_planned=planned_list[-1]), diff --git a/slayer/ir/bound.py b/slayer/ir/bound.py index 573ca369..2e395a8b 100644 --- a/slayer/ir/bound.py +++ b/slayer/ir/bound.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Optional, Tuple +from typing import Literal, Optional, Tuple from pydantic import BaseModel, ConfigDict, Field @@ -92,6 +92,7 @@ class OrderSpec(BaseModel): bound: BoundExpr direction: str = "asc" + nulls: Literal["default", "first", "last"] = "default" class GranularityConflict(BaseModel): diff --git a/slayer/ir/planned.py b/slayer/ir/planned.py index 7ec6b587..e9b3d88e 100644 --- a/slayer/ir/planned.py +++ b/slayer/ir/planned.py @@ -20,6 +20,7 @@ AggregateKey, LiteralKey, Phase, + CastKey, ScalarCallKey, TransformKey, ValueKey, @@ -775,6 +776,16 @@ def plan_value_keys(planned) -> Iterator[ValueKey]: yield from _walk_plan_keys(planned, set()) +def plans_try_cast(planned_list) -> bool: + """Whether any plan renders a ``TRY_CAST``.""" + return any( + isinstance(k, CastKey) and k.try_ + for planned in planned_list + for root in plan_value_keys(planned) + for k in walk_value_keys(root) + ) + + def plans_read_clock(planned_list) -> bool: """Whether any plan evaluates ``now()`` / ``current_date()``.""" return any( diff --git a/slayer/ir/variables.py b/slayer/ir/variables.py index e76d189d..7efef8d6 100644 --- a/slayer/ir/variables.py +++ b/slayer/ir/variables.py @@ -2,18 +2,20 @@ - :func:`merge_query_variables` collapses the four variable layers into the effective dict (runtime > stage > outer > model_defaults). -- :func:`apply_variables_to_query` substitutes ``{var}`` (Mode-B regime) into a - query's filters, measure formulas, computed-dimension expressions, order - expressions and ``date_range`` bounds. +- :func:`apply_variables_to_query` binds ``{var}`` as literals into a query's + filters, measure formulas, computed-dimension expressions and order + expressions, and substitutes its ``date_range`` bounds. - :func:`substitute_model_sql_surfaces` substitutes a model's four Mode-A - surfaces (SQL regime) and its saved measure formulas (Mode-B regime). + surfaces (SQL regime) and binds its saved measure formulas. """ from __future__ import annotations +import re from typing import Any, Dict, Optional from slayer.core.models import ModelMeasure, SlayerModel +from slayer.core.syntax import bind_variables from slayer.core.query import ( ComputedDimension, OrderItem, @@ -50,34 +52,53 @@ def merge_query_variables( } -def _python_sub(text: str, variables: Dict[str, Any]) -> str: - # Mode-B takes the python regime: SQL quote-doubling would concatenate adjacent literals. - return substitute_variables(filter_str=text, variables=variables, escape="python") - - def _substituted_measure(m: ModelMeasure, variables: Dict[str, Any]) -> ModelMeasure: - text = _python_sub(m.formula, variables) + text = bind_variables(m.formula, variables) return m if text == m.formula else m.substituted(text) def _substituted_dimension(d: Any, variables: Dict[str, Any]) -> Any: if not isinstance(d, ComputedDimension): return d - text = _python_sub(d.expression, variables) - return d if text == d.expression else d.substituted(text) + text = bind_variables(d.expression, variables) + if text == d.expression: + return d + bound = d.model_copy(update={"expression": text}) + bound._template = d.template + return bound def _substituted_order(o: OrderItem, variables: Dict[str, Any]) -> OrderItem: if not o.raw_formula: return o - text = _python_sub(o.raw_formula, variables) + text = bind_variables(o.raw_formula, variables) return o if text == o.raw_formula else o.model_copy(update={"raw_formula": text}) +_BOUND_VARIABLE_RE = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}") + + +def _date_range_bound(text: str, variables: Dict[str, Any]) -> str: + """A ``date_range`` bound with each ``{name}`` replaced by its value's text (a string or a whole number); + the result is then checked as a time point.""" + def _value(match: "re.Match[str]") -> str: + name = match.group(1) + if name not in variables: + raise ValueError( + f"Undefined variable '{name}' in date_range bound {text!r}. Available variables: {sorted(variables)}" + ) + value = variables[name] + if isinstance(value, bool) or not isinstance(value, (str, int)): + raise ValueError(f"Variable '{name}' in a date_range bound must be a string or an integer year.") + return str(value) + + return _BOUND_VARIABLE_RE.sub(_value, text) + + def _substituted_time_dimension(td: TimeDimension, variables: Dict[str, Any]) -> TimeDimension: if not td.date_range: return td - bounds = [None if b is None else _python_sub(b, variables) for b in td.date_range] + bounds = [None if b is None else _date_range_bound(b, variables) for b in td.date_range] if bounds == td.date_range: return td # Revalidated so a substituted bound is shape-checked like a literal one. @@ -99,7 +120,7 @@ def apply_variables_to_query( effective: Dict[str, Any] = dict(variables or {}) changed: Dict[str, Any] = {} if query.filters is not None: - changed["filters"] = [_python_sub(f, effective) for f in query.filters] + changed["filters"] = [bind_variables(f, effective) for f in query.filters] surfaces = ( ("measures", lambda m: _substituted_measure(m, effective)), ("dimensions", lambda d: _substituted_dimension(d, effective)), @@ -152,7 +173,7 @@ def substitute_model_sql_surfaces( *, model: SlayerModel, variables: Dict[str, Any], backslash_escapes: bool, ) -> SlayerModel: """Copy of ``model`` with ``{var}`` substituted into its four Mode-A surfaces (SQL regime) - and its saved measure formulas (python regime); no-op when unneeded.""" + and bound into its saved measure formulas; no-op when unneeded.""" if not variables and not model_needs_substitution_pass(model): return model variables = coerce_declared_list_variables( @@ -180,7 +201,7 @@ def _sub(text: str) -> str: update["filters"] = [_sub(f) for f in model.filters] if model.measures: update["measures"] = [ - m.model_copy(update={"formula": _python_sub(m.formula, variables)}) if _measure_needs_pass(m) else m + m.model_copy(update={"formula": bind_variables(m.formula, variables)}) if _measure_needs_pass(m) else m for m in model.measures ] return model.model_copy(update=update) diff --git a/slayer/osi/converter.py b/slayer/osi/converter.py index 01d05e0c..0faa3efd 100644 --- a/slayer/osi/converter.py +++ b/slayer/osi/converter.py @@ -20,7 +20,7 @@ from slayer.core.enums import DataType, JoinCardinality from slayer.core.errors import AmbiguousJoinPathError -from slayer.core.formula import parse_formula +from slayer.core.syntax import parse_expr from slayer.core.join_walker import resolve_hop from slayer.core.models import Column, ModelJoin, ModelMeasure, SlayerModel, join_key_error from slayer.core.refs import IDENTIFIER_RE as _IDENTIFIER_RE @@ -795,7 +795,7 @@ def _build_measure(self, metric: OSIMetric, sm_model_names: list[str]) -> None: return try: - parse_formula(result.formula) + parse_expr(result.formula) except Exception as exc: # noqa: BLE001 — reject un-parseable emission self._unconv( f"Metric {metric.name!r}: emitted formula {result.formula!r} is " diff --git a/slayer/sql/dialects/_tier2.py b/slayer/sql/dialects/_tier2.py index 50d9f96a..507fc143 100644 --- a/slayer/sql/dialects/_tier2.py +++ b/slayer/sql/dialects/_tier2.py @@ -15,6 +15,7 @@ from typing import ClassVar from sqlglot import exp +from sqlglot.expressions.datatypes import DType from slayer.sql.dialects.base import SqlDialect from slayer.sql.dialects.trino import PrestoFamilyDialect @@ -23,7 +24,7 @@ class RedshiftDialect(SqlDialect): sqlglot_name: str = "redshift" ilike_native: ClassVar[bool] = True - vendor_cast_types: ClassVar[frozenset[exp.DataType.Type]] = frozenset({exp.DataType.Type.MONEY}) + vendor_cast_types: ClassVar[frozenset[DType]] = frozenset({exp.DataType.Type.MONEY}) ds_type_aliases: frozenset[str] = frozenset({"redshift"}) explain_prefix: str | None = "EXPLAIN" explain_postfix: str = "" diff --git a/slayer/sql/dialects/base.py b/slayer/sql/dialects/base.py index ef09d61d..95eb0ee4 100644 --- a/slayer/sql/dialects/base.py +++ b/slayer/sql/dialects/base.py @@ -13,6 +13,7 @@ from sqlalchemy.engine.url import make_url from sqlalchemy.exc import ArgumentError from sqlglot import exp +from sqlglot.expressions.datatypes import DType from sqlglot.expressions.core import Expression from sqlglot.dialects.dialect import Dialect as _SqlglotDialect from sqlglot.errors import SqlglotError @@ -133,7 +134,7 @@ def cast_target(type_text: str, *, dialect: "SqlDialect") -> exp.DataType: #: Types only some databases have; a dialect without one rejects a CAST to it at plan time. -VENDOR_ONLY_CAST_TYPES: frozenset[exp.DataType.Type] = frozenset({ +VENDOR_ONLY_CAST_TYPES: frozenset[DType] = frozenset({ exp.DataType.Type.MONEY, exp.DataType.Type.SMALLMONEY, }) @@ -326,6 +327,8 @@ class SqlDialect(BaseModel): # The server's profile, when SQL generation needs it (Postgres TRY_CAST); unknown otherwise. server_profile: ServerProfile = ServerProfile() + # Whether rendering a TRY_CAST reads ``server_profile``. + try_cast_reads_profile: bool = False # An ``IN`` subquery and its joins run once on the initiator (``GLOBAL``): exact over shards. global_in_subqueries: bool = False @@ -370,7 +373,7 @@ def declared_cast_type(self, dt: Optional[DataType]) -> Optional[DataType]: return dt # Vendor-only types sqlglot parses for every dialect; a dialect lists the ones it has. - vendor_cast_types: ClassVar[frozenset[exp.DataType.Type]] = frozenset() + vendor_cast_types: ClassVar[frozenset[DType]] = frozenset() def cast_type_supported(self, data_type: exp.DataType) -> bool: """Whether this dialect can express a CAST to ``data_type``.""" @@ -409,6 +412,18 @@ def build_ordered( kwargs["nulls_first"] = False return exp.Ordered(**kwargs) + def build_order_terms( + self, + order_col: Expression, + *, + descending: bool, + nulls: Literal["default", "first", "last"] = "default", + resolved: Optional[Expression] = None, # NOSONAR(S1172) — read by the T-SQL emulation + ) -> list[exp.Ordered]: + """The ``ORDER BY`` terms of one entry; ``resolved`` is the expression ``order_col`` names, when it is + an alias of the SELECT being ordered.""" + return [self.build_ordered(order_col, descending=descending, nulls=nulls)] + def native_nulls_first(self, *, descending: bool) -> bool: """Where NULLs natively sort for ``descending``, read from sqlglot's emitter.""" ordering = getattr( diff --git a/slayer/sql/dialects/postgres.py b/slayer/sql/dialects/postgres.py index d80819b0..9d71ee49 100644 --- a/slayer/sql/dialects/postgres.py +++ b/slayer/sql/dialects/postgres.py @@ -6,6 +6,7 @@ from typing import Any, ClassVar from sqlglot import exp +from sqlglot.expressions.datatypes import DType from sqlglot.expressions.core import Expression from slayer.sql.dialects.base import ServerProfile, SqlDialect, as_text @@ -43,7 +44,7 @@ def _cast_round_arg_to_numeric(node: exp.Expression) -> exp.Expression: class PostgresDialect(SqlDialect): sqlglot_name: str = "postgres" ds_type_aliases: frozenset[str] = frozenset({"postgres", "postgresql"}) - vendor_cast_types: ClassVar[frozenset[exp.DataType.Type]] = frozenset({exp.DataType.Type.MONEY}) + vendor_cast_types: ClassVar[frozenset[DType]] = frozenset({exp.DataType.Type.MONEY}) ilike_native: ClassVar[bool] = True explain_prefix: str | None = "EXPLAIN ANALYZE" explain_postfix: str = "" @@ -52,6 +53,7 @@ class PostgresDialect(SqlDialect): # NAMEDATALEN: 63 usable bytes; over-length names are SILENTLY truncated. max_identifier_bytes: int | None = 63 statement_timeout_best_effort: bool = True + try_cast_reads_profile: bool = True url_scheme: str | None = "postgresql+psycopg2" async_driver: str | None = "asyncpg" install_extra: str | None = "postgres" diff --git a/slayer/sql/dialects/sqlite.py b/slayer/sql/dialects/sqlite.py index dfb45fb2..f632a18b 100644 --- a/slayer/sql/dialects/sqlite.py +++ b/slayer/sql/dialects/sqlite.py @@ -8,6 +8,7 @@ from datetime import date, datetime, timedelta from sqlglot import exp +from sqlglot.expressions.datatypes import DType from sqlglot.expressions.core import Expression from typing import ClassVar, Optional @@ -416,7 +417,7 @@ def register_sqlite_udfs(dbapi_connection) -> None: class SqliteDialect(SqlDialect): sqlglot_name: str = "sqlite" # Any type name is a column affinity on SQLite. - vendor_cast_types: ClassVar[frozenset[exp.DataType.Type]] = VENDOR_ONLY_CAST_TYPES + vendor_cast_types: ClassVar[frozenset[DType]] = VENDOR_ONLY_CAST_TYPES ds_type_aliases: frozenset[str] = frozenset({"sqlite"}) explain_prefix: str | None = "EXPLAIN QUERY PLAN" explain_postfix: str = "" diff --git a/slayer/sql/dialects/tsql.py b/slayer/sql/dialects/tsql.py index 05a66f35..c7e5a792 100644 --- a/slayer/sql/dialects/tsql.py +++ b/slayer/sql/dialects/tsql.py @@ -7,15 +7,17 @@ from __future__ import annotations import re -from typing import ClassVar, Literal +from typing import ClassVar, Literal, Optional from sqlglot import exp +from sqlglot.expressions.datatypes import DType from sqlglot.expressions.core import Expr, Expression from slayer.core.enums import SUB_DAY_GRANULARITIES, DataType, DatePart, TimeGranularity from slayer.sql.dialects.base import ( DottedAliasManglingMixin, SqlDialect, + operand_copy, StatAgg1Name, StatAgg2Name, _build_covar_decomposition, @@ -96,7 +98,7 @@ def _bit_value(predicate: Expr) -> Expression: class TsqlDialect(DottedAliasManglingMixin, SqlDialect): sqlglot_name: str = "tsql" ds_type_aliases: frozenset[str] = frozenset({"mssql", "sqlserver", "tsql"}) - vendor_cast_types: ClassVar[frozenset[exp.DataType.Type]] = frozenset({ + vendor_cast_types: ClassVar[frozenset[DType]] = frozenset({ exp.DataType.Type.MONEY, exp.DataType.Type.SMALLMONEY, }) explain_prefix: str | None = "SET SHOWPLAN_ALL ON;" @@ -138,6 +140,30 @@ def build_ordered( order_col, descending=descending, nulls=nulls, ) + def build_order_terms( + self, + order_col: Expression, + *, + descending: bool, + nulls: Literal["default", "first", "last"] = "default", + resolved: Optional[Expression] = None, + ) -> list[exp.Ordered]: + """An explicit NULL placement is a leading ``CASE WHEN IS NULL`` key over the resolved + expression (an alias is not visible inside an ORDER BY expression), then the native term.""" + if nulls == "default": + return [self.build_ordered(order_col, descending=descending)] + null_rank = exp.Case( + ifs=[exp.If( + this=exp.Is(this=operand_copy(resolved if resolved is not None else order_col), expression=exp.Null()), + true=exp.Literal.number(0 if nulls == "first" else 1), + )], + default=exp.Literal.number(1 if nulls == "first" else 0), + ) + return [ + self.build_ordered(null_rank, descending=False), + self.build_ordered(order_col, descending=descending), + ] + def build_date_trunc( self, col_expr: Expression, diff --git a/slayer/sql/generator.py b/slayer/sql/generator.py index 17d738bd..58931531 100644 --- a/slayer/sql/generator.py +++ b/slayer/sql/generator.py @@ -50,7 +50,7 @@ resolved_terminal, ) from slayer.core.models import VALUE_PLACEHOLDER, aggregation_definition, rendered_formula, reserved_value_param_message -from slayer.core.refs import EXPRESSION_SOURCE_KINDS as _EXPRESSION_SOURCE_KINDS +from slayer.core.refs import EXPRESSION_SOURCE_KINDS as _EXPRESSION_SOURCE_KINDS, key_display from slayer.core.window_duration import WINDOW_UNIT_GRANULARITY from slayer.sql.column_expansion import ( is_trivial_base, @@ -2025,9 +2025,14 @@ def _resolve_first_last_time_arg(key) -> None: return scope.resolve(arg) + def _resolve_filter(key) -> None: + if key.filter is not None: + scope.resolve(key.filter) + _for_each_local_agg(_resolve_source) _for_each_local_agg(_resolve_kwargs) _for_each_local_agg(_resolve_first_last_time_arg) + _for_each_local_agg(_resolve_filter) return resolved def _throwaway_frame( @@ -2238,6 +2243,10 @@ def _defer_composite(sid: str, slot, full_alias: str) -> None: select_columns.append(dim_expr.copy().as_(full_alias)) group_by_keys.setdefault(sid, dim_expr) _record_alias(sid, full_alias) + elif isinstance(key, LiteralKey) and slot.is_dimension: + # A constant needs no GROUP BY entry (a constant one is refused or read as a position). + select_columns.append(_literal(key.value, dialect=self._dialect).as_(full_alias)) + _record_alias(sid, full_alias) elif isinstance(key, SLOT_COMPOSITE_KINDS) or ( isinstance(key, LiteralKey) and not slot.is_dimension ): @@ -2870,13 +2879,19 @@ def _frame(*, attached=None) -> ScopeFrame: ), )) - value_ref = ranked_scope.materialize_for( - self._ranked_value_expr( - key=local_key, root_model=root_model, - root_relation=root_relation, bundle=bundle, scope=ranked_scope, - ), - consumer=cte_scope, + value_expr = self._ranked_value_expr( + key=local_key, root_model=root_model, + root_relation=root_relation, bundle=bundle, scope=ranked_scope, ) + # A FILTER ranks the passing rows first and picks only a passing row's value. + passing_first: List[exp.Ordered] = [] + if local_key.filter is not None: + value_expr = self._passing_value(local_key.filter, value=value_expr, scope=ranked_scope) + passing_first.append(exp.Ordered(this=exp.Case( + ifs=[exp.If(this=value_expr.args["ifs"][0].this.copy(), true=exp.Literal.number(0))], + default=exp.Literal.number(1), + ))) + value_ref = ranked_scope.materialize_for(value_expr, consumer=cte_scope) ranking_time = self._ranked_scope_expr( key=plan.ranking_time_key, root_model=root_model, root_relation=root_relation, bundle=bundle, scope=ranked_scope, @@ -2909,6 +2924,7 @@ def _frame(*, attached=None) -> ScopeFrame: descending=plan.agg == "last", ), ), + leading=passing_first, )) inner = inner.from_(from_expr) inner = _apply_joins(select=inner, joins=joins) @@ -3112,6 +3128,11 @@ def _render_association_producer_body( # NOSONAR(S3776) — one cohesive two-le # the input is root-determined, constant per entity); ``count(*)`` keeps no # value column — level 2 counts the entity rows. is_star = isinstance(agg_slot.key.source, StarKey) + if agg_slot.key.filter is not None: + raise RuntimeError( + f"{key_display(agg_slot.key)}: a key-carried FILTER reached an association producer; binding " + f"carries one only on a local star count or a ranked pick.", + ) picked_alias = "_v" # Parameters the grain determines are picked once per cell as # _p (below) and read by level 2 as _base._p; the level-1 value pick @@ -3872,8 +3893,7 @@ def _render_outer_composite(cslot) -> Expression: else grain_alias_column(alias=_full_alias, table="_base") ) order_terms = [ - resolve_order_term(entry=entry, env=order_env) - for entry in lowered.order + term for entry in lowered.order for term in resolve_order_term(entry=entry, env=order_env) ] if order_terms: combined_statement.set("order", exp.Order(expressions=order_terms)) @@ -4742,8 +4762,8 @@ def _planned_order_terms( dialect=self._dialect, ) return [ - resolve_order_term(entry=entry, env=env) - for entry in _lower_positions(planned_query).order + term for entry in _lower_positions(planned_query).order + for term in resolve_order_term(entry=entry, env=env) ] def _series_shift_cte( @@ -5671,21 +5691,24 @@ def _build_agg_render_spec_from_planned( # NOSONAR(S3776) — sequential isinst f"'count(*)' takes no args or kwargs; got " f"args={key.args!r}, kwargs={key.kwargs!r}." ) + star_scope = scope or self._throwaway_frame( + model=source_model, relation=source_relation, + bundle=bundle or ResolvedSourceBundle(dialect=self.dialect, source_model=source_model), + ) return AggRenderSpec( name="", sql=None, + # A key-carried FILTER counts the passing rows. + value=None if key.filter is None else self._passing_value( + key.filter, value=exp.Literal.number(1), scope=star_scope, + ), aggregation=key.agg, alias=full_alias, model_name=source_relation, type=slot_type, formula=formula, agg_kwargs=self._agg_param_exprs( - key=key, formula=formula, resolved=resolved_agg_kwargs, - scope=scope or self._throwaway_frame( - model=source_model, relation=source_relation, - bundle=bundle or ResolvedSourceBundle( - dialect=self.dialect, source_model=source_model), - ), + key=key, formula=formula, resolved=resolved_agg_kwargs, scope=star_scope, ), ) param_scope = scope or self._throwaway_frame( @@ -5804,6 +5827,11 @@ def _build_agg_render_spec_from_planned( # NOSONAR(S3776) — sequential isinst f"AggregateKey source {type(source).__name__} not supported.", ) + def _passing_value(self, condition, *, value: Expression, scope: ScopeFrame) -> exp.Case: + """``CASE WHEN THEN END``: ``value`` on the rows a FILTER keeps.""" + passing = render_value_key(key=condition, ctx=RenderContext(scope=scope, dialect=self._dialect)) + return exp.Case(ifs=[exp.If(this=passing, true=value)]) + def _semi_join_exists_conditions( self, *, planned_query, source_model, source_relation: str, bundle, ) -> "List[Expression]": @@ -6131,6 +6159,7 @@ def _host_order_terms( source_model=None, bundle=None, aliases_by_slot_id: Optional[Dict[str, List[str]]] = None, + select: Optional[exp.Select] = None, ) -> List[exp.Ordered]: """ORDER BY terms resolved against a base SELECT with no CTE chain.""" scoped_order = _lower_positions(planned_query).order @@ -6142,7 +6171,11 @@ def _host_order_terms( bundle=bundle, aliases_by_slot_id=aliases_by_slot_id, ) - return [resolve_order_term(entry=entry, env=env) for entry in scoped_order] + if select is not None: + env.projection = { + a.alias: a.this for a in select.expressions if isinstance(a, exp.Alias) and a.alias + } + return [term for entry in scoped_order for term in resolve_order_term(entry=entry, env=env)] def _apply_planned_order_limit( self, @@ -6165,6 +6198,7 @@ def _apply_planned_order_limit( source_model=source_model, bundle=bundle, aliases_by_slot_id=aliases_by_slot_id, + select=select, ), planned_query=planned_query, ) @@ -6381,11 +6415,15 @@ def read_models(statement: Expression) -> Set[str]: } +def _as_dialect(dialect: "str | SqlDialect") -> SqlDialect: + return dialect if isinstance(dialect, SqlDialect) else get_dialect(dialect) + + def generate_planned_stages( planned_queries, *, bundle, - dialect: str = "postgres", + dialect: "str | SqlDialect" = "postgres", projection_aliases: "Sequence[str]" = (), kept_stages: "Optional[Set[str]]" = None, read_set: "Optional[Set[str]]" = None, @@ -6403,10 +6441,10 @@ def generate_planned_stages( read_set.update(read_models(statement)) spine_sizes = [p.spine.size for p in planned_queries if p.spine is not None] if spine_sizes: - get_dialect(dialect).attach_sequence_setting(statement, size=max(spine_sizes)) + _as_dialect(dialect).attach_sequence_setting(statement, size=max(spine_sizes)) # Semi-join pushdown emits a correlated EXISTS; attached unconditionally so SQL never depends on server state. if any(plan_has_semi_join_filters(p) for p in emitted_plans(planned_queries, kept_stages=kept)): - get_dialect(dialect).attach_correlated_setting(statement) + _as_dialect(dialect).attach_correlated_setting(statement) # Length-fit over-limit projection aliases from the plan-derived canonical keys, not parsed off the SQL — # BigQuery can't parse a backticked dotted alias. return _finish_statement( @@ -6418,7 +6456,7 @@ def generate_planned_stages( def _build_planned_stages_ast( - planned_queries, *, bundle, dialect: str, kept_stages: "Optional[Set[str]]" = None, + planned_queries, *, bundle, dialect: "str | SqlDialect", kept_stages: "Optional[Set[str]]" = None, ) -> exp.Select: """Compose a multi-stage DAG (``plan_stages`` output) as one statement AST.""" if not planned_queries: @@ -6485,7 +6523,7 @@ def _build_planned_stages_ast( source_relation=planned.source_relation, inner=body, expected_columns=[c.name for c in planned.stage_schema.columns], - dialect=dialect, + dialect=_as_dialect(dialect).sqlglot_name, ), depends_on=_merged_deps( [h.name for h in hoisted], @@ -6518,14 +6556,14 @@ def _build_planned_stages_ast( def _fit_overlimit_identifiers( - tree, *, dialect: str, exempt: AbstractSet[str], stage_columns: AbstractSet[str], + tree, *, dialect: "str | SqlDialect", exempt: AbstractSet[str], stage_columns: AbstractSet[str], ) -> None: """Length-fit over-limit SLayer-minted identifiers in place the way the emission pass fits quoted ones: every unquoted one (stage identities, stage column references) and the stage column names wherever they occur, so a definition and its references agree; user-authored ``exempt`` names pass unless they name a stage column.""" - d = get_dialect(dialect) + d = _as_dialect(dialect) if d.max_identifier_bytes is None: return for ident in tree.find_all(exp.Identifier): diff --git a/slayer/sql/render/order_terms.py b/slayer/sql/render/order_terms.py index 7fbdb826..ff5d1c06 100644 --- a/slayer/sql/render/order_terms.py +++ b/slayer/sql/render/order_terms.py @@ -29,10 +29,11 @@ from __future__ import annotations from enum import Enum -from typing import Dict, Literal +from typing import Dict, List, Literal from pydantic import BaseModel, ConfigDict, Field from sqlglot import exp +from sqlglot.expressions.core import Expression from slayer.ir.planned import SlotId from slayer.sql.dialects.base import SqlDialect @@ -106,6 +107,8 @@ class OrderEnv(BaseModel): ranked_cte: Dict[str, exp.Expression] = Field(default_factory=dict) transform_step: Dict[str, exp.Expression] = Field(default_factory=dict) outer_composite: Dict[str, exp.Expression] = Field(default_factory=dict) + #: The ordered SELECT's own projection (alias → expression), where the ORDER BY attaches to it. + projection: Dict[str, Expression] = Field(default_factory=dict) #: Owns the null-ordering spelling (P-H). Required so no render site can #: silently fall back to a default dialect's null ordering. dialect: SqlDialect @@ -139,8 +142,8 @@ def uniform( ) -def resolve_order_term(*, entry: ScopedOrder, env: OrderEnv) -> exp.Ordered: - """One scoped order entry → one ``ORDER BY`` term. +def resolve_order_term(*, entry: ScopedOrder, env: OrderEnv) -> List[exp.Ordered]: + """One scoped order entry → its ``ORDER BY`` terms. Raises :class:`OrderSlotNotMaterialisedError` when the entry's scope did not materialise the slot. Returning an unsorted result instead is the one @@ -154,6 +157,8 @@ def resolve_order_term(*, entry: ScopedOrder, env: OrderEnv) -> exp.Ordered: f"{entry.scope.value} scope did not materialise " f"(it carries {sorted(refs)}).", ) - return env.dialect.build_ordered( + resolved = env.projection.get(ref.name) if isinstance(ref, exp.Column) and not ref.table else None + return env.dialect.build_order_terms( ref.copy(), descending=entry.direction == "desc", nulls=entry.nulls, + resolved=None if resolved is None else resolved.copy(), ) diff --git a/slayer/sql/render/ranked.py b/slayer/sql/render/ranked.py index 01e45302..5feec749 100644 --- a/slayer/sql/render/ranked.py +++ b/slayer/sql/render/ranked.py @@ -82,6 +82,7 @@ def build_rank_column( *, partition_by: Sequence[Expression], ranking_time: exp.Ordered, + leading: Sequence[exp.Ordered] = (), ) -> Expression: """``ROW_NUMBER() OVER (PARTITION BY ORDER BY