diff --git a/.basedpyright/baseline.json b/.basedpyright/baseline.json index 0955fbbaa..c9de491d0 100644 --- a/.basedpyright/baseline.json +++ b/.basedpyright/baseline.json @@ -18,96 +18,6 @@ } } ], - "./slayer/core/formula.py": [ - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 21, - "endColumn": 23, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 20, - "endColumn": 64, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 21, - "endColumn": 66, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 70, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 20, - "endColumn": 64, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 13, - "endColumn": 24, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 13, - "endColumn": 19, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 13, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 66, - "lineCount": 1 - } - }, - { - "code": "reportOperatorIssue", - "range": { - "startColumn": 15, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportOptionalOperand", - "range": { - "startColumn": 16, - "endColumn": 34, - "lineCount": 1 - } - } - ], "./slayer/demo/jaffle_shop.py": [ { "code": "reportOptionalSubscript", @@ -188,16 +98,6 @@ } } ], - "./slayer/engine/syntax.py": [ - { - "code": "reportArgumentType", - "range": { - "startColumn": 8, - "endColumn": 30, - "lineCount": 1 - } - } - ], "./slayer/facade/catalog_sql.py": [ { "code": "reportPrivateImportUsage", @@ -1815,14 +1715,6 @@ "lineCount": 1 } }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 26, - "endColumn": 40, - "lineCount": 1 - } - }, { "code": "reportArgumentType", "range": { @@ -10578,32 +10470,6 @@ } } ], - "./tests/test_dev1576_heals.py": [ - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 22, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 22, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 22, - "endColumn": 38, - "lineCount": 1 - } - } - ], "./tests/test_dev1645_invalid_postgres_sql.py": [ { "code": "reportArgumentType", @@ -11363,22 +11229,6 @@ "lineCount": 1 } }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 32, - "endColumn": 55, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 29, - "endColumn": 40, - "lineCount": 1 - } - }, { "code": "reportArgumentType", "range": { @@ -11963,14 +11813,6 @@ "lineCount": 1 } }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 29, - "endColumn": 40, - "lineCount": 1 - } - }, { "code": "reportArgumentType", "range": { @@ -15239,78 +15081,6 @@ "lineCount": 1 } }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 59, - "endColumn": 63, - "lineCount": 1 - } - }, { "code": "reportArgumentType", "range": { @@ -22221,14 +21991,6 @@ "lineCount": 1 } }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 28, - "lineCount": 1 - } - }, { "code": "reportArgumentType", "range": { @@ -22317,22 +22079,6 @@ "lineCount": 1 } }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 24, - "endColumn": 35, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 18, - "endColumn": 23, - "lineCount": 1 - } - }, { "code": "reportCallIssue", "range": { @@ -32177,22 +31923,6 @@ "lineCount": 1 } }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 32, - "endColumn": 56, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 29, - "endColumn": 45, - "lineCount": 1 - } - }, { "code": "reportArgumentType", "range": { @@ -34058,96 +33788,6 @@ } } ], - "./tests/test_syntax.py": [ - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - } - ], "./tests/test_time_dimensions_filters.py": [ { "code": "reportArgumentType", diff --git a/architecture/core.arc42.md b/architecture/core.arc42.md index 3e807d57c..f47c6d1c7 100644 --- a/architecture/core.arc42.md +++ b/architecture/core.arc42.md @@ -6,7 +6,7 @@ family, and the typed error/warning vocabulary. It is the bottom layer: it is meant to import no other SLayer node; the remaining crossings are the grandfathered child-level `#legacy` doors (`core.query`/`core.models` → -`engine.syntax`, `sql.*`, `storage.migrations`), slated to die (system +`sql.*`, `storage.migrations`), slated to die (system principle 2). ## 2. Building blocks @@ -20,6 +20,7 @@ flowchart TD subgraph core["Core domain models"] core__query["Query"] core__models["Models"] + core__syntax["Syntax"] end subgraph sql["SQL generation"] sql__dialects["Dialects"] @@ -27,7 +28,8 @@ flowchart TD sql__window_detect["Window detect"] end subgraph engine["Query engine"] - engine__syntax["Syntax"] + engine__binding["Binding"] + engine__bind_inputs["Bind inputs"] end ir["Intermediate representation"] subgraph storage["Storage backends"] @@ -41,7 +43,6 @@ flowchart TD usage_reporting__features["Features"] end surfaces("User-facing surfaces") - core__query -.-> engine__syntax core__models -.-> sql__dialects core__models -.-> sql__sql_predicate core__models -.-> sql__window_detect @@ -50,6 +51,9 @@ flowchart TD core__query -.-> storage__migrations core__models -.-> core__query core__query --> core__models + core__query --> core__syntax + engine__binding --> core__syntax + engine__bind_inputs --> core__syntax engine --> core importers --> core ir --> core @@ -61,14 +65,14 @@ flowchart TD surfaces --> core usage_reporting__features --> core classDef leaf fill:none; - class core__query,core__models,sql__dialects,sql__sql_predicate,sql__window_detect,engine__syntax,ir,storage__migrations,importers,search,memories,protocols,usage_reporting__features,surfaces leaf; + class core__query,core__models,core__syntax,sql__dialects,sql__sql_predicate,sql__window_detect,engine__binding,engine__bind_inputs,ir,storage__migrations,importers,search,memories,protocols,usage_reporting__features,surfaces leaf; ``` *Dashed arrows: legacy edges slated to die.* Identity: `keys.py`; scopes: `scope.py`; errors/warnings: `errors.py`, `warnings.py`; Pydantic domain -models alongside. +models alongside; the Mode B grammar: `syntax.py`. ## 3. Principles diff --git a/architecture/engine.arc42.md b/architecture/engine.arc42.md index 27402334e..531d2ab8c 100644 --- a/architecture/engine.arc42.md +++ b/architecture/engine.arc42.md @@ -18,6 +18,7 @@ flowchart TD subgraph core["Core domain models"] core__query["Query"] core__models["Models"] + core__syntax["Syntax"] end subgraph sql["SQL generation"] sql__render["Render"] @@ -26,7 +27,6 @@ flowchart TD sql__window_detect["Window detect"] end subgraph engine["Query engine"] - engine__syntax["Syntax"] engine__binding["Binding"] engine__bind_inputs["Bind inputs"] engine__elaborate_env["Checker"] @@ -38,7 +38,6 @@ flowchart TD subgraph storage["Storage backends"] storage__migrations["Migrations"] end - core__query -.-> engine__syntax core__models -.-> sql__dialects core__models -.-> sql__sql_predicate core__models -.-> sql__window_detect @@ -47,12 +46,13 @@ flowchart TD core__query -.-> storage__migrations core__models -.-> core__query core__query --> core__models + core__query --> core__syntax sql__render --> sql__dialects sql__sql_predicate --> sql__window_detect - engine__binding --> engine__syntax + engine__binding --> core__syntax engine__bind_inputs --> engine__binding engine__bind_inputs --> engine__elaborate_env - engine__bind_inputs --> engine__syntax + engine__bind_inputs --> core__syntax engine__compile --> engine__elaborate engine__compile --> engine__elaborate_env engine__elaborate --> engine__bind_inputs @@ -70,7 +70,7 @@ flowchart TD storage --> engine storage --> sql classDef leaf fill:none; - class core__query,core__models,sql__render,sql__dialects,sql__sql_predicate,sql__window_detect,engine__syntax,engine__binding,engine__bind_inputs,engine__elaborate_env,engine__elaborate,engine__compile,engine__plan,ir,storage__migrations leaf; + class core__query,core__models,core__syntax,sql__render,sql__dialects,sql__sql_predicate,sql__window_detect,engine__binding,engine__bind_inputs,engine__elaborate_env,engine__elaborate,engine__compile,engine__plan,ir,storage__migrations leaf; ``` *Dashed arrows: legacy edges slated to die.* @@ -83,7 +83,6 @@ only permitted arrows among them (DEV-1897 child-level model-truth): flowchart TD %% engine_focus: Engine seam subgraph engine["Query engine"] - engine__syntax["Syntax"] engine__binding["Binding"] engine__bind_inputs["Bind inputs"] engine__elaborate_env["Checker"] @@ -91,10 +90,8 @@ flowchart TD engine__compile["Compile"] engine__plan["Plan"] end - engine__binding --> engine__syntax engine__bind_inputs --> engine__binding engine__bind_inputs --> engine__elaborate_env - engine__bind_inputs --> engine__syntax engine__compile --> engine__elaborate engine__compile --> engine__elaborate_env engine__elaborate --> engine__bind_inputs @@ -102,12 +99,12 @@ flowchart TD engine__plan --> engine__compile engine__plan --> engine__elaborate classDef leaf fill:none; - class engine__syntax,engine__binding,engine__bind_inputs,engine__elaborate_env,engine__elaborate,engine__compile,engine__plan leaf; + class engine__binding,engine__bind_inputs,engine__elaborate_env,engine__elaborate,engine__compile,engine__plan leaf; ``` Stages: -`normalization.py` → `syntax.py` (parse) → `binding.py` + `bind_inputs.py` +`normalization.py` → `core/syntax.py` (parse) → `binding.py` + `bind_inputs.py` (bind) → `elaborate.py` / `elaborate_env.py` (type) → `compile/` (plan) → `slayer/ir/planned.py` (the typed hand-off to `sql`), fed by `bundle_builder.py` (builds the `ir` source bundle); `plan.py` composes @@ -155,11 +152,11 @@ elaborate→compile and `query_engine.py` orchestrates. unchanged; normalization applies to queries at execute time only. The one exception: a saved query stores each stage's inferred population as its `source_model`. [review] -9. **The term-interface boundary**: `syntax`, `binding`, `bind_inputs`, +9. **The term-interface boundary**: `binding`, `bind_inputs`, `elaborate`, `elaborate_env` (THE checker), `compile` and `plan` are declared children with only the modeled arrows among them — the compiler - consults the checker, never syntax or binding; every user-facing algebra - type error raises in the checker, as a `QueryTypeError`. + consults the checker, never `core.syntax` or binding; every user-facing + algebra type error raises in the checker, as a `QueryTypeError`. [enforced: arch_check:model-truth] [enforced: test:tests/test_dev1871_raise_parity.py] 10. **Dependencies are closures**: every predicate that classifies a column diff --git a/architecture/index.yaml b/architecture/index.yaml index abdbefc63..2650b5bc5 100644 --- a/architecture/index.yaml +++ b/architecture/index.yaml @@ -6,7 +6,7 @@ python: # Count of `#legacy` arrows in the model; arch_check fails if the model's count # differs. Only ever lower this — every legacy arrow is a door slated to die. -legacy_arrows: {baseline: 8} +legacy_arrows: {baseline: 7} # Fail-closed deferral-site count (tests/test_law_guard_ratchet.py). Only ever lower. x-guards: {baseline: 0} diff --git a/architecture/model.c4 b/architecture/model.c4 index cada409ea..96f9e1795 100644 --- a/architecture/model.c4 +++ b/architecture/model.c4 @@ -23,6 +23,7 @@ model { } query = node 'Query' models = node 'Models' + syntax = node 'Syntax' } sql = node 'SQL generation' { metadata { @@ -40,7 +41,6 @@ model { package 'slayer.engine' arc42 'architecture/engine.arc42.md' } - syntax = node 'Syntax' binding = node 'Binding' bind_inputs = node 'Bind inputs' elaborate_env = node 'Checker' @@ -98,7 +98,6 @@ model { } } - core.query -> engine.syntax #legacy core.models -> sql.dialects #legacy core.models -> sql.sql_predicate #legacy core.models -> sql.window_detect #legacy @@ -107,12 +106,13 @@ model { core.query -> storage.migrations #legacy core.models -> core.query #legacy core.query -> core.models + core.query -> core.syntax sql.render -> sql.dialects sql.sql_predicate -> sql.window_detect - engine.binding -> engine.syntax + engine.binding -> core.syntax engine.bind_inputs -> engine.binding engine.bind_inputs -> engine.elaborate_env - engine.bind_inputs -> engine.syntax + engine.bind_inputs -> core.syntax engine.compile -> engine.elaborate engine.compile -> engine.elaborate_env engine.elaborate -> engine.bind_inputs diff --git a/architecture/system.arc42.md b/architecture/system.arc42.md index 50fa687c2..eb83b460d 100644 --- a/architecture/system.arc42.md +++ b/architecture/system.arc42.md @@ -28,7 +28,6 @@ flowchart TD protocols("BI wire protocols") usage_reporting["Usage reporting"] surfaces("User-facing surfaces") - core -.-> engine core -.-> sql core -.-> storage engine --> core @@ -91,7 +90,7 @@ All code, old and new, MUST obey these. [enforced: arch_check:model-truth] 2. **`core` imports no other SLayer node.** The remaining crossings are the grandfathered child-level `#legacy` doors (`core.query`/`core.models` → - `engine.syntax`, `sql.*`, `storage.migrations`), slated to die. + `sql.*`, `storage.migrations`), slated to die. [enforced: arch_check:model-truth] 3. **Every top-level `slayer.*` package/module belongs to exactly one node.** A new top-level package must be claimed in a node's model metadata in the same change. @@ -125,8 +124,9 @@ All code, old and new, MUST obey these. [enforced: test:tests/test_dev1743_resolution.py] [enforced: test:tests/test_dev1954_canonical_paths.py] 10. **Two expression layers**: Mode A free SQL (`Column.sql`, model `filters`) - vs Mode B Python-AST DSL (formulas, query fields, scalar allowlist only) — - one canonical scalar allowlist, extended never forked. [review] + vs Mode B, a SQL-compatible grammar (formulas, query fields) — one + canonical function library, extended never forked. [review] Every Mode B + construct means the same on every dialect. [target: DEV-2120] 11. **Versioned persistence**: models/queries/datasource configs carry `version`; migrations run automatically on load. [review] 12. **Pydantic v2 for all models; never dataclasses.** [review] diff --git a/docs/concepts/formulas.md b/docs/concepts/formulas.md index d3419e16a..523bee8c7 100644 --- a/docs/concepts/formulas.md +++ b/docs/concepts/formulas.md @@ -77,6 +77,10 @@ new derived quantity owned by the query author. Global validation still applies: the aggregation name must be known, and numeric-only aggregations reject a confidently non-numeric expression (`sum(lower(name))`). +### Filtered aggregation (`FILTER (WHERE …)`) + +Any aggregation takes a SQL `FILTER (WHERE c)` clause with a row-level condition `c` and aggregates only the rows passing it, every row-level input included (`count(*) FILTER (WHERE status = 'paid')`, `last(amount) FILTER (WHERE amount > 0)` picks the newest passing row). + ### Windowed aggregations Every aggregation — built-in or custom — accepts an optional `window` parameter for @@ -265,7 +269,8 @@ When a measure is renamed via `name`, query filters and ORDER BY entries in the | `-` | `"sum(revenue) - sum(cost)"` | `SUM(revenue) - SUM(cost)` | | `*` | `"avg(price) * sum(quantity)"` | `AVG(price) * SUM(quantity)` | | `/` | `"sum(revenue) / count(*)"` | `SUM(revenue) / NULLIF(COUNT(*), 0)` | -| `**` | `"sum(value) ** 2"` | `SUM(value) ** 2` | +| `**` | `"sum(value) ** 2"` | `POWER(SUM(value), 2)` | +| `%` | `"sum(quantity) % 12"` | `SUM(quantity) % 12` | Parentheses work as expected: `"(sum(revenue) - sum(cost)) / count(*)"`. Division by zero is NULL on every backend. @@ -449,7 +454,7 @@ Multiple partition columns: `partition_by=[region, channel]`. Cross-model dotted The rank family's `partition_by=` may name a computed dimension, as an aggregation's can (one that itself wraps an aggregation is for now accepted only in filters and order). Each key must be a member of the transform's operand grain — the union of its inner aggregates' grains (an ungrained inner contributes the query's dimensions) — so `rank(sum(revenue, partition_by=[city, region]), partition_by=product, direction='desc')` is an error naming the remedy. -> **Note:** SLayer's formula parser is Python-AST-based and rejects raw `OVER (...)` SQL in `ModelMeasure.formula` and filter strings. Use the rank-family transforms (`rank`, `percent_rank`, `dense_rank`, `ntile`) for ranking instead of `row_number() over (...) <= N`. If you need a non-standard window expression, define it on a `Column.sql` (e.g., `{"name": "rn", "sql": "row_number() over (order by mass desc)", "type": "NUMBER"}`) and filter on the column — SLayer auto-promotes the predicate to a post-aggregation outer `WHERE`. +> **Note:** SLayer's formula grammar rejects raw `OVER (...)` SQL in `ModelMeasure.formula` and filter strings. Use the rank-family transforms (`rank`, `percent_rank`, `dense_rank`, `ntile`) for ranking instead of `row_number() over (...) <= N`. If you need a non-standard window expression, define it on a `Column.sql` (e.g., `{"name": "rn", "sql": "row_number() over (order by mass desc)", "type": "NUMBER"}`) and filter on the column — SLayer auto-promotes the predicate to a post-aggregation outer `WHERE`. ### First and Last Functions @@ -543,13 +548,4 @@ Any formula, filter, or field expression can branch with SQL `CASE`: ## Parsing Internals -Both field and filter formulas are parsed by `slayer/core/formula.py` using Python's `ast` module. - -**Field formulas** are classified into: - -- **AggregatedMeasureRef** — an aggregation over a measure (`"sum(revenue)"`, `"count(*)"`) -- **ArithmeticField** — arithmetic on aggregated measures (`"sum(revenue) / count(*)"`) -- **TransformField** — function call, possibly nested (`"cumsum(sum(revenue))"`) -- **MixedArithmeticField** — arithmetic containing function calls. Covers both transform calls (`"cumsum(sum(revenue)) / count(*)"`) and non-transform SQL function calls wrapping aggregated refs, e.g. `"count(*) / nullif(max(revenue), 0)"` or `"coalesce(sum(revenue), 0) + avg(amount)"`. Aggregated refs nested inside non-transform calls are resolved as their own measure aliases; the call passes through to emitted SQL unchanged. - -The query engine binds and expands field formulas into ordered planned stages, and the SQL generator translates them into stacked CTEs. +Every Mode-B field is parsed by one SQL-compatible grammar (`slayer/core/syntax.py`) into an expression tree that the query engine binds, checks and expands into planned stages, and the SQL generator translates into stacked CTEs. diff --git a/docs/concepts/queries.md b/docs/concepts/queries.md index 05fb1d9ce..f24c575f7 100644 --- a/docs/concepts/queries.md +++ b/docs/concepts/queries.md @@ -196,6 +196,7 @@ A downstream stage of a multi-stage query may declare a time dimension on any DA ## OrderItem A sort specification: `column` names a dimension (`status`), a declared measure's short alias (`revenue_sum`), or a formula (`count(*)`); `direction` is `asc` or `desc`. +An optional `nulls` (`first` or `last`) places NULL values first or last on every dialect; without it the database's default placement applies. ```json {"column": "count(*)", "direction": "desc"} @@ -225,7 +226,7 @@ What each shape of an *undeclared* order target does: | An inline **composite** (`sum(revenue) / sum(cnt)`, `abs(sum(amount))`, `change(sum(amount)) / 2`) | Computed hidden, sorted on, stripped. | | A **windowed** aggregate (`sum(amount, window='90d')`), alone or inside a composite | Computed hidden in its own rolling-window CTE, sorted on, stripped. | | A raw row column, in a **raw-rows** query (`distinct_dimension_values: false`, no measures) | Sorted on directly (`ORDER BY orders.created_at`). Applies to a **joined** column (`customers.regions.name`) and to a derived column whose `sql` reaches through a join — the join is pulled in for the sort. | -| A raw row column, in an **aggregated / dedup** query | Sorted on **per group**: ASC by each group's minimum, DESC by each group's maximum. The wrap is implicit — you do not write `min(created_at)`. | +| A raw row column or row-level expression (`amount - fee`), in an **aggregated / dedup** query | Sorted on **per group**: ASC by each group's minimum, DESC by each group's maximum. The wrap is implicit — you do not write `min(created_at)`. | | A **joined** row column (`customers.regions.name`) in an aggregated query | Same per-group wrap, computed in a CTE rooted at the source model with the join pulled inside, so each group gets its own extreme rather than one global value. | Ordering by an undeclared row column in a grouped query is **not** the same as @@ -238,17 +239,14 @@ NULLs sort **last** in both directions, on every database, so the same query returns the same row order regardless of backend. (SQL Server is the one exception: its native ordering is used because the portable emulation makes the statement fail there.) +An explicit `nulls` (`first` or `last`) places NULLs there on every database, SQL Server included. An order target that names nothing SLayer can resolve is an error, never a silently unsorted result. Transform and composite order targets accept the full formula syntax, so `{"column": "sum(revenue) / sum(cnt)"}` and `{"column": "change(sum(revenue))"}` both -work without declaring a measure. One limit: the operands must be written as -formulas, not as the *names* of measures you declared in the same query — -`{"column": "rev / cnt"}` is rejected at validation, because referencing a -declared measure by its alias inside an expression is not supported anywhere in -SLayer. Write `{"column": "sum(revenue) / sum(cnt)"}` instead. +work without declaring a measure, and their operands may also be the names of measures declared in the same query (`{"column": "rev / cnt"}`), as in filters. Windowed measures work in declared composites and transforms too — see [formulas](formulas.md#windowed-aggregations). @@ -319,6 +317,9 @@ Filter formulas define conditions for the query. They go in the `filters` parame | `IS NOT NULL` | `"discount IS NOT NULL"` | | `like` | `"name like '%acme%'"` | | `not like` | `"name not like '%test%'"` | +| `ilike` | `"name ilike '%acme%'"` | +| `between` | `"amount between 100 and 1000"` | +| `is distinct from` | `"discount is distinct from 0"` | The right-hand side of `in` / `not in` must be a non-empty tuple of literal values (strings, numbers, or booleans) — references and expressions on the @@ -464,7 +465,7 @@ The same placeholders work in measure formulas, computed-dimension expressions, This produces the filter `status = 'completed' AND amount > 100`. - Variable names must be alphanumeric + underscore (`[a-zA-Z_][a-zA-Z0-9_]*`) -- Values must be strings, numbers, or **lists**. **You write the surrounding quotes** in the template (`status = '{status}'`); the string value is then automatically escaped so an embedded quote (e.g. `O'Brien`), backslash, or control character (newline, tab) can't break out of that literal or the filter parser (DEV-1727). Numbers (including booleans) are inserted verbatim; non-finite floats (`nan`/`inf`) are rejected. +- Values must be strings, numbers, or **lists**, and every value binds as a literal: **you write the surrounding quotes** for a string (`status = '{status}'`, an embedded `'` is doubled), an unquoted placeholder takes a number (a numeric string included), and a placeholder in a name position (`sum({col})`, `customers.{c}`, `{f}(x)`, `partition_by={x}`) is rejected, so no value can change the expression's structure; non-finite floats (`nan`/`inf`) are rejected. - **List values render an `IN`-list.** A list variable powers an `in` / `not in` filter — write the parentheses and omit per-element quotes (each string element is auto-quoted): ```json @@ -476,13 +477,13 @@ This produces the filter `status = 'completed' AND amount > 100`. } ``` - An empty list raises (`IN ()` is invalid SQL). Under the hood a list renders as a Python tuple (with a trailing comma, e.g. `('US',)`) so that even a single-element list is treated as membership rather than a bare value — you may see that trailing comma in `show_sql` output; it is not a typo. + An empty list raises (`IN ()` is invalid SQL). - `{{` and `}}` produce literal `{` and `}` - Undefined variables raise an error - The same `{variable}` mechanism also works in the **raw-SQL (Mode A) surfaces** of a model — see [Variables in model SQL](models.md#variables-in-model-sql). !!! note "Trusted input" - Substituted values are treated as trusted. The automatic escaping keeps a string value inside the quoted literal you wrote, but a `{var}` placed in an **unquoted** position is still raw SQL/expression substitution — do not feed untrusted end-user input through `variables`. + In query fields a value is always bound as a literal, but in a model's raw-SQL (Mode A) surfaces it is substituted as text, so do not feed untrusted end-user input through `variables` there. #### Variables passed as a runtime kwarg diff --git a/docs/concepts/references.md b/docs/concepts/references.md index 2a8e4e7f1..321798f4b 100644 --- a/docs/concepts/references.md +++ b/docs/concepts/references.md @@ -7,7 +7,7 @@ SLayer has two distinct expression layers and the rules for what each one accept | Mode | Fields | Parser | Accepts | Rejects | |---|---|---|---|---| | **A — SQL** | `Column.sql`, `Column.filter`, each entry of `SlayerModel.filters` | sqlglot | Any valid SQL expression for the underlying dialect — function calls (`json_extract`, `coalesce`, `nullif`, `lower`, `length`, …), arithmetic, `CASE WHEN`, string literals, comparison and boolean operators in SQL spelling (`=`, `<>`, `IS NULL`, `AND`, `OR`, `NOT`, `IN`, `LIKE`). Bare names and dotted join paths (`customers.regions.name`). | Aggregations (`sum(revenue)`); SLayer transform calls (`cumsum`, `change`, `rank`, …); references to `ModelMeasure` formulas; raw `OVER (...)` window functions inside `Column.filter` / `SlayerModel.filters` (allowed only in `Column.sql`); the legacy `__`-delimited split-alias qualifier (`customers__regions.name`) — now a hard error (`LegacyDunderAliasError`). | -| **B — DSL** | `ModelMeasure.formula`, `SlayerQuery.measures`, `SlayerQuery.filters`, `SlayerQuery.dimensions`, `SlayerQuery.time_dimensions`, `SlayerQuery.order`, `SlayerQuery.main_time_dimension` | Python AST formula parser | Bare names that resolve to a `Column` or `ModelMeasure` on the model; single-dot dotted paths through joins (`customers.regions.name`, `sum(customers.revenue)`); aggregations (`sum(revenue)`, `count(*)`, `percentile(price, p=0.9)`, parametric forms), including same-model expression sources (`sum(amount - cost)`); transform calls (`cumsum(sum(revenue))`, `rank(sum(revenue), partition_by=region, direction='desc')`); arithmetic / boolean / comparison operators; the SQL `\|\|` concat operator (folded into `concat(...)`); pattern matching via the `like(value, pattern)` scalar (emits the SQL `LIKE` operator — wrap in `not (...)` for `NOT LIKE`); a closed allowlist of scalar functions (matched case-insensitively) — null handling (`nullif`, `coalesce`, `ifnull`), math (`ln`, `log10`, `log2`, `log`, `exp`, `sqrt`, `pow`, `power`, `abs`, `floor`, `ceil`, `ceiling`, `round`, `sign`, `trunc`, `mod`), scalar min/max (`greatest`, `least`), string hygiene (`lower`, `upper`, `trim`, `ltrim`, `rtrim`, `replace`, `substr`, `substring`, `instr`, `length`, `concat`), `like`, and [date functions](#date-and-time-functions) (`date_part`, `date_diff`, `date_add`, `current_date`, `now`), each with a declared argument count that is validated (`coalesce`, `concat`, `greatest` and `least` are variadic; `trunc` takes exactly one argument); `{variable}` placeholders (filters only). | identifiers using the reserved `__slayer_` prefix; raw SQL function calls outside that allowlist (`json_extract`, `date_trunc`, …), and any allowlisted call with the wrong number of arguments; raw `OVER (...)`; bare names that don't resolve to a Column / ModelMeasure / custom aggregation / query alias; `NULL` inside an `in` / `not in` list (use `is null` / `is not null` instead — see below). | +| **B — DSL** | `ModelMeasure.formula`, `SlayerQuery.measures`, `SlayerQuery.filters`, `SlayerQuery.dimensions`, `SlayerQuery.time_dimensions`, `SlayerQuery.order`, `SlayerQuery.main_time_dimension` | SQL-compatible grammar | Bare names that resolve to a `Column` or `ModelMeasure` on the model; single-dot dotted paths through joins (`customers.regions.name`, `sum(customers.revenue)`); aggregations (`sum(revenue)`, `count(*)`, `percentile(price, p=0.9)`, parametric forms), including same-model expression sources (`sum(amount - cost)`); transform calls (`cumsum(sum(revenue))`, `rank(sum(revenue), partition_by=region, direction='desc')`); arithmetic / boolean / comparison operators; the SQL `\|\|` concat operator (folded into `concat(...)`); SQL forms — `CASE`, `CAST` / `TRY_CAST` / `::`, `BETWEEN`, `IS [NOT] DISTINCT FROM`, `[NOT] LIKE` / `ILIKE`, `EXTRACT`, `POSITION`, `SUBSTRING … FROM … FOR`, `INTERVAL` / `DATE` / `TIMESTAMP` literals, `CURRENT_DATE`, `FILTER (WHERE …)`, plus the `like(value, pattern)` scalar; a closed allowlist of scalar functions (matched case-insensitively) — null handling (`nullif`, `coalesce`, `ifnull`), math (`ln`, `log10`, `log2`, `log`, `exp`, `sqrt`, `pow`, `power`, `abs`, `floor`, `ceil`, `ceiling`, `round`, `sign`, `trunc`, `mod`), scalar min/max (`greatest`, `least`), string hygiene (`lower`, `upper`, `trim`, `ltrim`, `rtrim`, `replace`, `substr`, `substring`, `instr`, `length`, `concat`), `like`, and [date functions](#date-and-time-functions) (`date_part`, `date_diff`, `date_add`, `current_date`, `now`), each with a declared argument count that is validated (`coalesce`, `concat`, `greatest` and `least` are variadic; `trunc` takes exactly one argument); `{variable}` placeholders supplying literal values. | identifiers using the reserved `__slayer_` prefix; raw SQL function calls outside that allowlist (`json_extract`, `date_trunc`, …), and any allowlisted call with the wrong number of arguments; raw `OVER (...)`; bare names that don't resolve to a Column / ModelMeasure / custom aggregation / query alias; `NULL` inside an `in` / `not in` list (use `is null` / `is not null` instead — see below); `\|` (use `\|\|`), `//`, `SIMILAR TO`, `REGEXP` / `RLIKE`, `TIME '…'` literals. | ## Identifier resolution @@ -31,6 +31,9 @@ SLayer has two distinct expression layers and the rules for what each one accept * Aggregations: `()` (e.g. `sum(revenue)`), `count(*)`, `(, )` (e.g. `weighted_avg(price, weight=quantity)`), and `()` for cross-model aggregations — see [Aggregation syntax](#aggregation-syntax). * Transform calls wrap aggregated refs: `cumsum(sum(revenue))`, `rank(sum(revenue), partition_by=region, direction='desc')`, `change(sum(customers.revenue))`, etc. * A `__` token in a name is matched by **exact name**, not split into a join walk — write a single-dot DSL path (`customers.region`) for a join. Only the reserved `__slayer_` prefix is rejected. +* Strings are single-quoted with `''` for an embedded quote and a backslash is an ordinary character (`'O''Brien'`, `'C:\temp'`), while a double-quoted name is an identifier (`sum("order total")`, `customers."Region"`). +* `CAST(x AS type)` and `x::type` are the database's own cast of the type you write, while `TRY_CAST(x AS type)` yields NULL for an input it cannot convert on every dialect except Postgres older than 16 (or of unknown version), which gets a plain `CAST`. +* `INTERVAL '7 days'` (one unit) means `interval(7, 'day')`, `DATE '…'` / `TIMESTAMP '…'` are typed temporal literals, and `CURRENT_DATE` / `CURRENT_TIMESTAMP` mean `current_date()` / `now()`. ## Aggregation syntax diff --git a/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/customers.yaml b/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/customers.yaml index 5e034fc72..79f23b019 100644 --- a/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/customers.yaml +++ b/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/customers.yaml @@ -1,4 +1,4 @@ -version: 15 +version: 16 name: customers sql_table: customers source_kind: table diff --git a/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/items.yaml b/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/items.yaml index bc9b1582f..caa382ebf 100644 --- a/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/items.yaml +++ b/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/items.yaml @@ -1,4 +1,4 @@ -version: 15 +version: 16 name: items sql_table: items source_kind: table diff --git a/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/orders.yaml b/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/orders.yaml index 23075ddbb..d767bde11 100644 --- a/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/orders.yaml +++ b/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/orders.yaml @@ -1,4 +1,4 @@ -version: 15 +version: 16 name: orders sql_table: orders source_kind: table diff --git a/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/products.yaml b/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/products.yaml index 0e3d029d4..c542b1bef 100644 --- a/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/products.yaml +++ b/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/products.yaml @@ -1,4 +1,4 @@ -version: 15 +version: 16 name: products sql_table: products source_kind: table diff --git a/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/stores.yaml b/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/stores.yaml index bd76d34d6..a8faac726 100644 --- a/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/stores.yaml +++ b/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/stores.yaml @@ -1,4 +1,4 @@ -version: 15 +version: 16 name: stores sql_table: stores source_kind: table diff --git a/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/supplies.yaml b/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/supplies.yaml index ab9e509c4..bdaf14802 100644 --- a/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/supplies.yaml +++ b/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/supplies.yaml @@ -1,4 +1,4 @@ -version: 15 +version: 16 name: supplies sql_table: supplies source_kind: table diff --git a/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/tweets.yaml b/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/tweets.yaml index 381646531..e9ac223d7 100644 --- a/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/tweets.yaml +++ b/docs/examples/09_lightning_talk/slayer_models/models/jaffle_shop/tweets.yaml @@ -1,4 +1,4 @@ -version: 15 +version: 16 name: tweets sql_table: tweets source_kind: table diff --git a/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/.openspec.yaml b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/.openspec.yaml new file mode 100644 index 000000000..9076a6609 --- /dev/null +++ b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-10-20 diff --git a/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/design.md b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/design.md new file mode 100644 index 000000000..864b63e84 --- /dev/null +++ b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/design.md @@ -0,0 +1,65 @@ +## Context + +See proposal.md for motivation. Today `slayer/engine/syntax.py` runs about eight text pre-passes (colon, CASE, extract, count-distinct, star-args, LIKE, filter operators, `||`) and then `ast.parse`; filters alone go through `parse_filter_expr`. `slayer/core/formula.py` holds a second, older Python-AST parser (`parse_formula` / `FieldSpec`) used only by the cube, dbt and OSI importers (and, outside the repo, by Storyline). `{var}` substitution is textual (`escape="python"`), run before parsing. The IR already carries `OrderEntry.nulls`; nothing sets it. SQL generation already probes a cached per-engine `ServerProfile` (ClickHouse uses it to refuse correlated subqueries). + +This change is PR 1 of three: DEV-2119 (expressions in every column position) and DEV-2120 (portable function library, regex, JSON, coercion, collation, C9) stack on it. + +## Goals / Non-Goals + +**Goals:** one Mode-B grammar and one parse tree for every position and every consumer; SQL forms in every position; SQL string / identifier quoting; literal-only, structural `{var}` binding; casts; `NULLS FIRST/LAST`; stored text upgraded so no stored meaning changes. + +**Non-Goals:** new scalar functions, regex, JSON, TEXT↔number coercion, collation (DEV-2120); expressions in column-only positions (DEV-2119); removing query-time Mode A (DEV-2021); a `DECIMAL` member of `DataType`. + +## Decisions + +### D1. The parser lives in `core` and is pure syntax + +`slayer/engine/syntax.py` moves to `slayer/core/syntax.py` (declared child `core.syntax`) with `la:deterministic-refactor`. It imports only `slayer.core.*`: the transform tables (`ALL_TRANSFORMS`, `TIME_TRANSFORMS`, `TIMELESS_TRANSFORMS`) move from `core/formula.py` to `core/enums.py` next to `RANK_FAMILY_TRANSFORMS`. Engine P2 ("parsing is pure syntax") keeps governing it; `la-arch-check` model-truth enforces the boundary. Approved harness edits H1–H7: `model.c4` (`core.syntax` child; arrows `core.query → core.syntax`, `engine.binding → core.syntax`, `engine.bind_inputs → core.syntax`; legacy `core.query → engine.syntax` removed), `index.yaml` baseline 8 → 7, `system.arc42.md` P2 and P10, `core.arc42.md` §1 / §2, `engine.arc42.md` §2 / P9, regenerated diagrams. P10's "every Mode B construct means the same on every dialect" carries `[target: DEV-2120]`. + +### D2. pyparsing grammar, one entry point + +`Forward()` recursion; `infix_notation` ladder (loosest first) `OR`, `AND`, `NOT`, comparison / `IS` / `IN` / `BETWEEN` / `LIKE` / `ILIKE`, `||`, `+ -`, `* / %`, unary `-`, `**` (right-associative, right operand may be unary-negated: Python semantics, `-2**2 = -4`), `::`, postfix (call, `FILTER`, `.`); operator alternations as `Regex`; `lru_cache` on `parse_expr` (packrat measured slower). Keywords are case-insensitive, ASCII-exact (the DEV-1833 Unicode rules carry over) and reserved only in their slot, so `case`, `end`, `with`, `model`, `date`, … stay identifiers elsewhere; `CASE` is a conditional only when `WHEN` follows. CASE still lowers to `iif`, `EXTRACT` to `date_part`, `count(distinct x)` to `count_distinct`, `POSITION` / `SUBSTRING … FROM … FOR` to `instr` / `substr`, `BETWEEN` to its conjunction, `CURRENT_DATE` / `CURRENT_TIMESTAMP` to the clock calls, interval literals to `interval(n, unit)` — so every spelling collapses to one node and downstream stays spelling-insensitive (engine P2). `parse_filter_expr` is deleted; the kwarg-vs-comparison rule (an aggregation / transform / unknown call reads `name =` after `(` or `,` as a kwarg; a scalar call and everything else reads `=` as comparison) is the one rule for every position. Syntax errors carry the pyparsing location. + +Alternatives: keep `ast.parse` with more pre-passes (the status quo the issue retires); sqlglot as the parser (it would read Mode-B constructs — kwargs, colon, transforms, `{var}` — as SQL and need its own rewrite layer). + +### D3. New node and key kinds + +Casts are a new `Cast` parsed node and a new `CastKey(source, type_text, data_type, try_)` (system P13: a cast is not a scalar call). The key implements `children()` / `map_children()`, equality, phase (its source's), binding, checker typing (`data_type` per `queries/casts`; UNKNOWN for unmapped types) and render dispatch. `FILTER (WHERE c)` attaches to the `AggCall` node as an optional `filter` predicate and to its aggregate key; `IS [NOT] DISTINCT FROM` reuses the dialect's null-safe-equality builder; `x IS TRUE` lowers to `x IS NOT NULL AND x` (`IS FALSE`: `x IS NOT NULL AND NOT x`), and the `NOT` forms negate those, so NULL never propagates. + +### D4. Casts: the database's cast, plus a portable `TRY_CAST` + +`CAST` / `::` render `exp.Cast` with the author's `exp.DataType` (sqlglot-parsed, transpiled by the target dialect's generator). The plan-time "type the dialect cannot express" check is: sqlglot raises, or the round-trip of the type through the target dialect loses it (unsupported-type warning) → typed error. `TRY_CAST` lowers per dialect: native `TRY_CAST` (DuckDB, SQL Server, Snowflake, Trino), `SAFE_CAST` (BigQuery), `accurateCastOrNull` (ClickHouse); Postgres ≥ 16 `CASE WHEN pg_input_is_valid(CAST(x AS TEXT), '') THEN CAST(x AS ) END`; SQLite a registered `slayer_try_cast(value, target)` connection function; MySQL a numeric `REGEXP` guard (dates already yield NULL). The Postgres form needs the server version at render time: the Postgres dialect gains `server_profile_sql()` (`SELECT current_setting('server_version_num')`) and `parse_server_profile`; the engine fetches the cached profile before rendering a Postgres query that contains a `TRY_CAST` and passes it to the dialect; no profile (dry run without a connection, failed probe) → the plain `CAST`, which runs on every version. Guards cast the input to TEXT first so a non-text source never meets a regex. + +Rejected alternatives (user rulings): normalising `CAST` value semantics (rounding, boolean words, ISO-only, NULL-on-error) — `CAST` must be the author's database cast; a calendar-exact regex on old Postgres — performance on recent versions; refusing on old Postgres — a plain cast is strictly better. + +### D5. Strings, identifiers and `{var}` binding + +Strings are SQL: `'…'`, `''` escape, backslash literal. `"…"` is a quoted identifier everywhere (`""` escape), producing the same `Ref` / `DottedRef` as a bare name. A quoted name that fails to resolve gets the "strings use single quotes" hint in the binder's unresolved-reference error. + +Placeholders are masked before parsing with offset-preserving sentinels of the placeholder's exact length, so error positions and names stay true. The grammar admits a placeholder in three slots only: inside a string literal (a template literal of text and placeholder parts), a bare literal slot, and the whole `IN` list; anything else is a parse error naming "variables supply literal values". Values bind at the existing substitution points (`ir/variables.py`, `core/query.py`), now into the parse tree: a template literal renders the value's text; a bare slot takes a number or boolean (a numeric string reads as a number; a string value errors with the quote hint); the `IN` slot takes a list. Result names are computed from the text as written, before binding, for measures and computed dimensions alike (the `_template` / `template` plumbing stays, `ComputedDimension.substituted`'s text splice goes). `date_range` bounds are not Mode B: they keep their own literal-only substitution, then time-point validation. `escape="python"` for Mode B is deleted; Mode-A substitution is untouched. + +### D6. Stored-text migration + +New stored-only steps at SlayerModel 15 → 16, SlayerQuery 6 → 7 and Memory 4 → 5, registered in `slayer/storage/migrations.py` alongside `rank_direction_migration.py`, recursing like it does (`source_queries`, saved queries, inline `ModelExtension.measures`, memories' queries). The rewrite uses Python's `tokenize` over the legacy text: each STRING token is decoded with the old grammar's rules (`ast.literal_eval`) and re-emitted as a SQL single-quoted literal; an `OP` token `|` becomes `||`; everything else is copied byte-for-byte; untokenisable text is left unchanged. It runs before the existing load-path repairs (legacy time literals, aliases), which parse filter text with sqlglot and therefore already read `"…"` as an identifier. Mode-A fields are never visited. + +### D7. NULL ordering + +`OrderItem.nulls: Literal["first", "last"] | None` (validated; serialised; preserved by `_strip_order`, saved-query refinement, variable binding and every other path that rebuilds an order item) feeds `OrderEntry.nulls` at `engine/compile/stages.py`. `SqlDialect.build_ordered` returns a list of order terms; base returns one `exp.Ordered` with `nulls_first`; T-SQL emits a leading `CASE WHEN IS NULL THEN … END` key (NULLs-first or NULLs-last per the policy) followed by the expression, built over the resolved ordering expression rather than its bracketed alias, fixing the alias mis-resolution noted in `tsql.py`. + +### D8. One parser repo-wide (absorbs DEV-1831) + +`parse_formula`, the `FieldSpec` classes, `_rewrite_funcstyle_aggregations`, `_preprocess_agg_refs` and `_expand_named_measures` are deleted. The importers call `parse_expr`; the dbt importer's dangling-reference and cycle checks are re-implemented as a namespace-aware dependency walk over `ParsedExpr` (named measures and custom-aggregation names as the namespace, with cycle detection), shared by the three importers. The facade translator's and the importers' emitted Mode-B text is checked against the new grammar (single-quoted strings, no Python escapes, no `|`). + +## Risks / Trade-offs + +- [The parser swap touches every query] → SQL golden baselines must regenerate byte-identically for every pre-existing shape; the existing parser tests port to the new parser and stay green. +- [pyparsing performance on long formulas] → `lru_cache`; packrat off (measured slower); a benchmark test over the longest formulas in the suite and docs, compared with the old parser. +- [Live payloads with `"…"` strings or backslash escapes break] → typed errors with the single-quote hint; release note; the migration covers everything stored. +- [Storyline imports `parse_formula`] → accepted break, recorded on DEV-2034; the port is `parse_expr` plus `walk_parsed_refs`. +- [Postgres before 16: `TRY_CAST` raises on invalid input] → documented; a plain cast is the only form that runs there. +- [Render-time server profile on Postgres] → read from the existing per-engine cache; probed only when the query contains a `TRY_CAST`. +- [Retargeting tests that assert on deleted helpers] → listed individually in tasks.md, approved by the user. + +## Migration Plan + +Stored data upgrades on load (D6) and re-saves in the new spelling. No rollback step beyond the usual: an older SLayer opening a store re-saved by this version reads `'…'` strings correctly but fails on `"…"` identifiers, as for any forward-written store. diff --git a/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/proposal.md b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/proposal.md new file mode 100644 index 000000000..85820d3fc --- /dev/null +++ b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/proposal.md @@ -0,0 +1,36 @@ +## Why + +Mode B is parsed by Python's `ast` behind a stack of text pre-passes, so SQL spellings work only in some positions (`=`, `IS NULL`, `AND`, `LIKE` only in filters), `"abc"` is a string, and there is no `CAST`, `BETWEEN`, `ILIKE`, `FILTER (WHERE …)` or `NULLS FIRST/LAST`. The SLayer SQL programme (DEV-1768) needs Mode B to be a SQL-compatible grammar, and replacing query-time Mode-A columns (DEV-2021) needs Mode B to express what agents put there — `CAST` alone blocks a third of those expressions. + +## What Changes + +- Mode B gets its own grammar (pyparsing), replacing `ast.parse` and every text pre-pass; one entry point for every position. The parser moves to `slayer/core/syntax.py` (declared child `core.syntax`), killing the legacy arrow `core.query → engine.syntax`. +- SQL forms are valid in every position (measures, dimensions, order, filters), with every existing spelling kept: `=`, `<>`, `IS [NOT] NULL`, `[NOT] IN`, `AND`/`OR`/`NOT`, `[NOT] LIKE` over any operand, `[NOT] ILIKE`, `||`, `[NOT] BETWEEN`, `IS [NOT] DISTINCT FROM`, `IS [NOT] TRUE|FALSE`, `count(distinct x)`, `CAST(x AS t)` / `x::t`, `EXTRACT(f FROM x)`, `INTERVAL '3 day'` / `INTERVAL '3' DAY` / `INTERVAL 3 DAY`, `DATE '…'` / `TIMESTAMP '…'`, `CURRENT_DATE` / `CURRENT_TIMESTAMP`, `POSITION(a IN b)`, `SUBSTRING(x FROM a [FOR b])`, `agg(x) FILTER (WHERE c)`. +- `CAST` / `::` are the database's own cast of the author's type (spelling transpiled to the dialect, parameters kept); `TRY_CAST(x AS t)` is the portable cast that yields NULL for invalid input on every dialect (Postgres before 16: a plain cast). +- `ORDER BY … NULLS FIRST|LAST` through `OrderItem.nulls`, honoured on every dialect. +- **BREAKING** `"…"` is a quoted identifier everywhere; strings are `'…'` with `''` as the only escape (backslash is literal). Stored Mode-B text is migrated; a live double-quoted name that resolves to nothing errors with a "use single quotes for strings" hint. +- **BREAKING** `{var}` placeholders bind into the parse tree and supply literal values only: inside a string literal, in a bare number / boolean position, or as a whole `IN` list. Anything else is an error. +- **BREAKING** `|` as concatenation and `//` are rejected with hints (`||`; `div(a, b)` / `floor(a / b)`); stored `|` is migrated to `||`. `**` is lowered to `pow` (it crashed before). +- The importers' legacy parser (`parse_formula`, `FieldSpec`, the colon rewriters in `slayer/core/formula.py`) is deleted; cube / dbt / OSI importers use `parse_expr` with a namespace-aware dependency check (absorbs DEV-1831). **BREAKING** for Python callers of `parse_formula` (Storyline ports on its next bump). + +## Capabilities + +### New Capabilities +- `queries/expression-grammar`: the Mode-B grammar — SQL forms in every position, identifiers and quoting, string literals, precedence, legacy spellings and the forms rejected with hints. +- `queries/casts`: `CAST` / `::` as the database's cast of the author's type, the inferred result type, and the portable `TRY_CAST`. + +### Modified Capabilities +- `queries/expression-keywords`: CASE conditions accept SQL spellings natively; LIKE rewriting is replaced by the grammar's LIKE / ILIKE operators. +- `queries/variables`: placeholders bind as literals in three positions; the textual Mode-B escaping regime is gone. +- `aggregations/functional-form`: `agg(x) FILTER (WHERE c)`. +- `queries/date-functions`: SQL interval and typed-literal spellings; `CURRENT_DATE` / `CURRENT_TIMESTAMP`. +- `queries/positions`: order items carry an explicit NULL placement. +- `models/stored-upgrade`: stored Mode-B text is upgraded to the new string and identifier rules. + +## Impact + +- Code: `slayer/engine/syntax.py` → `slayer/core/syntax.py`; `slayer/core/query.py`, `slayer/core/keys.py` (cast key), `slayer/core/formula.py` (parser deleted; transform tables to `slayer/core/enums.py`), `slayer/engine/binding.py`, `bind_inputs.py`, `population.py`, `model_reads.py`, `schema_drift.py`, `unrelated.py`, `query_engine.py`, `slayer/memories/resolver.py`, `slayer/ir/variables.py`, `slayer/sql/render/*`, `slayer/sql/dialects/*` (casts, NULL ordering, server profile on Postgres), `slayer/sql/client.py`, `slayer/storage/migrations.py` + a new migration module, `slayer/dbt|cube|osi/converter.py`, `slayer/facade/translator.py` (emitted Mode-B text). +- Dependencies: `pyparsing` added. +- Architecture: `model.c4`, `index.yaml` (legacy baseline 8 → 7), `system.arc42.md` P2 / P10, `core.arc42.md`, `engine.arc42.md`. +- External: Storyline's `parse_formula` use (`storyline/api/adapters/models.py`); agents writing double-quoted strings or backslash escapes. +- Follow-ups stacked on this change: DEV-2119 (expressions in every column position), DEV-2120 (portable function library). diff --git a/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/aggregations/functional-form/spec.md b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/aggregations/functional-form/spec.md new file mode 100644 index 000000000..517afe4b4 --- /dev/null +++ b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/aggregations/functional-form/spec.md @@ -0,0 +1,28 @@ +## ADDED Requirements + +### Requirement: An aggregation accepts a FILTER clause +`agg(…) FILTER (WHERE c)` SHALL be accepted on every aggregation — built-in, aliased and model-defined custom — in every aggregation position, and SHALL mean the aggregation computed over only the rows where `c` is true. `c` SHALL be a row-level predicate over the aggregation's dataset; an aggregate, a transform or an attached value inside `c` SHALL be rejected with a typed query error naming the clause. Every row-level input of the aggregation SHALL be restricted by `c`: the source value, every column-valued parameter (`weight=`, `other=`), and the ranking key of `first` / `last`; literal parameters are unaffected. The clause SHALL compose with `partition_by=` and `window=`. An unnamed filtered aggregation SHALL take its result key from its full text by the existing expression-naming rule, so it never collides with the unfiltered aggregation. + +#### Scenario: Filtered count +- **WHEN** a query on `orders` by `region` has measure `count(*) FILTER (WHERE status = 'paid')` +- **THEN** each region's value is its number of paid orders, 0 where it has none + +#### Scenario: Filtered sum equals a masked sum +- **WHEN** one measure is `sum(amount) FILTER (WHERE status = 'paid')` and another `sum(iif(status = 'paid', amount, NULL))` +- **THEN** both return the same values + +#### Scenario: Filtered weighted average uses only passing rows +- **WHEN** a measure is `weighted_avg(price, weight=quantity) FILTER (WHERE status = 'paid')` +- **THEN** it equals the weighted average computed over the paid rows alone, numerator and denominator both + +#### Scenario: Filtered last picks the newest passing row +- **WHEN** a query by `customer_id` has measure `last(balance, updated_at) FILTER (WHERE status = 'active')` and a customer's newest row is inactive +- **THEN** the value is the balance of that customer's newest active row + +#### Scenario: Aggregate inside the condition rejected +- **WHEN** a measure is `sum(amount) FILTER (WHERE amount > avg(amount))` +- **THEN** it is rejected with the typed error naming the FILTER clause + +#### Scenario: With partition_by +- **WHEN** a query by `region, city` has measure `sum(amount, partition_by=region) FILTER (WHERE status = 'paid')` +- **THEN** each row carries its region's paid total diff --git a/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/models/stored-upgrade/spec.md b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/models/stored-upgrade/spec.md new file mode 100644 index 000000000..700c0a330 --- /dev/null +++ b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/models/stored-upgrade/spec.md @@ -0,0 +1,35 @@ +## ADDED Requirements + +### Requirement: Stored Mode-B text is upgraded to the SQL string and identifier rules + +On load, every Mode-B surface of a stored model, saved query or memory written before this change — `ModelMeasure.formula`, and a stored query's measures, dimensions, computed-dimension expressions, time-dimension columns, order columns and expressions, `main_time_dimension` and filters, recursively through stored `source_queries`, saved queries and inline `ModelExtension.measures` — SHALL be rewritten so that it means under the current grammar what it meant when stored: a double-quoted string literal becomes the equivalent single-quoted literal, a string literal using backslash escapes is re-encoded with its decoded value, and a `|` concatenation becomes `||`. The rewrite SHALL be token-exact: text inside string literals is never otherwise touched, and stored text that cannot be tokenised is left unchanged. Mode-A surfaces SHALL NOT be rewritten, and a client query without a version SHALL NOT be rewritten. + +#### Scenario: Double-quoted strings in a stored query + +- **WHEN** a stored query written before this change filters `status == "paid"` and has measure `sum(iif(region == "EU", amount, 0))` +- **THEN** it loads as `status == 'paid'` and `sum(iif(region == 'EU', amount, 0))` and returns the rows it returned before + +#### Scenario: Escapes and embedded quotes + +- **WHEN** a stored filter is `name == "O'Brien"`, another `name == 'O\'Brien'`, and a stored measure formula compares with `"tab\there"` +- **THEN** they load as `name == 'O''Brien'` (twice) and a single-quoted literal containing a real tab character, and match the same rows as before + +#### Scenario: Pipe concatenation + +- **WHEN** a stored computed dimension is `status | region` +- **THEN** it loads as `status || region` and returns the same values + +#### Scenario: Mode-A text untouched + +- **WHEN** a stored model has `Column.sql` `"Order Total" * 2` and a model filter `status = "x"` +- **THEN** both load byte-for-byte unchanged + +#### Scenario: Client queries are not rewritten + +- **WHEN** a client sends a query without a version that filters `status = "paid"` +- **THEN** it is not rewritten and fails with the double-quoted-name hint unless a column named `paid` exists + +#### Scenario: Released stores still open with their results + +- **WHEN** the committed 0.10.2 store is opened +- **THEN** every model and memory loads and every recorded query returns the rows recorded from 0.10.2 diff --git a/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/casts/spec.md b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/casts/spec.md new file mode 100644 index 000000000..3840f7a9f --- /dev/null +++ b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/casts/spec.md @@ -0,0 +1,67 @@ +## Purpose + +Defines casts in Mode-B expressions: `CAST` / `::` as the database's own cast of the author's type, and `TRY_CAST` as the one portable cast that turns an invalid input into NULL on every dialect. + +## ADDED Requirements + +### Requirement: CAST is the database's cast of the author's type + +`CAST(x AS )` and `x::` SHALL be accepted in every Mode-B position, chains included (`x::int::text`). The cast SHALL render the author's type, parameters included (`DECIMAL(10,2)`, `VARCHAR(3)`), changing only its spelling to the target dialect's; its value semantics — rounding, parsing, invalid input — SHALL be the database's own. A type the target dialect cannot express SHALL be rejected at plan time with a typed query error naming the type and the dialect. + +#### Scenario: Author's parameterised type is kept + +- **WHEN** a measure on Postgres is `sum(CAST(amount AS DECIMAL(10,2)))` +- **THEN** the generated SQL casts to `DECIMAL(10, 2)` and the value is Postgres's own + +#### Scenario: Spelling transpiled to the dialect + +- **WHEN** a dimension is `amount::text` on MySQL +- **THEN** the generated SQL is valid MySQL casting to its character type + +#### Scenario: Chained casts + +- **WHEN** a dimension is `amount::int::text` +- **THEN** it casts to INT, then to TEXT + +#### Scenario: Invalid input follows the database + +- **WHEN** a dimension is `CAST(status AS INT)` over a row holding `'abc'` +- **THEN** Postgres fails the query with its own error and SQLite returns 0, as each database's `CAST` does + +### Requirement: A cast's result type is inferred + +A cast SHALL type as SLayer's data type for its target: the integer family as INT; floating-point, `REAL`, `DECIMAL` and `NUMERIC` as DOUBLE; character types as TEXT; `BOOLEAN` as BOOLEAN; `DATE` as DATE; `TIMESTAMP` and `DATETIME` as TIMESTAMP; any other type as UNKNOWN. The inferred type SHALL drive checking and response types exactly as a column's declared type does. + +#### Scenario: Cast to DATE is a date operand + +- **WHEN** a dimension is `date_diff('day', CAST(created_txt AS DATE), DATE '2025-01-01')` over a TEXT column `created_txt` +- **THEN** it binds without a date-operand type error and the response reports INT + +#### Scenario: Unknown target type + +- **WHEN** a dimension is `CAST(payload AS JSON)` on Postgres +- **THEN** it renders and its response type is UNKNOWN + +### Requirement: TRY_CAST turns invalid input into NULL on every dialect + +`TRY_CAST(x AS )` SHALL render the author's type like `CAST` and SHALL yield NULL, never an error or a substitute value, for an input the cast cannot convert, on every Tier-1 dialect. Where the database cannot itself tell a valid input from an invalid one (SQLite, MySQL numeric targets), a number SHALL be valid when it is an optionally signed decimal with an optional fraction and exponent, surrounding spaces ignored; a boolean when it is one of `true`, `false`, `t`, `f`, `yes`, `no`, `1`, `0` in any case; a date or timestamp when it is an ISO time-point shape naming a real calendar value. On Postgres, a server older than 16 — or one whose version cannot be determined, including a dry run without a connection — SHALL get a plain `CAST`, so an invalid input fails the query there; this is documented. + +#### Scenario: Invalid text becomes NULL + +- **WHEN** a dimension is `TRY_CAST(status AS INT)` over rows holding `'12'` and `'abc'` +- **THEN** they yield 12 and NULL on SQLite, DuckDB, MySQL and Postgres 16 + +#### Scenario: SQLite does not truncate to a prefix + +- **WHEN** a dimension is `TRY_CAST(status AS INT)` over a row holding `'12abc'` on SQLite +- **THEN** it yields NULL, not 12 + +#### Scenario: Postgres before 16 + +- **WHEN** the same query runs against a Postgres server reporting version 15 +- **THEN** the generated SQL is a plain `CAST`, and a row holding `'abc'` fails the query with Postgres's error + +#### Scenario: Postgres 16 uses the native validity check + +- **WHEN** the query runs against a Postgres server reporting version 16 or later +- **THEN** the generated SQL guards the cast with Postgres's own input-validity check and `'abc'` yields NULL diff --git a/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/date-functions/spec.md b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/date-functions/spec.md new file mode 100644 index 000000000..ab5c339b3 --- /dev/null +++ b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/date-functions/spec.md @@ -0,0 +1,58 @@ +## MODIFIED Requirements + +### Requirement: The interval spelling is date_add +`x + interval(n, unit)`, `interval(n, unit) + x` and `x - interval(n, unit)` SHALL mean exactly +`date_add(x, n, unit)` (for subtraction, `date_add(x, -n, unit)`) and SHALL resolve to the same +value as that call, sharing its result key and SQL. The SQL interval literals `INTERVAL ' '`, +`INTERVAL '' ` and `INTERVAL ` (one unit; keyword and unit case-insensitive; a +unit's plural form accepted) SHALL mean `interval(n, unit)` in all of these positions. Chains SHALL +apply left to right. `interval` SHALL follow the same count and unit rules as `date_add`. An interval +in any other position — standalone, as the left side of `-`, negated, multiplied, compared, nested in +another interval, added to another interval, or as a function argument — SHALL be rejected before +binding with an error pointing to `date_add`, and an interval literal naming more than one unit +(`INTERVAL '1 day 2 hours'`) SHALL be rejected with an error suggesting a chain. + +#### Scenario: Operator spelling equals date_add +- **WHEN** one measure uses `max(shipped_at - interval(2, 'day'))` and another `max(date_add(shipped_at, -2, 'day'))` +- **THEN** both resolve to the same value and render identical SQL + +#### Scenario: Chained intervals +- **WHEN** a filter uses `created_at + interval(1, 'month') + interval(2, 'hour') < now()` +- **THEN** it binds as `date_add(date_add(created_at, 1, 'month'), 2, 'hour') < now()` + +#### Scenario: Commuted addition +- **WHEN** a filter uses `interval(7, 'day') + order_date > current_date()` +- **THEN** it binds as `date_add(order_date, 7, 'day') > current_date()` + +#### Scenario: Stray interval rejected +- **WHEN** a filter uses `interval(7, 'day') - order_date > 0`, `-interval(7, 'day') + order_date > order_date`, or `order_date + interval(1, 'day') * 2 > order_date` +- **THEN** each is rejected with an error pointing to `date_add` + +#### Scenario: SQL interval literals equal date_add +- **WHEN** filters use `order_date + INTERVAL '7 days' > CURRENT_DATE`, `order_date + INTERVAL '7' DAY > CURRENT_DATE` and `order_date + INTERVAL 7 DAY > CURRENT_DATE` +- **THEN** each binds as `date_add(order_date, 7, 'day') > current_date()` + +#### Scenario: Multi-unit interval literal rejected +- **WHEN** a filter uses `order_date + INTERVAL '1 day 2 hours' > now()` +- **THEN** it is rejected with an error suggesting `+ INTERVAL '1 day' + INTERVAL '2 hours'` + +## ADDED Requirements + +### Requirement: SQL date literals and clock keywords +`DATE ''` SHALL be a DATE value and `TIMESTAMP ''` a TIMESTAMP value, accepting +exactly the shapes and calendar checks of ISO literals in date positions, in every position; a +malformed or impossible value SHALL be rejected with the same typed query error. `CURRENT_DATE` SHALL +mean `current_date()` and `CURRENT_TIMESTAMP` SHALL mean `now()`, sharing their result keys, SQL and +result-cache bypass. + +#### Scenario: Typed literals +- **WHEN** a filter uses `created_at >= TIMESTAMP '2024-01-01 00:00'` and a dimension uses `date_diff('day', DATE '2024-01-01', created_at)` +- **THEN** both bind with typed temporal literals and execute + +#### Scenario: Impossible typed literal +- **WHEN** a filter uses `created_at >= DATE '2024-02-30'` +- **THEN** the query is rejected with the type error naming the literal + +#### Scenario: Clock keywords +- **WHEN** one filter uses `created_at >= CURRENT_DATE` and another `created_at >= current_date()` +- **THEN** both render identical SQL and neither query is served from or stored in the result cache diff --git a/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/expression-grammar/spec.md b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/expression-grammar/spec.md new file mode 100644 index 000000000..3a69a306b --- /dev/null +++ b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/expression-grammar/spec.md @@ -0,0 +1,190 @@ +## Purpose + +Defines the Mode-B expression grammar: one SQL-compatible syntax shared by every query and model formula position — its operators and predicates, identifiers and quoting, string literals, precedence, the legacy spellings it keeps, and the forms it rejects with a hint. + +## ADDED Requirements + +### Requirement: One grammar in every Mode-B position + +Query measures, dimensions (string and object form), filters, order targets, `ModelMeasure.formula` and `ModelExtension.measures` SHALL all be parsed by the same grammar: any construct accepted in one of these positions SHALL be accepted syntactically in all of them, and whether it is legal there SHALL be decided only by typing (field / measure / position rules), never by the position's parser. + +#### Scenario: SQL predicate spellings in a measure + +- **WHEN** a query on `orders` has measure `sum(iif(status = 'paid' AND amount IS NOT NULL, amount, 0))` +- **THEN** it parses, binds and returns the paid total — the same value as `sum(iif(status == 'paid' and amount is not None, amount, 0))` + +#### Scenario: Same text in filter, measure and order + +- **WHEN** `amount BETWEEN 10 AND 20` is used as a filter, inside `sum(iif(amount BETWEEN 10 AND 20, 1, 0))` as a measure, and `iif(amount BETWEEN 10 AND 20, 0, 1)` as an order target on a raw-rows query +- **THEN** each parses and evaluates the same predicate + +### Requirement: SQL operators and predicates + +The grammar SHALL accept, with keywords matched case-insensitively: + +- comparison `=`, `<>`, `!=`, `==`, `<`, `<=`, `>`, `>=`; +- `IS [NOT] NULL`, `IS [NOT] DISTINCT FROM`, `IS [NOT] TRUE`, `IS [NOT] FALSE`; +- `[NOT] IN (, …)` and `[NOT] IN [, …]`; +- `[NOT] BETWEEN a AND b`, meaning `x >= a AND x <= b` (negated: `NOT (x >= a AND x <= b)`); +- `AND`, `OR`, `NOT`; +- `||` string concatenation; +- arithmetic `+`, `-`, `*`, `/`, `%` (the result's sign is the dividend's on every dialect), `**` (power) and unary `-`. + +`IS [NOT] DISTINCT FROM` SHALL be a null-safe comparison on every dialect. + +#### Scenario: BETWEEN and NOT BETWEEN + +- **WHEN** a query on `orders` filters `amount BETWEEN 10 AND 20`, and another filters `amount NOT BETWEEN 10 AND 20` +- **THEN** the first keeps exactly the rows with `10 <= amount <= 20` and the second exactly the rows with a non-NULL amount outside that range + +#### Scenario: Null-safe comparison + +- **WHEN** a query filters `region IS NOT DISTINCT FROM NULL` +- **THEN** it keeps exactly the rows whose region is NULL, on every Tier-1 dialect + +#### Scenario: Boolean tests + +- **WHEN** a query filters `(amount > 100) IS TRUE` and another `(amount > 100) IS NOT TRUE` +- **THEN** the second keeps the rows the first drops, including rows whose amount is NULL + +#### Scenario: Concatenation operator + +- **WHEN** a dimension is `{"expression": "status || '-' || region", "name": "tag"}` +- **THEN** each row's `tag` is the two values joined by `-`, and NULL when either is NULL + +#### Scenario: Modulo sign + +- **WHEN** a measure is `min(-7 % 3)` +- **THEN** it returns `-1` on every Tier-1 dialect + +### Requirement: LIKE and ILIKE are operators over any operand + +`x [NOT] LIKE p` and `x [NOT] ILIKE p` SHALL accept any expression on either side. `ILIKE` SHALL match case-insensitively on every dialect. + +#### Scenario: LIKE over a nested call + +- **WHEN** a query filters `lower(trim(status)) LIKE 'pa%'` +- **THEN** it parses and keeps the rows whose trimmed, lower-cased status starts with `pa` + +#### Scenario: ILIKE + +- **WHEN** a query filters `status ILIKE 'PA%'` on data holding `paid` and `Pending` +- **THEN** it keeps `paid` and drops `Pending`, on every Tier-1 dialect + +#### Scenario: LIKE inside a string literal is text + +- **WHEN** a filter is `note = 'we like cats'` +- **THEN** the literal is compared byte-for-byte and no LIKE is formed + +### Requirement: SQL-standard call forms + +The grammar SHALL accept `POSITION(a IN b)` (meaning `instr(b, a)`) and `SUBSTRING(x FROM a [FOR b])` (meaning `substr(x, a[, b])`). + +#### Scenario: SQL substring and position + +- **WHEN** a dimension is `SUBSTRING(status FROM 1 FOR 3)` and another `POSITION('a' IN status)` +- **THEN** they equal `substr(status, 1, 3)` and `instr(status, 'a')`, with the same result keys and SQL + +### Requirement: Precedence and associativity + +Operators SHALL bind, loosest first: `OR`; `AND`; `NOT`; comparison, `IS`, `IN`, `BETWEEN`, `LIKE`, `ILIKE`; `||`; `+` `-`; `*` `/` `%`; unary `-`; `**`; `::`; postfix (call, `FILTER`, `.`). `**` SHALL be right-associative and its right operand MAY carry a unary minus. Parentheses override everything. + +#### Scenario: NOT and comparison + +- **WHEN** a filter is `NOT amount > 100 AND status = 'paid'` +- **THEN** it means `(NOT (amount > 100)) AND (status = 'paid')` + +#### Scenario: Power + +- **WHEN** measures are `min(2 ** 3 ** 2)`, `min(-2 ** 2)`, `min((-2) ** 2)` and `min(2 ** -2)` +- **THEN** they return 512, -4, 4 and 0.25 + +#### Scenario: Concatenation binds tighter than comparison + +- **WHEN** a filter is `status || 'x' = 'paidx'` +- **THEN** it means `(status || 'x') = 'paidx'` + +### Requirement: Identifiers and quoted identifiers + +A bare identifier or dotted path SHALL name a column, measure, alias or join path as before. A double-quoted name (`"order total"`, `customers."Region"`) SHALL be an identifier in every identifier position, with `""` standing for one `"` inside it. A name beginning `__slayer_` SHALL be rejected, quoted or not. A double-quoted name that resolves to nothing SHALL fail with the unresolved-reference error plus a hint that strings use single quotes. + +#### Scenario: Quoted column name + +- **WHEN** model `orders` has a column named `order total` and a query has measure `sum("order total")` +- **THEN** it sums that column + +#### Scenario: Quoted segment in a dotted path + +- **WHEN** a query on `orders` has dimension `customers."Region"` and `customers` has a column named `Region` +- **THEN** it groups by that column + +#### Scenario: Double-quoted string hint + +- **WHEN** a query on `orders` filters `status = "paid"` and no column or alias is named `paid` +- **THEN** it fails with the unresolved-reference error naming `paid` and stating that strings use single quotes + +### Requirement: String literals follow SQL rules + +A string literal SHALL be single-quoted, with `''` standing for one `'`; a backslash SHALL be an ordinary character. + +#### Scenario: Doubled quote + +- **WHEN** a query filters `customer_name = 'O''Brien'` +- **THEN** it compares against `O'Brien` + +#### Scenario: Backslash is literal + +- **WHEN** a query filters `path = 'C:\temp'` +- **THEN** it compares against the six characters `C:\temp` + +### Requirement: Legacy spellings keep their meaning + +`==`, `!=`, `is None`, `is not None`, `True`, `False`, `None`, `[…]` lists after `in`, the method-call aggregation form (`customers.sum(revenue)`) and the colon aggregation form (`revenue:sum`) SHALL keep their meaning, result keys and SQL. Every expression that parsed before this change and contains none of the forms this change removes SHALL bind to the same value with byte-identical generated SQL. + +#### Scenario: Python spellings + +- **WHEN** a filter is `status == 'paid' and region is not None` and another is `status = 'paid' AND region IS NOT NULL` +- **THEN** both render identical SQL + +#### Scenario: Pre-existing expressions are unchanged + +- **WHEN** the SQL golden baselines committed before this change are regenerated +- **THEN** they are byte-identical + +### Requirement: Keyword arguments versus comparison + +Inside an aggregation, a transform or an unknown call, `name = value` directly after `(` or `,` SHALL be a keyword argument. Inside a scalar function call and everywhere else, `=` SHALL be a comparison. + +#### Scenario: Keyword argument + +- **WHEN** a measure is `percentile(amount, p=0.9)` +- **THEN** `p` is a keyword argument + +#### Scenario: Comparison inside a scalar call + +- **WHEN** a measure is `sum(iif(status = 'paid', amount, 0))` +- **THEN** `status = 'paid'` is the condition of `iif` + +### Requirement: Forms rejected with a hint + +The following SHALL fail at parse time with an error naming the form and the remedy: `a | b` (use `||`); `a // b` (use `div(a, b)` or `floor(a / b)`); `TRIM(BOTH | LEADING | TRAILING …)`; a scalar `min(a, b)` / `max(a, b)` with two positional values (use `greatest` / `least`); `TIME '…'` literals; `SIMILAR TO`; `REGEXP` / `RLIKE` operators; raw `OVER (…)`. A syntax error SHALL report the character position where parsing failed. + +#### Scenario: Bitwise-or spelling + +- **WHEN** a measure is `min(status | region)` +- **THEN** it fails naming `|` and suggesting `||` + +#### Scenario: Floor-division spelling + +- **WHEN** a measure is `sum(amount) // 2` +- **THEN** it fails suggesting `div(a, b)` or `floor(a / b)` + +#### Scenario: Two-argument scalar max + +- **WHEN** a dimension is `max(amount, 10)` +- **THEN** it fails suggesting `greatest(amount, 10)` + +#### Scenario: Syntax error position + +- **WHEN** a filter is `amount > > 5` +- **THEN** the error names the position of the second `>` diff --git a/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/expression-keywords/spec.md b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/expression-keywords/spec.md new file mode 100644 index 000000000..950d8a1b8 --- /dev/null +++ b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/expression-keywords/spec.md @@ -0,0 +1,44 @@ +## MODIFIED Requirements + +### Requirement: CASE WHEN lowering + +A Mode-B expression SHALL accept SQL `CASE … END` conditionals — searched +(`CASE WHEN cond THEN val … [ELSE val] END`) and simple +(`CASE operand WHEN val THEN val … [ELSE val] END`) — lowering them to nested +`iif(cond, then, otherwise)` calls with `None` as the default otherwise. WHEN +conditions, THEN values and ELSE values SHALL accept the full Mode-B grammar in every +expression position, including measures. A `CASE` that is recognized as a conditional +(a `WHEN` follows it) but is malformed MUST raise a specific malformed-CASE error naming +the defect. + +#### Scenario: searched CASE lowers to iif + +- WHEN `CASE WHEN amount > 100 THEN 'big' ELSE 'small' END` is parsed as a Mode-B expression +- THEN it parses as `iif(amount > 100, 'big', 'small')` — a scalar call, usable wherever a scalar expression is legal + +#### Scenario: simple CASE compares the operand per branch + +- WHEN `CASE status WHEN 'a' THEN 1 WHEN 'b' THEN 2 END` is parsed +- THEN it parses as `iif(status == 'a', 1, iif(status == 'b', 2, None))` + +#### Scenario: nested CASE in THEN and ELSE values + +- WHEN a THEN or ELSE value itself contains a complete `CASE … END` +- THEN the nested conditional is lowered recursively and the enclosing branches are unaffected + +#### Scenario: SQL operator spellings inside WHEN conditions + +- WHEN `CASE WHEN region = 'EU' AND amount IS NOT NULL THEN 1 ELSE 0 END` appears in a measure formula +- THEN the WHEN condition parses with the same meaning as `region == 'EU' and amount is not None` + +#### Scenario: recognized-but-malformed CASE still errors specifically + +- WHEN `CASE WHEN a THEN 1` (missing END) or `CASE WHEN a 1 END` (missing THEN) is parsed +- THEN a malformed-CASE error is raised naming the missing keyword, not a generic syntax error + +## REMOVED Requirements + +### Requirement: LIKE operator rewriting + +**Reason**: LIKE is no longer a filter-only text rewrite to a `like()` call over a restricted left side; the grammar's `[NOT] LIKE` / `[NOT] ILIKE` operators accept any operand in every position, string literals follow SQL rules, and a double-quoted operand is an identifier. +**Migration**: See `queries/expression-grammar` › "LIKE and ILIKE are operators over any operand" and "String literals follow SQL rules". Stored formulas are upgraded on load (`models/stored-upgrade`). diff --git a/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/positions/spec.md b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/positions/spec.md new file mode 100644 index 000000000..13c12a927 --- /dev/null +++ b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/positions/spec.md @@ -0,0 +1,24 @@ +## ADDED Requirements + +### Requirement: Order items carry an explicit NULL placement +An order item SHALL accept `nulls` set to `first` or `last`. With it, the sorted rows SHALL place NULL values first or last on every Tier-1 dialect, SQL Server included, for every kind of order target — a projected field, a hidden filter- or order-only value, a measure, a transform or an expression. Without it, NULL placement SHALL be unchanged. `nulls` SHALL survive serialisation, saved-query refinement and the stage rewrites that rebuild order items. + +#### Scenario: NULLS FIRST on a descending order +- **WHEN** a raw-rows query on `orders` orders by `{"column": "discount", "direction": "desc", "nulls": "first"}` +- **THEN** the rows with a NULL discount come first, then the rest by discount descending, on every Tier-1 dialect + +#### Scenario: NULLS LAST on a hidden aggregate +- **WHEN** a query groups by `region` and orders by `{"column": "max(discount)", "direction": "asc", "nulls": "last"}` without projecting `max(discount)` +- **THEN** regions whose `max(discount)` is NULL come last, and the hidden value is not returned + +#### Scenario: SQL Server honours the policy over an aliased target +- **WHEN** the same two queries run on SQL Server +- **THEN** they return the same row order as on Postgres + +#### Scenario: Refinement keeps the policy +- **WHEN** a saved query ordering with `"nulls": "first"` is run with a refinement adding a filter +- **THEN** the refined result keeps NULLs first + +#### Scenario: Invalid policy +- **WHEN** an order item has `"nulls": "middle"` +- **THEN** the query is rejected at construction naming the allowed values diff --git a/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/variables/spec.md b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/variables/spec.md new file mode 100644 index 000000000..1c3e7cd5f --- /dev/null +++ b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/specs/queries/variables/spec.md @@ -0,0 +1,75 @@ +## MODIFIED Requirements + +### Requirement: Every Mode-B formula surface substitutes variables + +`{var}` placeholders in a query's filters, measure formulas, computed-dimension expressions (string and object form) and order expressions SHALL be bound from the query's effective variables as literal values, never as text spliced into the expression. A placeholder inside a string literal, wholly or partly (`'{status}'`, `'%{q}%'`), SHALL contribute its value's text to that literal; a placeholder in a bare literal position SHALL take a number or boolean value, a numeric string reading as its number; a placeholder forming the whole parenthesised list of `IN` SHALL take a list value. `{{` and `}}` inside a string literal SHALL render literal braces. A query carrying placeholders on these surfaces SHALL construct successfully and SHALL execute with the bound values. + +#### Scenario: Measure formula + +- **WHEN** a query on `orders` has measure `sum(amount) * {k} / 100` and `variables={"k": 10}` +- **THEN** it executes and returns the total amount multiplied by 10 and divided by 100, by executed values on SQLite and DuckDB + +#### Scenario: Computed dimension in string and object form + +- **WHEN** a query has dimension `"amount * {k}"`, or dimension `{"expression": "amount * {k}"}`, with `variables={"k": 10}` +- **THEN** both construct and group by `amount * 10` + +#### Scenario: Order expression in colon and call form + +- **WHEN** a query orders by `sum(amount) * {k}` descending, with `variables={"k": -1}` +- **THEN** both construct and the rows come back in ascending total-amount order + +#### Scenario: Placeholder in operand positions + +- **WHEN** a placeholder appears as an aggregation argument (`sum({k})`), a transform argument, a scalar-function argument, a CASE branch, or the whole right-hand side of `in` (`region in ({regions})` with a list value) +- **THEN** the query constructs and executes with the substituted value + +#### Scenario: Quoted string value is escaped + +- **WHEN** a measure contains `'{name}'` inside a quoted literal and `variables={"name": "O'Brien"}` +- **THEN** the literal's value is `O'Brien` and the query executes without a parse error + +### Requirement: Saved measure formulas substitute on the models whose SQL surfaces substitute + +A saved `ModelMeasure.formula` SHALL bind `{var}` placeholders as literal values, by the same rules as query formula surfaces, on exactly the models whose Mode-A surfaces substitute (the query's direct source model and each stage's own source model), using that model's effective variables. Its placeholders SHALL be reported by model variable inspection as required unless the model defaults them. A saved measure reached through a join or a cross-model reference SHALL NOT substitute, and a placeholder in it SHALL raise the unresolved-placeholder error. + +#### Scenario: Saved measure on the source model + +- **WHEN** model `orders` has saved measure `amt_scaled` with formula `sum(amount) * {k}` and a query requests `amt_scaled` with `variables={"k": 10}` +- **THEN** it returns ten times the total amount + +#### Scenario: Saved measure through a join + +- **WHEN** model `customers`, joined from `orders`, has saved measure `spend_scaled` with formula `sum(spend) * {k}` and a query on `orders` requests `customers.spend_scaled` with `variables={"k": 10}` +- **THEN** the query fails with the unresolved-placeholder error naming `{k}` + +#### Scenario: Inspection lists saved-measure variables + +- **WHEN** model `orders` has saved measure formula `sum(amount) * {k}` and no default for `k` +- **THEN** model variable inspection reports `k` as required + +## ADDED Requirements + +### Requirement: Variables supply literal values only + +A placeholder in any other position — an identifier or path segment (`{col}:sum`, `t.{c}`), a function name (`{f}(x)`), a keyword argument's name, or a value that must be a reference (`partition_by={x}`) — SHALL be rejected with an error stating that variables supply literal values. A string value bound into a bare literal position SHALL be rejected with the same error plus a hint to put the placeholder inside quotes. No variable value SHALL ever change the structure of the expression it is bound into. + +#### Scenario: Identifier injection rejected + +- **WHEN** a query on `orders` has measure `sum({col})` with `variables={"col": "amount"}` +- **THEN** it fails with the literal-values error and the hint to quote the placeholder + +#### Scenario: Function and path positions rejected + +- **WHEN** a measure is `{f}(amount)`, or a dimension is `customers.{c}`, or a measure is `sum(amount, partition_by={x})` +- **THEN** each fails with the literal-values error + +#### Scenario: Numeric string in a bare position + +- **WHEN** a filter is `amount > {min}` with `variables={"min": "10"}` +- **THEN** it keeps the rows with amount above 10 + +#### Scenario: Hostile string stays a literal + +- **WHEN** a filter is `status = '{s}'` with `variables={"s": "x' OR '1' = '1"}` +- **THEN** it keeps only the rows whose status equals that whole string — none, on data without it diff --git a/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/tasks.md b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/tasks.md new file mode 100644 index 000000000..f124729c1 --- /dev/null +++ b/openspec/changes/archive/2026-10-22-dev-2101-mode-b-as-a-sql-compatible-grammar-custom-parser-sql-forms/tasks.md @@ -0,0 +1,81 @@ +## 1. Tracker records (pr-plan) + +- [x] 1.1 DEV-2101 body: point to this change and list the scope added in planning (casts with `TRY_CAST`, `FILTER` row semantics, placeholder binding, DEV-1831 absorption, split into DEV-2119 / DEV-2120); verify by reading the issue +- [x] 1.2 Comments (not body edits): DEV-2021 (re-scope: remove query-time Mode A for every caller after DEV-2102; `ModelExtension.columns` deleted outright, replaced by Mode-B expressions via DEV-2119 / DEV-2120; inline-model removal waits on DEV-1768 stages), DEV-1768 (EXTEND column items die with DEV-2021; `MODEL … SOURCE (raw SQL)` waits on it; EXTEND / MODEL items stay Mode A until then), DEV-2044 (Mode-B half absorbed here, Mode-A half stays), DEV-1831 (absorbed here), DEV-1320 (absorbed by DEV-2120), DEV-2034 (Storyline must port `storyline/api/adapters/models.py` from `parse_formula` / `FieldSpec` to `parse_expr` + `walk_parsed_refs`), DEV-2043 (eventual replacement for Storyline-side formula validation), DEV-1370 (superseded: one parser, no `FieldSpec`); verify each comment exists + +## 2. Failing tests (pr-tests) + +- [x] 2.1 Grammar tests for every `queries/expression-grammar` scenario, plus the ambiguity table: `f(x=1)` kwarg vs `=` in scalar calls, `x::t` vs `x:sum`, `a::int::text`, predicates inside transform arguments, `"x"` identifier, `CASE` only before `WHEN`, `date '…'` vs a column named `date`, `**` cases `2**3**2`, `-2**2`, `(-2)**2`, `2**-2`; verify they fail on the current parser +- [x] 2.2 SQL-form-in-every-position matrix: each form of the proposal as measure, dimension, filter and order target, executed on SQLite and DuckDB, emission-checked on every Tier-1 dialect; verify failing +- [x] 2.3 `queries/casts` scenarios, including the Postgres 15 / 16 / unknown-version emissions (fake server profile), SQLite `'12abc'`, MySQL emission, and a concrete type a dialect cannot express; verify failing +- [x] 2.4 `aggregations/functional-form` FILTER scenarios (count, sum ≡ masked sum, weighted_avg numerator and denominator, `last` newest passing row, `corr` / `covar_*` pairs, custom aggregation, rejected aggregate inside `c`, with `partition_by` and `window`); verify failing +- [x] 2.5 `queries/variables` scenarios incl. injection rejections, numeric-string binding, hostile string, template-derived names for measures and computed dimensions with different runtime values, `date_range` bounds (`{start}`, `{end}`, one-sided, missing, non-string); verify failing +- [x] 2.6 `queries/date-functions` scenarios (interval literal forms, multi-unit rejection, typed literals, clock keywords and cache bypass); verify failing +- [x] 2.7 `queries/positions` NULL-placement scenarios on every Tier-1 dialect (SQL Server and Postgres in their integration suites), incl. hidden targets, bracketed aliases, refinement and stage rewrites; verify failing +- [x] 2.8 `models/stored-upgrade` scenarios: tokenizer migration cases (embedded `"` and `'`, backslashes, control escapes, Unicode, adjacent literals, malformed text, `|`), Mode-A untouched, unversioned client query untouched, 0.10.2 corpus still green; verify failing +- [x] 2.9 Importer dependency check over `ParsedExpr`: transitive dangling reference and cyclic saved measures for dbt and Cube; for OSI, the emitted formula is validated by the new grammar; verify failing where behaviour is new, green where it is preserved +- [x] 2.10 Regression: `sum(a) ** 2` executes (today `NotImplementedError`); `%` sign per dialect; verify failing +- [x] 2.11 Parser benchmark over the longest formulas in the test suite and docs vs the old parser (recorded, not gating); verify it runs +- [x] 2.12 Retarget, with the user's approval already given, the tests asserting on deleted helpers to equivalent `parse_expr` / `ParsedExpr` assertions: `tests/test_dev1833_keyword_lexing.py`, `tests/test_dev1450fix_group2_correctness.py`, `tests/test_dev1740_case_rewrite.py`, `tests/test_dev1743_validators.py` (pre-pass helpers `_rewrite_case_when`, `_rewrite_sql_like`, `_preprocess_colons`, `_normalize_sql_filter_operators`), and `tests/test_formula.py`, `tests/test_functional_remedies.py`, `tests/test_dev1576_heals.py`, `tests/test_dev1737_scalar_specs.py`, `tests/test_rank_direction.py` (`parse_formula` / `FieldSpec`); flip tests asserting `"abc"` is a string to the identifier reading; ruling 2026-10-21: `apply_variables_to_query` keeps returning substituted text, rendered from the bound tree with SQL quoting (`'O''Brien'`, `('a', 'b')`) — `tests/test_variables_planner.py` expectations switch to SQL quoting and the `escape="python"` tests in `tests/test_mode_a_variable_substitution.py` are deleted; list each changed test in the commit message +- [x] 2.13 Codex review of the tests against this change (when `reviewers.codex` is on) + +## 3. Architecture move + +- [x] 3.1 Move `slayer/engine/syntax.py` → `slayer/core/syntax.py` with `la:deterministic-refactor`; move `ALL_TRANSFORMS` / `TIME_TRANSFORMS` / `TIMELESS_TRANSFORMS` to `slayer/core/enums.py`; verify `la-typecheck`, `dr-mock-lint` and the full unit suite +- [x] 3.2 Apply the approved harness edits H1–H6 exactly as in the "Approved harness edits" section below, then `la-arch-diagrams` (H7); verify `la-arch-check` and `npx -y likec4@1.47.0 validate architecture` + +## 4. Grammar + +- [x] 4.1 Add `pyparsing` to `pyproject.toml` / `poetry.lock`; verify `poetry install -E all` +- [x] 4.2 Implement the grammar (design D2) producing the existing `ParsedExpr` nodes plus `Cast` and the `FILTER` predicate; delete `ast.parse`, every pre-pass and `parse_filter_expr`; repoint every caller (`core/query.py`, `engine/bind_inputs.py`, `binding.py`, `population.py`, `model_reads.py`, `schema_drift.py`, `unrelated.py`, `query_engine.py`, `memories/resolver.py`); verify 2.1 / 2.2 parse-level tests and the SQL golden baselines byte-identical +- [x] 4.3 Quoted identifiers end to end and the single-quote hint in the unresolved-reference error; verify the expression-grammar identifier scenarios +- [x] 4.4 Rejections with hints (`|`, `//`, `TRIM(BOTH…)`, two-argument scalar `min` / `max`, `TIME '…'`, `SIMILAR TO`, `REGEXP` / `RLIKE`, raw `OVER`) and syntax-error positions; verify their scenarios + +## 5. Keys, binding, rendering + +- [x] 5.1 `CastKey` (design D3): traversal protocol, binding, checker typing, render via `exp.Cast` with the author's type and the dialect-expressibility check; verify the CAST scenarios +- [x] 5.2 `TRY_CAST` lowering per dialect, SQLite `slayer_try_cast` registration, MySQL guard, Postgres server profile (`server_profile_sql`, `parse_server_profile`) and render-time profile plumbing; verify 2.3 +- [x] 5.3 `FILTER (WHERE c)` binding (row-level check) and lowering by masking every row-level input incl. the `first` / `last` ranking key; verify 2.4 +- [x] 5.4 `BETWEEN`, `IS [NOT] DISTINCT FROM`, `IS [NOT] TRUE|FALSE`, `ILIKE`, `||`, `POSITION`, `SUBSTRING FROM FOR`, interval literals, typed literals, clock keywords; `**` → `pow`; verify 2.2, 2.6, 2.10 +- [x] 5.5 `OrderItem.nulls` plumbing (design D7) and the multi-term `build_ordered` with the T-SQL emulation over the resolved expression; verify 2.7 +- [x] 5.6 The binder rejects a `round` `ndigits` that is not an integer literal (`round(x, 2.5)`, `round(x, col)`) with an error naming `round` (ruling 2026-10-21: the check moves from the deleted importer parser); verify the `tests/test_dev1576_heals.py` arity cases + +## 6. Variables + +- [x] 6.1 Offset-preserving placeholder masking, the three grammar slots, binding into the tree at the existing substitution points, names from the written text, `date_range` literal-only substitution; delete the Mode-B `escape="python"` path and `ComputedDimension.substituted`'s text splice; verify 2.5 + +## 7. Migration + +- [x] 7.1 Stored-only steps SlayerModel 15 → 16, SlayerQuery 6 → 7, Memory 4 → 5 (design D6), ordered before the legacy load-path repairs; verify 2.8 and the 0.10.2 corpus + +## 8. One parser (DEV-1831) + +- [x] 8.1 Port the dbt, Cube and OSI importers to `parse_expr` with the shared namespace-aware dependency walk; delete `parse_formula`, `FieldSpec`, `_rewrite_funcstyle_aggregations`, `_preprocess_agg_refs`, `_expand_named_measures`; verify 2.9 and the importer fixture suites +- [x] 8.2 Check the Mode-B text the facade translator and the importers emit against the new grammar (single quotes, no backslash escapes, no `|`); fix emitters; verify the facade and importer suites + +## 9. Docs and agent surfaces + +- [x] 9.1 `docs/concepts/formulas.md` (Parsing Internals, operators incl. `**` / `%`, FILTER, the CASE section), `references.md` (two-mode table, string and identifier rules, casts and `TRY_CAST`, interval and typed literals), `queries.md` (filters operator table, Variables / "Trusted input", OrderItem `nulls`), `time.md` if touched; `zensical.toml` only if a page is added; verify `grep` finds no "Python AST" / "Python-AST" claims and no double-quoted string examples in Mode-B docs +- [x] 9.2 MCP `Field` descriptions in `slayer/core/query.py` / `slayer/mcp/server.py` and `slayer/memories/help_content/*` (strings single-quoted, SQL forms, `nulls`); verify `tests/test_mcp_surface_budget.py` and the documented-idioms tests +- [x] 9.3 Release-notes draft (untracked, never staged): double-quoted strings, backslash escapes, `|` / `//`, `{var}` literal-only, `parse_formula` removal, `CAST` / `TRY_CAST`, Postgres < 16 `TRY_CAST` + +## 10. Gates + +- [x] 10.1 `poetry run pytest -m "not integration"`, the CI integration invocation, `poetry run ruff check slayer/ tests/`, `poetry run basedpyright` (no new errors vs baseline), `la-arch-check`; verify all green + +## Approved harness edits (exact text, approved 2026-10-20) + +H1 `architecture/model.c4`: add `syntax = node 'Syntax'` under `core` after `models = node 'Models'`; remove `syntax = node 'Syntax'` from `engine`; remove `core.query -> engine.syntax #legacy`; add `core.query -> core.syntax` after `core.query -> core.models`; `engine.binding -> engine.syntax` → `engine.binding -> core.syntax`; `engine.bind_inputs -> engine.syntax` → `engine.bind_inputs -> core.syntax`. + +H2 `architecture/index.yaml`: `legacy_arrows: {baseline: 8}` → `legacy_arrows: {baseline: 7}`. + +H3 `architecture/system.arc42.md` principle 2: the door list `(`core.query`/`core.models` → `engine.syntax`, `sql.*`, `storage.migrations`)` → `(`core.query`/`core.models` → `sql.*`, `storage.migrations`)`. + +H4 `architecture/system.arc42.md` principle 10 becomes: +"10. **Two expression layers**: Mode A free SQL (`Column.sql`, model `filters`) vs Mode B, a SQL-compatible grammar (formulas, query fields) — one canonical function library, extended never forked. [review] Every Mode B construct means the same on every dialect. [target: DEV-2120]" + +H5 `architecture/core.arc42.md`: §1 door list as in H3; §2 "Pydantic domain models alongside." → "Pydantic domain models alongside; the Mode B grammar: `syntax.py`." + +H6 `architecture/engine.arc42.md`: §2 "`normalization.py` → `syntax.py` (parse)" → "`normalization.py` → `core/syntax.py` (parse)"; principle 9 becomes "**The term-interface boundary**: `binding`, `bind_inputs`, `elaborate`, `elaborate_env` (THE checker), `compile` and `plan` are declared children with only the modeled arrows among them — the compiler consults the checker, never `core.syntax` or binding; every user-facing algebra type error raises in the checker, as a `QueryTypeError`." (status tags unchanged). + +H7: regenerate the embedded mermaid with `la-arch-diagrams`. diff --git a/openspec/specs/aggregations/functional-form/spec.md b/openspec/specs/aggregations/functional-form/spec.md index f60f1ac4f..fc9fc0ab3 100644 --- a/openspec/specs/aggregations/functional-form/spec.md +++ b/openspec/specs/aggregations/functional-form/spec.md @@ -341,3 +341,30 @@ function or a transform-only name. #### Scenario: Ambiguous method-call shapes are rejected - **WHEN** a measure is written `order_items.weighted_avg(price, qty)`, `order_items.sum(products.price)`, `order_items.sum(price, partition_by=region)` or `order_items.round(price)` - **THEN** each is rejected with an error showing the functional spelling to write + +### Requirement: An aggregation accepts a FILTER clause +`agg(…) FILTER (WHERE c)` SHALL be accepted on every aggregation — built-in, aliased and model-defined custom — in every aggregation position, and SHALL mean the aggregation computed over only the rows where `c` is true. `c` SHALL be a row-level predicate over the aggregation's dataset; an aggregate, a transform or an attached value inside `c` SHALL be rejected with a typed query error naming the clause. Every row-level input of the aggregation SHALL be restricted by `c`: the source value, every column-valued parameter (`weight=`, `other=`), and the ranking key of `first` / `last`; literal parameters are unaffected. The clause SHALL compose with `partition_by=` and `window=`. An unnamed filtered aggregation SHALL take its result key from its full text by the existing expression-naming rule, so it never collides with the unfiltered aggregation. + +#### Scenario: Filtered count +- **WHEN** a query on `orders` by `region` has measure `count(*) FILTER (WHERE status = 'paid')` +- **THEN** each region's value is its number of paid orders, 0 where it has none + +#### Scenario: Filtered sum equals a masked sum +- **WHEN** one measure is `sum(amount) FILTER (WHERE status = 'paid')` and another `sum(iif(status = 'paid', amount, NULL))` +- **THEN** both return the same values + +#### Scenario: Filtered weighted average uses only passing rows +- **WHEN** a measure is `weighted_avg(price, weight=quantity) FILTER (WHERE status = 'paid')` +- **THEN** it equals the weighted average computed over the paid rows alone, numerator and denominator both + +#### Scenario: Filtered last picks the newest passing row +- **WHEN** a query by `customer_id` has measure `last(balance, updated_at) FILTER (WHERE status = 'active')` and a customer's newest row is inactive +- **THEN** the value is the balance of that customer's newest active row + +#### Scenario: Aggregate inside the condition rejected +- **WHEN** a measure is `sum(amount) FILTER (WHERE amount > avg(amount))` +- **THEN** it is rejected with the typed error naming the FILTER clause + +#### Scenario: With partition_by +- **WHEN** a query by `region, city` has measure `sum(amount, partition_by=region) FILTER (WHERE status = 'paid')` +- **THEN** each row carries its region's paid total diff --git a/openspec/specs/models/stored-upgrade/spec.md b/openspec/specs/models/stored-upgrade/spec.md index 43561666e..edabacba5 100644 --- a/openspec/specs/models/stored-upgrade/spec.md +++ b/openspec/specs/models/stored-upgrade/spec.md @@ -138,3 +138,37 @@ A model SHALL NOT store the rendered SQL of its backing query. On load, a stored - **WHEN** a query-backed model is inspected with `show_sql=True` - **THEN** the output has no backing-SQL section, and a `dry_run` query on the model returns its SQL + +### Requirement: Stored Mode-B text is upgraded to the SQL string and identifier rules + +On load, every Mode-B surface of a stored model, saved query or memory written before this change — `ModelMeasure.formula`, and a stored query's measures, dimensions, computed-dimension expressions, time-dimension columns, order columns and expressions, `main_time_dimension` and filters, recursively through stored `source_queries`, saved queries and inline `ModelExtension.measures` — SHALL be rewritten so that it means under the current grammar what it meant when stored: a double-quoted string literal becomes the equivalent single-quoted literal, a string literal using backslash escapes is re-encoded with its decoded value, and a `|` concatenation becomes `||`. The rewrite SHALL be token-exact: text inside string literals is never otherwise touched, and stored text that cannot be tokenised is left unchanged. Mode-A surfaces SHALL NOT be rewritten, and a client query without a version SHALL NOT be rewritten. + +#### Scenario: Double-quoted strings in a stored query + +- **WHEN** a stored query written before this change filters `status == "paid"` and has measure `sum(iif(region == "EU", amount, 0))` +- **THEN** it loads as `status == 'paid'` and `sum(iif(region == 'EU', amount, 0))` and returns the rows it returned before + +#### Scenario: Escapes and embedded quotes + +- **WHEN** a stored filter is `name == "O'Brien"`, another `name == 'O\'Brien'`, and a stored measure formula compares with `"tab\there"` +- **THEN** they load as `name == 'O''Brien'` (twice) and a single-quoted literal containing a real tab character, and match the same rows as before + +#### Scenario: Pipe concatenation + +- **WHEN** a stored computed dimension is `status | region` +- **THEN** it loads as `status || region` and returns the same values + +#### Scenario: Mode-A text untouched + +- **WHEN** a stored model has `Column.sql` `"Order Total" * 2` and a model filter `status = "x"` +- **THEN** both load byte-for-byte unchanged + +#### Scenario: Client queries are not rewritten + +- **WHEN** a client sends a query without a version that filters `status = "paid"` +- **THEN** it is not rewritten and fails with the double-quoted-name hint unless a column named `paid` exists + +#### Scenario: Released stores still open with their results + +- **WHEN** the committed 0.10.2 store is opened +- **THEN** every model and memory loads and every recorded query returns the rows recorded from 0.10.2 diff --git a/openspec/specs/queries/casts/spec.md b/openspec/specs/queries/casts/spec.md new file mode 100644 index 000000000..19e7266dc --- /dev/null +++ b/openspec/specs/queries/casts/spec.md @@ -0,0 +1,68 @@ +# queries/casts Specification + +## Purpose +Defines casts in Mode-B expressions: `CAST` / `::` as the database's own cast of the author's type, and `TRY_CAST` as the one portable cast that turns an invalid input into NULL on every dialect. + +## Requirements + +### Requirement: CAST is the database's cast of the author's type + +`CAST(x AS )` and `x::` SHALL be accepted in every Mode-B position, chains included (`x::int::text`). The cast SHALL render the author's type, parameters included (`DECIMAL(10,2)`, `VARCHAR(3)`), changing only its spelling to the target dialect's; its value semantics — rounding, parsing, invalid input — SHALL be the database's own. A type the target dialect cannot express SHALL be rejected at plan time with a typed query error naming the type and the dialect. + +#### Scenario: Author's parameterised type is kept + +- **WHEN** a measure on Postgres is `sum(CAST(amount AS DECIMAL(10,2)))` +- **THEN** the generated SQL casts to `DECIMAL(10, 2)` and the value is Postgres's own + +#### Scenario: Spelling transpiled to the dialect + +- **WHEN** a dimension is `amount::text` on MySQL +- **THEN** the generated SQL is valid MySQL casting to its character type + +#### Scenario: Chained casts + +- **WHEN** a dimension is `amount::int::text` +- **THEN** it casts to INT, then to TEXT + +#### Scenario: Invalid input follows the database + +- **WHEN** a dimension is `CAST(status AS INT)` over a row holding `'abc'` +- **THEN** Postgres fails the query with its own error and SQLite returns 0, as each database's `CAST` does + +### Requirement: A cast's result type is inferred + +A cast SHALL type as SLayer's data type for its target: the integer family as INT; floating-point, `REAL`, `DECIMAL` and `NUMERIC` as DOUBLE; character types as TEXT; `BOOLEAN` as BOOLEAN; `DATE` as DATE; `TIMESTAMP` and `DATETIME` as TIMESTAMP; any other type as UNKNOWN. The inferred type SHALL drive checking and response types exactly as a column's declared type does. + +#### Scenario: Cast to DATE is a date operand + +- **WHEN** a dimension is `date_diff('day', CAST(created_txt AS DATE), DATE '2025-01-01')` over a TEXT column `created_txt` +- **THEN** it binds without a date-operand type error and the response reports INT + +#### Scenario: Unknown target type + +- **WHEN** a dimension is `CAST(payload AS JSON)` on Postgres +- **THEN** it renders and its response type is UNKNOWN + +### Requirement: TRY_CAST turns invalid input into NULL on every dialect + +`TRY_CAST(x AS )` SHALL render the author's type like `CAST` and SHALL yield NULL, never an error or a substitute value, for an input the cast cannot convert, on every Tier-1 dialect. Where the database cannot itself tell a valid input from an invalid one (SQLite, MySQL numeric targets), a number SHALL be valid when it is an optionally signed decimal with an optional fraction and exponent, surrounding spaces ignored; a boolean when it is one of `true`, `false`, `t`, `f`, `yes`, `no`, `1`, `0` in any case; a date or timestamp when it is an ISO time-point shape naming a real calendar value. On Postgres, a server older than 16 — or one whose version cannot be determined, including a dry run without a connection — SHALL get a plain `CAST`, so an invalid input fails the query there; this is documented. + +#### Scenario: Invalid text becomes NULL + +- **WHEN** a dimension is `TRY_CAST(status AS INT)` over rows holding `'12'` and `'abc'` +- **THEN** they yield 12 and NULL on SQLite, DuckDB, MySQL and Postgres 16 + +#### Scenario: SQLite does not truncate to a prefix + +- **WHEN** a dimension is `TRY_CAST(status AS INT)` over a row holding `'12abc'` on SQLite +- **THEN** it yields NULL, not 12 + +#### Scenario: Postgres before 16 + +- **WHEN** the same query runs against a Postgres server reporting version 15 +- **THEN** the generated SQL is a plain `CAST`, and a row holding `'abc'` fails the query with Postgres's error + +#### Scenario: Postgres 16 uses the native validity check + +- **WHEN** the query runs against a Postgres server reporting version 16 or later +- **THEN** the generated SQL guards the cast with Postgres's own input-validity check and `'abc'` yields NULL diff --git a/openspec/specs/queries/date-functions/spec.md b/openspec/specs/queries/date-functions/spec.md index 45cc84069..476334aa9 100644 --- a/openspec/specs/queries/date-functions/spec.md +++ b/openspec/specs/queries/date-functions/spec.md @@ -179,11 +179,14 @@ to NULL SHALL yield NULL. ### Requirement: The interval spelling is date_add `x + interval(n, unit)`, `interval(n, unit) + x` and `x - interval(n, unit)` SHALL mean exactly `date_add(x, n, unit)` (for subtraction, `date_add(x, -n, unit)`) and SHALL resolve to the same -value as that call, sharing its result key and SQL. Chains SHALL apply left to right. `interval` -SHALL follow the same count and unit rules as `date_add`. `interval(...)` in any other position — -standalone, as the left side of `-`, negated, multiplied, compared, nested in another `interval`, -added to another `interval`, or as a function argument — SHALL be rejected before binding with an -error pointing to `date_add`. +value as that call, sharing its result key and SQL. The SQL interval literals `INTERVAL ' '`, +`INTERVAL '' ` and `INTERVAL ` (one unit; keyword and unit case-insensitive; a +unit's plural form accepted) SHALL mean `interval(n, unit)` in all of these positions. Chains SHALL +apply left to right. `interval` SHALL follow the same count and unit rules as `date_add`. An interval +in any other position — standalone, as the left side of `-`, negated, multiplied, compared, nested in +another interval, added to another interval, or as a function argument — SHALL be rejected before +binding with an error pointing to `date_add`, and an interval literal naming more than one unit +(`INTERVAL '1 day 2 hours'`) SHALL be rejected with an error suggesting a chain. #### Scenario: Operator spelling equals date_add - **WHEN** one measure uses `max(shipped_at - interval(2, 'day'))` and another `max(date_add(shipped_at, -2, 'day'))` @@ -201,6 +204,14 @@ error pointing to `date_add`. - **WHEN** a filter uses `interval(7, 'day') - order_date > 0`, `-interval(7, 'day') + order_date > order_date`, or `order_date + interval(1, 'day') * 2 > order_date` - **THEN** each is rejected with an error pointing to `date_add` +#### Scenario: SQL interval literals equal date_add +- **WHEN** filters use `order_date + INTERVAL '7 days' > CURRENT_DATE`, `order_date + INTERVAL '7' DAY > CURRENT_DATE` and `order_date + INTERVAL 7 DAY > CURRENT_DATE` +- **THEN** each binds as `date_add(order_date, 7, 'day') > current_date()` + +#### Scenario: Multi-unit interval literal rejected +- **WHEN** a filter uses `order_date + INTERVAL '1 day 2 hours' > now()` +- **THEN** it is rejected with an error suggesting `+ INTERVAL '1 day' + INTERVAL '2 hours'` + ### Requirement: Date operands must be temporal Every date operand (`ts`, `start`, `end`) SHALL have a determinable DATE or TIMESTAMP type. A temporal operand SHALL be one of: a column (local, joined or stage) declared DATE or TIMESTAMP; @@ -410,3 +421,22 @@ Monday = 1 … Sunday = 7) and `date_part('day_of_week', x) % 7` (Sunday = 0 … #### Scenario: Custom aggregation named weekday still works - **WHEN** a model declares a custom aggregation `weekday` and a measure uses `weekday(amount)` - **THEN** the measure renders the model's `weekday` aggregation + +### Requirement: SQL date literals and clock keywords +`DATE ''` SHALL be a DATE value and `TIMESTAMP ''` a TIMESTAMP value, accepting +exactly the shapes and calendar checks of ISO literals in date positions, in every position; a +malformed or impossible value SHALL be rejected with the same typed query error. `CURRENT_DATE` SHALL +mean `current_date()` and `CURRENT_TIMESTAMP` SHALL mean `now()`, sharing their result keys, SQL and +result-cache bypass. + +#### Scenario: Typed literals +- **WHEN** a filter uses `created_at >= TIMESTAMP '2024-01-01 00:00'` and a dimension uses `date_diff('day', DATE '2024-01-01', created_at)` +- **THEN** both bind with typed temporal literals and execute + +#### Scenario: Impossible typed literal +- **WHEN** a filter uses `created_at >= DATE '2024-02-30'` +- **THEN** the query is rejected with the type error naming the literal + +#### Scenario: Clock keywords +- **WHEN** one filter uses `created_at >= CURRENT_DATE` and another `created_at >= current_date()` +- **THEN** both render identical SQL and neither query is served from or stored in the result cache diff --git a/openspec/specs/queries/expression-grammar/spec.md b/openspec/specs/queries/expression-grammar/spec.md new file mode 100644 index 000000000..1661455c0 --- /dev/null +++ b/openspec/specs/queries/expression-grammar/spec.md @@ -0,0 +1,191 @@ +# queries/expression-grammar Specification + +## Purpose +Defines the Mode-B expression grammar: one SQL-compatible syntax shared by every query and model formula position — its operators and predicates, identifiers and quoting, string literals, precedence, the legacy spellings it keeps, and the forms it rejects with a hint. + +## Requirements + +### Requirement: One grammar in every Mode-B position + +Query measures, dimensions (string and object form), filters, order targets, `ModelMeasure.formula` and `ModelExtension.measures` SHALL all be parsed by the same grammar: any construct accepted in one of these positions SHALL be accepted syntactically in all of them, and whether it is legal there SHALL be decided only by typing (field / measure / position rules), never by the position's parser. + +#### Scenario: SQL predicate spellings in a measure + +- **WHEN** a query on `orders` has measure `sum(iif(status = 'paid' AND amount IS NOT NULL, amount, 0))` +- **THEN** it parses, binds and returns the paid total — the same value as `sum(iif(status == 'paid' and amount is not None, amount, 0))` + +#### Scenario: Same text in filter, measure and order + +- **WHEN** `amount BETWEEN 10 AND 20` is used as a filter, inside `sum(iif(amount BETWEEN 10 AND 20, 1, 0))` as a measure, and `iif(amount BETWEEN 10 AND 20, 0, 1)` as an order target on a raw-rows query +- **THEN** each parses and evaluates the same predicate + +### Requirement: SQL operators and predicates + +The grammar SHALL accept, with keywords matched case-insensitively: + +- comparison `=`, `<>`, `!=`, `==`, `<`, `<=`, `>`, `>=`; +- `IS [NOT] NULL`, `IS [NOT] DISTINCT FROM`, `IS [NOT] TRUE`, `IS [NOT] FALSE`; +- `[NOT] IN (, …)` and `[NOT] IN [, …]`; +- `[NOT] BETWEEN a AND b`, meaning `x >= a AND x <= b` (negated: `NOT (x >= a AND x <= b)`); +- `AND`, `OR`, `NOT`; +- `||` string concatenation; +- arithmetic `+`, `-`, `*`, `/`, `%` (the result's sign is the dividend's on every dialect), `**` (power) and unary `-`. + +`IS [NOT] DISTINCT FROM` SHALL be a null-safe comparison on every dialect. + +#### Scenario: BETWEEN and NOT BETWEEN + +- **WHEN** a query on `orders` filters `amount BETWEEN 10 AND 20`, and another filters `amount NOT BETWEEN 10 AND 20` +- **THEN** the first keeps exactly the rows with `10 <= amount <= 20` and the second exactly the rows with a non-NULL amount outside that range + +#### Scenario: Null-safe comparison + +- **WHEN** a query filters `region IS NOT DISTINCT FROM NULL` +- **THEN** it keeps exactly the rows whose region is NULL, on every Tier-1 dialect + +#### Scenario: Boolean tests + +- **WHEN** a query filters `(amount > 100) IS TRUE` and another `(amount > 100) IS NOT TRUE` +- **THEN** the second keeps the rows the first drops, including rows whose amount is NULL + +#### Scenario: Concatenation operator + +- **WHEN** a dimension is `{"expression": "status || '-' || region", "name": "tag"}` +- **THEN** each row's `tag` is the two values joined by `-`, and NULL when either is NULL + +#### Scenario: Modulo sign + +- **WHEN** a measure is `min(-7 % 3)` +- **THEN** it returns `-1` on every Tier-1 dialect + +### Requirement: LIKE and ILIKE are operators over any operand + +`x [NOT] LIKE p` and `x [NOT] ILIKE p` SHALL accept any expression on either side. `ILIKE` SHALL match case-insensitively on every dialect. + +#### Scenario: LIKE over a nested call + +- **WHEN** a query filters `lower(trim(status)) LIKE 'pa%'` +- **THEN** it parses and keeps the rows whose trimmed, lower-cased status starts with `pa` + +#### Scenario: ILIKE + +- **WHEN** a query filters `status ILIKE 'PA%'` on data holding `paid` and `Pending` +- **THEN** it keeps `paid` and drops `Pending`, on every Tier-1 dialect + +#### Scenario: LIKE inside a string literal is text + +- **WHEN** a filter is `note = 'we like cats'` +- **THEN** the literal is compared byte-for-byte and no LIKE is formed + +### Requirement: SQL-standard call forms + +The grammar SHALL accept `POSITION(a IN b)` (meaning `instr(b, a)`) and `SUBSTRING(x FROM a [FOR b])` (meaning `substr(x, a[, b])`). + +#### Scenario: SQL substring and position + +- **WHEN** a dimension is `SUBSTRING(status FROM 1 FOR 3)` and another `POSITION('a' IN status)` +- **THEN** they equal `substr(status, 1, 3)` and `instr(status, 'a')`, with the same result keys and SQL + +### Requirement: Precedence and associativity + +Operators SHALL bind, loosest first: `OR`; `AND`; `NOT`; comparison, `IS`, `IN`, `BETWEEN`, `LIKE`, `ILIKE`; `||`; `+` `-`; `*` `/` `%`; unary `-`; `**`; `::`; postfix (call, `FILTER`, `.`). `**` SHALL be right-associative and its right operand MAY carry a unary minus. Parentheses override everything. + +#### Scenario: NOT and comparison + +- **WHEN** a filter is `NOT amount > 100 AND status = 'paid'` +- **THEN** it means `(NOT (amount > 100)) AND (status = 'paid')` + +#### Scenario: Power + +- **WHEN** measures are `min(2 ** 3 ** 2)`, `min(-2 ** 2)`, `min((-2) ** 2)` and `min(2 ** -2)` +- **THEN** they return 512, -4, 4 and 0.25 + +#### Scenario: Concatenation binds tighter than comparison + +- **WHEN** a filter is `status || 'x' = 'paidx'` +- **THEN** it means `(status || 'x') = 'paidx'` + +### Requirement: Identifiers and quoted identifiers + +A bare identifier or dotted path SHALL name a column, measure, alias or join path as before. A double-quoted name (`"order total"`, `customers."Region"`) SHALL be an identifier in every identifier position, with `""` standing for one `"` inside it. A name beginning `__slayer_` SHALL be rejected, quoted or not. A double-quoted name that resolves to nothing SHALL fail with the unresolved-reference error plus a hint that strings use single quotes. + +#### Scenario: Quoted column name + +- **WHEN** model `orders` has a column named `order total` and a query has measure `sum("order total")` +- **THEN** it sums that column + +#### Scenario: Quoted segment in a dotted path + +- **WHEN** a query on `orders` has dimension `customers."Region"` and `customers` has a column named `Region` +- **THEN** it groups by that column + +#### Scenario: Double-quoted string hint + +- **WHEN** a query on `orders` filters `status = "paid"` and no column or alias is named `paid` +- **THEN** it fails with the unresolved-reference error naming `paid` and stating that strings use single quotes + +### Requirement: String literals follow SQL rules + +A string literal SHALL be single-quoted, with `''` standing for one `'`; a backslash SHALL be an ordinary character. + +#### Scenario: Doubled quote + +- **WHEN** a query filters `customer_name = 'O''Brien'` +- **THEN** it compares against `O'Brien` + +#### Scenario: Backslash is literal + +- **WHEN** a query filters `path = 'C:\temp'` +- **THEN** it compares against the six characters `C:\temp` + +### Requirement: Legacy spellings keep their meaning + +`==`, `!=`, `is None`, `is not None`, `True`, `False`, `None`, `[…]` lists after `in`, the method-call aggregation form (`customers.sum(revenue)`) and the colon aggregation form (`revenue:sum`) SHALL keep their meaning, result keys and SQL. Every expression that parsed before this change and contains none of the forms this change removes SHALL bind to the same value with byte-identical generated SQL. + +#### Scenario: Python spellings + +- **WHEN** a filter is `status == 'paid' and region is not None` and another is `status = 'paid' AND region IS NOT NULL` +- **THEN** both render identical SQL + +#### Scenario: Pre-existing expressions are unchanged + +- **WHEN** the SQL golden baselines committed before this change are regenerated +- **THEN** they are byte-identical + +### Requirement: Keyword arguments versus comparison + +Inside an aggregation, a transform or an unknown call, `name = value` directly after `(` or `,` SHALL be a keyword argument. Inside a scalar function call and everywhere else, `=` SHALL be a comparison. + +#### Scenario: Keyword argument + +- **WHEN** a measure is `percentile(amount, p=0.9)` +- **THEN** `p` is a keyword argument + +#### Scenario: Comparison inside a scalar call + +- **WHEN** a measure is `sum(iif(status = 'paid', amount, 0))` +- **THEN** `status = 'paid'` is the condition of `iif` + +### Requirement: Forms rejected with a hint + +The following SHALL fail at parse time with an error naming the form and the remedy: `a | b` (use `||`); `a // b` (use `div(a, b)` or `floor(a / b)`); `TRIM(BOTH | LEADING | TRAILING …)`; a scalar `min(a, b)` / `max(a, b)` with two positional values (use `greatest` / `least`); `TIME '…'` literals; `SIMILAR TO`; `REGEXP` / `RLIKE` operators; raw `OVER (…)`. A syntax error SHALL report the character position where parsing failed. + +#### Scenario: Bitwise-or spelling + +- **WHEN** a measure is `min(status | region)` +- **THEN** it fails naming `|` and suggesting `||` + +#### Scenario: Floor-division spelling + +- **WHEN** a measure is `sum(amount) // 2` +- **THEN** it fails suggesting `div(a, b)` or `floor(a / b)` + +#### Scenario: Two-argument scalar max + +- **WHEN** a dimension is `max(amount, 10)` +- **THEN** it fails suggesting `greatest(amount, 10)` + +#### Scenario: Syntax error position + +- **WHEN** a filter is `amount > > 5` +- **THEN** the error names the position of the second `>` diff --git a/openspec/specs/queries/expression-keywords/spec.md b/openspec/specs/queries/expression-keywords/spec.md index 6bbf96d1b..24686d3e3 100644 --- a/openspec/specs/queries/expression-keywords/spec.md +++ b/openspec/specs/queries/expression-keywords/spec.md @@ -14,10 +14,10 @@ A Mode-B expression SHALL accept SQL `CASE … END` conditionals — searched (`CASE WHEN cond THEN val … [ELSE val] END`) and simple (`CASE operand WHEN val THEN val … [ELSE val] END`) — lowering them to nested `iif(cond, then, otherwise)` calls with `None` as the default otherwise. WHEN -conditions SHALL accept SQL operator spellings (`=`, `<>`, `AND`/`OR`/`NOT`, -`IS [NOT] NULL`, `[NOT] IN`, `[NOT] LIKE`) in every expression position, including -measures. A `CASE` that is recognized as a conditional (a `WHEN` follows it) but is -malformed MUST raise a specific malformed-CASE error naming the defect. +conditions, THEN values and ELSE values SHALL accept the full Mode-B grammar in every +expression position, including measures. A `CASE` that is recognized as a conditional +(a `WHEN` follows it) but is malformed MUST raise a specific malformed-CASE error naming +the defect. #### Scenario: searched CASE lowers to iif @@ -37,7 +37,7 @@ malformed MUST raise a specific malformed-CASE error naming the defect. #### Scenario: SQL operator spellings inside WHEN conditions - WHEN `CASE WHEN region = 'EU' AND amount IS NOT NULL THEN 1 ELSE 0 END` appears in a measure formula -- THEN the WHEN condition is normalized (`==`, `and`, `is not None`) and the expression parses +- THEN the WHEN condition parses with the same meaning as `region == 'EU' and amount is not None` #### Scenario: recognized-but-malformed CASE still errors specifically @@ -95,38 +95,3 @@ leaf regardless of whitespace around the dots. - WHEN `CASE case WHEN 1 THEN 2 END` is parsed - THEN the ambiguous bare keyword-named operand raises an error (never silent corruption), and the parenthesized form `CASE (case) WHEN 1 THEN 2 END` parses correctly with `case` as the operand reference - -### Requirement: LIKE operator rewriting - -A Mode-B filter SHALL accept `lhs [NOT] LIKE 'pattern'` — LHS a bare or dotted -identifier or single scalar call, pattern a single-quoted string literal with -backslash-escape support — rewriting it to the `like(lhs, pattern)` scalar (negated: -`not like(...)`). The keyword match SHALL be ASCII-exact (any ASCII casing; never via -Unicode case folding) and SHALL never apply inside a string literal. A double-quoted -pattern is NOT rewritten (in SQL sources double quotes denote identifiers), so it fails -loudly rather than silently changing meaning. - -#### Scenario: basic LIKE and NOT LIKE - -- WHEN `name LIKE 'a%'` / `name NOT LIKE 'a%'` / `lower(customers.email) like '%@x.io'` appear in a filter -- THEN each rewrites to the corresponding `like(...)` / `not like(...)` scalar call - -#### Scenario: escaped quote inside the pattern - -- WHEN `col LIKE 'It\'s%'` appears in a filter -- THEN the full pattern including the escaped quote is preserved as the second argument - -#### Scenario: LIKE inside a string literal is untouched - -- WHEN a filter contains ` like ` only inside a string literal, e.g. `note == "we like 'cats'"` -- THEN the literal is preserved byte-for-byte and no rewrite occurs - -#### Scenario: case-fold keyword spoofs are not LIKE - -- WHEN a filter contains `x lıke 'p%'` or `x liKe 'p%'` (dotless ı / KELVIN SIGN fold to `like`) -- THEN no rewrite occurs (the token is an ordinary identifier), while ASCII `x LiKe 'p%'` still rewrites - -#### Scenario: double-quoted pattern is rejected loudly - -- WHEN `col like "p%"` appears in a filter -- THEN parsing fails with an invalid-expression error rather than rewriting to a string match diff --git a/openspec/specs/queries/positions/spec.md b/openspec/specs/queries/positions/spec.md index aaa9a04fd..917d21775 100644 --- a/openspec/specs/queries/positions/spec.md +++ b/openspec/specs/queries/positions/spec.md @@ -162,3 +162,26 @@ the invalid-name error. #### Scenario: Object with a model keeps the error - **WHEN** an order entry is `{"column": {"name": "sum(amount)", "model": "orders"}}` - **THEN** the query is rejected with the invalid-name error + +### Requirement: Order items carry an explicit NULL placement +An order item SHALL accept `nulls` set to `first` or `last`. With it, the sorted rows SHALL place NULL values first or last on every Tier-1 dialect, SQL Server included, for every kind of order target — a projected field, a hidden filter- or order-only value, a measure, a transform or an expression. Without it, NULL placement SHALL be unchanged. `nulls` SHALL survive serialisation, saved-query refinement and the stage rewrites that rebuild order items. + +#### Scenario: NULLS FIRST on a descending order +- **WHEN** a raw-rows query on `orders` orders by `{"column": "discount", "direction": "desc", "nulls": "first"}` +- **THEN** the rows with a NULL discount come first, then the rest by discount descending, on every Tier-1 dialect + +#### Scenario: NULLS LAST on a hidden aggregate +- **WHEN** a query groups by `region` and orders by `{"column": "max(discount)", "direction": "asc", "nulls": "last"}` without projecting `max(discount)` +- **THEN** regions whose `max(discount)` is NULL come last, and the hidden value is not returned + +#### Scenario: SQL Server honours the policy over an aliased target +- **WHEN** the same two queries run on SQL Server +- **THEN** they return the same row order as on Postgres + +#### Scenario: Refinement keeps the policy +- **WHEN** a saved query ordering with `"nulls": "first"` is run with a refinement adding a filter +- **THEN** the refined result keeps NULLs first + +#### Scenario: Invalid policy +- **WHEN** an order item has `"nulls": "middle"` +- **THEN** the query is rejected at construction naming the allowed values diff --git a/openspec/specs/queries/variables/spec.md b/openspec/specs/queries/variables/spec.md index beff0f529..d169444d2 100644 --- a/openspec/specs/queries/variables/spec.md +++ b/openspec/specs/queries/variables/spec.md @@ -7,7 +7,7 @@ Governs `{var}` placeholder substitution across query and model surfaces: which ### Requirement: Every Mode-B formula surface substitutes variables -`{var}` placeholders SHALL be substituted from the query's effective variables in a query's filters, measure formulas, computed-dimension expressions (string and object form), and order expressions, using the Mode-B escaping regime: a string value is escaped so it stays inside the quoted literal the author wrote, a number or boolean is inserted verbatim, a list renders a Mode-B tuple, and `{{` / `}}` render literal braces. A query carrying placeholders on these surfaces SHALL construct successfully and SHALL execute with the substituted values. +`{var}` placeholders in a query's filters, measure formulas, computed-dimension expressions (string and object form) and order expressions SHALL be bound from the query's effective variables as literal values, never as text spliced into the expression. A placeholder inside a string literal, wholly or partly (`'{status}'`, `'%{q}%'`), SHALL contribute its value's text to that literal; a placeholder in a bare literal position SHALL take a number or boolean value, a numeric string reading as its number; a placeholder forming the whole parenthesised list of `IN` SHALL take a list value. `{{` and `}}` inside a string literal SHALL render literal braces. A query carrying placeholders on these surfaces SHALL construct successfully and SHALL execute with the bound values. #### Scenario: Measure formula @@ -32,11 +32,11 @@ Governs `{var}` placeholder substitution across query and model surfaces: which #### Scenario: Quoted string value is escaped - **WHEN** a measure contains `'{name}'` inside a quoted literal and `variables={"name": "O'Brien"}` -- **THEN** the value stays inside the literal and the query executes without a parse error +- **THEN** the literal's value is `O'Brien` and the query executes without a parse error ### Requirement: Saved measure formulas substitute on the models whose SQL surfaces substitute -A saved `ModelMeasure.formula` SHALL substitute `{var}` under the Mode-B regime on exactly the models whose Mode-A surfaces substitute (the query's direct source model and each stage's own source model), using that model's effective variables. Its placeholders SHALL be reported by model variable inspection as required unless the model defaults them. A saved measure reached through a join or a cross-model reference SHALL NOT substitute, and a placeholder in it SHALL raise the unresolved-placeholder error. +A saved `ModelMeasure.formula` SHALL bind `{var}` placeholders as literal values, by the same rules as query formula surfaces, on exactly the models whose Mode-A surfaces substitute (the query's direct source model and each stage's own source model), using that model's effective variables. Its placeholders SHALL be reported by model variable inspection as required unless the model defaults them. A saved measure reached through a join or a cross-model reference SHALL NOT substitute, and a placeholder in it SHALL raise the unresolved-placeholder error. #### Scenario: Saved measure on the source model @@ -147,3 +147,27 @@ A `{name}` placeholder that reaches query binding unsubstituted SHALL raise a ty - **WHEN** a measure is `sum(amount) * {a, b}` - **THEN** parsing it fails as unsupported syntax, and executing it fails with the invalid-variable-name error, as in filters + +### Requirement: Variables supply literal values only + +A placeholder in any other position — an identifier or path segment (`{col}:sum`, `t.{c}`), a function name (`{f}(x)`), a keyword argument's name, or a value that must be a reference (`partition_by={x}`) — SHALL be rejected with an error stating that variables supply literal values. A string value bound into a bare literal position SHALL be rejected with the same error plus a hint to put the placeholder inside quotes. No variable value SHALL ever change the structure of the expression it is bound into. + +#### Scenario: Identifier injection rejected + +- **WHEN** a query on `orders` has measure `sum({col})` with `variables={"col": "amount"}` +- **THEN** it fails with the literal-values error and the hint to quote the placeholder + +#### Scenario: Function and path positions rejected + +- **WHEN** a measure is `{f}(amount)`, or a dimension is `customers.{c}`, or a measure is `sum(amount, partition_by={x})` +- **THEN** each fails with the literal-values error + +#### Scenario: Numeric string in a bare position + +- **WHEN** a filter is `amount > {min}` with `variables={"min": "10"}` +- **THEN** it keeps the rows with amount above 10 + +#### Scenario: Hostile string stays a literal + +- **WHEN** a filter is `status = '{s}'` with `variables={"s": "x' OR '1' = '1"}` +- **THEN** it keeps only the rows whose status equals that whole string — none, on data without it diff --git a/poetry.lock b/poetry.lock index df678aacb..4ab1a1c80 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5741,6 +5741,21 @@ typing-extensions = {version = ">=4.9", markers = "python_version < \"3.13\" and docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx_rtd_theme"] test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"] +[[package]] +name = "pyparsing" +version = "3.3.3" +description = "pyparsing - Classes and methods to define and execute parsing grammars" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pyparsing-3.3.3-py3-none-any.whl", hash = "sha256:ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4"}, + {file = "pyparsing-3.3.3.tar.gz", hash = "sha256:928ae7e20211f3b6f3915a72f06a0cfd29ab9d24279dd6346b6b1a7146397d36"}, +] + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + [[package]] name = "pytest" version = "9.0.3" @@ -8282,4 +8297,4 @@ trino = ["trino"] [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "e318b76bd724a2ca9aa8658ba05c53396639fff8ceb2dc4523df6097f292af38" +content-hash = "bea59f700fb0ee40d6f20183dbe0341ac5238cb3fceaa945925dc8922864df8b" diff --git a/pyproject.toml b/pyproject.toml index 1834a4d4e..6547e3a00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ sqlglot = {version = ">=30.17", extras = ["c"]} sqlalchemy = ">=2.0,<2.1" greenlet = ">=1" pydantic = ">=2.0" +pyparsing = ">=3.1" python-dateutil = ">=2.8" pyyaml = ">=6.0" fastapi = ">=0.100" diff --git a/slayer/core/enums.py b/slayer/core/enums.py index 15c10c210..405fd6096 100644 --- a/slayer/core/enums.py +++ b/slayer/core/enums.py @@ -272,6 +272,24 @@ def invert_cardinality( # Named once so classifier, planner and renderer agree. RANKED_AGGREGATIONS = ("first", "last") +# Transforms that require a time dimension for ORDER BY +TIME_TRANSFORMS = { + "cumsum", + "change", + "change_pct", + "time_shift", + "first", + "last", + "lag", + "lead", + "consecutive_periods", +} + +# Transforms that don't need time ordering +TIMELESS_TRANSFORMS = {"rank", "percent_rank", "dense_rank", "ntile"} + +ALL_TRANSFORMS = TIME_TRANSFORMS | TIMELESS_TRANSFORMS + # Default partition is the whole result set, not the group-by dimensions; # ``partition_by=`` opts into per-partition ranking. RANK_FAMILY_TRANSFORMS = {"rank", "percent_rank", "dense_rank", "ntile"} diff --git a/slayer/core/errors.py b/slayer/core/errors.py index 9b44c61e2..b5c9f1fb4 100644 --- a/slayer/core/errors.py +++ b/slayer/core/errors.py @@ -407,6 +407,14 @@ class UnrelatedDatasetError(QueryTypeError): """A row-level read of a dataset no join path reaches, outside an aggregation or a one-sided filter.""" +class CastTypeError(QueryTypeError): + """A CAST target type the query's dialect cannot express.""" + + +class AggregationFilterError(QueryTypeError): + """A ``FILTER (WHERE …)`` condition that is not a row-level predicate of its aggregation.""" + + class UnreachableDefinitionError(SlayerError, ValueError): """A stored ``Column.sql`` / ``Column.filter`` / model filter naming a dataset no join path reaches from its model.""" diff --git a/slayer/core/formula.py b/slayer/core/formula.py deleted file mode 100644 index 9ed75004b..000000000 --- a/slayer/core/formula.py +++ /dev/null @@ -1,924 +0,0 @@ -"""Formula parser for SLayer measure formulas. - -Parses formula strings into structured FieldSpec objects using Python's ast module. - -A formula can be: -- An aggregated measure ref: "sum(revenue)" → AggregatedMeasureRef -- Star count: "count(*)" → AggregatedMeasureRef("*", "count") -- With agg args: "weighted_avg(price, weight=quantity)" → AggregatedMeasureRef with kwargs -- Arithmetic: "sum(revenue) / count(*)" → ArithmeticField -- A transform: "cumsum(sum(revenue))" → TransformField wrapping AggregatedMeasureRef -- Nested transforms: "change(cumsum(sum(revenue)))" → TransformField wrapping TransformField -- Arithmetic on transforms: "cumsum(sum(revenue)) / count(*)" → MixedArithmeticField - -Functional aggregations are rewritten internally to the legacy colon spelling -(``revenue:sum``), which this parser also accepts as input. -""" - -import ast -import difflib -import io -import re -import tokenize -from typing import Any, Literal -from collections.abc import Mapping - -from pydantic import BaseModel, Field - -from slayer.core.direction import rank_direction, with_direction_kwarg -from slayer.core.enums import ( - BUILTIN_AGGREGATIONS, - RANK_FAMILY_TRANSFORMS, - normalize_aggregation_name, -) -from slayer.core.keys import SCALAR_FUNCTIONS -from slayer.core.refs import ( - AGG_REF_RE as _AGG_REF_RE, - IDENT_OR_PATH_RE as _IDENT_OR_PATH_RE, -) - -# Transforms that require a time dimension for ORDER BY -TIME_TRANSFORMS = { - "cumsum", - "change", - "change_pct", - "time_shift", - "first", - "last", - "lag", - "lead", - "consecutive_periods", -} - -# Transforms that don't need time ordering -TIMELESS_TRANSFORMS = {"rank", "percent_rank", "dense_rank", "ntile"} - -ALL_TRANSFORMS = TIME_TRANSFORMS | TIMELESS_TRANSFORMS - -CallCategory = Literal["transform", "scalar", "unknown"] - - -def _classify_call_name(name: str) -> CallCategory: - """Categorize a Mode B function-call identifier.""" - if name in ALL_TRANSFORMS: - return "transform" - if name.lower() in SCALAR_FUNCTIONS: - return "scalar" - return "unknown" - - -def _validate_scalar_call(node: ast.Call, original: str) -> None: - """Arity check for top-level scalar calls. ``round`` / ``abs`` have specific - shapes worth catching early; other scalars in ``SCALAR_FUNCTIONS`` have - variable arity (``coalesce``, ``greatest``, …) and are validated by sqlglot - at SQL-emission time.""" - name = node.func.id.lower() - if node.keywords: - raise ValueError( - f"'{name}' does not accept keyword arguments. Formula: {original!r}" - ) - if name == "abs": - if len(node.args) != 1: - raise ValueError( - f"'abs' requires exactly one argument. Formula: {original!r}" - ) - elif name == "round": - if len(node.args) not in (1, 2): - raise ValueError( - f"'round' accepts 1 or 2 arguments (expression[, ndigits]). " - f"Formula: {original!r}" - ) - if len(node.args) == 2 and not _is_integer_literal(node.args[1]): - raise ValueError( - f"'round' ndigits (2nd argument) must be an integer literal. " - f"Formula: {original!r}" - ) - - -def _is_integer_literal(node: ast.AST) -> bool: - """True for an integer constant or a negated integer constant (``-2``). - - Booleans are rejected — ``True``/``False`` are ``int`` subclasses but are - not meaningful ndigits values. - """ - if isinstance(node, ast.Constant): - return isinstance(node.value, int) and not isinstance(node.value, bool) - if ( - isinstance(node, ast.UnaryOp) - and isinstance(node.op, ast.USub) - and isinstance(node.operand, ast.Constant) - ): - v = node.operand.value - return isinstance(v, int) and not isinstance(v, bool) - return False - - -class AggregatedMeasureRef(BaseModel): - """A measure reference with explicit aggregation. - - Examples: - "sum(revenue)" → AggregatedMeasureRef("revenue", "sum") - "count(*)" → AggregatedMeasureRef("*", "count") - "sum(customers.revenue)" → AggregatedMeasureRef("customers.revenue", "sum") - "weighted_avg(price, weight=quantity)" → AggregatedMeasureRef("price", "weighted_avg", - agg_kwargs={"weight": "quantity"}) - "last(revenue, ordered_at)" → AggregatedMeasureRef("revenue", "last", - agg_args=["ordered_at"]) - """ - measure_name: str = Field(description="Measure name, e.g. 'revenue', 'customers.revenue', '*'") - aggregation_name: str = Field(description="Aggregation name, e.g. 'sum', 'weighted_avg'") - agg_args: list[str] = Field(default_factory=list, description="Positional aggregation args") - agg_kwargs: dict[str, str] = Field(default_factory=dict, description="Keyword aggregation args") - - -class ArithmeticField(BaseModel): - """An arithmetic expression over measures only (no transform calls inside).""" - sql: str = Field(description="Preprocessed formula with placeholders for aggregated refs") - measure_names: list[str] = Field(description="Placeholder IDs or bare measure names") - agg_refs: dict[str, AggregatedMeasureRef] = Field(default_factory=dict) - is_predicate: bool = Field( - default=False, - description="True when the top-level AST node is a comparison or boolean op " - "(so the field renders as a boolean expression). Drives boolean-aware SQL " - "generation for transforms like consecutive_periods.", - ) - - -class TransformField(BaseModel): - """A transform function call, possibly wrapping another transform or arithmetic.""" - transform: str = Field(description="Transform name: cumsum, lag, lead, change, change_pct, rank, percent_rank, dense_rank, ntile, time_shift, first, last, consecutive_periods") - inner: "FieldSpec" = Field(description="The measure or expression being transformed") - args: list[Any] = Field(default_factory=list, description="Extra transform args (offset, granularity, etc.)") - kwargs: dict[str, Any] = Field( - default_factory=dict, - description=( - "Keyword args from the call site, e.g. partition_by=[...] for the " - "rank family, direction='desc' for rank / dense_rank or n=4 for ntile. " - "Validated per-transform at parse time." - ), - ) - - -class MixedArithmeticField(BaseModel): - """Arithmetic that contains transform sub-expressions. - - E.g., "cumsum(sum(revenue)) / count(*)" — the cumsum needs to be computed first - as a CTE step, then the arithmetic references its result. - """ - sql: str = Field(description="Preprocessed formula with placeholders") - measure_names: list[str] = Field(description="Placeholder IDs or bare measure names") - sub_transforms: list[tuple] = Field(description="List of (placeholder_name, TransformField)") - agg_refs: dict[str, AggregatedMeasureRef] = Field(default_factory=dict) - is_predicate: bool = Field( - default=False, - description="True when the top-level AST node is a comparison or boolean op " - "(so the field renders as a boolean expression). Drives boolean-aware SQL " - "generation for transforms like consecutive_periods.", - ) - - -# The parsed result of a single field -FieldSpec = AggregatedMeasureRef | ArithmeticField | TransformField | MixedArithmeticField - -# Rebuild TransformField which uses a forward reference to FieldSpec -TransformField.model_rebuild() - - -# --------------------------------------------------------------------------- -# Function-style aggregation rewrite -# --------------------------------------------------------------------------- - -# Aggregation names that are also transform names — ambiguous, need special handling -_AMBIGUOUS_AGG_TRANSFORMS = BUILTIN_AGGREGATIONS & ALL_TRANSFORMS # {"first", "last"} - -_STRING_LITERAL_RE = re.compile(r"'(?:[^'\\]|\\.)*'") - - -def _find_balanced_close(s: str, start: int) -> int: - """Find the index of the balanced closing paren starting after the open paren at `start`.""" - depth = 1 - i = start + 1 - in_string = False - string_char = "" - while i < len(s): - ch = s[i] - if in_string: - if ch == "\\" and i + 1 < len(s): - i += 2 # skip escaped character - continue - if ch == string_char: - in_string = False - elif ch in ("'", '"'): - in_string = True - string_char = ch - elif ch == "(": - depth += 1 - elif ch == ")": - depth -= 1 - if depth == 0: - return i - i += 1 - return -1 # unbalanced - - -def _rewrite_funcstyle_aggregations( - formula: str, - extra_agg_names: frozenset[str] | None = None, -) -> str: - """Rewrite function-style aggregation calls to the legacy colon spelling (internal). - - E.g., ``sum(revenue)`` → ``revenue:sum``, ``count(customers.*)`` → ``customers.*:count``. - - For aggregation names that are also transform names (``first``, ``last``), - the rewrite only fires when the first argument holds no colon, so a - transform over an already-rewritten aggregate (``last(revenue:sum)``) is left alone. - - Args: - formula: The formula string to rewrite. - extra_agg_names: Additional (custom) aggregation names to recognise. - """ - agg_names = BUILTIN_AGGREGATIONS | (extra_agg_names or frozenset()) - - # Build word-boundary regex for known aggregation names. - # Sort by length descending so longer names match first (e.g., count_distinct before count). - # Negative lookbehind for ':' avoids matching a legacy colon suffix (e.g., revenue:last(...)). - sorted_names = sorted(agg_names, key=len, reverse=True) - pattern = re.compile( - r'(?.*``. - if first_arg != "*" and not _IDENT_OR_PATH_RE.fullmatch(first_arg.removesuffix(".*")): - search_start = close_paren + 1 - continue - - remaining_args = [p.strip() for p in parts[1:]] - if remaining_args: - replacement = f"{first_arg}:{agg_name}({', '.join(remaining_args)})" - else: - replacement = f"{first_arg}:{agg_name}" - - formula = formula[:match.start()] + replacement + formula[close_paren + 1:] - rewritten = True - break # restart scanning from the beginning after a rewrite - - if not rewritten: - break - - return formula - - -def _split_args(s: str) -> list[str]: - """Split a comma-separated argument string respecting parentheses and quotes.""" - parts = [] - depth = 0 - current: list[str] = [] - in_string = False - string_char = "" - i = 0 - while i < len(s): - ch = s[i] - if in_string: - if ch == "\\" and i + 1 < len(s): - current.append(ch) - current.append(s[i + 1]) - i += 2 - continue - current.append(ch) - if ch == string_char: - in_string = False - elif ch in ("'", '"'): - current.append(ch) - in_string = True - string_char = ch - elif ch == "(": - depth += 1 - current.append(ch) - elif ch == ")": - depth -= 1 - current.append(ch) - elif ch == "," and depth == 0: - parts.append("".join(current)) - current = [] - else: - current.append(ch) - i += 1 - if current: - parts.append("".join(current)) - return parts - - -# --------------------------------------------------------------------------- -# Legacy colon-spelling preprocessing (regex source-of-truth lives in slayer.core.refs) -# --------------------------------------------------------------------------- - - -def _split_agg_arglist(args_str: str | None) -> tuple[list[str], dict[str, str]]: - """Parse a legacy colon-spelling ``(...)`` arglist (with parens, or ``None``) into - (positional, keyword): ``(weight=quantity)`` → ``([], {"weight": "quantity"})``.""" - agg_args: list[str] = [] - agg_kwargs: dict[str, str] = {} - if not args_str: - return agg_args, agg_kwargs - inner = args_str[1:-1].strip() - if not inner: - return agg_args, agg_kwargs - for part in inner.split(","): - part = part.strip() - if "=" in part: - key, val = part.split("=", 1) - agg_kwargs[key.strip()] = val.strip() - else: - agg_args.append(part) - return agg_args, agg_kwargs - - -def _preprocess_agg_refs( - formula: str, - custom_agg_names: frozenset[str] = frozenset(), -) -> tuple[str, dict[str, AggregatedMeasureRef]]: - """Replace legacy colon-spelling aggregated refs with placeholder identifiers. - - Returns (preprocessed_formula, {placeholder: AggregatedMeasureRef}). - An aggregation name exactly matching one of ``custom_agg_names`` (the reachable - custom aggregations) skips alias/casing healing, so the custom aggregation wins. - """ - refs: dict[str, AggregatedMeasureRef] = {} - counter = [0] - - def _replace(match: re.Match) -> str: - measure_name = match.group(1) - # Heal aggregation aliases/casing; unknown names pass through to the enrichment error. - raw_agg = match.group(2) - agg_name = ( - raw_agg if raw_agg in custom_agg_names - else normalize_aggregation_name(raw_agg) - ) - agg_args, agg_kwargs = _split_agg_arglist(match.group(3)) - - placeholder = f"__agg{counter[0]}__" - counter[0] += 1 - refs[placeholder] = AggregatedMeasureRef( - measure_name=measure_name, - aggregation_name=agg_name, - agg_args=agg_args, - agg_kwargs=agg_kwargs, - ) - return placeholder - - processed = _AGG_REF_RE.sub(_replace, formula) - return processed, refs - - -def _expand_named_measures( - formula: str, - named_measures: Mapping[str, str], - _visited: frozenset[str] = frozenset(), -) -> str: - """Inline-expand bare references to ``ModelMeasure`` saved formulas. - - For each NAME token whose value is a key in ``named_measures``, replaces it - with ``()``. The expansion is a textual - substitution that runs *before* the rest of the formula pipeline (function- - style rewrite, legacy colon-spelling preprocessing, AST parsing), so the expanded - string is parsed as if the user had written the inlined formula directly. - - Substitution is **skipped** when the NAME token is: - - - Preceded by ``.`` — right-hand side of a cross-model reference - (e.g. ``customers.aov``); cross-model resolution is handled separately. - - Followed by ``.`` — left-hand side of a cross-model reference - (e.g. ``customers.aov`` where ``customers`` happens to be a saved name). - - Preceded or followed by ``:`` — part of a legacy colon-spelling aggregation - (``revenue:sum``). - - Followed by ``(`` — function call (transform like ``cumsum(...)``). - - Followed by ``=`` — keyword argument name in an aggregation call - (e.g. ``weight`` in ``weighted_avg(price, weight=quantity)``). - - Cycles (``a → b → a``) raise ``ValueError`` with the chain in the message. - - Saved-measure names are pre-validated to be Python identifiers - (``slayer/core/models.py:_NAME_PATTERN``), so plain Python tokenization is - sufficient. If tokenization fails for any reason, returns the formula - unchanged so the downstream parser can produce a clear error. - """ - if not named_measures: - return formula - - try: - tokens = list(tokenize.generate_tokens(io.StringIO(formula).readline)) - except (tokenize.TokenError, SyntaxError): # IndentationError is a SyntaxError - return formula - - def _significant(idx: int, step: int) -> tokenize.TokenInfo | None: - skip_types = {tokenize.NEWLINE, tokenize.NL, tokenize.ENCODING, - tokenize.ENDMARKER, tokenize.COMMENT, tokenize.INDENT, - tokenize.DEDENT} - j = idx + step - while 0 <= j < len(tokens): - t = tokens[j] - if t.type in skip_types: - j += step - continue - return t - return None - - replacements: list[tuple] = [] # (start_pos, end_pos, replacement_text) - for i, tok in enumerate(tokens): - if tok.type != tokenize.NAME or tok.string not in named_measures: - continue - prev_tok = _significant(i, -1) - next_tok = _significant(i, 1) - if prev_tok is not None and prev_tok.type == tokenize.OP and prev_tok.string in (".", ":"): - continue - if next_tok is not None and next_tok.type == tokenize.OP and next_tok.string in (".", ":", "(", "="): - continue - - if tok.string in _visited: - chain = " → ".join([*_visited, tok.string]) - raise ValueError( - f"Saved measure '{tok.string}' has a cyclic reference: {chain}" - ) - inner_expanded = _expand_named_measures( - named_measures[tok.string], - named_measures, - _visited | {tok.string}, - ) - replacements.append((tok.start, tok.end, f"({inner_expanded})")) - - if not replacements: - return formula - - # Position-based substitution assumes a single-line formula. Multi-line - # formulas are not part of the supported grammar; if encountered, fall back - # to no expansion and let the downstream parser produce a sensible error. - if any(start[0] != end[0] or start[0] != 1 for start, end, _ in replacements): - return formula - - result = formula - for (_, start_col), (_, end_col), repl in sorted( - replacements, key=lambda r: -r[0][1] - ): - result = result[:start_col] + repl + result[end_col:] - return result - - -def parse_formula( - formula: str, - extra_agg_names: frozenset[str] | None = None, - named_measures: Mapping[str, str] | None = None, -) -> FieldSpec: - """Parse a formula string into a FieldSpec. - - Examples: - "sum(revenue)" → AggregatedMeasureRef("revenue", "sum") - "count(*)" → AggregatedMeasureRef("*", "count") - "sum(revenue) / count(*)" → ArithmeticField(...) - "cumsum(sum(revenue))" → TransformField("cumsum", AggregatedMeasureRef(...)) - "weighted_avg(price, weight=qty)" → AggregatedMeasureRef(..., agg_kwargs={"weight": "qty"}) - "last(revenue, ordered_at)" → AggregatedMeasureRef(..., agg_args=["ordered_at"]) - - Bare measure names (e.g., "revenue") are valid only when ``named_measures`` - is supplied and contains the name — they are inline-expanded to the saved - formula. Otherwise bare names raise. - - Args: - formula: The formula string to parse. - extra_agg_names: Additional aggregation names for function-style rewriting. - named_measures: Mapping of saved-measure name → its formula. Bare - references to these names are inline-expanded (recursively, with - cycle detection) before the rest of parsing. - """ - if named_measures: - formula = _expand_named_measures(formula, named_measures) - # Functional aggregations → legacy colon spelling → ast-parseable placeholders. - formula = _rewrite_funcstyle_aggregations(formula, extra_agg_names) - processed, agg_refs = _preprocess_agg_refs( - formula=formula, custom_agg_names=extra_agg_names or frozenset() - ) - - try: - tree = ast.parse(processed, mode="eval") - except SyntaxError as e: - raise ValueError(f"Invalid formula syntax: {formula!r} — {e}") - - return _parse_node( - tree.body, - original=formula, - agg_refs=agg_refs, - known_measures=frozenset(named_measures or ()), - ) - - -def _bare_name_message(*, name: str, known_measures: frozenset[str]) -> str: - """Explain a bare name that expanded to no saved measure. - - A bare name is only ever a saved measure, and by this point expansion has - already failed, so the name is either misspelled or a column. Suggesting a - real measure first stops the reader from wrapping a measure in an - aggregation it would reject anyway. - """ - suggestion = difflib.get_close_matches( - word=name, possibilities=sorted(known_measures), n=1 - ) - hint = f" Did you mean '{suggestion[0]}'?" if suggestion else "" - return ( - f"'{name}' is not a saved measure.{hint} Reference a saved measure by its " - f"bare name, or aggregate a column (e.g., 'sum({name})'). " - f"For COUNT(*), use 'count(*)'." - ) - - -def _parse_node( - node: ast.AST, - original: str, - agg_refs: dict[str, AggregatedMeasureRef] | None = None, - known_measures: frozenset[str] = frozenset(), -) -> FieldSpec: - """Recursively parse an AST node into a FieldSpec.""" - if agg_refs is None: - agg_refs = {} - - # Simple name → aggregation placeholder or error for bare names - if isinstance(node, ast.Name): - if node.id in agg_refs: - return agg_refs[node.id] - raise ValueError( - _bare_name_message(name=node.id, known_measures=known_measures) - ) - - # Dotted name → cross-model measure must include aggregation - if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name): - name = f"{node.value.id}.{node.attr}" - raise ValueError( - f"Cross-model measure '{name}' must include an aggregation " - f"(e.g., 'sum({name})')." - ) - - # Function call → transform - if isinstance(node, ast.Call): - if not isinstance(node.func, ast.Name): - raise ValueError(f"Unsupported function call in formula: {original!r}") - - func_name = node.func.id - category = _classify_call_name(func_name) - if category == "scalar": - # Allowlisted scalar call. Route through - # _parse_mixed_arithmetic so inner aggregated refs and nested - # transforms are registered/extracted. - _validate_scalar_call(node, original) - return _parse_mixed_arithmetic(node, original, agg_refs, known_measures) - if category != "transform": - raise ValueError( - f"Unknown function '{func_name}' in formula {original!r}. " - f"Supported scalar functions: " - f"{', '.join(sorted(SCALAR_FUNCTIONS))}. " - f"Transforms: {', '.join(sorted(ALL_TRANSFORMS))}." - ) - - if not node.args: - raise ValueError(f"Transform '{func_name}' requires at least one argument (the measure)") - - # First arg is the measure/expression being transformed - inner = _parse_node(node.args[0], original, agg_refs, known_measures) - - # Remaining positional args are transform parameters (offset, granularity, etc.) - # The rank family is keyword-only after the measure; reject extra positionals - # so calls like `rank(sum(revenue), 'desc')` or `ntile(sum(revenue), 4, n=2)` fail - # fast instead of silently dropping the extra arg downstream. - if func_name in RANK_FAMILY_TRANSFORMS and len(node.args) > 1: - raise ValueError( - f"Transform '{func_name}' does not accept positional arguments " - f"beyond the measure; use keyword args (e.g. partition_by=, direction=, n=). " - f"Formula: {original!r}" - ) - extra_args = [] - for arg in node.args[1:]: - extra_args.append(_parse_literal(node=arg, original=original)) - - # Keyword args, validated per-transform. - kwargs = _parse_transform_kwargs( - transform=func_name, keywords=node.keywords, original=original - ) - - return TransformField(transform=func_name, inner=inner, args=extra_args, kwargs=kwargs) - - # Binary/unary/comparison/boolean operation → check if it contains transform calls - if isinstance(node, (ast.BinOp, ast.UnaryOp, ast.Compare, ast.BoolOp)): - if _contains_call(node): - return _parse_mixed_arithmetic(node, original, agg_refs, known_measures) - measure_names = _collect_names(node) - # Reject bare names (aggregations were preprocessed into placeholders) - for mname in measure_names: - if mname not in agg_refs: - if "." in mname: - raise ValueError( - f"Cross-model measure '{mname}' must include an aggregation " - f"(e.g., 'sum({mname})')." - ) - raise ValueError( - _bare_name_message( - name=mname, known_measures=known_measures - ) - ) - field_agg_refs = {n: agg_refs[n] for n in measure_names if n in agg_refs} - return ArithmeticField( - sql=ast.unparse(node), - measure_names=measure_names, - agg_refs=field_agg_refs, - is_predicate=isinstance(node, (ast.Compare, ast.BoolOp)), - ) - - # Constant (bare number) - if isinstance(node, ast.Constant): - return ArithmeticField(sql=ast.unparse(node), measure_names=[]) - - raise ValueError(f"Unsupported formula syntax: {original!r}") - - -def _contains_call(node: ast.AST) -> bool: - """Check if an AST subtree contains any function Call nodes.""" - for child in ast.walk(node): - if isinstance(child, ast.Call): - return True - return False - - -def _replace_calls_in_arith( - node: ast.AST, - *, - sub_transforms: list[tuple], - measure_names: list[str], - counter: list[int], - agg_refs: dict[str, AggregatedMeasureRef], - original: str, - known_measures: frozenset[str] = frozenset(), -) -> ast.AST: - """Walk the AST, replacing transform Call nodes with Name placeholders.""" - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id in ALL_TRANSFORMS: - placeholder = f"_t{counter[0]}" - counter[0] += 1 - transform = _parse_node(node, original, agg_refs, known_measures) - sub_transforms.append((placeholder, transform)) - return ast.Name(id=placeholder, ctx=ast.Load()) - - kwargs = { - "sub_transforms": sub_transforms, - "measure_names": measure_names, - "counter": counter, - "agg_refs": agg_refs, - "original": original, - "known_measures": known_measures, - } - - if isinstance(node, ast.Name): - if node.id not in [p for p, _ in sub_transforms]: - measure_names.append(node.id) - return node - - if isinstance(node, ast.BinOp): - node.left = _replace_calls_in_arith(node.left, **kwargs) - node.right = _replace_calls_in_arith(node.right, **kwargs) - return node - - if isinstance(node, ast.UnaryOp): - node.operand = _replace_calls_in_arith(node.operand, **kwargs) - return node - - if isinstance(node, ast.Compare): - node.left = _replace_calls_in_arith(node.left, **kwargs) - node.comparators = [_replace_calls_in_arith(c, **kwargs) for c in node.comparators] - return node - - if isinstance(node, ast.BoolOp): - node.values = [_replace_calls_in_arith(v, **kwargs) for v in node.values] - return node - - if isinstance(node, ast.Call): - # Non-transform scalar call wrapping aggregated refs (e.g. nullif, - # coalesce, abs). Validate the name against the canonical Mode B - # allowlist — previously any name passed through, which was a - # consistency bug (top-level had a strict allowlist while - # inside-arithmetic accepted anything). - if isinstance(node.func, ast.Name): - name = node.func.id - if name not in ALL_TRANSFORMS and name.lower() not in SCALAR_FUNCTIONS: - raise ValueError( - f"Unknown function {name!r} in formula {kwargs['original']!r}. " - f"Supported scalar functions: " - f"{', '.join(sorted(SCALAR_FUNCTIONS))}. " - f"Transforms: {', '.join(sorted(ALL_TRANSFORMS))}." - ) - node.args = [_replace_calls_in_arith(a, **kwargs) for a in node.args] - for kw in node.keywords: - kw.value = _replace_calls_in_arith(kw.value, **kwargs) - return node - - return node - - -def _parse_mixed_arithmetic( - node: ast.AST, - original: str, - agg_refs: dict[str, AggregatedMeasureRef] | None = None, - known_measures: frozenset[str] = frozenset(), -) -> MixedArithmeticField: - """Parse arithmetic that contains transform calls. - - Extracts transform calls, replaces them with placeholder names, - and returns a MixedArithmeticField. - """ - if agg_refs is None: - agg_refs = {} - - sub_transforms: list[tuple] = [] - measure_names: list[str] = [] - counter = [0] - - modified = _replace_calls_in_arith( - node, - sub_transforms=sub_transforms, - measure_names=measure_names, - counter=counter, - agg_refs=agg_refs, - original=original, - known_measures=known_measures, - ) - modified_sql = ast.unparse(modified) - - field_agg_refs = {n: agg_refs[n] for n in measure_names if n in agg_refs} - is_predicate = isinstance(node, (ast.Compare, ast.BoolOp)) - - return MixedArithmeticField( - sql=modified_sql, - measure_names=measure_names, - sub_transforms=sub_transforms, - agg_refs=field_agg_refs, - is_predicate=is_predicate, - ) - - -def _parse_literal(node: ast.AST, original: str) -> Any: - """Parse a literal value from an AST node (number, string, negative number).""" - if isinstance(node, ast.Constant): - return node.value - if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub) and isinstance(node.operand, ast.Constant): - return -node.operand.value - raise ValueError(f"Expected a literal value (number or string) in formula: {original!r}") - - -# Per-transform kwarg whitelist. Empty set means the transform takes no kwargs. -_ALLOWED_TRANSFORM_KWARGS: dict[str, frozenset] = { - "rank": frozenset({"partition_by"}), - "percent_rank": frozenset({"partition_by"}), - "dense_rank": frozenset({"partition_by"}), - "ntile": frozenset({"partition_by", "n"}), -} - - -def _parse_dotted_name(node: ast.AST, original: str) -> str: - """Render a Name / Attribute node back into a dotted-path string. - - ``region`` → ``"region"``; ``customers.region`` → ``"customers.region"``. - """ - if isinstance(node, ast.Name): - return node.id - if isinstance(node, ast.Attribute): - return f"{_parse_dotted_name(node=node.value, original=original)}.{node.attr}" - raise ValueError( - f"Expected a column name or dotted path in formula {original!r}, " - f"got {ast.dump(node)}" - ) - - -def _parse_transform_kwargs( # NOSONAR S3776 — straight-line whitelist + per-kwarg validation; splitting into helpers would force threading transform/original through every call just to preserve the error-message context - transform: str, keywords: list[ast.keyword], original: str -) -> dict[str, Any]: - """Parse and validate a transform's keyword arguments. - - Each transform has a fixed kwarg whitelist (see ``_ALLOWED_TRANSFORM_KWARGS``). - Unknown kwargs raise with the accepted set in the message. ``ntile`` requires - ``n`` (positive integer); ``partition_by`` accepts a column name, dotted - path, or list of those. - """ - allowed = _ALLOWED_TRANSFORM_KWARGS.get(transform, frozenset()) - parsed: dict[str, Any] = {} - direction_kw: ast.keyword | None = None - - for kw in keywords: - if kw.arg is None: - raise ValueError( - f"Transform '{transform}' does not accept **kwargs in formula {original!r}" - ) - if kw.arg == "direction" and transform in RANK_FAMILY_TRANSFORMS: - direction_kw = kw - continue - if kw.arg not in allowed: - if not allowed: - raise ValueError( - f"Transform '{transform}' does not accept keyword arguments; " - f"got '{kw.arg}=' in formula {original!r}" - ) - advertised = with_direction_kwarg(op=transform, accepted=allowed) - raise ValueError( - f"Transform '{transform}' does not accept keyword '{kw.arg}'. " - f"Accepted kwargs: {', '.join(sorted(advertised))}. " - f"Formula: {original!r}" - ) - - if kw.arg == "partition_by": - value = kw.value - if isinstance(value, ast.List): - cols = [ - _parse_dotted_name(node=elt, original=original) for elt in value.elts - ] - else: - cols = [_parse_dotted_name(node=value, original=original)] - if not cols: - raise ValueError( - f"Transform '{transform}': partition_by must reference at " - f"least one column in formula {original!r}" - ) - parsed["partition_by"] = cols - elif kw.arg == "n": - n_val = _parse_literal(node=kw.value, original=original) - if not isinstance(n_val, int) or isinstance(n_val, bool) or n_val <= 0: - raise ValueError( - f"Transform '{transform}': n must be a positive integer, " - f"got {n_val!r} in formula {original!r}" - ) - parsed["n"] = n_val - else: # pragma: no cover — guarded by the whitelist check above - parsed[kw.arg] = _parse_literal(node=kw.value, original=original) - - literal = direction_kw.value if direction_kw is not None else None - direction = rank_direction( - op=transform, given=direction_kw is not None, - value=literal.value if isinstance(literal, ast.Constant) else None, - ) - if direction is not None: - parsed["direction"] = direction - - if transform == "ntile" and "n" not in parsed: - raise ValueError( - f"Transform 'ntile' requires keyword argument 'n' (positive integer) " - f"in formula {original!r}" - ) - - return parsed - - -def _collect_names(node: ast.AST) -> list[str]: - """Collect all Name and dotted Attribute references from an AST subtree.""" - names = [] - # Collect dotted names (model.measure) first to avoid also collecting the bare Name part - dotted = set() - for child in ast.walk(node): - if isinstance(child, ast.Attribute) and isinstance(child.value, ast.Name): - dotted_name = f"{child.value.id}.{child.attr}" - names.append(dotted_name) - dotted.add(id(child.value)) # Mark the Name node as consumed - for child in ast.walk(node): - if isinstance(child, ast.Name) and id(child) not in dotted: - names.append(child.id) - return names diff --git a/slayer/core/keys.py b/slayer/core/keys.py index fc81d9e8f..dd0ebeea3 100644 --- a/slayer/core/keys.py +++ b/slayer/core/keys.py @@ -88,6 +88,7 @@ def _spec(low: int, high: Optional[int], *, sql: bool = True) -> ScalarSpec: "replace": _spec(3, 3), "substr": _spec(2, 3), "substring": _spec(2, 3), "instr": _spec(2, 2), "concat": _spec(1, None), "like": _spec(2, 2, sql=False), # emits the LIKE operator + "ilike": _spec(2, 2, sql=False), # case-insensitive LIKE on every dialect "iif": _spec(3, 3, sql=False), # emits CASE WHEN "date_part": _spec(2, 2, sql=False), "date_diff": _spec(3, 3, sql=False), @@ -402,7 +403,7 @@ def __eq__(self, other: object) -> bool: AggregateSource = Union[ ColumnKey, ColumnSqlKey, StarKey, TimeTruncKey, "ArithmeticKey", "ScalarCallKey", "LiteralKey", "AggregateKey", "TransformKey", - "InKey", "TimePointCmpKey", + "InKey", "TimePointCmpKey", "CastKey", ] # Positional and kwarg arg values share one union: `last(created_at)` binds an # identifier column, `weighted_avg(weight=qty)` a column, @@ -411,6 +412,8 @@ def __eq__(self, other: object) -> bool: # all via `_bind_agg_arg`. _AggregateArgValue = Union[ ColumnKey, ColumnSqlKey, SqlFragmentKey, "AggregateKey", "TransformKey", + # A FILTER-masked row parameter. + "ScalarCallKey", Decimal, str, bool, None, ] _AggregateKwargValue = _AggregateArgValue @@ -447,6 +450,8 @@ class AggregateKey(_FrozenKey, frozen=True): locus: Literal["target", "host"] = "target" # None = grain inherited from context; Grain.EMPTY = explicitly scalar. partition_keys: Optional["Grain"] = None + # A ranked (first / last) aggregate's ``FILTER (WHERE …)``; other families mask their inputs at binding. + filter: Optional["ValueKey"] = None @field_validator("kwargs", mode="before") @classmethod @@ -465,6 +470,8 @@ def children(self) -> Tuple["ValueKey", ...]: ] if self.partition_keys is not None: embedded.extend(self.partition_keys) + if self.filter is not None: + embedded.append(self.filter) return tuple(embedded) def map_children( @@ -479,6 +486,7 @@ def map_children( None if self.partition_keys is None else Grain.of(m(p) for p in self.partition_keys) ), + "filter": None if self.filter is None else m(self.filter), } return self.model_copy(update=update) if m.changed else self @@ -491,6 +499,7 @@ def __hash__(self) -> int: _typed_kwargs(self.kwargs), self.locus, self.partition_keys, + self.filter, )) def __eq__(self, other: object) -> bool: @@ -503,6 +512,7 @@ def __eq__(self, other: object) -> bool: and _typed_kwargs(self.kwargs) == _typed_kwargs(other.kwargs) and self.locus == other.locus and self.partition_keys == other.partition_keys + and self.filter == other.filter ) @@ -677,6 +687,50 @@ def __eq__(self, other: object) -> bool: ) +# A cast target's SLayer type, by the type's leading word. +_CAST_TYPE_WORDS: Mapping[str, DataType] = MappingProxyType({ + **{w: DataType.INT for w in ("INT", "INTEGER", "BIGINT", "SMALLINT", "TINYINT", "MEDIUMINT", "HUGEINT", + "INT2", "INT4", "INT8", "INT64", "LONG")}, + **{w: DataType.DOUBLE for w in ("FLOAT", "FLOAT4", "FLOAT8", "FLOAT64", "DOUBLE", "REAL", "DECIMAL", + "NUMERIC", "NUMBER", "BIGNUMERIC", "DEC")}, + **{w: DataType.TEXT for w in ("TEXT", "VARCHAR", "CHAR", "CHARACTER", "NVARCHAR", "NCHAR", "STRING", + "CLOB", "NTEXT", "BPCHAR")}, + **{w: DataType.BOOLEAN for w in ("BOOLEAN", "BOOL")}, + "DATE": DataType.DATE, + **{w: DataType.TIMESTAMP for w in ("TIMESTAMP", "DATETIME", "DATETIME2", "TIMESTAMPTZ", "SMALLDATETIME")}, +}) + + +def cast_data_type(type_text: str) -> DataType: + """SLayer's type for a cast to ``type_text`` (UNKNOWN for any unmapped type).""" + word = type_text.split("(")[0].split()[0].upper() if type_text.strip() else "" + return _CAST_TYPE_WORDS.get(word, DataType.UNKNOWN) + + +class CastKey(_FrozenKey, frozen=True): + """Identity for ``CAST(source AS type_text)`` (``try_``: ``TRY_CAST``) — the database's + own cast of the author's type; ``data_type`` is SLayer's type for the target.""" + + source: "ValueKey" + type_text: str + data_type: DataType + try_: bool = False + + @property + def phase(self) -> Phase: + return self.source.phase + + def children(self) -> Tuple["ValueKey", ...]: + return (self.source,) + + def map_children( + self, fn: Callable[["ValueKey"], "ValueKey"], + ) -> "CastKey": + m = _ChildMapper(fn) + source = m(self.source) + return self.model_copy(update={"source": source}) if m.changed else self + + TimePointOp = Literal["=", "!=", "<", "<=", ">", ">=", "in", "not in"] MIRRORED_TIME_POINT_OP: Dict[TimePointOp, TimePointOp] = {"=": "=", "!=": "!=", "<": ">", "<=": ">=", ">": "<", ">=": "<="} @@ -765,6 +819,7 @@ def map_children( TimePointCmpKey, InKey, SqlFragmentKey, + CastKey, ] @@ -871,6 +926,7 @@ def __sub__(self, other: object) -> "Grain": ScalarCallKey.model_rebuild() TimePointCmpKey.model_rebuild() InKey.model_rebuild() +CastKey.model_rebuild() TimeTruncKey.model_rebuild() # AggregateKey.source forward-references the expression composites. AggregateKey.model_rebuild() @@ -879,6 +935,8 @@ def __sub__(self, other: object) -> "Grain": VALUE_KEY_TYPES: Tuple[type, ...] = get_args(ValueKey) +# Key fields spelled only when set, so a key without one keeps its historical spelling. +OPTIONAL_KEY_FIELDS = frozenset({"filter"}) class KindPolicy(BaseModel): @@ -908,6 +966,7 @@ class KindPolicy(BaseModel): TimePointCmpKey: KindPolicy(), InKey: KindPolicy(slot_composite=True, materialised_order=True), SqlFragmentKey: KindPolicy(), + CastKey: KindPolicy(slot_composite=True, materialised_order=True), } SLOT_COMPOSITE_KINDS: Tuple[type, ...] = tuple(k for k, p in KIND_POLICY.items() if p.slot_composite) @@ -1238,6 +1297,8 @@ def temporal_type(key: object, *, column_type: ColumnTypeFn) -> Optional[DataTyp return temporal_type(key.source, column_type=column_type) if from_source else None if isinstance(key, ScalarCallKey): return _scalar_temporal_type(key, column_type=column_type) + if isinstance(key, CastKey): + return key.data_type if key.data_type in TEMPORAL_TYPES else None return None @@ -1282,18 +1343,24 @@ def boolean_valued(key: object, *, column_type: ColumnTypeFn) -> bool: return key.agg.lower() in _BOOLEAN_PRESERVING_AGGS and boolean_valued(key.source, column_type=column_type) if isinstance(key, TransformKey): return key.op in AXIS_COLLAPSING_TRANSFORMS and boolean_valued(key.input, column_type=column_type) + if isinstance(key, CastKey): + return key.data_type is DataType.BOOLEAN if isinstance(key, ScalarCallKey): - if key.name == "like": - return True - if key.name not in _BOOLEAN_BRANCH_SCALARS: - return False - positions = _BOOLEAN_BRANCH_SCALARS[key.name] - returned = key.args if positions is None else [key.args[i] for i in positions if i < len(key.args)] - branches = [a for a in returned if not _is_null_arg(a)] - return bool(branches) and all(boolean_valued(a, column_type=column_type) for a in branches) + return _scalar_boolean_valued(key, column_type=column_type) return False +def _scalar_boolean_valued(key: ScalarCallKey, *, column_type: ColumnTypeFn) -> bool: + if key.name in ("like", "ilike"): + return True + if key.name not in _BOOLEAN_BRANCH_SCALARS: + return False + positions = _BOOLEAN_BRANCH_SCALARS[key.name] + returned = key.args if positions is None else [key.args[i] for i in positions if i < len(key.args)] + branches = [a for a in returned if not _is_null_arg(a)] + return bool(branches) and all(boolean_valued(a, column_type=column_type) for a in branches) + + # Scalars that read every argument as a number. _NUMERIC_SCALARS = frozenset({ "ln", "log10", "log2", "log", "exp", "sqrt", "pow", "power", "abs", "floor", "ceil", "ceiling", @@ -1340,15 +1407,21 @@ def numeric_valued(key: object, *, column_type: ColumnTypeFn) -> bool: return agg in _BOOLEAN_PRESERVING_AGGS and numeric_valued(key.source, column_type=column_type) if isinstance(key, TransformKey): return key.op in _NUMBER_RESULT_TRANSFORMS or numeric_valued(key.input, column_type=column_type) + if isinstance(key, CastKey): + return key.data_type in _NUMERIC_TYPES if isinstance(key, ScalarCallKey): - if key.name in _NUMBER_RESULT_SCALARS: - return True - branches = [key.args[i] for i in value_arg_positions(key.name, len(key.args)) if not _is_null_arg(key.args[i])] - numbers = [numeric_valued(a, column_type=column_type) for a in branches] - return any(numbers) and all(n or boolean_valued(a, column_type=column_type) for a, n in zip(branches, numbers)) + return _scalar_numeric_valued(key, column_type=column_type) return False +def _scalar_numeric_valued(key: ScalarCallKey, *, column_type: ColumnTypeFn) -> bool: + if key.name in _NUMBER_RESULT_SCALARS: + return True + branches = [key.args[i] for i in value_arg_positions(key.name, len(key.args)) if not _is_null_arg(key.args[i])] + numbers = [numeric_valued(a, column_type=column_type) for a in branches] + return any(numbers) and all(n or boolean_valued(a, column_type=column_type) for a, n in zip(branches, numbers)) + + def _boolean_positions( children: Sequence[object], positions: Iterable[int], *, column_type: ColumnTypeFn, ) -> FrozenSet[int]: @@ -1418,6 +1491,8 @@ def aggregation_source_type(source: object, *, column_type: ColumnTypeFn) -> Opt return DataType.INT if isinstance(source, (ColumnKey, ColumnSqlKey)): return column_type(source) + if isinstance(source, CastKey) and source.data_type is not DataType.UNKNOWN: + return source.data_type return None @@ -1493,8 +1568,10 @@ def is_local_combined_regroup_ref( ) +# Null-safe comparisons: never UNKNOWN. +NULL_SAFE_COMPARISON_OPS = frozenset({"is distinct from", "is not distinct from"}) PREDICATE_COMPARISON_OPS = frozenset( - {"==", "=", "!=", "<>", "<", "<=", ">", ">=", "is", "is not"} + {"==", "=", "!=", "<>", "<", "<=", ">", ">=", "is", "is not", *NULL_SAFE_COMPARISON_OPS} ) BOOL_CONNECTIVE_OPS = frozenset({"and", "or", "not"}) diff --git a/slayer/core/models.py b/slayer/core/models.py index a0befba73..48cb8119b 100644 --- a/slayer/core/models.py +++ b/slayer/core/models.py @@ -15,24 +15,9 @@ ) from sqlalchemy.engine import URL as _SA_URL -from slayer.core.enums import ( - BUILTIN_AGGREGATION_FORMULAS, - BUILTIN_AGGREGATIONS, - DEFAULT_AGGREGATIONS_BY_TYPE, - GRANULARITY_NAMES, - DataType, - JoinCardinality, - JoinType, - ObjectKind, - PRIMARY_KEY_AGGREGATIONS, - RANKED_AGGREGATIONS, - RESERVED_DATE_SYNTAX_NAMES, - _coerce_legacy_datatype, - builtin_empty_value, -) +from slayer.core.enums import BUILTIN_AGGREGATION_FORMULAS, BUILTIN_AGGREGATIONS, DEFAULT_AGGREGATIONS_BY_TYPE, GRANULARITY_NAMES, DataType, JoinCardinality, JoinType, ObjectKind, PRIMARY_KEY_AGGREGATIONS, RANKED_AGGREGATIONS, RESERVED_DATE_SYNTAX_NAMES, _coerce_legacy_datatype, builtin_empty_value, ALL_TRANSFORMS from slayer.core.errors import GranularityDefinitionError, JoinKeyError from slayer.core.format import NumberFormat -from slayer.core.formula import ALL_TRANSFORMS from slayer.core.granularity import CustomGranularity, GranularitySpec, definition_error, granularity_key from slayer.core.keys import SCALAR_FUNCTIONS from slayer.core.refs import IDENTIFIER_RE diff --git a/slayer/core/query.py b/slayer/core/query.py index d94dd4d90..a165ebe68 100644 --- a/slayer/core/query.py +++ b/slayer/core/query.py @@ -23,8 +23,7 @@ ) from slayer.core.direction import normalize_direction -from slayer.core.enums import BUILTIN_AGGREGATIONS, GRANULARITY_NAMES, TimeGranularity, normalize_aggregation_name -from slayer.core.formula import ALL_TRANSFORMS +from slayer.core.enums import BUILTIN_AGGREGATIONS, GRANULARITY_NAMES, TimeGranularity, normalize_aggregation_name, ALL_TRANSFORMS from slayer.core.keys import SCALAR_FUNCTIONS from slayer.core.errors import DistinctDimensionValuesError, GranularityCallError, RefinementConflictError from slayer.core.granularity import GranularitySpec, granularity_key @@ -37,7 +36,7 @@ ) from slayer.core.refs import auto_name_from_expression from slayer.core.time_points import TIME_POINT_FORMS, is_time_point_shape -from slayer.engine.syntax import AggCall, DottedRef, Ref, parse_expr, walk_parsed_refs +from slayer.core.syntax import AggCall, DottedRef, Ref, canonical_measure_text, parse_expr from slayer.sql.window_detect import WINDOW_IN_FILTER_ERROR, has_window_function from slayer.storage.migrations import CURRENT_VERSIONS, migrate as _migrate_schema @@ -663,20 +662,24 @@ def template(self) -> str: """The expression as written, before ``{var}`` substitution.""" return self._template or self.expression - def substituted(self, expression: str) -> "ComputedDimension": - """A copy carrying the substituted ``expression`` and remembering its template.""" - out = self.model_copy(update={"expression": expression}) - out._template = self.template - return out - @model_validator(mode="after") def _fill_name(self) -> "ComputedDimension": if self.name is None: - self.name = auto_name_from_expression(self.expression) + self.name = computed_dimension_name(self.expression) _validate_model_name(self.name, "Computed dimension") return self +def computed_dimension_name(expression: str) -> str: + """An unnamed computed dimension's name: its expression's canonical text, sanitised, so every + spelling of one expression names it alike.""" + try: + text = canonical_measure_text(parse_expr(expression)) + except Exception: # noqa: BLE001 — an unparseable expression is reported at binding + text = expression + return auto_name_from_expression(text) + + def _coerce_column_ref(v: Any) -> Any: """Allow plain string where a ColumnRef is expected: "x" → {"name": "x"}.""" if isinstance(v, str): @@ -697,24 +700,18 @@ def _coerce_column_ref(v: Any) -> Any: def _order_formula_candidate(v: str) -> str | None: - """``v`` verbatim if it carries a measure expression (legacy colon spelling, - call-style text, or an expression containing an aggregation), else - ``None``; shared by ``_capture_raw_formula`` and ``_coerce_order_column`` - so they can't drift. The author's spelling is preserved — resolution - happens at binding via the native parser. Bare-alias - arithmetic (``rev / cnt``) is deliberately NOT a candidate — it falls - through to ColumnRef validation and fails fast.""" + """``v`` verbatim if it is an expression rather than a column reference, else ``None``; + shared by ``_capture_raw_formula`` and ``_coerce_order_column`` so they can't drift. + Binding judges whether the expression is legal there.""" if ":" in v or _FUNCSTYLE_CALL_PATTERN.match(v): return v if _is_valid_column_ref_name(v): return None try: - parsed = parse_expr(v) + parse_expr(v) except Exception: return None - if any(isinstance(node, AggCall) for node in walk_parsed_refs(parsed)): - return v - return None + return v def _is_valid_column_ref_name(name: str) -> bool: @@ -859,6 +856,7 @@ class OrderItem(BaseModel): column: Annotated[ColumnRef, BeforeValidator(_coerce_order_column)] direction: str = "asc" + nulls: Literal["first", "last"] | None = None raw_formula: str | None = Field( default=None, description="Internal — captured automatically from expression strings; do not set.", @@ -1068,7 +1066,9 @@ def _strip_order(order: list[OrderItem] | None, model_name: str, pattern: re.Pat if stripped is item.column and raw_formula == item.raw_formula: new_order.append(item) else: - new_order.append(OrderItem(column=stripped, direction=item.direction, raw_formula=raw_formula)) + new_order.append(OrderItem( + column=stripped, direction=item.direction, nulls=item.nulls, raw_formula=raw_formula, + )) return None if all(n is o for n, o in zip(new_order, order)) else new_order @@ -1253,9 +1253,9 @@ def _validate_query_name(cls, v: str | None) -> str | None: default=None, description=( "Conditions, AND-ed; each routes automatically to row filtering (WHERE), aggregate " - "filtering (HAVING) or filtering after transforms. Operators: == != < <= > >=, in, " - "like, is null / is not null, and / or / not; build text literals from the column's " - "sampled values.\n" + "filtering (HAVING) or filtering after transforms. SQL operators: = <> < <= > >=, in, " + "between, like / ilike, is null / is not null, and / or / not; strings are " + "single-quoted ('paid'); build them from the column's sampled values.\n" "- Aggregates and transforms filter too, by formula or by a measure's name: " "sum(amount) > 100; top N per group: " "rank(sum(amount), partition_by=customers.regions.name, direction='desc') <= 1.\n" @@ -1266,7 +1266,7 @@ def _validate_query_name(cls, v: str | None) -> str | None: "relative: 'today', 'last month', 'last 7 days' (excludes the current day), " "'year to date'. order_date >= '2025-Q1' starts at the period, <= ends with it, " "order_date in '2025-Q1' is inside it.\n" - "- {variable} placeholders take values from variables.\n" + "- {variable} placeholders take literal values from variables.\n" 'More: inspect(reference=["memory:help.time", "memory:help.joins"], ' 'entity_type="memory").' ), @@ -1281,8 +1281,9 @@ def _validate_query_name(cls, v: str | None) -> str | None: order: OrderField = Field( default=None, description=( - 'Sort keys, [{"column": ..., "direction": "asc" | "desc"}]. column is a result ' - "column name or any aggregate or transform expression, displayed or not: with " + 'Sort keys, [{"column": ..., "direction": "asc" | "desc", "nulls": "first" | "last"}] ' + "(nulls optional). column is a result column name or any aggregate or transform " + "expression, displayed or not: with " 'measures ["count(*)"], [{"column": "sum(amount)", "direction": "desc"}] sorts by ' "an amount it does not return. Add limit for the top N. Return only the columns the " "question asks for: never add the sort value to measures just to sort by it." diff --git a/slayer/core/refs.py b/slayer/core/refs.py index 9f2ab0e88..d077bb260 100644 --- a/slayer/core/refs.py +++ b/slayer/core/refs.py @@ -11,8 +11,10 @@ from slayer.core.granularity import CustomGranularity from slayer.core.keys import ( MIRRORED_TIME_POINT_OP, + OPTIONAL_KEY_FIELDS, AggregateKey, ArithmeticKey, + CastKey, ColumnKey, ColumnSqlKey, InKey, @@ -65,7 +67,7 @@ def auto_name_from_expression(expression: str) -> str: # The row-level ``ValueKey`` kinds an ``AggregateKey.source`` may take when it # is a same-model scalar EXPRESSION rather than a column / star. -EXPRESSION_SOURCE_KINDS = (ArithmeticKey, ScalarCallKey, LiteralKey, TimeTruncKey, InKey, TimePointCmpKey) +EXPRESSION_SOURCE_KINDS = (ArithmeticKey, ScalarCallKey, LiteralKey, TimeTruncKey, InKey, TimePointCmpKey, CastKey) # The pinned legacy key spelling: the historical Pydantic @@ -94,6 +96,7 @@ def auto_name_from_expression(expression: str) -> str: ("partition_keys", lambda k: ( None if k.partition_keys is None else k.partition_keys.keys )), + ("filter", attrgetter("filter")), )), TransformKey: ("TransformKey", ( ("op", attrgetter("op")), ("input", attrgetter("input")), @@ -120,6 +123,10 @@ def auto_name_from_expression(expression: str) -> str: SqlFragmentKey: ("SqlFragmentKey", ( ("template", attrgetter("template")), ("refs", attrgetter("refs")), )), + CastKey: ("CastKey", ( + ("source", attrgetter("source")), ("type_text", attrgetter("type_text")), + ("try_", attrgetter("try_")), + )), } @@ -146,6 +153,7 @@ def _legacy_key_fields(key: Any) -> tuple[str, list[tuple[str, str]]]: spelled_cls, fields = _LEGACY_KEY_SPELLINGS[type(key)] return spelled_cls, [ (name, _legacy_value_spelling(get(key))) for name, get in fields + if not (name in OPTIONAL_KEY_FIELDS and get(key) is None) ] @@ -231,7 +239,8 @@ def _aggregate_display(key: AggregateKey) -> str: args.extend(f"{k}={_arg_display(v)}" for k, v in key.kwargs) if key.partition_keys is not None: args.append(f"partition_by={_grain_display_list(key.partition_keys)}") - return _call_display(key.agg, args) + text = _call_display(key.agg, args) + return text if key.filter is None else f"{text} FILTER (WHERE {key_display(key.filter)})" def _transform_display(key: TransformKey) -> str: @@ -265,18 +274,25 @@ def _fragment_display(key: SqlFragmentKey) -> str: return text -def key_display(key: ValueKey) -> str: - """User-facing formula text of any key kind.""" +def _leaf_display(key: ValueKey) -> str | None: if isinstance(key, ColumnKey): return ".".join((*key.path, key.leaf)) if isinstance(key, ColumnSqlKey): return ".".join((*key.path, key.column_name)) - if isinstance(key, TimeTruncKey): - return f"{key.granularity}({key_display(key.column)})" if isinstance(key, StarKey): return ".".join((*key.path, "*")) if isinstance(key, LiteralKey): return _scalar_display(key.value) + return None + + +def key_display(key: ValueKey) -> str: + """User-facing formula text of any key kind.""" + leaf = _leaf_display(key) + if leaf is not None: + return leaf + if isinstance(key, TimeTruncKey): + return f"{key.granularity}({key_display(key.column)})" if isinstance(key, AggregateKey): return _aggregate_display(key) if isinstance(key, TransformKey): @@ -285,6 +301,8 @@ def key_display(key: ValueKey) -> str: return _arithmetic_display(key) if isinstance(key, ScalarCallKey): return _call_display(key.name, [_arg_display(a) for a in key.args]) + if isinstance(key, CastKey): + return f"{'TRY_CAST' if key.try_ else 'CAST'}({key_display(key.source)} AS {key.type_text})" if isinstance(key, TimePointCmpKey): return _time_point_cmp_display(key) if isinstance(key, InKey): @@ -331,20 +349,17 @@ def agg_kwarg_canonical_str(value: Any) -> str: return _decimal_to_plain_str(Decimal(str(value))) if isinstance(value, str): return value - if isinstance(value, ColumnKey): - if value.path: - return ".".join(value.path) + "." + value.leaf - return value.leaf - if isinstance(value, ColumnSqlKey): - if value.path: - return ".".join(value.path) + "." + value.column_name - return value.column_name + if isinstance(value, (ColumnKey, ColumnSqlKey)): + return ".".join((*value.path, value.leaf if isinstance(value, ColumnKey) else value.column_name)) if isinstance(value, AggregateKey): return _agg_key_canonical_str(value) if isinstance(value, TransformKey): return _transform_key_canonical_str(value) if isinstance(value, SqlFragmentKey): return sql_fragment_digest(value) + if isinstance(value, ScalarCallKey): + # A FILTER-masked row parameter. + return auto_name_from_expression(key_display(value)) raise TypeError( f"AggregateKey kwarg value of type {type(value).__name__!r} " f"is not supported: {value!r}", diff --git a/slayer/core/syntax.py b/slayer/core/syntax.py new file mode 100644 index 000000000..f44b3f4ec --- /dev/null +++ b/slayer/core/syntax.py @@ -0,0 +1,1678 @@ +"""The Mode-B grammar. + +``parse_expr(text) -> ParsedExpr`` parses a Mode-B expression (``ModelMeasure.formula``, +query measures / dimensions / filters / order) with one SQL-compatible pyparsing grammar — +pure syntax, no scope resolution or named-measure expansion (binder's job). Every spelling +of a construct collapses to one node: CASE → ``iif``, ``EXTRACT`` → ``date_part``, +``count(distinct x)`` → ``count_distinct``, ``BETWEEN`` → its conjunction, ``x::t`` → +``Cast``, interval literals → ``date_add``. +""" + +from __future__ import annotations + +import difflib +import math +import re +import threading +import unicodedata +from decimal import Decimal +from functools import lru_cache, partial +from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional, Tuple, Union, cast + +import pyparsing as pp +from pydantic import BaseModel, ConfigDict + +from slayer.core.direction import normalize_direction +from slayer.core.enums import ( + ALL_TRANSFORMS, + AMBIGUOUS_WEEKDAY_NAMES, + BUILTIN_AGGREGATIONS, + DATE_PART_ALIASES, + GRANULARITY_NAMES, + TimeGranularity, + ambiguous_weekday_message, + normalize_aggregation_name, +) +from slayer.core.errors import GranularityCallError, IllegalWindowInFilterError, UnknownFunctionError +from slayer.core.keys import DATE_UNIT_ARGS, SCALAR_FUNCTIONS, check_scalar_arity +from slayer.core.refs import split_agg_suffix + + +# --------------------------------------------------------------------------- +# ParsedExpr family +# --------------------------------------------------------------------------- + + +class _BaseNode(BaseModel): + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + +class Ref(_BaseNode): + name: str + # Written double-quoted; not part of identity. + quoted: bool = False + + def __eq__(self, other: object) -> bool: + return isinstance(other, Ref) and self.name == other.name + + def __hash__(self) -> int: + return hash(("Ref", self.name)) + + +class DottedRef(_BaseNode): + parts: Tuple[str, ...] + # Some segment was written double-quoted; not part of identity. + quoted: bool = False + + def __eq__(self, other: object) -> bool: + return isinstance(other, DottedRef) and self.parts == other.parts + + def __hash__(self) -> int: + return hash(("DottedRef", self.parts)) + + +class StarSource(_BaseNode): + pass + + +class Literal(_BaseNode): + value: Decimal | str | bool | None = None + + +class TemporalLiteral(_BaseNode): + """``DATE '…'`` / ``TIMESTAMP '…'``; the text is validated at binding.""" + + kind: str + text: str + + +class Placeholder(_BaseNode): + """An unsubstituted ``{name}`` variable in a literal slot.""" + + name: str + + +class TupleLit(_BaseNode): + """Literal-only list RHS for ``IN`` / ``NOT IN``.""" + + elements: Tuple[Literal, ...] + + +class AggCall(_BaseNode): + source: Any + agg: str + args: Tuple[Any, ...] = () + kwargs: Tuple[Tuple[str, Any], ...] = () + # ``FILTER (WHERE …)`` predicate. + filter: Any = None + + +class TransformCall(_BaseNode): + op: str + input: Any + args: Tuple[Any, ...] = () + kwargs: Tuple[Tuple[str, Any], ...] = () + + +class ScalarCall(_BaseNode): + name: str + args: Tuple[Any, ...] = () + + +class Cast(_BaseNode): + """``CAST(x AS t)`` / ``x::t`` / ``TRY_CAST(x AS t)``; ``type_text`` is the normalised author type.""" + + source: Any + type_text: str + try_: bool = False + + +class Arith(_BaseNode): + op: str + left: Any + right: Any + + +class UnaryOp(_BaseNode): + op: str + operand: Any + + +class Cmp(_BaseNode): + op: str + left: Any + right: Any + # Written as a single `=`; not part of identity. + single_eq: bool = False + + def __eq__(self, other: object) -> bool: + return isinstance(other, Cmp) and (self.op, self.left, self.right) == (other.op, other.left, other.right) + + def __hash__(self) -> int: + return hash(("Cmp", self.op, self.left, self.right)) + + +class BoolOp(_BaseNode): + op: str + operands: Tuple[Any, ...] + + +ParsedExpr = Union[ + Ref, DottedRef, StarSource, Literal, TemporalLiteral, Placeholder, TupleLit, + AggCall, TransformCall, ScalarCall, Cast, + Arith, UnaryOp, Cmp, BoolOp, +] + + +# --------------------------------------------------------------------------- +# Lexical helpers +# --------------------------------------------------------------------------- + + +# The ``__slayer_`` prefix is reserved from user input (P3). +_RESERVED_EXPR_PREFIX = "__slayer_" +# Masked ``{name}`` placeholders: the braces become private-use sentinels, so offsets hold. +_PH_OPEN, _PH_CLOSE = "", "" +_VAR_NAME = r"(?a:[A-Za-z_]\w*)" +_CODE_PLACEHOLDER_RE = re.compile(r"\{\{|\}\}|\{(" + _VAR_NAME + r")\}") +_LITERAL_VALUES = "variables supply literal values" +_OPTIONAL_BLOCKS = ( + "Optional blocks '{? ... ?}' are supported only on raw-SQL model surfaces, not in Mode-B expressions." +) +_DISTINCT_ONLY = "DISTINCT is supported only as count(distinct x), i.e. count_distinct(x)" + + +class _Problem(ValueError): + """A grammar-level rejection; ``parse_expr`` prefixes the expression text.""" + + def __init__(self, detail: str) -> None: + super().__init__(detail) + self.detail = detail + + +class _Region(BaseModel): + kind: str # "code" | "string" | "ident" + start: int + end: int + + +def _quoted_end(text: str, i: int) -> int: + """End offset of the quoted span opening at ``i`` (a doubled quote escapes; unterminated runs to the end).""" + quote, j, n = text[i], i + 1, len(text) + while j < n: + if text[j] != quote: + j += 1 + elif j + 1 < n and text[j + 1] == quote: + j += 2 + else: + return j + 1 + return n + + +def _regions(text: str) -> List[_Region]: + """Split ``text`` into code, ``'…'`` string and ``"…"`` identifier spans (doubled quotes escape).""" + out: List[_Region] = [] + i = start = 0 + n = len(text) + while i < n: + quote = text[i] + if quote not in ("'", '"'): + i += 1 + continue + if i > start: + out.append(_Region(kind="code", start=start, end=i)) + end = _quoted_end(text, i) + out.append(_Region(kind="string" if quote == "'" else "ident", start=i, end=end)) + i = start = end + if start < n: + out.append(_Region(kind="code", start=start, end=n)) + return out + + +def _mask_code(chunk: str) -> str: + if "{?" in chunk or "?}" in chunk: + raise _Problem(_OPTIONAL_BLOCKS) + chunk = _CODE_PLACEHOLDER_RE.sub( + lambda m: f"{_PH_OPEN}{m.group(1)}{_PH_CLOSE}" if m.group(1) else m.group(0), chunk, + ) + if "{" in chunk or "}" in chunk: + raise _Problem( + "unsupported AST node Set: a brace expression is not a variable placeholder; " + "write a variable as {name}." + ) + return chunk + + +def _mask_placeholders(text: str) -> str: + """``{name}`` outside string literals → sentinel-bracketed name of the same length.""" + if _PH_OPEN in text or _PH_CLOSE in text: + raise _Problem("unsupported character in expression.") + parts: List[str] = [] + for region in _regions(text): + chunk = text[region.start:region.end] + if region.kind == "code": + chunk = _mask_code(chunk) + elif region.kind == "ident" and re.search(r"\{" + _VAR_NAME + r"\}", chunk): + raise _Problem(f"{_LITERAL_VALUES}, never an identifier.") + parts.append(chunk) + return "".join(parts) + + +class _State: + """Per-parse scratch: the expression text and the farthest failure offset.""" + + def __init__(self) -> None: + self.text = "" + self.farthest = 0 + + +_STATE = _State() +_PARSE_LOCK = threading.Lock() + + +def _record_failure(_s: str, loc: int, _expr: Any, _err: Any) -> None: + if loc > _STATE.farthest: + _STATE.farthest = loc + + +def _on(element: pp.ParserElement, fn: Callable[[List[Any]], Any]) -> pp.ParserElement: + """``element`` producing ``fn(tokens)``, groups as plain lists.""" + def action(toks: pp.ParseResults) -> Any: + return fn(toks.as_list()) + return element.set_parse_action(action) + + +def _first(toks: pp.ParseResults) -> str: + """The first token of a terminal match, as text.""" + return str(toks[0]) + + +def _t(element: pp.ParserElement) -> pp.ParserElement: + """A terminal whose failures feed the farthest-failure error position.""" + element.set_fail_action(_record_failure) + return element + + +def _fail(detail: str) -> pp.ParserElement: + """Matches nothing and raises ``detail`` — the committed branch of a construct.""" + def _raise(_s: str, _loc: int, _toks: pp.ParseResults) -> None: + raise _Problem(detail) + return pp.Empty().set_parse_action(_raise) + + +def _need(element: pp.ParserElement, detail: str) -> pp.ParserElement: + return element | _fail(detail) + + +_RUN_CHAR = r"[^\x00-\x20\x7f!-/:-@\[-^`{-~\s]" +_RUN = r"(?!\d)" + _RUN_CHAR + r"+" + + +class _Word: + """A lexed identifier: the raw spelling, its NFKC name, and whether it was double-quoted.""" + + __slots__ = ("raw", "name", "quoted") + + def __init__(self, raw: str, name: str, quoted: bool) -> None: + self.raw = raw + self.name = name + self.quoted = quoted + + +def _check_reserved(name: str) -> None: + if name.startswith(_RESERVED_EXPR_PREFIX): + raise _Problem( + f"the identifier {name!r} uses the reserved {_RESERVED_EXPR_PREFIX!r} prefix, which is " + f"reserved for SLayer-internal names." + ) + + +def _bare_word(s: str, loc: int, toks: pp.ParseResults) -> _Word: + raw = _first(toks) + name = unicodedata.normalize("NFKC", raw) + if not name.isidentifier(): + raise pp.ParseException(s, loc, f"{raw!r} is not an identifier") + _check_reserved(name) + return _Word(raw=raw, name=name, quoted=False) + + +def _quoted_word(toks: List[Any]) -> _Word: + raw = toks[0] + name = raw[1:-1].replace('""', '"') + if not name: + raise _Problem('an empty quoted identifier "" names nothing.') + _check_reserved(name) + return _Word(raw=raw, name=name, quoted=True) + + +def _kw(*words: str) -> pp.ParserElement: + """An ASCII-exact, case-insensitive keyword (a Unicode case-fold spoof is never one).""" + wanted = frozenset(words) + def is_keyword(toks: pp.ParseResults) -> bool: + raw = _first(toks) + return raw.isascii() and raw.lower() in wanted + return _t(pp.Regex(_RUN).add_condition(is_keyword).set_name("/".join(w.upper() for w in words))) + + +def _sym(pattern: str, name: str) -> pp.ParserElement: + return _t(pp.Regex(pattern).set_name(name)) + + +LPAR, RPAR = pp.Suppress(_sym(r"\(", "'('")), pp.Suppress(_sym(r"\)", "')'")) +LBRACK, RBRACK = pp.Suppress(_sym(r"\[", "'['")), pp.Suppress(_sym(r"\]", "']'")) +COMMA = pp.Suppress(_sym(",", "','")) +DOT = pp.Suppress(_sym(r"\.", "'.'")) +STAR = _sym(r"\*(?!\*)", "'*'") +COLON = pp.Suppress(_sym(r":(?!:)", "':'")) +DCOLON = pp.Suppress(_sym("::", "'::'")) +EQ = pp.Suppress(_sym(r"=(?!=)", "'='")) + +_BARE = _t(pp.Regex(_RUN).set_name("identifier")).set_parse_action(_bare_word) +_QUOTED = _on(_t(pp.Regex(r'"(?:[^"]|"")*"').set_name("quoted identifier")), _quoted_word) +_PLACEHOLDER = _t(pp.Regex(_PH_OPEN + "(" + _VAR_NAME + ")" + _PH_CLOSE).set_name("variable")) + + +def _number(toks: List[Any]) -> Literal: + text = toks[0] + return Literal(value=Decimal(text)) + + +def _string(toks: List[Any]) -> Literal: + return Literal(value=toks[0][1:-1].replace("''", "'")) + + +_NUMBER = _on(_t(pp.Regex(r"(?:\d+\.\d*|\.\d+|\d+)(?:[eE][+-]?\d+)?").set_name("number")), _number) +_STRING = _on(_t(pp.Regex(r"'(?:[^']|'')*'").set_name("string")), _string) +_CONSTANT_WORDS = {"True": True, "False": False, "None": None} +_SQL_CONSTANT_WORDS = {"true": True, "false": False, "null": None} + + +def _constant_value(raw: str) -> Tuple[bool, Any]: + if raw in _CONSTANT_WORDS: + return True, _CONSTANT_WORDS[raw] + if raw.isascii() and raw.lower() in _SQL_CONSTANT_WORDS: + return True, _SQL_CONSTANT_WORDS[raw.lower()] + return False, None + + +def _constant(s: str, loc: int, toks: pp.ParseResults) -> Literal: + known, value = _constant_value(_first(toks)) + if not known: + raise pp.ParseException(s, loc, "not a constant") + return Literal(value=value) + + +_CONSTANT = _t(pp.Regex(_RUN).set_name("constant")).set_parse_action(_constant) + + +def _placeholder_node(s: str, loc: int, toks: pp.ParseResults) -> Placeholder: + raw = _first(toks) + end = loc + len(raw) + rest = s[end:].lstrip() + if rest.startswith(("(", ".")) or (rest.startswith(":") and not rest.startswith("::")): + raise _Problem(f"{_LITERAL_VALUES}; a variable cannot name a function, column or path.") + return Placeholder(name=raw[1:-1]) + + +def _reject_placeholder(_s: str, _loc: int, _toks: pp.ParseResults) -> None: + raise _Problem(f"{_LITERAL_VALUES}; a variable cannot name a column or path segment.") + + +def _reject_placeholder_kwarg(_s: str, _loc: int, _toks: pp.ParseResults) -> None: + raise _Problem(f"{_LITERAL_VALUES}; a variable cannot name a keyword argument.") + + +# --------------------------------------------------------------------------- +# Grammar +# --------------------------------------------------------------------------- + + +expr = pp.Forward() +expr.set_name("expression") +_concat_level = pp.Forward() + +_SEGMENT = _BARE | _QUOTED +_PATH = pp.Group(_SEGMENT + pp.ZeroOrMore(DOT + (_SEGMENT | _PLACEHOLDER.copy().set_parse_action(_reject_placeholder)))) + + +def _path_node(words: List[_Word]) -> Ref | DottedRef: + quoted = any(w.quoted for w in words) + if len(words) == 1: + return Ref(name=words[0].name, quoted=quoted) + return DottedRef(parts=tuple(w.name for w in words), quoted=quoted) + + +_PATH_STAR = pp.Group(_PATH + DOT + pp.Suppress(STAR)) + + +class _Kw: + __slots__ = ("name", "value") + + def __init__(self, name: str, value: Any) -> None: + self.name = name + self.value = value + + +class _Distinct: + __slots__ = ("value",) + + def __init__(self, value: Any) -> None: + self.value = value + + +_ARG_END = pp.FollowedBy(_sym(r"[,)]", "',' or ')'")) +_STAR_ARG = _on((pp.Suppress(STAR) + _ARG_END), lambda _toks: StarSource()) +_PATH_STAR_ARG = _on((_PATH_STAR + _ARG_END), lambda toks: DottedRef(parts=(*(w.name for w in toks[0][0]), "*"), quoted=any(w.quoted for w in toks[0][0]))) +_SEQ_ITEMS = pp.Group(pp.Optional(pp.DelimitedList(expr, allow_trailing_delim=True))) +_LIST_VALUE = _on((LBRACK + _SEQ_ITEMS + RBRACK), lambda toks: tuple(toks[0])) +_TUPLE_VALUE = _on((LPAR + pp.Group(expr + pp.OneOrMore(COMMA + expr) + pp.Optional(COMMA)) + RPAR), lambda toks: tuple(toks[0])) +_DISTINCT_VALUE = _kw("distinct") + _fail(f"{_DISTINCT_ONLY}.") +_KW_VALUE = _DISTINCT_VALUE | _LIST_VALUE | _TUPLE_VALUE | _STAR_ARG | _PATH_STAR_ARG | expr +_KWARG = _on((_BARE + EQ + _KW_VALUE), lambda toks: _Kw(name=toks[0].name, value=toks[1])) +_PH_KWARG = (_PLACEHOLDER + EQ).set_parse_action(_reject_placeholder_kwarg) +_VALUE_ARG = _STAR_ARG | _PATH_STAR_ARG | expr +_DISTINCT_ARG = _on((pp.Suppress(_kw("distinct")) + _need(_VALUE_ARG, f"{_DISTINCT_ONLY}.")), lambda toks: _Distinct(value=toks[0])) +_UNPACKING = _sym(r"\*\*", "'**'") + _fail("dictionary unpacking (**kwargs) is not supported in calls.") +_ANY_ARG = _UNPACKING | _PH_KWARG | _KWARG | _DISTINCT_ARG | _VALUE_ARG +_POSITIONAL_ARG = _UNPACKING | _PH_KWARG | _DISTINCT_ARG | _VALUE_ARG + + +def _arg_list(first: pp.ParserElement, rest: pp.ParserElement) -> pp.ParserElement: + return pp.Group(pp.Optional(first + pp.ZeroOrMore(COMMA + rest) + pp.Optional(COMMA))) + + +def _callee_is(kind: str): + def _check(toks: pp.ParseResults) -> bool: + name = toks.as_list()[0][-1].name + if kind == "scalar": + return name.lower() in SCALAR_FUNCTIONS or name.lower() in DATE_PART_ALIASES + return normalize_aggregation_name(name) in BUILTIN_AGGREGATIONS or name in ALL_TRANSFORMS + return _check + + +_SCALAR_CALL = _PATH.copy().add_condition(_callee_is("scalar")) + LPAR + _arg_list(_POSITIONAL_ARG, _POSITIONAL_ARG) + RPAR +_VALUE_FIRST_CALL = _PATH.copy().add_condition(_callee_is("value")) + LPAR + _arg_list(_POSITIONAL_ARG, _ANY_ARG) + RPAR +_OTHER_CALL = _PATH + LPAR + _arg_list(_ANY_ARG, _ANY_ARG) + RPAR +_CALL = _SCALAR_CALL | _VALUE_FIRST_CALL | _OTHER_CALL + + +def _split_args(items: List[Any], *, call: str) -> Tuple[Tuple[Any, ...], Tuple[Tuple[str, Any], ...], bool]: + positional: List[Any] = [] + kwargs: List[Tuple[str, Any]] = [] + distinct = False + for i, item in enumerate(items): + if isinstance(item, _Kw): + if any(k == item.name for k, _ in kwargs): + raise _Problem(f"{call}() got keyword argument {item.name!r} more than once.") + kwargs.append((item.name, item.value)) + continue + if kwargs: + raise _Problem(f"{call}(): a positional argument follows a keyword argument.") + if isinstance(item, _Distinct): + if i != 0: + raise _Problem(f"{_DISTINCT_ONLY}.") + distinct = True + item = item.value + if isinstance(item, tuple): + raise _Problem(f"{call}(): a list is only valid as a keyword-argument value.") + positional.append(item) + return tuple(positional), tuple(kwargs), distinct + + +class _PendingCall: + """A parsed call, dispatched once its postfix is known (a trailing ``OVER (`` rejects first).""" + + __slots__ = ("words", "items") + + def __init__(self, words: List[_Word], items: List[Any]) -> None: + self.words = words + self.items = items + + +def _resolve(node: Any) -> Any: + return _dispatch_pending(node) if isinstance(node, _PendingCall) else node + + +def _dispatch_pending(call: _PendingCall) -> Any: + words = call.words + name = words[-1].name + args, kwargs, distinct = _split_args(call.items, call=name) + if distinct: + if name.lower() != "count" or len(words) > 1 or len(args) != 1 or isinstance(args[0], StarSource): + raise _Problem(f"{_DISTINCT_ONLY}.") + name = "count_distinct" + if len(words) > 1: + return _method_call(path=tuple(w.name for w in words[:-1]), name=name, args=args, kwargs=kwargs) + return _dispatch_call(name, args=args, kwargs=kwargs, original=_STATE.text) + + +_on(_CALL, lambda toks: _PendingCall(words=list(toks[0]), items=list(toks[1]))) + +_COLON_SOURCE = (_on(pp.Suppress(STAR), lambda _toks: StarSource()) + | _on(_PATH_STAR.copy(), lambda toks: DottedRef(parts=(*(w.name for w in toks[0][0]), "*"), + quoted=any(w.quoted for w in toks[0][0]))) + | _on(_PATH.copy(), lambda toks: _path_node(list(toks[0])))) +_COLON_AGG = _COLON_SOURCE + COLON + _BARE + pp.Optional(LPAR + _arg_list(_ANY_ARG, _ANY_ARG) + RPAR) + + +def _colon_node(toks: List[Any]) -> AggCall: + source, agg = toks[0], toks[1].name + args, kwargs, distinct = _split_args(list(toks[2]) if len(toks) > 2 else [], call=agg) + if distinct: + raise _Problem(f"{_DISTINCT_ONLY}.") + _reject_bare_star_args(args, kwargs, func_name=agg, original=_STATE.text) + return AggCall(source=source, agg=agg, args=args, kwargs=kwargs) + + +_on(_COLON_AGG, _colon_node) +_REF = _on(_PATH.copy(), lambda toks: _path_node(list(toks[0]))) + +# -- CASE ---------------------------------------------------------------------- + +_CASE_WORDS = ("case", "when", "then", "else", "end") +WHEN, THEN, ELSE, END = (pp.Suppress(_kw(w)) for w in _CASE_WORDS[1:]) +_MISSING_THEN = "Malformed CASE expression: a WHEN branch is missing its THEN." +_WHEN_CLAUSE = pp.Group(WHEN + _need(expr, "Malformed CASE expression: a WHEN needs a condition.") + + _need(THEN, _MISSING_THEN) + + _need(expr, "Malformed CASE expression: a THEN needs a value.")) +_CASE_TAIL = (pp.Group(pp.OneOrMore(_WHEN_CLAUSE)) + + pp.Group(pp.Optional(ELSE + _need(expr, "Malformed CASE expression: an ELSE needs a value."))) + + _need(END, "Malformed CASE expression: missing END.")) +_SEARCHED_CASE = pp.Suppress(_kw("case")) + pp.FollowedBy(WHEN) + pp.Group(pp.Empty()) + _CASE_TAIL +# A bare keyword-named simple-CASE operand (``CASE case WHEN …``) is ambiguous: no CASE. +_SIMPLE_CASE = (pp.Suppress(_kw("case")) + ~(_kw(*_CASE_WORDS) + WHEN) + + pp.Group(expr) + pp.FollowedBy(WHEN) + _CASE_TAIL) + + +def _case_node(toks: List[Any]) -> Any: + operand = toks[0][0] if len(toks[0]) else None + branches = [(b[0], b[1]) for b in toks[1]] + result = toks[2][0] if len(toks[2]) else Literal(value=None) + for cond, value in reversed(branches): + test = Cmp(op="==", left=operand, right=cond) if operand is not None else cond + result = ScalarCall(name="iif", args=(test, value, result)) + return result + + +_CASE = _on(_SEARCHED_CASE | _SIMPLE_CASE, _case_node) + +# -- CAST / EXTRACT / POSITION / SUBSTRING / TRIM ------------------------------ + +_TYPE_EXTRA_WORDS = ("precision", "varying", "with", "without", "time", "zone", "local", "unsigned", "signed") +_TYPE_WORD = _t(pp.Regex(r"[A-Za-z_][A-Za-z0-9_]*").set_name("type name")) +_TYPE_SPEC = (pp.Group(_TYPE_WORD + pp.ZeroOrMore(_kw(*_TYPE_EXTRA_WORDS))) + + pp.Group(pp.Optional(LPAR + pp.DelimitedList(_sym(r"\d+", "type parameter")) + RPAR))) + + +def _type_text(toks: List[Any]) -> str: + words, params = toks + text = " ".join(w.upper() for w in words) + return f"{text}({','.join(params)})" if len(params) else text + + +_on(_TYPE_SPEC, _type_text) +_CAST = ((pp.Suppress(_kw("cast")) | _kw("try_cast")) + LPAR + + _need(expr, "CAST needs a value: CAST(x AS ).") + + _need(pp.Suppress(_kw("as")), "CAST needs AS : CAST(x AS ).") + + _need(_TYPE_SPEC, "CAST needs a type after AS.") + + _need(RPAR, "CAST(x AS ) is missing its ')'.")) + + +def _cast_node(toks: List[Any]) -> Cast: + try_ = len(toks) == 3 + return Cast(source=toks[-2], type_text=toks[-1], try_=try_) + + +_on(_CAST, _cast_node) + +_EXTRACT = (pp.Suppress(_kw("extract")) + LPAR + (_STRING | _BARE) + pp.Suppress(_kw("from")) + + expr + RPAR) + + +def _extract_node(toks: List[Any]) -> ScalarCall: + part, value = toks + word = part.value if isinstance(part, Literal) else part.name + return ScalarCall(name="date_part", args=_date_args( + name="date_part", args=(Literal(value=word), value), original=_STATE.text, + )) + + +_on(_EXTRACT, _extract_node) +_POSITION = _on((pp.Suppress(_kw("position")) + LPAR + _concat_level + pp.Suppress(_kw("in")) + expr + RPAR), lambda toks: ScalarCall(name="instr", args=(toks[1], toks[0]))) +_SUBSTRING = _on((pp.Suppress(_kw("substring")) + LPAR + expr + pp.Suppress(_kw("from")) + expr + + pp.Optional(pp.Suppress(_kw("for")) + expr) + RPAR), lambda toks: ScalarCall(name="substr", args=tuple(toks))) +_TRIM_FORM = (_kw("trim") + LPAR + _kw("both", "leading", "trailing")) + _fail( + "TRIM(BOTH | LEADING | TRAILING …) is not supported; use trim(x), ltrim(x) or rtrim(x).", +) + +# -- typed literals, intervals, clock keywords ---------------------------------- + +_TEMPORAL = _on((_kw("date", "timestamp") + _STRING), lambda toks: TemporalLiteral(kind=toks[0].lower(), text=toks[1].value)) +_TIME_LITERAL = (_kw("time") + _STRING) + _fail( + "TIME '…' literals are not supported (there is no time-of-day type); compare with a TIMESTAMP '…' literal.", +) +_UNIT_WORDS = frozenset(g.value for g in TimeGranularity) + + +def _unit_of(word: str) -> Optional[str]: + low = word.lower() + if low in _UNIT_WORDS: + return low + if low.endswith("s") and low[:-1] in _UNIT_WORDS: + return low[:-1] + return None + + +def _is_unit(toks: pp.ParseResults) -> bool: + raw = _first(toks) + return raw.isascii() and _unit_of(raw) is not None + + +_UNIT = _t(pp.Regex(_RUN).add_condition(_is_unit)) +_INTERVAL = (pp.Suppress(_kw("interval")) + (_STRING | _NUMBER) + pp.Optional(_UNIT)) + + +def _interval_count(text: str) -> Literal: + if not re.fullmatch(r"[+-]?\d+", text.strip()): + raise _Problem(f"an interval count must be a whole number; got {text.strip()!r}.") + return Literal(value=Decimal(int(text))) + + +def _interval_node(toks: List[Any]) -> ScalarCall: + quantity = toks[0] + unit_word = toks[1] if len(toks) > 1 else None + if isinstance(quantity.value, str): + pieces = quantity.value.split() + else: + pieces = [str(quantity.value)] + if unit_word is not None: + pieces.append(unit_word) + if len(pieces) > 2 and len(pieces) % 2 == 0 and unit_word is None: + chain = " ".join(f"+ INTERVAL '{n} {u}'" for n, u in zip(pieces[::2], pieces[1::2])) + raise _Problem(f"an interval literal names one unit; write a chain: x {chain}.") + if len(pieces) != 2: + raise _Problem( + "write an interval literal as INTERVAL ' ', INTERVAL '' or INTERVAL ." + ) + count, word = pieces + unit = _unit_of(word) + if unit is None: + accepted = ", ".join(g.value for g in TimeGranularity) + raise _Problem(f"unknown interval unit {word!r}; accepted: {accepted}.") + return ScalarCall(name="interval", args=(_interval_count(count), Literal(value=unit))) + + +_on(_INTERVAL, _interval_node) +_CLOCK = _on((_kw("current_date", "current_timestamp") + ~pp.FollowedBy(LPAR)), lambda toks: ScalarCall(name="current_date" if toks[0].lower() == "current_date" else "now")) + +# -- primary, postfix ---------------------------------------------------------- + +_PAREN = LPAR + expr + RPAR +_PLACEHOLDER_ATOM = _PLACEHOLDER.copy().set_parse_action(_placeholder_node) +_KEYWORD_LED = pp.FollowedBy(pp.Regex( + r"(?i:case|cast|try_cast|extract|position|substring|trim|date|timestamp|time|interval|current_date|current_timestamp)" + r"(?![A-Za-z0-9_])" +)) + (_CASE | _CAST | _EXTRACT | _POSITION | _SUBSTRING | _TRIM_FORM | _TEMPORAL | _TIME_LITERAL | _INTERVAL | _CLOCK) +_ATOM = (_KEYWORD_LED | _CONSTANT | _PLACEHOLDER_ATOM | _NUMBER | _STRING | _PAREN + | _CALL | _COLON_AGG | _REF) + +_FILTER_CLAUSE = (pp.Suppress(_kw("filter")) + LPAR + + _need(pp.Suppress(_kw("where")), "FILTER needs (WHERE ).") + + _need(expr, "FILTER (WHERE …) needs a condition.") + + _need(RPAR, "FILTER (WHERE ) is missing its ')'.")) + + +def _reject_over(_s: str, _loc: int, _toks: pp.ParseResults) -> None: + raise IllegalWindowInFilterError( + filter_expr=_STATE.text, + source="raw OVER(...) is not allowed in Mode-B DSL", + suggestion="use a transform instead (rank, percent_rank, dense_rank, ntile, cumsum, lag, lead, time_shift, …).", + ) + + +_OVER = (_kw("over") + pp.FollowedBy(LPAR)).set_parse_action(_reject_over) +_PRIMARY = _ATOM + pp.Group(pp.ZeroOrMore(pp.Group(_FILTER_CLAUSE) | _OVER)) + + +def _primary_node(toks: List[Any]) -> Any: + node = _resolve(toks[0]) + for clause in toks[1]: + if not isinstance(node, AggCall): + raise _Problem("FILTER (WHERE …) applies only to an aggregation, e.g. sum(x) FILTER (WHERE c).") + if node.filter is not None: + raise _Problem("an aggregation takes one FILTER (WHERE …) clause.") + node = node.model_copy(update={"filter": clause[0]}) + return node + + +_on(_PRIMARY, _primary_node) + + +def _fold_casts(toks: pp.ParseResults) -> Any: + items = list(toks[0]) + node = items[0] + for type_text in items[1:]: + node = Cast(source=node, type_text=type_text) + return node + + +def _signed(toks: pp.ParseResults) -> Any: + sign, node = toks[0] + return UnaryOp(op="-", operand=node) if sign == "-" else node + + +def _pow(toks: pp.ParseResults) -> Any: + left, right = toks[0] + return ScalarCall(name="pow", args=(left, right)) + + +def _fold_arith(toks: pp.ParseResults) -> Any: + items = list(toks[0]) + node = items[0] + for op, right in zip(items[1::2], items[2::2]): + node = _interval_as_date_add(op=op, left=node, right=right) or Arith(op=op, left=node, right=right) + return node + + +def _fold_concat(toks: pp.ParseResults) -> Any: + items = list(toks[0]) + node = items[0] + for right in items[1:]: + node = ScalarCall(name="concat", args=(node, right)) + return node + + +_FLOORDIV = _sym("//", "'//'") + _fail("'//' is not an operator; use div(a, b) or floor(a / b).") +_PIPE = _sym(r"\|(?!\|)", "'|'") + _fail("'|' is not an operator; use || to concatenate strings.") +# Parentheses are atoms (_PAREN), so infix_notation's own grouping is disabled. +_concat_level <<= pp.infix_notation(_PRIMARY, [ + (DCOLON + _need(_TYPE_SPEC, "'::' needs a type."), 1, pp.OpAssoc.LEFT, _fold_casts), + (pp.Suppress(_sym(r"\*\*", "'**'")), 2, pp.OpAssoc.RIGHT, _pow), + (_sym("[+-]", "'-' or '+'"), 1, pp.OpAssoc.RIGHT, _signed), + (_FLOORDIV | _sym(r"\*(?!\*)|/(?!/)|%", "'*', '/' or '%'"), 2, pp.OpAssoc.LEFT, _fold_arith), + (_sym("[+-]", "'+' or '-'"), 2, pp.OpAssoc.LEFT, _fold_arith), + (pp.Suppress(_PIPE | _sym(r"\|\|", "'||'")), 2, pp.OpAssoc.LEFT, _fold_concat), +], lpar=pp.NoMatch(), rpar=pp.NoMatch()) # pyright: ignore[reportArgumentType] — unannotated; documented to accept a ParserElement + +# -- comparison level ---------------------------------------------------------- + +_CMP_OPS = {"=": "==", "==": "==", "<>": "!=", "!=": "!=", "<": "<", "<=": "<=", ">": ">", ">=": ">="} +_IS = pp.Suppress(_kw("is")) +_NOT_FLAG = _on(pp.Optional(_kw("not")), lambda toks: bool(len(toks))) + + +class _Suffix: + """A comparison-level suffix: builds the predicate node from its left operand.""" + + __slots__ = ("build",) + + def __init__(self, build) -> None: + self.build = build + + +def _suffix(element: pp.ParserElement, build) -> pp.ParserElement: + return _on(element, lambda toks: _Suffix(lambda left: build(left, toks))) + + +def _is_op(negate: bool) -> str: + return "is not" if negate else "is" + + +def _negated(node: Any, negate: bool) -> Any: + return UnaryOp(op="not", operand=node) if negate else node + + +def _truth_test(left: Any, toks: List[Any]) -> Any: + negate, word = toks[0], toks[1].lower() + known = Cmp(op=_is_op(True), left=left, right=Literal(value=None)) + test = BoolOp(op="and", operands=(known, left if word == "true" else UnaryOp(op="not", operand=left))) + return _negated(test, negate) + + +def _in_rhs_elements(items: List[Any]) -> TupleLit: + if not items: + raise _Problem( + "an empty list is not allowed on the right-hand side of in / not in (dialect-dependent SQL); " + "use a non-empty literal list." + ) + elements: List[Literal] = [] + for item in items: + if (isinstance(item, UnaryOp) and item.op == "-" and isinstance(item.operand, Literal) + and isinstance(item.operand.value, Decimal)): + item = Literal(value=-item.operand.value) + if not isinstance(item, Literal): + raise _Problem( + "every element on the right-hand side of in / not in must be a literal (string, number, or " + f"boolean); got {type(item).__name__}." + ) + elements.append(item) + return TupleLit(elements=tuple(elements)) + + +_IN_LIST = ((LPAR + _PLACEHOLDER_ATOM + RPAR) + | _PLACEHOLDER_ATOM + | _on((LPAR + _SEQ_ITEMS + RPAR), lambda toks: _in_rhs_elements(list(toks[0]))) + | _on((LBRACK + _SEQ_ITEMS + RBRACK), lambda toks: _in_rhs_elements(list(toks[0]))) + | _STRING + | _fail("the right-hand side of in / not in must be a literal list, e.g. status in ('a', 'b').")) + +_CMP_SUFFIX = ( + _suffix(_sym(r"==|!=|<>|<=|>=|=|<|>", "comparison") + _concat_level, + lambda left, t: Cmp(op=_CMP_OPS[t[0]], left=left, right=t[1], single_eq=t[0] == "=")) + | _suffix(_IS + _NOT_FLAG + pp.Suppress(_kw("distinct")) + pp.Suppress(_kw("from")) + _concat_level, + lambda left, t: Cmp(op="is not distinct from" if t[0] else "is distinct from", left=left, right=t[1])) + | _suffix(_IS + _NOT_FLAG + pp.Suppress(_kw("null") | pp.Regex(r"None\b")), + lambda left, t: Cmp(op=_is_op(t[0]), left=left, right=Literal(value=None))) + | _suffix(_IS + _NOT_FLAG + (_kw("true", "false")), _truth_test) + | _suffix(_IS + _NOT_FLAG + _concat_level, + lambda left, t: Cmp(op=_is_op(t[0]), left=left, right=t[1])) + | _suffix(_NOT_FLAG + pp.Suppress(_kw("in")) + _IN_LIST, + lambda left, t: Cmp(op="not in" if t[0] else "in", left=left, right=t[1])) + | _suffix(_NOT_FLAG + pp.Suppress(_kw("between")) + _concat_level + pp.Suppress(_kw("and")) + _concat_level, + lambda left, t: _negated(BoolOp(op="and", operands=( + Cmp(op=">=", left=left, right=t[1]), Cmp(op="<=", left=left, right=t[2]), + )), t[0])) + | _suffix(_NOT_FLAG + _kw("like", "ilike") + _concat_level, + lambda left, t: _negated(ScalarCall(name=t[1].lower(), args=(left, t[2])), t[0])) + | (_NOT_FLAG + _kw("similar") + _kw("to")) + _fail("SIMILAR TO is not supported; use LIKE or ILIKE.") + | (_kw("regexp", "rlike") + _fail("the REGEXP / RLIKE operators are not supported; use LIKE or ILIKE.")) +) + + +def _comparison_node(toks: pp.ParseResults) -> Any: + left, *suffixes = toks[0] + if len(suffixes) > 1: + raise _Problem( + "chained comparisons are not supported. Each comparison must be a single comparison; split " + "(e.g.) `1 < x < 10` into `1 < x and x < 10`." + ) + return suffixes[0].build(left) if suffixes else left + + + + +def _not(toks: pp.ParseResults) -> Any: + return UnaryOp(op="not", operand=toks[0][1]) + + +def _bool(word: str): + return lambda toks: BoolOp(op=word, operands=tuple(toks[0])) + + +_PREDICATE = pp.infix_notation(_concat_level, [ + (_CMP_SUFFIX, 1, pp.OpAssoc.LEFT, _comparison_node), + (_kw("not"), 1, pp.OpAssoc.RIGHT, _not), + (pp.Suppress(_kw("and")), 2, pp.OpAssoc.LEFT, _bool("and")), + (pp.Suppress(_kw("or")), 2, pp.OpAssoc.LEFT, _bool("or")), +], lpar=pp.NoMatch(), rpar=pp.NoMatch()) # pyright: ignore[reportArgumentType] — unannotated; documented to accept a ParserElement +_TERNARY = _kw("if") + _fail( + "the Python conditional 'x if cond else y' is not supported; use CASE WHEN cond THEN x ELSE y END or iif(cond, x, y).", +) +expr <<= _PREDICATE + pp.Optional(_TERNARY) +_EXPRESSION = (expr + pp.StringEnd().set_fail_action(_record_failure)).parse_with_tabs() + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +@lru_cache(maxsize=8192) +def parse_expr(text: str) -> ParsedExpr: + """Parse a Mode-B expression string into a ``ParsedExpr``. + + Raises ``ValueError`` (empty / syntax / unsupported form / reserved prefix), + ``UnknownFunctionError``, or ``IllegalWindowInFilterError`` (raw ``OVER(...)``). + """ + if not text or not text.strip(): + raise ValueError("Empty Mode-B expression.") + with _PARSE_LOCK: + _STATE.text, _STATE.farthest = text, 0 + try: + masked = _mask_placeholders(text) + parsed = cast(ParsedExpr, _EXPRESSION.parse_string(masked, parse_all=True)[0]) + except _Problem as problem: + raise ValueError(f"Invalid Mode-B expression {text!r}: {problem.detail}") from None + except pp.ParseBaseException as e: + raise ValueError(f"Invalid Mode-B expression {text!r}: {_syntax_message(text, e)}") from None + if any(_is_interval(n) for n in _walk_parsed(parsed)): + raise ValueError( + f"Invalid Mode-B expression {text!r}: interval(n, unit) is only valid as " + f"`x + interval(n, unit)`, `interval(n, unit) + x` or `x - interval(n, unit)`; " + f"otherwise use date_add(x, n, unit)." + ) + return parsed + + +def _syntax_message(text: str, error: pp.ParseBaseException) -> str: + pos = max(_STATE.farthest, error.loc) + if pos >= len(text.rstrip()): + return f"unexpected end of expression at column {pos + 1}." + return f"syntax error at column {pos + 1}, near {text[pos:pos + 12]!r}." + + +# --------------------------------------------------------------------------- +# Variables +# --------------------------------------------------------------------------- + + +def placeholder_names(parsed: Any) -> List[str]: + """Names of every ``Placeholder`` in a parsed tree, in walk order.""" + return [n.name for n in _walk_parsed(parsed) if isinstance(n, Placeholder)] + + +_ANY_BRACE_RE = re.compile(r"\{\{|\}\}|\{([^}]*)\}") +_NUMBER_TEXT_RE = re.compile(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?") + + +def bind_variables(text: str, variables: Mapping[str, Any]) -> str: + """``text`` with every ``{name}`` bound as a literal value: inside a string literal its text, + in a bare literal slot a number or boolean, as a whole ``IN`` list a list. The template is + parsed first, so no value can change the expression's structure.""" + if not _ANY_BRACE_RE.search(text): + return text + regions = [r for r in _regions(text) if r.kind != "ident"] + for region in regions: + chunk = text[region.start:region.end] + if region.kind == "code" and ("{?" in chunk or "?}" in chunk): + raise ValueError(f"{_OPTIONAL_BLOCKS} Got: {text!r}.") + for m in _ANY_BRACE_RE.finditer(chunk): + _check_variable(m, text=text, variables=variables) + # Strings bound and ``{{`` / ``}}`` outside strings read as braces: the grammar judges the variable slots. + parse_expr(_spliced(text, variables=variables, regions=regions, kinds=("string",), code_braces=True)) + return _spliced(text, variables=variables, regions=regions, kinds=("string", "code"), code_braces=False) + + +def _spliced( + text: str, *, variables: Mapping[str, Any], regions: List[_Region], kinds: Tuple[str, ...], code_braces: bool, +) -> str: + out: List[str] = [] + cursor = 0 + for region in regions: + out.append(text[cursor:region.start]) + chunk = text[region.start:region.end] + if region.kind in kinds: + in_string = region.kind == "string" + chunk = _ANY_BRACE_RE.sub( + partial(_bound_text, text=text, region_start=region.start, in_string=in_string, variables=variables), + chunk, + ) + elif code_braces: + chunk = chunk.replace("{{", "{").replace("}}", "}") + out.append(chunk) + cursor = region.end + out.append(text[cursor:]) + return "".join(out) + + +def _check_variable(match: "re.Match[str]", *, text: str, variables: Mapping[str, Any]) -> None: + if match.group(0) in ("{{", "}}"): + return + name = match.group(1) + if not re.fullmatch(_VAR_NAME, name): + raise ValueError( + f"Invalid variable name '{name}' in filter: {text!r}. " + f"Variable names must contain only letters, digits, and underscores." + ) + if name not in variables: + raise ValueError( + f"Undefined variable '{name}' in filter: {text!r}. Available variables: {sorted(variables)}" + ) + + +def _bound_text( + match: "re.Match[str]", *, text: str, region_start: int, in_string: bool, variables: Mapping[str, Any], +) -> str: + whole = match.group(0) + if whole in ("{{", "}}"): + return whole[0] + name = match.group(1) + value = variables[name] + if in_string: + return _value_text(name, value).replace("'", "''") + start = region_start + match.start() + if _opens_in_list(text, start=start, end=region_start + match.end()): + values = value if isinstance(value, (list, tuple)) else [value] + if not values: + raise ValueError( + f"Variable '{name}' cannot be an empty list; 'IN ()' is invalid SQL. " + f"For 'no filter' semantics, use a sentinel default." + ) + return ", ".join(_literal_text(name, v) for v in values) + return _bare_literal_text(name, value) + + +def _opens_in_list(text: str, *, start: int, end: int) -> bool: + """Whether the placeholder at ``[start, end)`` is the whole list of an ``IN``.""" + before = text[:start].rstrip() + if before.endswith("(") and text[end:].lstrip().startswith(")"): + before = before[:-1].rstrip() + tail = before[-3:] + return tail[-2:].lower() == "in" and not (len(tail) == 3 and tail[0].isascii() and tail[0].isalpha()) + + +def _value_text(name: str, value: Any) -> str: + if isinstance(value, bool): + return str(value) + if isinstance(value, str): + return value + if isinstance(value, (int, float)): + return _number_text(name, value) + raise ValueError(f"Variable '{name}' must be a string, number, or list/tuple, got {type(value).__name__}") + + +def _number_text(name: str, value: int | float) -> str: + if isinstance(value, float) and not math.isfinite(value): + raise ValueError(f"Variable '{name}' must be finite, got {value!r}") + return str(value) + + +def _literal_text(name: str, value: Any) -> str: + """One value as a literal: a string quoted, a number or boolean as itself.""" + if isinstance(value, str): + return "'" + value.replace("'", "''") + "'" + if isinstance(value, (list, tuple)): + raise ValueError(f"{_LITERAL_VALUES}: variable '{name}' holds a list, which fills only a whole IN list.") + return _value_text(name, value) + + +def _bare_literal_text(name: str, value: Any) -> str: + if isinstance(value, (list, tuple)): + raise ValueError( + f"{_LITERAL_VALUES}: variable '{name}' holds a list, which fills only a whole IN list, e.g. x in ({{{name}}})." + ) + if isinstance(value, str): + stripped = value.strip() + if _NUMBER_TEXT_RE.fullmatch(stripped): + return stripped + raise ValueError( + f"{_LITERAL_VALUES}: variable '{name}' holds the text {value!r} in a number or boolean slot; " + f"put the placeholder inside quotes ('{{{name}}}') to use it as a string." + ) + return _value_text(name, value) + + +# --------------------------------------------------------------------------- +# Saved-measure dependencies +# --------------------------------------------------------------------------- + + +def measure_dependency_problems( + formulas: List[Tuple[Optional[str], str]], *, aggregations: frozenset[str] = frozenset(), +) -> List[Optional[str]]: + """Per ``(name, formula)``: why the saved measure cannot resolve among its peers, else ``None``. + + Outside an aggregation a bare name must be a peer measure; an aggregation must be built-in + or one of ``aggregations``; a measure on a cycle, or reading one that cannot resolve, cannot + resolve either.""" + names = {name for name, _ in formulas if name} + own: Dict[int, Optional[str]] = {} + deps: Dict[int, set] = {} + for i, (_name, formula) in enumerate(formulas): + try: + refs, problem = _measure_refs(parse_expr(formula), names=names, aggregations=aggregations) + except ValueError as e: + refs, problem = set(), str(e) + own[i], deps[i] = problem, refs + index = {name: i for i, (name, _) in enumerate(formulas) if name} + out: List[Optional[str]] = [] + for i, (name, _) in enumerate(formulas): + out.append(_resolution_problem(i, own=own, deps=deps, index=index, path=(name or "",))) + return out + + +def _resolution_problem( + i: int, *, own: Dict[int, Optional[str]], deps: Dict[int, set], index: Dict[str, int], path: Tuple[str, ...], +) -> Optional[str]: + if own[i] is not None: + return own[i] + for dep in sorted(deps[i]): + if dep in path: + return f"saved measure {dep!r} has a cyclic reference: {' → '.join((*path, dep))}" + problem = _resolution_problem(index[dep], own=own, deps=deps, index=index, path=(*path, dep)) + if problem is not None: + return problem if dep == path[-1] else f"{dep!r} cannot resolve: {problem}" + return None + + +def _measure_refs(node: Any, *, names: set, aggregations: frozenset[str]) -> Tuple[set, Optional[str]]: + """The peer measures ``node`` reads outside aggregations, and the first problem found.""" + if isinstance(node, Ref): + if node.name in names: + return {node.name}, None + match = difflib.get_close_matches(word=node.name, possibilities=sorted(names), n=1) + hint = f" Did you mean '{match[0]}'?" if match else "" + return set(), ( + f"'{node.name}' is not a saved measure.{hint} Reference a saved measure by its bare name, or " + f"aggregate a column (e.g., 'sum({node.name})'). For COUNT(*), use 'count(*)'." + ) + if isinstance(node, DottedRef): + dotted = ".".join(node.parts) + return set(), f"Cross-model measure '{dotted}' must include an aggregation (e.g., 'sum({dotted})')." + if isinstance(node, AggCall): + known = normalize_aggregation_name(node.agg) in BUILTIN_AGGREGATIONS or node.agg in aggregations + return set(), None if known else f"unknown aggregation {node.agg!r}" + children = [node.input] if isinstance(node, TransformCall) else _operands(node) + refs: set = set() + for child in children: + found, problem = _measure_refs(child, names=names, aggregations=aggregations) + if problem is not None: + return refs, problem + refs |= found + return refs, None + + +# --------------------------------------------------------------------------- +# Reference walk +# --------------------------------------------------------------------------- + + +def walk_parsed_refs( + parsed: ParsedExpr, +) -> Iterator[Ref | DottedRef | AggCall]: + """Yield the reference-bearing leaves (``Ref`` / ``DottedRef`` / ``AggCall``) + of a tree — scope-free name extraction for schema-drift / memory tagging. + + ``AggCall`` is yielded whole (args/kwargs/filter opaque); ``TransformCall`` + descends ``input`` only; every other composite descends its operands. + """ + if isinstance(parsed, (Ref, DottedRef, AggCall)): + yield parsed + return + if isinstance(parsed, TransformCall): + yield from walk_parsed_refs(parsed.input) + return + if isinstance(parsed, ScalarCall): + for a in parsed.args: + yield from walk_parsed_refs(a) + return + if isinstance(parsed, Cast): + yield from walk_parsed_refs(parsed.source) + return + if isinstance(parsed, (Arith, Cmp)): + yield from walk_parsed_refs(parsed.left) + yield from walk_parsed_refs(parsed.right) + return + if isinstance(parsed, UnaryOp): + yield from walk_parsed_refs(parsed.operand) + return + if isinstance(parsed, BoolOp): + for op in parsed.operands: + yield from walk_parsed_refs(op) + + +# --------------------------------------------------------------------------- +# Call dispatch +# --------------------------------------------------------------------------- + + +# The node kinds an ``AggCall.source`` may take: a column, star, or an aggregation-free +# scalar expression, predicates included (a bare TupleLit is not a value). +_AGG_SOURCE_KINDS = ( + Ref, DottedRef, StarSource, Literal, TemporalLiteral, Placeholder, ScalarCall, Cast, Arith, UnaryOp, Cmp, BoolOp, +) + + +def _contains_agg_or_transform(node: Any) -> bool: + """Whether a parsed subtree contains any AggCall / TransformCall.""" + if isinstance(node, (AggCall, TransformCall)): + return True + return any(_contains_agg_or_transform(c) for c in _operands(node)) + + +def _operands(node: Any) -> List[Any]: + """The value operands of a row-level composite (empty for leaves and calls with their own scope).""" + if isinstance(node, ScalarCall): + return list(node.args) + if isinstance(node, Cast): + return [node.source] + if isinstance(node, (Arith, Cmp)): + return [node.left, node.right] + if isinstance(node, UnaryOp): + return [node.operand] + if isinstance(node, BoolOp): + return list(node.operands) + return [] + + +def _source_leaves(node: Any): + """Composition leaves of an agg source, treating an inner AggCall / + TransformCall as opaque.""" + children = _operands(node) + if not children: + yield node + return + for child in children: + yield from _source_leaves(child) + + +def _validated_agg_source(source: Any, *, func_name: str, original: str) -> Any: + """Validate a functional aggregation's first argument as its source; an attached + (re-aggregation) or mixed source is accepted.""" + leaves = list(_source_leaves(source)) + if any(isinstance(leaf, (AggCall, TransformCall)) for leaf in leaves): + return source + if not isinstance(source, _AGG_SOURCE_KINDS): + raise ValueError( + f"Invalid Mode-B expression {original!r}: {func_name!r} cannot " + f"aggregate a {type(source).__name__}; the aggregated expression " + f"must be built from columns, literals, arithmetic, comparisons, " + f"and scalar functions." + ) + return source + + +def _reject_bare_star_args( + args: Tuple[Any, ...], + kwargs: Tuple[Tuple[str, Any], ...], + *, + func_name: str, + original: str, +) -> None: + """``*`` is only an aggregation source — reject it as a plain call arg.""" + values = list(args) + [v for _, v in kwargs] + if any(isinstance(v, StarSource) for v in values): + raise ValueError( + f"Invalid Mode-B expression {original!r}: '*' is only valid as " + f"an aggregation source (e.g. count(*)), not as an argument to " + f"{func_name!r}." + ) + + +def _onto_path(node: Any, *, path: Tuple[str, ...]) -> Any: + """``node`` with every column leaf (and ``*``) read under the join ``path``.""" + if isinstance(node, Ref): + return DottedRef(parts=(*path, node.name), quoted=node.quoted) + if isinstance(node, DottedRef): + return DottedRef(parts=(*path, *node.parts), quoted=node.quoted) + if isinstance(node, StarSource): + return DottedRef(parts=(*path, "*")) + if isinstance(node, ScalarCall): + return node.model_copy(update={"args": tuple(_onto_path(a, path=path) for a in node.args)}) + if isinstance(node, Cast): + return node.model_copy(update={"source": _onto_path(node.source, path=path)}) + if isinstance(node, (Arith, Cmp)): + return node.model_copy(update={"left": _onto_path(node.left, path=path), "right": _onto_path(node.right, path=path)}) + if isinstance(node, UnaryOp): + return node.model_copy(update={"operand": _onto_path(node.operand, path=path)}) + if isinstance(node, BoolOp): + return node.model_copy(update={"operands": tuple(_onto_path(o, path=path) for o in node.operands)}) + return node + + +def _is_literal_arg(node: Any) -> bool: + if isinstance(node, UnaryOp) and node.op == "-": + return isinstance(node.operand, Literal) + return isinstance(node, (Literal, Placeholder)) + + +def _method_call_refusal(name: str, *, args: Tuple[Any, ...], kwargs: Tuple[Tuple[str, Any], ...]) -> Optional[str]: + """Why ``.(args)`` has no single functional reading, else ``None``.""" + lowered = name.lower() + transform_only = name in ALL_TRANSFORMS and normalize_aggregation_name(name) not in BUILTIN_AGGREGATIONS + if lowered in SCALAR_FUNCTIONS or lowered in DATE_PART_ALIASES or transform_only: + return f"calls {name!r}, which is not an aggregation" + if not args: + return "needs the aggregated value as its first argument" + if any(isinstance(leaf, (DottedRef, AggCall, TransformCall)) for leaf in _source_leaves(args[0])): + return "may read only bare column names of that path" + if any(k == "partition_by" for k, _ in kwargs): + return "cannot take partition_by=" + if not all(_is_literal_arg(v) for v in (*args[1:], *(v for _, v in kwargs))): + return "takes only literal further arguments" + return None + + +def _method_call(*, path: Tuple[str, ...], name: str, args: Tuple[Any, ...], kwargs: Tuple[Tuple[str, Any], ...]) -> ParsedExpr: + """``.(, )`` as its functional twin ``(., …)``.""" + original = _STATE.text + twin_args = (_onto_path(args[0], path=path), *args[1:]) if args else () + reason = _method_call_refusal(name, args=args, kwargs=kwargs) + if reason is not None: + raise ValueError( + f"Invalid Mode-B expression {original!r}: the method call {'.'.join(path)}.{name}(...) " + f"{reason}; write the functional form, e.g. {name}{_canonical_call_params(twin_args, kwargs)}." + ) + return _dispatch_call(name, args=twin_args, kwargs=kwargs, original=original) + + +_SCALAR_MIN_MAX = {"min": "least", "max": "greatest"} + + +def _dispatch_call( # NOSONAR(S3776) — the one call-dispatch ladder (builtin functional aggregation → transform → scalar → date-part alias → unknown-name AggCall deferral); each rung IS the documented dispatch order and splitting them hides it. + func_name: str, *, args: Tuple[Any, ...], kwargs: Tuple[Tuple[str, Any], ...], original: str, +) -> ParsedExpr: + # ``agg`` stores the RAW token so colon and functional spellings collapse to one + # AggCall; ``first``/``last`` dispatch at bind by the operand's type. + canonical_agg = normalize_aggregation_name(func_name) + if canonical_agg in BUILTIN_AGGREGATIONS: + if not args: + raise ValueError( + f"Invalid Mode-B expression {original!r}: aggregation " + f"{func_name!r} requires a value argument, e.g. " + f"{func_name}(column)." + ) + if canonical_agg in _SCALAR_MIN_MAX and len(args) == 2 and not kwargs: + scalar = _SCALAR_MIN_MAX[canonical_agg] + raise ValueError( + f"Invalid Mode-B expression {original!r}: {func_name}() with two values is the " + f"aggregation, not a scalar; use {scalar}{_canonical_call_params(args, ())}." + ) + source = _validated_agg_source( + args[0], func_name=func_name, original=original, + ) + return AggCall(source=source, agg=func_name, args=args[1:], kwargs=kwargs) + + if func_name in ALL_TRANSFORMS: + if not args: + raise ValueError( + f"Invalid Mode-B expression {original!r}: transform " + f"{func_name!r} requires at least one positional argument " + f"(the value to transform)." + ) + _reject_bare_star_args(args, kwargs, func_name=func_name, original=original) + return TransformCall(op=func_name, input=args[0], args=args[1:], kwargs=kwargs) + + # Normalised to lower case so ``COALESCE`` / ``coalesce`` intern to one key. + if func_name.lower() in SCALAR_FUNCTIONS: + if kwargs: + raise ValueError( + f"Invalid Mode-B expression {original!r}: scalar function " + f"{func_name!r} does not accept keyword arguments. Pass " + f"values positionally." + ) + _reject_bare_star_args(args, kwargs, func_name=func_name, original=original) + name = func_name.lower() + return ScalarCall(name=name, args=_date_args(name=name, args=args, original=original)) + + part = DATE_PART_ALIASES.get(func_name.lower()) + if part is not None: + if kwargs or len(args) != 1: + raise ValueError( + f"Invalid Mode-B expression {original!r}: date-part function {func_name!r} " + f"takes 1 argument; got {len(args) + len(kwargs)}." + ) + _reject_bare_star_args(args, kwargs, func_name=func_name, original=original) + return ScalarCall(name="date_part", args=(Literal(value=part.value), args[0])) + + # Unknown name with an aggregatable first arg → AggCall candidate, validated at binding. + if args and isinstance(args[0], _AGG_SOURCE_KINDS) and not _contains_agg_or_transform(args[0]): + return AggCall(source=args[0], agg=func_name, args=args[1:], kwargs=kwargs) + if args and any(isinstance(leaf, AggCall) for leaf in _source_leaves(args[0])): + source = _validated_agg_source(args[0], func_name=func_name, original=original) + return AggCall(source=source, agg=func_name, args=args[1:], kwargs=kwargs) + + if func_name.lower() in GRANULARITY_NAMES: + raise GranularityCallError.wrong_shape(original) + raise UnknownFunctionError( + name=func_name, + location=original, + suggestion=ambiguous_weekday_message(func_name) if func_name.lower() in AMBIGUOUS_WEEKDAY_NAMES else ( + f"Mode-B accepts the closed scalar allowlist " + f"({sorted(SCALAR_FUNCTIONS)}), transforms " + f"({sorted(ALL_TRANSFORMS)}), and aggregations " + f"(`sum(revenue)`); custom " + f"aggregation names must be defined on the model." + ), + ) + + +def keyword_spelling(node: Any) -> Optional[str]: + """The name of a ``name = value`` argument, which a scalar call reads as a comparison.""" + if isinstance(node, Cmp) and node.single_eq and isinstance(node.left, Ref) and not node.left.quoted: + return node.left.name + return None + + +def positional_hint(name: str) -> str: + return f"scalar functions take positional arguments; `{name} = …` here is a comparison." + + +def comparison_hint(name: str) -> str: + return f"to pass a comparison instead, write `{name} == …` or `({name} = …)`." + + +# Position of the count argument of the date-offset calls. +_COUNT_ARG = {"date_add": 1, "interval": 0} + + +def _date_args(*, name: str, args: Tuple[Any, ...], original: str) -> Tuple[Any, ...]: + """Validate and lower-case a date scalar's unit word; fold a negated numeric count literal.""" + out = list(args) + unit = DATE_UNIT_ARGS.get(name) + if unit is not None and unit[0] < len(out): + pos, vocab = unit + words = tuple(v.value for v in vocab) + out[pos] = Literal(value=_unit_word(out[pos], name=name, words=words, original=original)) + count_pos = _COUNT_ARG.get(name) + if count_pos is not None and count_pos < len(out): + out[count_pos] = _folded(out[count_pos]) + return tuple(out) + + +def _unit_word(arg: Any, *, name: str, words: Tuple[str, ...], original: str) -> str: + """The canonical lower-case unit (or ``date_part`` part) word of ``arg``, one of ``words``.""" + accepted = ", ".join(words) + what = "part" if name == "date_part" else "unit" + if not (isinstance(arg, Literal) and isinstance(arg.value, str)): + spelled = keyword_spelling(arg) + raise ValueError( + f"Invalid Mode-B expression {original!r}: the {what} of {name}() must be a " + f"string literal, one of: {accepted}." + (f" Note: {positional_hint(spelled)}" if spelled else "") + ) + word = arg.value.lower() + if name == "date_part": + if word in AMBIGUOUS_WEEKDAY_NAMES: + raise ValueError(f"Invalid Mode-B expression {original!r}: {ambiguous_weekday_message(arg.value)}") + word = DATE_PART_ALIASES[word].value if word in DATE_PART_ALIASES else word + if word not in words: + raise ValueError( + f"Invalid Mode-B expression {original!r}: unknown {name}() {what} " + f"{arg.value!r}; accepted: {accepted}." + ) + return word + + +def _folded(node: Any) -> Any: + """``-`` as one negative literal, so both count spellings intern alike.""" + if ( + isinstance(node, UnaryOp) and node.op == "-" + and isinstance(node.operand, Literal) and isinstance(node.operand.value, Decimal) + ): + return Literal(value=-node.operand.value) + return node + + +def _is_interval(node: Any) -> bool: + return isinstance(node, ScalarCall) and node.name == "interval" + + +def _interval_as_date_add(*, op: str, left: Any, right: Any) -> Optional[ScalarCall]: + """``x + interval(n, u)`` / ``interval(n, u) + x`` / ``x - interval(n, u)`` as ``date_add``.""" + if _is_interval(left) == _is_interval(right) or op not in ("+", "-"): + return None + if op == "-" and _is_interval(left): + return None + interval, base = (right, left) if _is_interval(right) else (left, right) + if len(interval.args) != 2: + raise ValueError(check_scalar_arity(name="interval", argc=len(interval.args))) + count, unit = interval.args + if op == "-": + count = _folded(UnaryOp(op="-", operand=count)) + return ScalarCall(name="date_add", args=(base, count, unit)) + + +def _walk_parsed(node: Any) -> Iterator[Any]: + """Every node of a parsed tree, including call args, kwargs, agg sources and filters.""" + if isinstance(node, tuple): + for item in node: + yield from _walk_parsed(item) + return + if not isinstance(node, _BaseNode): + return + yield node + for field in type(node).model_fields: + yield from _walk_parsed(getattr(node, field)) + + +# --------------------------------------------------------------------------- +# Canonical text rendering + entity-ref splitting +# --------------------------------------------------------------------------- + + +def _canonical_call_params( + args: Tuple[Any, ...], kwargs: Tuple[Tuple[str, Any], ...], *, bare_direction: bool = False, +) -> str: + parts = [canonical_measure_text(a) for a in args] + parts += [_canonical_kwarg(k, v, bare_direction=bare_direction) for k, v in kwargs] + return f"({', '.join(parts)})" if parts else "" + + +def _canonical_kwarg(name: str, value: Any, *, bare_direction: bool) -> str: + """``name=value``; a transform's recognised ``direction`` renders as its bare normalised value.""" + if bare_direction and name == "direction" and isinstance(value, Literal): + direction = normalize_direction(value.value) + if direction is not None: + return direction + return f"{name}={_canonical_kwarg_text(value)}" + + +def _canonical_kwarg_text(value: Any) -> str: + if isinstance(value, tuple): + return f"[{', '.join(canonical_measure_text(v) for v in value)}]" + return canonical_measure_text(value) + + +def _identifier_text(name: str) -> str: + return name if name.isidentifier() else '"' + name.replace('"', '""') + '"' + + +def canonical_measure_text(parsed: Any) -> str: # NOSONAR(S3776) — flat per-node-kind rendering table; each branch is one spelling rule. + """Render a ``ParsedExpr`` back to canonical legacy colon-spelling text. + + Used for alias derivation so the functional and legacy colon spellings of one + formula sanitize to the SAME public name. Deterministic, not a + verbatim round-trip: grouping parens are dropped and spacing normalised. + """ + if isinstance(parsed, Ref): + return _identifier_text(parsed.name) + if isinstance(parsed, DottedRef): + return ".".join(p if p == "*" else _identifier_text(p) for p in parsed.parts) + if isinstance(parsed, StarSource): + return "*" + if isinstance(parsed, Literal): + if isinstance(parsed.value, str): + return "'" + parsed.value.replace("'", "''") + "'" + return str(parsed.value) + if isinstance(parsed, TemporalLiteral): + return f"{parsed.kind.upper()} '{parsed.text}'" + if isinstance(parsed, Placeholder): + return f"{{{parsed.name}}}" + if isinstance(parsed, TupleLit): + return f"({', '.join(canonical_measure_text(e) for e in parsed.elements)})" + if isinstance(parsed, AggCall): + return _canonical_agg_text(parsed) + if isinstance(parsed, TransformCall): + inner = canonical_measure_text(parsed.input) + params = _canonical_call_params(parsed.args, parsed.kwargs, bare_direction=True) + return f"{parsed.op}({inner}{', ' + params[1:-1] if params else ''})" + if isinstance(parsed, ScalarCall): + return f"{parsed.name}({', '.join(canonical_measure_text(a) for a in parsed.args)})" + if isinstance(parsed, Cast): + fn = "TRY_CAST" if parsed.try_ else "CAST" + return f"{fn}({canonical_measure_text(parsed.source)} AS {parsed.type_text})" + if isinstance(parsed, (Arith, Cmp)): + left = canonical_measure_text(parsed.left) + right = canonical_measure_text(parsed.right) + return f"{left} {parsed.op} {right}" + if isinstance(parsed, UnaryOp): + operand = canonical_measure_text(parsed.operand) + return f"not {operand}" if parsed.op == "not" else f"{parsed.op}{operand}" + if isinstance(parsed, BoolOp): + return f" {parsed.op} ".join( + canonical_measure_text(o) for o in parsed.operands + ) + return str(parsed) + + +def _canonical_agg_text(parsed: AggCall) -> str: + source = canonical_measure_text(parsed.source) + if isinstance(parsed.source, (Ref, DottedRef, StarSource)): + text = f"{source}:{parsed.agg}{_canonical_call_params(parsed.args, parsed.kwargs)}" + else: + # Expression source (``sum(amount - cost)``): render functionally so the + # text can't collide with a distinct parse tree like ``amount - cost:sum``. + parts = [source] + parts += [canonical_measure_text(a) for a in parsed.args] + parts += [f"{k}={_canonical_kwarg_text(v)}" for k, v in parsed.kwargs] + text = f"{parsed.agg}({', '.join(parts)})" + if parsed.filter is not None: + text += f" FILTER (WHERE {canonical_measure_text(parsed.filter)})" + return text + + +def _functional_suffix_text(raw: str, *, agg: str) -> str: + """Rebuild the ``agg`` / ``agg(rest-args)`` suffix of a functional entity + ref, preserving any extra-arg text verbatim (``last(balance, updated_at)`` + → ``last(updated_at)``).""" + open_idx = raw.index("(") + depth = 0 + split_idx = None + for i in range(open_idx, len(raw)): + ch = raw[i] + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0: + split_idx = i + break + elif ch == "," and depth == 1: + split_idx = i + break + if split_idx is None or raw[split_idx] == ")": + return agg + rest = raw[split_idx + 1:].strip() + if rest.endswith(")"): + rest = rest[:-1].strip() + return f"{agg}({rest})" + + +def split_entity_agg_ref(raw: str) -> Tuple[str, Optional[str]]: + """``(prefix, agg_suffix)`` of a single aggregated-column entity + reference: ``sum(orders.amount)`` and the legacy ``orders.amount:sum`` split + identically. + + Legacy colon and call-free text splits exactly like + :func:`slayer.core.refs.split_agg_suffix`. Functional text must parse to + an ``AggCall`` over a pure column / star source; multi-column expression + text (``sum(a - b)``) raises ``ValueError`` — an expression is not an + entity. + """ + prefix, suffix = split_agg_suffix(raw) + if suffix is not None or "(" not in raw: + return prefix, suffix + parsed = parse_expr(raw) + if not isinstance(parsed, AggCall) or parsed.filter is not None or not isinstance( + parsed.source, (Ref, DottedRef, StarSource) + ): + raise ValueError( + f"{raw!r} is not a single aggregated column reference; only " + f"the `agg(column)` form names an entity." + ) + source = parsed.source + if isinstance(source, StarSource): + prefix = "*" + elif isinstance(source, Ref): + prefix = source.name + else: + prefix = ".".join(source.parts) + return prefix, _functional_suffix_text(raw, agg=parsed.agg) + + +__all__ = [ + "AggCall", "Arith", "BoolOp", "Cast", "Cmp", "DottedRef", "Literal", "ParsedExpr", "Placeholder", "Ref", + "ScalarCall", "StarSource", "TemporalLiteral", "TransformCall", "TupleLit", "UnaryOp", + "bind_variables", "canonical_measure_text", "comparison_hint", "keyword_spelling", "measure_dependency_problems", + "parse_expr", "placeholder_names", "positional_hint", "split_entity_agg_ref", "walk_parsed_refs", +] diff --git a/slayer/cube/converter.py b/slayer/cube/converter.py index 5fadce04f..362139b91 100644 --- a/slayer/cube/converter.py +++ b/slayer/cube/converter.py @@ -19,7 +19,7 @@ invert_cardinality, ) from slayer.core.format import NumberFormat, NumberFormatType -from slayer.core.formula import parse_formula +from slayer.core.syntax import measure_dependency_problems from slayer.core.models import Column, ModelJoin, ModelMeasure, SlayerModel from slayer.core.query import render_probe_text from slayer.core.refs import IDENTIFIER_RE, functional_agg_text @@ -57,7 +57,6 @@ } _DURATION_UNITS = {"day": "d", "month": "m", "week": "w", "year": "y", "hour": "h", "minute": "min", "second": "s"} -_STAR_COUNT = "count(*)" _CUBE_INFRA_FIELDS = ("refresh_key", "calendar", "hierarchies", "access_policy", "sql_alias") @@ -842,27 +841,15 @@ def _drop_measures_over(self, *, cube_name, measures, info, dropped, report) -> return good_measures def _drop_unparseable_measures(self, cube_name, good_measures, report) -> list[ModelMeasure]: - # Fixpoint: dropping a measure can break a calc measure naming it. Only - # measures are bare-referenceable; a bare column needs an aggregation. - while True: - known = {m.name for m in good_measures if m.name} - broken = [m for m in good_measures if not self._formula_parses(m.formula, known)] - if not broken: - return good_measures - for m in broken: + # Only measures are bare-referenceable; a bare column needs an aggregation. + problems = measure_dependency_problems([(m.name, m.formula) for m in good_measures]) + for m, problem in zip(good_measures, problems): + if problem is not None: report.add(CubeConversionIssue( category=CubeIssueCategory.COMPLEX_MEASURE, severity="warning", cube=cube_name, member=m.name, - message=f"Measure formula does not parse; dropped: {m.formula!r}")) - good_measures = [m for m in good_measures if m not in broken] - - def _formula_parses(self, formula: str, known_names: set[str]) -> bool: - nm = dict.fromkeys(known_names, _STAR_COUNT) - try: - parse_formula(formula, named_measures=nm or None) - return True - except Exception: # noqa: BLE001 - return False + message=f"Measure formula does not resolve ({problem}); dropped: {m.formula!r}")) + return [m for m, problem in zip(good_measures, problems) if problem is None] def _dedisambiguate_namespace(self, columns, measures, report, *, cube) -> None: # The allocator already guarantees uniqueness; this is a defensive check. diff --git a/slayer/dbt/converter.py b/slayer/dbt/converter.py index ace0a2ab5..553e2f5c1 100644 --- a/slayer/dbt/converter.py +++ b/slayer/dbt/converter.py @@ -24,7 +24,7 @@ from slayer.core.enums import DataType from slayer.core.format import NumberFormat, NumberFormatType -from slayer.core.formula import parse_formula +from slayer.core.syntax import measure_dependency_problems from slayer.core.models import ( Column, ModelJoin, @@ -261,32 +261,24 @@ def _prune_dangling_measures(self) -> None: self._prune_model_measures(model) def _prune_model_measures(self, model: SlayerModel) -> None: - agg_names = frozenset(a.name for a in model.aggregations) - changed = True - while changed: - changed = False - named = {m.name: m.formula for m in model.measures if m.name} - survivors: list[ModelMeasure] = [] - for m in model.measures: - others = {k: v for k, v in named.items() if k != m.name} - try: - parse_formula(m.formula, extra_agg_names=agg_names, named_measures=others) - survivors.append(m) - except (ValueError, RecursionError) as exc: - self._unconverted.append(ConversionWarning( - model_name=model.name, - metric_name=m.name, - category="dangling_reference", - severity="dropped", - message=( - f"Measure '{m.name}' references a name that does not resolve " - f"on model '{model.name}' and was dropped: {exc}" - ), - suggestion="Ensure every referenced metric/measure converts successfully.", - )) - changed = True - if changed: - model.measures = survivors + problems = measure_dependency_problems( + [(m.name, m.formula) for m in model.measures], + aggregations=frozenset(a.name for a in model.aggregations), + ) + for m, problem in zip(model.measures, problems): + if problem is not None: + self._unconverted.append(ConversionWarning( + model_name=model.name, + metric_name=m.name, + category="dangling_reference", + severity="dropped", + message=( + f"Measure '{m.name}' references a name that does not resolve " + f"on model '{model.name}' and was dropped: {problem}" + ), + suggestion="Ensure every referenced metric/measure converts successfully.", + )) + model.measures = [m for m, problem in zip(model.measures, problems) if problem is None] def _convert_regular_models(self, existing_names: set) -> list[SlayerModel]: """Convert orphan dbt models (not wrapped by semantic_models) to hidden SLayer models.""" diff --git a/slayer/engine/bind_inputs.py b/slayer/engine/bind_inputs.py index a10a94e79..59a1b5ee9 100644 --- a/slayer/engine/bind_inputs.py +++ b/slayer/engine/bind_inputs.py @@ -8,7 +8,7 @@ from datetime import date, datetime from typing import Callable, Dict, FrozenSet, List, Optional, Tuple, cast -from slayer.core.enums import BUILTIN_AGGREGATIONS, DataType, TimeGranularity, normalize_aggregation_name +from slayer.core.enums import BUILTIN_AGGREGATIONS, DataType, TimeGranularity, normalize_aggregation_name, TIME_TRANSFORMS from slayer.core.errors import ( AmbiguousJoinPathError, AmbiguousReferenceError, @@ -17,7 +17,6 @@ UnknownReferenceError, ) from slayer.core.format import NumberFormat -from slayer.core.formula import TIME_TRANSFORMS from slayer.core.time_spine import is_spine, is_spine_query, names_spine from slayer.core.join_walker import resolve_hop, resolved_terminal, walk_terminal from slayer.core.keys import ( @@ -49,6 +48,7 @@ SlayerQuery, TimeDimension, call_callee, + computed_dimension_name, functional_call_column, granularity_call_parts, ) @@ -98,7 +98,7 @@ scope_column_type, stage_measure_type, ) -from slayer.engine.syntax import ( +from slayer.core.syntax import ( AggCall, DottedRef, ParsedExpr, @@ -106,7 +106,6 @@ TransformCall, canonical_measure_text, parse_expr, - parse_filter_expr, ) from slayer.ir.bound import ( GranularityConflict, @@ -203,7 +202,7 @@ def _reject_measure_refs_in_filters( if not isinstance(f, str): continue try: - parsed = parse_filter_expr(f) + parsed = parse_expr(f) except Exception: # noqa: BLE001 — binder reports parse errors properly continue check_raw_rows_filter_measure_ref( @@ -286,6 +285,7 @@ def _map_bound_keys( routed_dotted=spec.bound.routed_dotted, ), direction=spec.direction, + nulls=spec.nulls, ) for spec in order_specs ] @@ -376,7 +376,7 @@ def bind_query_inputs( # NOSONAR(S3776) — one cohesive bind pass. The stages if not isinstance(f, str): continue bf = bind_filter( - parsed=parse_filter_expr(f), + parsed=parse_expr(f), scope=scope, bundle=bundle, alias_map=filter_alias_map, @@ -431,6 +431,7 @@ def bind_query_inputs( # NOSONAR(S3776) — one cohesive bind pass. The stages granularity=_td_granularity(matching_td, bundle=bundle), ).bound, direction=o.direction, + nulls=o.nulls or "default", )) continue # A placeholder ColumnRef means the item is an EXPRESSION: bind raw_formula, skip alias lookups. @@ -440,9 +441,11 @@ def bind_query_inputs( # NOSONAR(S3776) — one cohesive bind pass. The stages parsed=parse_expr(o.raw_formula), scope=scope, bundle=bundle, + alias_map=filter_alias_map, dimension_alias_map=dim_alias_map, ), direction=o.direction, + nulls=o.nulls or "default", )) continue # An ORDER BY over a partition_by / window= aggregate must bind raw_formula (the alias shortcut would drop the partition/window). @@ -452,6 +455,7 @@ def bind_query_inputs( # NOSONAR(S3776) — one cohesive bind pass. The stages _part_bound = bind_expr( parsed=parse_expr(o.raw_formula), scope=scope, bundle=bundle, + alias_map=filter_alias_map, dimension_alias_map=dim_alias_map, ) if any( @@ -461,7 +465,7 @@ def bind_query_inputs( # NOSONAR(S3776) — one cohesive bind pass. The stages for k in walk_value_keys(_part_bound.value_key) ): order_specs.append(OrderSpec( - bound=_part_bound, direction=o.direction, + bound=_part_bound, direction=o.direction, nulls=o.nulls or "default", )) continue # A FOREIGN-qualified order ref must not resolve to a same-named local column via the bare-leaf shortcut. @@ -485,6 +489,7 @@ def bind_query_inputs( # NOSONAR(S3776) — one cohesive bind pass. The stages parsed=parse_expr(o.raw_formula), scope=scope, bundle=bundle, + alias_map=filter_alias_map, dimension_alias_map=dim_alias_map, ) else: @@ -494,7 +499,7 @@ def bind_query_inputs( # NOSONAR(S3776) — one cohesive bind pass. The stages scope=scope, bundle=bundle, ) - order_specs.append(OrderSpec(bound=bo, direction=o.direction)) + order_specs.append(OrderSpec(bound=bo, direction=o.direction, nulls=o.nulls or "default")) column_type = scope_column_type(scope=scope, bundle=bundle) check_date_operands( @@ -951,7 +956,7 @@ def _declared_computed_dimension( bound=bound, declared_name=name, public_name=name, - name_is_explicit=name != auto_name_from_expression(d.template), + name_is_explicit=name != computed_dimension_name(d.template), type=dim_type, is_dimension=True, ) @@ -1219,7 +1224,7 @@ def _resolve_granularity_calls(query: SlayerQuery, *, bundle: ResolvedSourceBund td = ( _datasource_time_dimension(item.expression, bundle=bundle) if isinstance(item, ComputedDimension) - and item.name == auto_name_from_expression(item.template) + and item.name == computed_dimension_name(item.template) else None ) if td is None: diff --git a/slayer/engine/binding.py b/slayer/engine/binding.py index 9b15d720a..d12063122 100644 --- a/slayer/engine/binding.py +++ b/slayer/engine/binding.py @@ -5,14 +5,18 @@ import difflib import os +from datetime import datetime from decimal import Decimal -from typing import Dict, Final, List, Mapping, Optional, Tuple, TypeGuard +from functools import partial +from typing import Callable, Dict, Final, List, Mapping, Optional, Tuple, TypeGuard from pydantic import BaseModel, ConfigDict from slayer.core.errors import ( AggregationArgumentError, + AggregationFilterError, AggregationNotAllowedError, + CastTypeError, CircularJoinPathError, GranularityCallError, IllegalScopeReferenceError, @@ -20,32 +24,19 @@ MeasureCycleError, MeasureRecursionLimitError, PartitionKeyError, + TimeLiteralError, TransformArgumentError, UnknownFunctionError, UnknownReferenceError, UnresolvableDimensionJoinError, UnresolvedPlaceholderError, ) -from slayer.core.enums import ( - AMBIGUOUS_WEEKDAY_NAMES, - BUILTIN_AGGREGATION_PARAM_ORDER, - BUILTIN_AGGREGATIONS, - DATE_PART_ALIASES, - DEFAULT_AGGREGATIONS_BY_TYPE, - GRANULARITY_NAMES, - NUMERIC_ONLY_AGGREGATIONS, - PRIMARY_KEY_AGGREGATIONS, - RANKED_AGGREGATIONS, - DataType, - ambiguous_weekday_message, - normalize_aggregation_name, -) +from slayer.core.enums import AMBIGUOUS_WEEKDAY_NAMES, BUILTIN_AGGREGATION_PARAM_ORDER, BUILTIN_AGGREGATIONS, DATE_PART_ALIASES, DEFAULT_AGGREGATIONS_BY_TYPE, GRANULARITY_NAMES, NUMERIC_ONLY_AGGREGATIONS, PRIMARY_KEY_AGGREGATIONS, RANKED_AGGREGATIONS, DataType, ambiguous_weekday_message, normalize_aggregation_name, ALL_TRANSFORMS from slayer.core.direction import rank_direction, with_direction_kwarg -from slayer.core.formula import ALL_TRANSFORMS from slayer.core.enums import RANK_FAMILY_TRANSFORMS from slayer.core.granularity import CustomGranularity, Granularity, resolve_granularity from slayer.core.refs import EXPRESSION_SOURCE_KINDS, key_display -from slayer.core.keys import DATE_ADD_COUNT_ARG, DATE_OPERAND_ARGS, MIRRORED_TIME_POINT_OP, SCALAR_FUNCTIONS, check_scalar_arity, type_date_values, AggregateKey, AggregateSource, ArithmeticKey, ColumnKey, ColumnSqlKey, Grain, InKey, LiteralKey, ScalarCallKey, SqlFragmentKey, StarKey, TimePointCmpKey, TimePointOp, TimeTruncKey, TransformKey, ValueKey, boolean_valued, column_leaf, column_path, is_attached_source, normalize_scalar, prepend_value_key, temporal_type, walk_value_keys +from slayer.core.keys import source_anchor_path, window_kwarg_of, DATE_ADD_COUNT_ARG, DATE_OPERAND_ARGS, MIRRORED_TIME_POINT_OP, SCALAR_FUNCTIONS, cast_data_type, check_scalar_arity, type_date_values, AggregateKey, AggregateSource, ArithmeticKey, CastKey, ColumnKey, ColumnSqlKey, Grain, InKey, LiteralKey, ScalarCallKey, SqlFragmentKey, StarKey, TimePointCmpKey, TimePointOp, TimeTruncKey, TransformKey, ValueKey, boolean_valued, column_leaf, column_path, is_attached_source, normalize_scalar, prepend_value_key, temporal_type, walk_value_keys from slayer.core.join_walker import ( aggregation_owner, canonical_path, @@ -66,12 +57,13 @@ from slayer.engine.param_binding import bind_aggregation_params from slayer.core.query import TimeDimension from slayer.core.scope import ModelScope, StageSchema, resolve_generated_column -from slayer.core.time_points import is_time_point +from slayer.core.time_points import is_time_point, parse_temporal_value from slayer.ir.source_bundle import ResolvedSourceBundle -from slayer.engine.syntax import ( +from slayer.core.syntax import ( AggCall, Arith, BoolOp, + Cast, Cmp, DottedRef, Literal, @@ -79,14 +71,18 @@ Ref, ScalarCall, StarSource, + TemporalLiteral, TransformCall, TupleLit, UnaryOp, canonical_measure_text, + comparison_hint, + keyword_spelling, parse_expr, placeholder_names, + positional_hint, ) -from slayer.sql.sql_expr import has_window_function +from slayer.sql.sql_expr import build_cast_type, has_window_function from slayer.sql.sql_template import SqlTemplateError, aggregation_reads from slayer.ir.bound import BoundExpr, BoundFilter, BoundTimeDimension @@ -138,6 +134,7 @@ def bind_expr( scope: ModelScope | StageSchema, bundle: ResolvedSourceBundle, allow_measures: bool = False, + alias_map: Optional[Dict[str, "ValueKey"]] = None, dimension_alias_map: Optional[Dict[str, "ValueKey"]] = None, ) -> BoundExpr: """Bind a parsed expression against a scope into a ``BoundExpr``. @@ -145,6 +142,7 @@ def bind_expr( ``allow_measures`` enables saved-measure resolution (bare and dotted) in the eligible positions — measure formulas and computed-dimension expressions; off everywhere else, so a saved-measure name there errors. + ``alias_map`` resolves query aliases as in :func:`bind_filter`. ``dimension_alias_map`` resolves ``partition_by=`` to the dimension's bound key; it applies ONLY there.""" _reject_placeholders(parsed) @@ -153,7 +151,7 @@ def bind_expr( if allow_measures else None ) value_key = _bind( - parsed, scope=scope, bundle=bundle, in_filter=False, measure_ctx=measure_ctx, + parsed, scope=scope, bundle=bundle, in_filter=False, alias_map=alias_map, measure_ctx=measure_ctx, dim_alias_map=dimension_alias_map, ) return BoundExpr( @@ -350,92 +348,138 @@ def _bind( # boundary — a measure is legal at value level but not inside an aggregation. # ``dim_alias_map`` rides every edge but resolves ONLY inside an # aggregation's ``partition_by``. - if isinstance(parsed, Literal): - return LiteralKey(value=normalize_scalar(parsed.value)) - - if isinstance(parsed, Ref): - return _resolve_ref( - parsed.name, scope=scope, bundle=bundle, alias_map=alias_map, - measure_ctx=measure_ctx, - ) - - if isinstance(parsed, DottedRef): - return _resolve_dotted( - parsed.parts, scope=scope, bundle=bundle, alias_map=alias_map, - measure_ctx=measure_ctx, - ) - - if isinstance(parsed, StarSource): - return StarKey() - + leaf = _bind_leaf(parsed, scope=scope, bundle=bundle, alias_map=alias_map, measure_ctx=measure_ctx) + if leaf is not None: + return leaf + sub = partial( + _bind, scope=scope, bundle=bundle, in_filter=in_filter, alias_map=alias_map, + measure_ctx=measure_ctx, dim_alias_map=dim_alias_map, + ) + if isinstance(parsed, Cast): + return _bind_cast(parsed, source=sub(parsed.source), bundle=bundle) if isinstance(parsed, AggCall): return _bind_agg_call( parsed, scope=scope, bundle=bundle, alias_map=alias_map, measure_ctx=measure_ctx, dim_alias_map=dim_alias_map, ) - if isinstance(parsed, TransformCall): return _bind_transform( parsed, scope=scope, bundle=bundle, alias_map=alias_map, measure_ctx=measure_ctx, dim_alias_map=dim_alias_map, ) - if isinstance(parsed, ScalarCall): return _bind_scalar( parsed, scope=scope, bundle=bundle, in_filter=in_filter, alias_map=alias_map, measure_ctx=measure_ctx, dim_alias_map=dim_alias_map, ) - - if isinstance(parsed, Arith): - return ArithmeticKey( - op=parsed.op, - operands=( - _bind(parsed.left, scope=scope, bundle=bundle, in_filter=in_filter, alias_map=alias_map, measure_ctx=measure_ctx, dim_alias_map=dim_alias_map), - _bind(parsed.right, scope=scope, bundle=bundle, in_filter=in_filter, alias_map=alias_map, measure_ctx=measure_ctx, dim_alias_map=dim_alias_map), - ), + if isinstance(parsed, Cmp): + return _bind_cmp( + parsed, sub=sub, scope=scope, bundle=bundle, in_filter=in_filter, + alias_map=alias_map, measure_ctx=measure_ctx, dim_alias_map=dim_alias_map, ) - + if isinstance(parsed, Arith): + return ArithmeticKey(op=parsed.op, operands=(sub(parsed.left), sub(parsed.right))) if isinstance(parsed, UnaryOp): - return ArithmeticKey( - op=parsed.op, - operands=(_bind(parsed.operand, scope=scope, bundle=bundle, in_filter=in_filter, alias_map=alias_map, measure_ctx=measure_ctx, dim_alias_map=dim_alias_map),), - ) + return ArithmeticKey(op=parsed.op, operands=(sub(parsed.operand),)) + if isinstance(parsed, BoolOp): + return ArithmeticKey(op=parsed.op, operands=tuple(sub(v) for v in parsed.operands)) + raise ValueError( + f"Unsupported ParsedExpr node: {type(parsed).__name__}" + ) - if isinstance(parsed, Cmp): - time_point = _time_point_comparison(parsed, units=bundle.granularities) - if time_point is not None: - operand, op, point, literal_on_left = time_point - return TimePointCmpKey( - op=op, point=point, literal_on_left=literal_on_left, - operand=_bind(operand, scope=scope, bundle=bundle, in_filter=in_filter, alias_map=alias_map, measure_ctx=measure_ctx, dim_alias_map=dim_alias_map), - ) - # ``IN`` / ``NOT IN`` fold into a single ``InKey`` (structured - # column + literal-tuple handle for the generator). - if parsed.op in _MEMBERSHIP_OP: - return _bind_in( - parsed, - scope=scope, bundle=bundle, in_filter=in_filter, - alias_map=alias_map, measure_ctx=measure_ctx, - dim_alias_map=dim_alias_map, + +def _bind_leaf( + parsed: ParsedExpr, + *, + scope: ModelScope | StageSchema, + bundle: ResolvedSourceBundle, + alias_map: Optional[Dict[str, "ValueKey"]], + measure_ctx: Optional[MeasureResolutionCtx], +) -> Optional[ValueKey]: + """A literal, reference or ``*`` leaf bound; ``None`` for a composite node.""" + if isinstance(parsed, Literal): + return LiteralKey(value=normalize_scalar(parsed.value)) + if isinstance(parsed, TemporalLiteral): + return LiteralKey(value=_temporal_literal_value(parsed)) + if isinstance(parsed, StarSource): + return StarKey() + if not isinstance(parsed, (Ref, DottedRef)): + return None + try: + if isinstance(parsed, Ref): + return _resolve_ref( + parsed.name, scope=scope, bundle=bundle, alias_map=alias_map, + measure_ctx=measure_ctx, ) - return ArithmeticKey( - op=parsed.op, - operands=( - _bind(parsed.left, scope=scope, bundle=bundle, in_filter=in_filter, alias_map=alias_map, measure_ctx=measure_ctx, dim_alias_map=dim_alias_map), - _bind(parsed.right, scope=scope, bundle=bundle, in_filter=in_filter, alias_map=alias_map, measure_ctx=measure_ctx, dim_alias_map=dim_alias_map), - ), + return _resolve_dotted( + parsed.parts, scope=scope, bundle=bundle, alias_map=alias_map, + measure_ctx=measure_ctx, ) + except UnknownReferenceError as e: + raise _with_hint(e, hint=_QUOTE_HINT.format(name=e.name)) if parsed.quoted else e - if isinstance(parsed, BoolOp): - operands = tuple( - _bind(v, scope=scope, bundle=bundle, in_filter=in_filter, alias_map=alias_map, measure_ctx=measure_ctx, dim_alias_map=dim_alias_map) - for v in parsed.operands + +def _bind_cmp( + parsed: Cmp, + *, + sub: Callable[[ParsedExpr], ValueKey], + scope: ModelScope | StageSchema, + bundle: ResolvedSourceBundle, + in_filter: bool, + alias_map: Optional[Dict[str, "ValueKey"]], + measure_ctx: Optional[MeasureResolutionCtx], + dim_alias_map: Optional[Dict[str, "ValueKey"]], +) -> ValueKey: + time_point = _time_point_comparison(parsed, units=bundle.granularities) + if time_point is not None: + operand, op, point, literal_on_left = time_point + return TimePointCmpKey(op=op, point=point, literal_on_left=literal_on_left, operand=sub(operand)) + # ``IN`` / ``NOT IN`` fold into a single ``InKey`` (structured + # column + literal-tuple handle for the generator). + if parsed.op in _MEMBERSHIP_OP: + return _bind_in( + parsed, + scope=scope, bundle=bundle, in_filter=in_filter, + alias_map=alias_map, measure_ctx=measure_ctx, + dim_alias_map=dim_alias_map, ) - return ArithmeticKey(op=parsed.op, operands=operands) + return ArithmeticKey(op=parsed.op, operands=(sub(parsed.left), sub(parsed.right))) - raise ValueError( - f"Unsupported ParsedExpr node: {type(parsed).__name__}" + +_QUOTE_HINT = "{name!r} was written in double quotes, which name a column; strings use single quotes ('…')." + + +def _with_hint(error: UnknownReferenceError, *, hint: str) -> UnknownReferenceError: + """``error`` with ``hint`` leading its suggestion.""" + return UnknownReferenceError( + name=error.name, scope_kind=error.scope_kind, scope_summary=error.scope_summary, + suggestion=f"{hint} {error.suggestion}" if error.suggestion else hint, + ) + + +def _temporal_literal_value(parsed: TemporalLiteral): + """A ``DATE '…'`` / ``TIMESTAMP '…'`` literal's value, checked like an ISO literal in a date position.""" + value = parse_temporal_value(parsed.text) + if parsed.kind == "date" and isinstance(value, datetime): + value = None + if value is None: + raise TimeLiteralError( + summary=f"{parsed.kind.upper()} '{parsed.text}' is not a valid {parsed.kind} literal.", + suggestion="write an ISO time point: DATE 'YYYY-MM-DD' or TIMESTAMP 'YYYY-MM-DD HH:MM[:SS]'.", + ) + if parsed.kind == "timestamp" and not isinstance(value, datetime): + value = datetime(value.year, value.month, value.day) + return value + + +def _bind_cast(parsed: Cast, *, source: ValueKey, bundle: ResolvedSourceBundle) -> CastKey: + try: + build_cast_type(parsed.type_text, dialect=bundle.dialect) + except ValueError as e: + raise CastTypeError(summary=str(e), suggestion="cast to a type this database has.") from None + return CastKey( + source=source, type_text=parsed.type_text, data_type=cast_data_type(parsed.type_text), try_=parsed.try_, ) @@ -984,7 +1028,7 @@ def _bind_expression_agg_source( agg: str, scope: ModelScope | StageSchema, bundle: ResolvedSourceBundle, -) -> ArithmeticKey | ScalarCallKey | LiteralKey | TimeTruncKey | InKey | TimePointCmpKey: +) -> ArithmeticKey | ScalarCallKey | LiteralKey | TimeTruncKey | InKey | TimePointCmpKey | CastKey: """Bind a scalar-expression aggregate source. Dotted joined-model leaves and operands carrying ``Column.filter`` are both @@ -1100,6 +1144,11 @@ def _bind_agg_call( ) -> AggregateKey | TransformKey | TimeTruncKey: """Bind an ``AggCall`` (a granularity callee is a row-level bucket); ``first`` / ``last`` dispatch by the bound operand's type — attached → the series transform, row grain → the ranked aggregation.""" + if parsed.filter is not None: + return _bind_filtered_agg( + parsed, scope=scope, bundle=bundle, alias_map=alias_map, measure_ctx=measure_ctx, + dim_alias_map=dim_alias_map, + ) granularity = _callee_granularity(parsed.agg, bundle=bundle) if granularity is not None: return _bind_granularity_call(parsed, scope=scope, bundle=bundle, granularity=granularity) @@ -1117,6 +1166,66 @@ def _bind_agg_call( return _bind_agg(parsed, scope=scope, bundle=bundle, dim_alias_map=dim_alias_map) +_FILTER_CLAUSE = "FILTER (WHERE …)" + + +def _filter_error(reason: str) -> AggregationFilterError: + return AggregationFilterError( + summary=f"{_FILTER_CLAUSE} {reason}", + location=_FILTER_CLAUSE, + suggestion="filter on columns of the aggregated rows, e.g. sum(amount) FILTER (WHERE status = 'paid').", + ) + + +def _bind_filtered_agg( + parsed: AggCall, *, + scope: ModelScope | StageSchema, + bundle: ResolvedSourceBundle, + alias_map: Optional[Dict[str, "ValueKey"]] = None, + measure_ctx: Optional[MeasureResolutionCtx] = None, + dim_alias_map: Optional[Dict[str, "ValueKey"]] = None, +) -> AggregateKey: + """``agg(…) FILTER (WHERE c)``: the aggregation over the rows where ``c`` holds. Every row-level + input is masked by ``c``; a star source and a ranked pick carry ``c`` on the key instead.""" + condition = _bind(parsed.filter, scope=scope, bundle=bundle, in_filter=False, dim_alias_map=dim_alias_map) + if any(isinstance(k, (AggregateKey, TransformKey)) for k in walk_value_keys(condition)): + raise _filter_error("needs a row-level condition; it cannot read an aggregate, a transform or a measure.") + key = _bind_agg_call( + parsed.model_copy(update={"filter": None}), scope=scope, bundle=bundle, alias_map=alias_map, + measure_ctx=measure_ctx, dim_alias_map=dim_alias_map, + ) + if not isinstance(key, AggregateKey) or is_attached_source(key.source): + raise _filter_error("applies only to an aggregation over rows.") + if key.agg in RANKED_AGGREGATIONS: + return _carrying_filter(key, condition=condition) + value = LiteralKey(value=normalize_scalar(1)) if isinstance(key.source, StarKey) else key.source + masked = key.model_copy(update={ + "source": _masked(value, condition=condition), + "kwargs": tuple((k, _masked(v, condition=condition) if isinstance(v, _ROW_INPUT_KINDS) else v) + for k, v in key.kwargs), + }) + if source_anchor_path(masked.source) == source_anchor_path(key.source): + return masked + if isinstance(key.source, StarKey) and not source_anchor_path(key.source): + # The condition reads a joined model: the star keeps the count's home. + return _carrying_filter(key, condition=condition) + raise _filter_error("must read the aggregated rows' own model or models joined to it one-to-one.") + + +def _carrying_filter(key: AggregateKey, *, condition: ValueKey) -> AggregateKey: + if window_kwarg_of(key) is not None: + raise _filter_error("cannot combine with window= here; filter the query's rows instead.") + return key.model_copy(update={"filter": condition}) + + +# Row-valued aggregation parameters a FILTER masks (literals and attached values are not rows). +_ROW_INPUT_KINDS = (ColumnKey, ColumnSqlKey, SqlFragmentKey) + + +def _masked(value: ValueKey, *, condition: ValueKey) -> ScalarCallKey: + return ScalarCallKey(name="iif", args=(condition, value, LiteralKey(value=None))) + + def _bind_granularity_call( parsed: AggCall, *, scope: ModelScope | StageSchema, bundle: ResolvedSourceBundle, granularity: Granularity, @@ -1195,6 +1304,7 @@ def _bind_agg( _check_agg_kwarg_names( agg=effective_agg, source=source, bundle=bundle, names=[*folded, *(k for k, _ in parsed.kwargs if k != "partition_by")], + keywords=frozenset(k for k, _ in parsed.kwargs), ) args = tuple( _bind_agg_arg(a, scope=scope, bundle=bundle, dim_alias_map=dim_alias_map) @@ -1271,7 +1381,7 @@ def _declared_agg_param_names( def _check_agg_kwarg_names( - *, agg: str, source, bundle: ResolvedSourceBundle, names: List[str], + *, agg: str, source, bundle: ResolvedSourceBundle, names: List[str], keywords: frozenset[str], ) -> None: """Reject an argument name the rendered ``agg`` never reads (only its placeholders or own parameters, plus ``window``).""" if not names: @@ -1290,13 +1400,14 @@ def _check_agg_kwarg_names( raise SqlTemplateError(reserved_value_param_message(agg)) if name in accepted: continue + hint = f" Note: {comparison_hint(name)}" if name in keywords else "" if accepted == {"window"}: raise AggregationArgumentError( - f"Aggregation '{agg}' takes no args or kwargs other than window; got '{name}'." + f"Aggregation '{agg}' takes no args or kwargs other than window; got '{name}'.{hint}" ) raise AggregationArgumentError( f"Aggregation '{agg}' does not accept argument '{name}'; " - f"accepted: {', '.join(sorted(accepted))}." + f"accepted: {', '.join(sorted(accepted))}.{hint}" ) @@ -1610,7 +1721,8 @@ def _bind_transform_params( if k not in allowed_kwargs: raise TransformArgumentError( summary=f"Transform {op!r} does not accept keyword " - f"argument {k!r}. Accepted: {sorted(advertised)}." + f"argument {k!r}. Accepted: {sorted(advertised)}.", + suggestion=comparison_hint(k), ) seen_kwargs.add(k) scalar = _fold_to_scalar(v) @@ -1695,9 +1807,12 @@ def _bind_scalar( ) # Check arity for every allowlisted scalar: sqlglot's handling is # inconsistent (silent arg-drop / DB-rejected SQL), so error clearly here. + spelled = [n for n in map(keyword_spelling, parsed.args) if n is not None] arity_error = check_scalar_arity( name=parsed.name, argc=len(parsed.args), ) + if arity_error is not None and spelled: + raise ValueError(f"{arity_error} Note: {positional_hint(spelled[0])}") if arity_error is not None: if parsed.name == "like": raise ValueError( @@ -1705,6 +1820,12 @@ def _bind_scalar( f"(value, pattern); got {len(parsed.args)}." ) raise ValueError(arity_error) + if parsed.name == "round" and len(parsed.args) == 2 and not _is_integer_literal(parsed.args[1]): + raise ValueError( + "'round' ndigits (2nd argument) must be an integer literal; got " + f"{canonical_measure_text(parsed.args[1])!r}." + ) + def bind(a: ParsedExpr, *, in_temporal_slot: bool) -> ValueKey: key = _bind( a, scope=scope, bundle=bundle, in_filter=in_filter, alias_map=alias_map, @@ -1714,12 +1835,27 @@ def bind(a: ParsedExpr, *, in_temporal_slot: bool) -> ValueKey: return type_date_values(key) if in_temporal_slot else key slots = set(DATE_OPERAND_ARGS.get(parsed.name, ())) - args = tuple(bind(a, in_temporal_slot=i in slots) for i, a in enumerate(parsed.args)) + try: + args = tuple(bind(a, in_temporal_slot=i in slots) for i, a in enumerate(parsed.args)) + except UnknownReferenceError as e: + if e.name not in spelled: + raise + raise _with_hint(e, hint=positional_hint(e.name)) from e if parsed.name == "date_add": _check_literal_count(args[DATE_ADD_COUNT_ARG]) return ScalarCallKey(name=parsed.name, args=args) +def _is_integer_literal(parsed: ParsedExpr) -> bool: + """An integer literal, signed or not (``2``, ``-2``).""" + if isinstance(parsed, UnaryOp) and parsed.op == "-": + parsed = parsed.operand + return ( + isinstance(parsed, Literal) and isinstance(parsed.value, Decimal) + and parsed.value == parsed.value.to_integral_value() + ) + + def _check_literal_count(count: ValueKey) -> None: if not isinstance(count, LiteralKey): return diff --git a/slayer/engine/compile/projection.py b/slayer/engine/compile/projection.py index 91c655da8..e5ef3f785 100644 --- a/slayer/engine/compile/projection.py +++ b/slayer/engine/compile/projection.py @@ -25,6 +25,7 @@ Phase, REGROUP_LEAF_PREFIX, SLOT_COMPOSITE_KINDS, + CastKey, ScalarCallKey, StarKey, TimeTruncKey, @@ -529,6 +530,8 @@ def _canonical_name(key: ValueKey) -> str: # NOSONAR(S3776) — sequential isin return f"_arith_{key.op}" if isinstance(key, ScalarCallKey): return f"_scalar_{key.name}" + if isinstance(key, CastKey): + return f"_cast_{_canonical_name(key.source)}" if isinstance(key, LiteralKey): return f"_lit_{key.value}" if isinstance(key, StarKey): diff --git a/slayer/engine/compile/shift.py b/slayer/engine/compile/shift.py index cc2d12bdb..65b9f13f9 100644 --- a/slayer/engine/compile/shift.py +++ b/slayer/engine/compile/shift.py @@ -10,6 +10,7 @@ ColumnKey, ColumnSqlKey, REGROUP_LEAF_PREFIX, + CastKey, ScalarCallKey, TimeTruncKey, TransformKey, @@ -57,7 +58,7 @@ def _series_mode(inner: ValueKey, *, to_original: Dict[ValueKey, ValueKey]) -> b transform or cross-model aggregate leaf.""" if isinstance(inner, TransformKey) or is_boolean_shaped(inner): return True - if isinstance(inner, (ArithmeticKey, ScalarCallKey)): + if isinstance(inner, (ArithmeticKey, ScalarCallKey, CastKey)): has_transform, has_cross_model = _series_flags(inner, to_original=to_original) return has_transform or has_cross_model return False diff --git a/slayer/engine/compile/stages.py b/slayer/engine/compile/stages.py index e0a41e8d4..a04d7bd52 100644 --- a/slayer/engine/compile/stages.py +++ b/slayer/engine/compile/stages.py @@ -30,9 +30,9 @@ from slayer.core.enums import DataType, JoinType, RANKED_AGGREGATIONS from slayer.core.errors import AmbiguousJoinPathError -from slayer.core.keys import SLOT_COMPOSITE_KINDS, AggregateKey, Grain, ArithmeticKey, ColumnKey, ColumnSqlKey, InKey, LiteralKey, Phase, PREDICATE_COMPARISON_OPS, StarKey, TimeTruncKey, TransformKey, ValueKey, column_leaf, effective_root_grain, constituent_grain, attached_parameter_grain, substitute_value_keys, substitute_consumer_keys, walk_value_keys, REGROUP_LEAF_PREFIX, is_cross_model_agg, is_kernel_requiring, split_top_level_and, window_kwarg_of, is_row_attach_root, attached_inputs, operand_aggregates, operand_constituents, parameter_row_leaves, source_anchor_path, source_row_leaves, shift_offset_of, VALUE_KEY_TYPES +from slayer.core.keys import SLOT_COMPOSITE_KINDS, AggregateKey, AggregateSource, Grain, ArithmeticKey, ColumnKey, ColumnSqlKey, InKey, LiteralKey, Phase, PREDICATE_COMPARISON_OPS, StarKey, TimeTruncKey, TransformKey, ValueKey, column_leaf, effective_root_grain, constituent_grain, attached_parameter_grain, substitute_value_keys, substitute_consumer_keys, walk_value_keys, REGROUP_LEAF_PREFIX, is_cross_model_agg, is_kernel_requiring, split_top_level_and, window_kwarg_of, is_row_attach_root, attached_inputs, NULL_SAFE_COMPARISON_OPS, OPTIONAL_KEY_FIELDS, operand_aggregates, operand_constituents, parameter_row_leaves, source_anchor_path, source_row_leaves, shift_offset_of, VALUE_KEY_TYPES from slayer.core.models import Column, SlayerModel, aggregation_definition, empty_value -from slayer.core.refs import key_display +from slayer.core.refs import EXPRESSION_SOURCE_KINDS, key_display from slayer.engine.reference_closure import ( aggregate_input_closure, column_default_key, @@ -79,6 +79,7 @@ from slayer.engine.elaborate_env import ( check_reserved_regroup_prefix, ConsumerPosition, + row_refs_off_grain, is_dimension_transform_root, position_classes, check_stage_flatten_collision, @@ -183,6 +184,23 @@ def _row_key_path(key: ValueKey) -> tuple: return tuple(getattr(key, "path", ())) +def _order_wrap_source(key: ValueKey, *, dim_keys: frozenset) -> Optional[Tuple[AggregateSource, tuple]]: + """(source, join path) of the MIN/MAX wrap a grouped ORDER BY row value needs, else ``None``.""" + if isinstance(key, (ColumnKey, ColumnSqlKey, TimeTruncKey)): + # A TimeTruncKey is not a legal aggregate source; wrap its underlying column (DATE_TRUNC is monotonic). + return (key.column if isinstance(key, TimeTruncKey) else key), _row_key_path(key) + if not isinstance(key, EXPRESSION_SOURCE_KINDS) or any( + isinstance(k, (AggregateKey, TransformKey)) + or (isinstance(k, ColumnKey) and k.leaf.startswith(REGROUP_LEAF_PREFIX)) + for k in walk_value_keys(key) + ): + return None + off_grain = row_refs_off_grain(key, dim_keys) + if not off_grain: + return None + return key, next((p for p in map(_row_key_path, off_grain) if p), ()) + + # Duration-windowed measures (``window='90d'``). @@ -249,7 +267,10 @@ def _windowed_slot_id_set( def _stable_text(obj: Any) -> str: """A process-independent spelling of a key (sets sorted, never hash order).""" if isinstance(obj, BaseModel): - fields = ",".join(_stable_text(getattr(obj, f)) for f in type(obj).model_fields) + fields = ",".join( + _stable_text(getattr(obj, f)) for f in type(obj).model_fields + if not (f in OPTIONAL_KEY_FIELDS and getattr(obj, f) is None) + ) return f"{type(obj).__name__}({fields})" if isinstance(obj, (set, frozenset)): return "{" + ",".join(sorted(_stable_text(x) for x in obj)) + "}" @@ -1652,6 +1673,8 @@ def _pred_value(cj: ValueKey, *, h: Tuple[str, ...], srcs: _NullSources) -> str: return _is_value( a=cj.operands[0], b=cj.operands[1], negated=op == "is not", h=h, srcs=srcs, ) + if op in NULL_SAFE_COMPARISON_OPS: + return _D if op in PREDICATE_COMPARISON_OPS: return _unknown_if_null(cj.operands, h=h, srcs=srcs) return _D @@ -3481,6 +3504,7 @@ def _rewrite_regrouped_prebound( value_key=substitute_value_keys(sp.bound.value_key, mapping), ), direction=sp.direction, + nulls=sp.nulls, ) for sp in prebound.order_specs ], @@ -3996,6 +4020,7 @@ def _emit_planned(routed: _Routed) -> PlannedQuery: # NOSONAR(S3776) — projec host_model_for_wraps = ( scope.source_model if isinstance(scope, ModelScope) else None ) + grain_dim_keys = frozenset(dm.bound.value_key for dm in declared_measures[:n_dims + n_tds]) for spec in order_specs: okey = spec.bound.value_key osid = projection.registry.find_by_key(okey) @@ -4006,12 +4031,11 @@ def _emit_planned(routed: _Routed) -> PlannedQuery: # NOSONAR(S3776) — projec if isinstance(okey, ColumnKey) and okey.leaf.startswith(REGROUP_LEAF_PREFIX): # A combined regroup placeholder resolves via its producer, not a hidden MIN/MAX. continue - if isinstance(okey, (ColumnKey, ColumnSqlKey, TimeTruncKey)): - if grain is None: - continue # raw-rows query -> split emission, no wrap needed - path = _row_key_path(okey) - # A TimeTruncKey is not a legal aggregate source; wrap its underlying column (DATE_TRUNC is monotonic). - src = okey.column if isinstance(okey, TimeTruncKey) else okey + if grain is None: + continue # raw-rows query -> split emission, no wrap needed + wrap = _order_wrap_source(okey, dim_keys=grain_dim_keys) + if wrap is not None: + src, path = wrap # ASC orders each group by its MIN, DESC by its MAX. wrap_key = AggregateKey( source=src, @@ -4186,6 +4210,7 @@ def _emit_planned(routed: _Routed) -> PlannedQuery: # NOSONAR(S3776) — projec slot_id=sid, direction=direction, phase=order_slot.key.phase, + nulls=spec.nulls, )) transform_layers = _emit_transform_layers(slots=projection.registry.slots) diff --git a/slayer/engine/elaborate_env.py b/slayer/engine/elaborate_env.py index 3f63a6f5e..58353744c 100644 --- a/slayer/engine/elaborate_env.py +++ b/slayer/engine/elaborate_env.py @@ -15,7 +15,7 @@ from pydantic import BaseModel, ConfigDict -from slayer.core.enums import RANK_FAMILY_TRANSFORMS, DataType, TimeGranularity +from slayer.core.enums import RANK_FAMILY_TRANSFORMS, DataType, TimeGranularity, TIME_TRANSFORMS from slayer.core.errors import ( AssociationError, CanonicalAliasShadowsColumnError, @@ -42,7 +42,6 @@ UnrelatedDatasetError, UnsafeJoinInputError, ) -from slayer.core.formula import TIME_TRANSFORMS from slayer.core.granularity import CustomGranularity, Granularity, nests_into, unknown_granularity_message from slayer.core.window_duration import parse_window_duration from slayer.core.keys import ( @@ -65,6 +64,7 @@ Grain, InKey, LiteralKey, + CastKey, ScalarCallKey, TimePointCmpKey, TimeTruncKey, @@ -135,7 +135,7 @@ def _walk(k: ValueKey) -> None: return blockers, attached -def _measure_blockers(cj: ValueKey, dim_keys: frozenset) -> List[ValueKey]: +def row_refs_off_grain(cj: ValueKey, dim_keys: frozenset) -> List[ValueKey]: """Row-level refs outside aggregate subtrees not available at query grain. A subtree equal to a query dimension's bound key IS the grouped value — available at query grain wholesale.""" @@ -187,7 +187,7 @@ def type_position_conjunct( if not field_blockers: return ConjunctTyping(MaskTyping.FIELD, 1 if attached else 0) if has_measure_position: - measure_blockers = _measure_blockers(cj, dim_keys) + measure_blockers = row_refs_off_grain(cj, dim_keys) if not measure_blockers: return ConjunctTyping(MaskTyping.MEASURE, 1) raise PositionTypingError( @@ -221,7 +221,7 @@ def check_measures_at_query_grain(prebound: PreboundQuery) -> None: if dm.is_dimension: continue root = dm.bound.value_key - blockers = _measure_blockers(root, dim_keys) + blockers = row_refs_off_grain(root, dim_keys) if not blockers: continue names = ", ".join(_key_display(k) for k in blockers) @@ -1024,6 +1024,8 @@ def _operand_display(k: object) -> str: return f"'{k}'" if isinstance(k, ScalarCallKey): return f"{k.name}({', '.join(_operand_display(a) for a in k.args)})" + if isinstance(k, CastKey): + return f"{'TRY_CAST' if k.try_ else 'CAST'}({_operand_display(k.source)} AS {k.type_text})" if isinstance(k, ArithmeticKey): if len(k.operands) == 1: return f"{k.op}{_operand_display(k.operands[0])}" @@ -1037,7 +1039,7 @@ def _operand_display(k: object) -> str: bool: DataType.BOOLEAN, str: DataType.TEXT, }) _SCALAR_RESULT_TYPES: Mapping[str, DataType] = MappingProxyType({ - **{name: DataType.TEXT for name in _TEXT_SCALARS}, "like": DataType.BOOLEAN, + **{name: DataType.TEXT for name in _TEXT_SCALARS}, "like": DataType.BOOLEAN, "ilike": DataType.BOOLEAN, }) @@ -1050,6 +1052,8 @@ def _leaf_count_type(key: object, *, column_type: ColumnTypeFn) -> Optional[Data return DataType.BOOLEAN if isinstance(key, ScalarCallKey): return _SCALAR_RESULT_TYPES.get(key.name) + if isinstance(key, CastKey) and key.data_type is not DataType.UNKNOWN: + return key.data_type return None @@ -1246,9 +1250,9 @@ def _time_search_children(key: ValueKey) -> List[ValueKey]: return [ key.source, *[a for a in key.args if isinstance( - a, (AggregateKey, TransformKey, ArithmeticKey, ScalarCallKey))], + a, (AggregateKey, TransformKey, ArithmeticKey, ScalarCallKey, CastKey))], *[v for _, v in key.kwargs if isinstance( - v, (AggregateKey, TransformKey, ArithmeticKey, ScalarCallKey))], + v, (AggregateKey, TransformKey, ArithmeticKey, ScalarCallKey, CastKey))], ] if isinstance(key, TransformKey): return [key.input] @@ -1258,9 +1262,11 @@ def _time_search_children(key: ValueKey) -> List[ValueKey]: return [ a for a in key.args if isinstance( - a, (AggregateKey, TransformKey, ArithmeticKey, ScalarCallKey, TimePointCmpKey, InKey), + a, (AggregateKey, TransformKey, ArithmeticKey, ScalarCallKey, TimePointCmpKey, InKey, CastKey), ) ] + if isinstance(key, CastKey): + return [key.source] if isinstance(key, TimePointCmpKey): return [key.operand] if isinstance(key, InKey): diff --git a/slayer/engine/key_metadata.py b/slayer/engine/key_metadata.py index 788c38729..fa31e9d1d 100644 --- a/slayer/engine/key_metadata.py +++ b/slayer/engine/key_metadata.py @@ -21,6 +21,7 @@ ColumnTypeFn, ColumnSqlKey, LiteralKey, + CastKey, ScalarCallKey, StarKey, TimeTruncKey, @@ -180,6 +181,8 @@ def measure_key_type( *, model: SlayerModel, key: ValueKey, bundle=None, ) -> Optional[DataType]: """``type`` for a measure slot, from its bound key alone.""" + if isinstance(key, CastKey): + return key.data_type date_type = date_scalar_type(key, column_type=_model_types(model=model, bundle=bundle)) if date_type is not None: return date_type @@ -249,6 +252,8 @@ def dimension_key_metadata( pre-existing: joined refs surface format / description through ``response_meta``, which resolves them against the owning model. """ + if isinstance(key, CastKey): + return key.data_type, None, None if not isinstance(key, (ColumnKey, ColumnSqlKey, TimeTruncKey)): date_type = date_scalar_type(key, column_type=_model_types(model=model, bundle=bundle)) if date_type is not None: diff --git a/slayer/engine/model_reads.py b/slayer/engine/model_reads.py index 18c1e54fe..d1d2d05c6 100644 --- a/slayer/engine/model_reads.py +++ b/slayer/engine/model_reads.py @@ -13,7 +13,7 @@ from slayer.core.query import ComputedDimension, ModelExtension, SlayerQuery, SourceSpec, render_probe_text from slayer.engine.param_binding import default_reference_sites from slayer.engine.reference_closure import parse_fragment_any_dialect -from slayer.engine.syntax import DottedRef, Ref, parse_expr, parse_filter_expr +from slayer.core.syntax import DottedRef, Ref, parse_expr from slayer.ir.source_bundle import apply_extension_overlay, follow_sibling_chain, source_name_if_sibling from slayer.sql.column_expansion import is_trivial_base, reference_sites, root_scope_column_ids @@ -98,15 +98,15 @@ def query_reference_parts(query: SlayerQuery) -> list[tuple[str, ...]]: if kind == "ref": out.append(tuple(text.split("."))) elif kind == "formula": - out.extend(_expr_parts(text, parse=parse_expr)) + out.extend(_expr_parts(text)) else: - out.extend(_expr_parts(render_probe_text(text), parse=parse_filter_expr)) + out.extend(_expr_parts(render_probe_text(text))) return out -def _expr_parts(text: str, *, parse) -> list[tuple[str, ...]]: +def _expr_parts(text: str) -> list[tuple[str, ...]]: try: - parsed = parse(text) + parsed = parse_expr(text) except Exception: # noqa: BLE001 — unparseable text reads nothing; the binder reports it return [] return list(_node_parts(parsed)) @@ -192,7 +192,7 @@ def measure(self, *, host: SlayerModel, name: str) -> None: measure = host.get_measure(name) if measure is None or not measure.formula or not self._first_visit(host, f"measure:{name}"): return - for parts in _expr_parts(measure.formula, parse=parse_expr): + for parts in _expr_parts(measure.formula): self.ref(parts, host=host) def _first_visit(self, host: SlayerModel, key: str) -> bool: diff --git a/slayer/engine/population.py b/slayer/engine/population.py index b8c054ee5..22f8d4da8 100644 --- a/slayer/engine/population.py +++ b/slayer/engine/population.py @@ -23,12 +23,11 @@ from slayer.core.query import ComputedDimension, SlayerQuery, render_probe_text from slayer.core.time_spine import TIME_SPINE_MODEL, mentions_spine, names_spine from slayer.engine.dimension_routing import resolve_route, safe_route_reachable -from slayer.engine.syntax import ( +from slayer.core.syntax import ( AggCall, DottedRef, Ref, parse_expr, - parse_filter_expr, walk_parsed_refs, ) from slayer.memories.resolver import _all_models_in_datasource @@ -121,14 +120,14 @@ def _parsed_filter_refs(filter_str: str) -> tuple[list[str], bool]: def _all_filter_refs(filter_str: str) -> tuple[list[str], bool]: """``(row-valued refs, ok)`` of one masked filter via the execution filter parser. - Uses ``parse_filter_expr`` (the same parser execution binds with), so SQL operator + Uses ``parse_expr`` (the same parser execution binds with), so SQL operator spellings and aggregations in either spelling (``sum(x)``) are all handled uniformly and string literals never surface as references. ``ok`` is False when the filter is aggregate-bearing or unparseable — either way it contributes nothing to determination. """ try: - parsed = parse_filter_expr(render_probe_text(filter_str)) + parsed = parse_expr(render_probe_text(filter_str)) except Exception: return [], False nodes = list(walk_parsed_refs(parsed)) diff --git a/slayer/engine/query_engine.py b/slayer/engine/query_engine.py index 3323eb221..c4c162c2d 100644 --- a/slayer/engine/query_engine.py +++ b/slayer/engine/query_engine.py @@ -98,7 +98,7 @@ ItemPath, RootModelRecommendation, ) -from slayer.engine.syntax import split_entity_agg_ref +from slayer.core.syntax import split_entity_agg_ref from slayer.engine.cache import ( CacheConfig, QueryCache, @@ -116,6 +116,7 @@ is_spliced, plan_has_semi_join_filters, plans_read_clock, + plans_try_cast, ) from slayer.engine.schema_drift import ( AppliedEntry, @@ -749,6 +750,20 @@ def _apply_policy( ), ) + async def _render_dialect( + self, *, dialect: str, datasource: DatasourceConfig, planned_list, + ) -> "str | SqlDialect": + """``dialect``, carrying the server profile when rendering a ``TRY_CAST`` reads it; an unreadable + profile renders as an unknown server.""" + sql_dialect = get_dialect(dialect) + if not sql_dialect.try_cast_reads_profile or not plans_try_cast(planned_list): + return dialect + try: + profile = await self._client_for(datasource).server_profile() + except Exception: # noqa: BLE001 — an unknown server gets the plain cast + profile = ServerProfile() + return sql_dialect.model_copy(update={"server_profile": profile}) + async def _require_correlated_subqueries( self, *, dialect: str, datasource: DatasourceConfig, planned_list, ) -> None: @@ -1171,6 +1186,7 @@ async def _plan_and_render( # NOSONAR S3776 — linear pipeline (localize→inf # Plan the DAG (root last) and render the whole chain to one SQL string. planned_list = plan_stages(queries=[*normed_named.values(), query], bundle=bundle) + render_dialect = await self._render_dialect(dialect=dialect, datasource=datasource, planned_list=planned_list) kept: Set[str] = set() sql: Optional[str] = None statement: Optional[Any] = None @@ -1178,12 +1194,12 @@ async def _plan_and_render( # NOSONAR S3776 — linear pipeline (localize→inf with collect_stale_spellings() as render_stale_spellings: if as_statement: statement = _build_planned_stages_ast( - planned_list, bundle=bundle, dialect=dialect, kept_stages=kept, + planned_list, bundle=bundle, dialect=render_dialect, kept_stages=kept, ) reads |= read_models(statement) else: sql = generate_planned_stages( - planned_queries=planned_list, bundle=bundle, dialect=dialect, + planned_queries=planned_list, bundle=bundle, dialect=render_dialect, # Plan-derived canonical projection keys drive the write-side length # fit; the read side decodes against the same set. projection_aliases=projection_result_keys(root_planned=planned_list[-1]), diff --git a/slayer/engine/reference_closure.py b/slayer/engine/reference_closure.py index f6faf2184..8e3ab9ada 100644 --- a/slayer/engine/reference_closure.py +++ b/slayer/engine/reference_closure.py @@ -24,6 +24,7 @@ from slayer.core.keys import ( AggregateKey, ArithmeticKey, + CastKey, ColumnKey, ColumnSqlKey, InKey, @@ -236,7 +237,7 @@ def _child_keys(node, *, descend_aggregates: bool = True) -> List: if isinstance(node, AggregateKey): if not descend_aggregates: return [] - return [node.source, *node.args, *(v for _name, v in node.kwargs)] + return [node.source, *node.args, *(v for _name, v in node.kwargs), *([node.filter] if node.filter else [])] if isinstance(node, TransformKey): return [node.input, *sorted(node.partition_keys, key=repr), node.time_key] if isinstance(node, ArithmeticKey): @@ -249,6 +250,8 @@ def _child_keys(node, *, descend_aggregates: bool = True) -> List: return [node.operand] if isinstance(node, SqlFragmentKey): return list(node.refs) + if isinstance(node, CastKey): + return [node.source] raise UnhandledValueKindError(node) diff --git a/slayer/engine/schema_drift.py b/slayer/engine/schema_drift.py index bc2c27270..d23680277 100644 --- a/slayer/engine/schema_drift.py +++ b/slayer/engine/schema_drift.py @@ -55,14 +55,13 @@ from slayer.core.join_walker import OrientedJoin, resolve_hop, walk from slayer.sql.column_expansion import resolve_ref_target from slayer.engine.dimension_routing import short_form_route_or_none -from slayer.engine.syntax import ( +from slayer.core.syntax import ( AggCall, DottedRef, ParsedExpr, Ref, StarSource, parse_expr, - parse_filter_expr, walk_parsed_refs, ) from slayer.sql import engine_factory, sqlite_introspect @@ -575,7 +574,7 @@ def _filter_refs_dsl(filter_str: str) -> list[str]: """Column/measure references in a DSL (Mode B) filter, in expression order (deduplicated); ``[]`` on parse failure.""" try: - parsed = parse_filter_expr(filter_str) + parsed = parse_expr(filter_str) except Exception: return [] return list(dict.fromkeys(_walk_ref_names(parsed))) diff --git a/slayer/engine/syntax.py b/slayer/engine/syntax.py deleted file mode 100644 index f8aac1715..000000000 --- a/slayer/engine/syntax.py +++ /dev/null @@ -1,1753 +0,0 @@ -"""Mode-B Python-AST parser. - -``parse_expr(text) -> ParsedExpr`` lowers a Mode-B DSL string -(``ModelMeasure.formula``, ``SlayerQuery.measures`` / ``.filters``) to a typed -tree — pure syntax, no scope resolution or named-measure expansion (binder's -job). Grammar: bare/dotted refs; functional or legacy colon aggregations, which -collapse to one ``AggCall``; transform calls; a closed scalar -allowlist; arithmetic / comparison / boolean / unary; grouping. Rejects -non-allowlisted calls, raw ``OVER(...)``, and chained comparisons. -""" - -from __future__ import annotations - -import ast -import re -from decimal import Decimal -from typing import Any, Dict, Iterator, List, Optional, Tuple, Union - -from pydantic import BaseModel, ConfigDict - -from slayer.core.direction import normalize_direction -from slayer.core.enums import ( - AMBIGUOUS_WEEKDAY_NAMES, - BUILTIN_AGGREGATIONS, - DATE_PART_ALIASES, - GRANULARITY_NAMES, - ambiguous_weekday_message, - normalize_aggregation_name, -) -from slayer.core.errors import GranularityCallError, IllegalWindowInFilterError, UnknownFunctionError -from slayer.core.formula import ALL_TRANSFORMS -from slayer.core.keys import DATE_UNIT_ARGS, SCALAR_FUNCTIONS, check_scalar_arity -from slayer.core.refs import split_agg_suffix - - -# --------------------------------------------------------------------------- -# ParsedExpr family -# --------------------------------------------------------------------------- - - -class _BaseNode(BaseModel): - model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) - - -class Ref(_BaseNode): - name: str - - -class DottedRef(_BaseNode): - parts: Tuple[str, ...] - - -class StarSource(_BaseNode): - pass - - -class Literal(_BaseNode): - value: Decimal | str | bool | None = None - - -class Placeholder(_BaseNode): - """An unsubstituted ``{name}`` variable placeholder; admitted wherever a ``Literal`` is.""" - - name: str - - -class TupleLit(_BaseNode): - """Literal-only tuple/list RHS for ``IN`` / ``NOT IN``; non-literal - elements and empty tuples are rejected at parse time.""" - - elements: Tuple[Literal, ...] - - -class AggCall(_BaseNode): - # source may also be an aggregation-free scalar expression (``sum(a - b)``), - # or — for a re-aggregation — a nested AggCall or a grained - # TransformCall, alone or composed. - source: ( - Ref | DottedRef | StarSource | Literal | Placeholder | ScalarCall | Arith | UnaryOp - | AggCall | TransformCall | Cmp | BoolOp - ) - agg: str - args: Tuple[Any, ...] = () - kwargs: Tuple[Tuple[str, Any], ...] = () - - -class TransformCall(_BaseNode): - op: str - input: Any - args: Tuple[Any, ...] = () - kwargs: Tuple[Tuple[str, Any], ...] = () - - -class ScalarCall(_BaseNode): - name: str - args: Tuple[Any, ...] = () - - -class Arith(_BaseNode): - op: str - left: Any - right: Any - - -class UnaryOp(_BaseNode): - op: str - operand: Any - - -class Cmp(_BaseNode): - op: str - left: Any - right: Any - - -class BoolOp(_BaseNode): - op: str - operands: Tuple[Any, ...] - - -ParsedExpr = Union[ - Ref, DottedRef, StarSource, Literal, Placeholder, TupleLit, - AggCall, TransformCall, ScalarCall, - Arith, UnaryOp, Cmp, BoolOp, -] - -# ``AggCall.source`` forward-references ScalarCall / Arith / UnaryOp (defined -# below it) for expression aggregation sources. -AggCall.model_rebuild() - - -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - - -# The ``__slayer_`` prefix is reserved from user input (P3). Matched only at an -# identifier boundary so a legal embedded name (``foo__slayer_bar``) stays -# referenceable; ``\w`` is Unicode-aware, covering ``é__slayer_bar`` too. -_RESERVED_EXPR_PREFIX = "__slayer_" -_RESERVED_EXPR_PREFIX_RE = re.compile(r"(? str: - """``col LIKE 'p%'`` → ``like(col, 'p%')``; ``col NOT LIKE 'p%'`` → - ``not like(col, 'p%')``. Matches starting inside string literals are - left untouched.""" - spans = [(m.start(), m.end()) for m in _PY_STRING_LITERAL_RE.finditer(text)] - - def _sub(m: "re.Match[str]") -> str: - # Skip a match inside a string literal, or one whose LHS is only part of - # a larger reference the ``\b``/``\w`` LHS token can't lex whole — a - # ``.``-rooted dotted path (``℘.name``) or a Unicode-fused prefix - # (``℘name``, decomposed ``éname``); that name is not a LIKE operand. - if any(s <= m.start() < e for s, e in spans) or _continues_ref( - text=text, pos=m.start() - 1 - ): - return m.group(0) - lhs, neg, pat = m.group(1), m.group(2), m.group(3) - call = f"like({lhs}, {pat})" - return f"not {call}" if neg else call - - return _SQL_LIKE_RE.sub(_sub, text) - - -# SQL ``CASE WHEN … END`` → nested ``iif(cond, then, otherwise)``. Lowered to the -# host language (a plain call) so binding + per-dialect emission reuse the whole -# scalar-call machinery. Only the WHEN-condition segments are operator-normalised -# (so ``CASE WHEN a = 5 …`` works in measures too); THEN/ELSE values are sliced -# verbatim and only recursed for nested CASE, so a value like ``'a AND b'`` is -# never rewritten. String literals are single tokens, so keywords inside them are -# invisible to the keyword scan. Identifiers lex complete (Unicode-aware start, -# dotted paths as ONE token), so a name containing or qualified by a keyword -# (``customers.case``, ``écase``) can never equal one. -_CASE_TOKEN_RE = re.compile( - r"(?P'(?:\\.|[^'\\])*'|\"(?:\\.|[^\"\\])*\")" - r"|(?P[^\W\d]\w*(?:\s*\.\s*[^\W\d]\w*)*)" - r"|(?P[(\[])" - r"|(?P[)\]])", - re.DOTALL, -) -_CASE_STOPS_OPERAND = frozenset({"WHEN", "THEN", "ELSE", "END"}) -_CASE_KEYWORDS = _CASE_STOPS_OPERAND | {"CASE"} - - -def _is_ident_adjacent(text: str, pos: int) -> bool: - """Whether ``text[pos]`` is identifier material the ``\\w`` token class - misses (combining marks, ``Other_ID_Start`` symbols like ``℘``).""" - return 0 <= pos < len(text) and ("a" + text[pos]).isidentifier() - - -def _continues_ref(text: str, pos: int) -> bool: - """Whether ``text[pos]`` extends a reference token past a keyword span — a - ``.`` join separator, or identifier material the ``\\w`` class misses. A - keyword touching such a char is a name component (``a.NULL``, ``℘case``), - not a keyword.""" - return _is_ident_adjacent(text=text, pos=pos) or ( - 0 <= pos < len(text) and text[pos] == "." - ) - - -def _sub_keyword_isolated(pattern: "re.Pattern[str]", repl: str, text: str) -> str: - """``pattern.sub(repl, text)`` but leaving a keyword match that is really a - reference component untouched — a dotted leaf/root (``a.NULL`` → keeps - ``NULL``) or one fused to ``Other_ID_Start`` / combining marks the ``\\b`` - boundary splits (``℘NULL`` → ``℘None``, ``℘AND`` → ``℘and``). Both would - silently rebind the reference (mirrors ``_case_keyword``'s guard).""" - def _guard(m: "re.Match[str]") -> str: - if _continues_ref(text=text, pos=m.start() - 1) or _continues_ref( - text=text, pos=m.end() - ): - return m.group(0) - return repl - return pattern.sub(_guard, text) - - -def _case_keyword(text: str, tok: Tuple[Optional[str], str, int, int]) -> Optional[str]: - """The CASE-grammar keyword a token spells, or None for an identifier. - - ASCII-exact (``caſe`` uppercases to ``CASE`` only via Unicode folding) and - rejected when adjacent raw text continues an identifier around the token. - """ - kind, val, start, end = tok - if kind != "id" or not val.isascii(): - return None - up = val.upper() - if up not in _CASE_KEYWORDS: - return None - if _is_ident_adjacent(text, start - 1) or _is_ident_adjacent(text, end): - return None - return up - - -def _case_has_when( - text: str, toks: List[Tuple[Optional[str], str, int, int]], i: int, -) -> bool: - """Whether the CASE at ``toks[i]`` opens a conditional: a depth-0 ``WHEN`` - follows before any other structural keyword, an unmatched ``)``, or end. - - Accepted mislex edge: a BARE keyword-named simple-CASE operand - (``CASE case WHEN 1 …``) is genuinely ambiguous and errors loudly — - parenthesize the operand (``CASE (case) WHEN 1 …``) to disambiguate. - """ - depth = 0 - for j in range(i + 1, len(toks)): - kind = toks[j][0] - if kind == "lp": - depth += 1 - elif kind == "rp": - if depth == 0: - return False - depth -= 1 - elif kind == "id" and depth == 0: - kw = _case_keyword(text, toks[j]) - if kw is not None: - return kw == "WHEN" - return False - - -def _rewrite_case_when(text: str) -> str: - if "case" not in text.lower(): - return text - toks = [ - (m.lastgroup, m.group(), m.start(), m.end()) - for m in _CASE_TOKEN_RE.finditer(text) - ] - if not any(_case_keyword(text, t) == "CASE" for t in toks): - return text - result, _ = _rw_value(text, toks, 0, frozenset(), 0) - return result - - -def _rw_value(text, toks, i, stop_kws, start_char): # NOSONAR(S3776) — one cohesive token-scan over a value/condition span: paren-depth tracking, the depth-0 stop-keyword break, and nested-CASE recursion are each one decision, and splitting them scatters the slice-and-recurse contract that preserves the original text. - """Rewrite one value/condition span → ``(rewritten, next_index)``. - - ``start_char`` is the span start (end of the keyword just consumed): bare - values (``1``) have no token, so non-CASE content is recovered by slicing the - original text (preserving colons/dots/spacing). Stops at a depth-0 stop - keyword or an unmatched ``)``. - """ - parts: List[str] = [] - depth = 0 - last = start_char - while i < len(toks): - kind, _, start, _ = toks[i] - if kind == "lp": - depth += 1 - elif kind == "rp": - if depth == 0: - break - depth -= 1 - elif kind == "id": - kw = _case_keyword(text, toks[i]) - if depth == 0 and kw in stop_kws: - break - if kw == "CASE" and _case_has_when(text, toks, i): - parts.append(text[last:start]) - nested, i = _rw_case(text, toks, i) - parts.append(nested) - last = toks[i - 1][3] - continue - i += 1 - stop_char = toks[i][2] if i < len(toks) else len(text) - parts.append(text[last:stop_char]) - return "".join(parts).strip(), i - - -def _rw_case(text, toks, i): - """Rewrite a ``CASE … END`` starting at ``toks[i]`` (the CASE keyword).""" - i += 1 # consume CASE - operand, i = _rw_value(text, toks, i, _CASE_STOPS_OPERAND, toks[i - 1][3]) - is_simple = bool(operand) - branches: List[Tuple[str, str]] = [] - while i < len(toks) and _case_keyword(text, toks[i]) == "WHEN": - i += 1 - cond, i = _rw_value(text, toks, i, frozenset({"THEN"}), toks[i - 1][3]) - if not (i < len(toks) and _case_keyword(text, toks[i]) == "THEN"): - raise ValueError( - f"Malformed CASE expression in {text!r}: a WHEN branch is " - f"missing its THEN." - ) - i += 1 - then_val, i = _rw_value( - text, toks, i, frozenset({"WHEN", "ELSE", "END"}), toks[i - 1][3], - ) - branches.append((cond, then_val)) - else_val = "None" - if i < len(toks) and _case_keyword(text, toks[i]) == "ELSE": - i += 1 - else_val, i = _rw_value(text, toks, i, frozenset({"END"}), toks[i - 1][3]) - else_val = else_val or "None" - if not (i < len(toks) and _case_keyword(text, toks[i]) == "END"): - raise ValueError( - f"Malformed CASE expression in {text!r}: missing END." - ) - i += 1 # consume END - if not branches: - raise ValueError( - f"Malformed CASE expression in {text!r}: needs at least one " - f"WHEN … THEN branch." - ) - result = else_val - for cond, then_val in reversed(branches): - if is_simple: - cond_expr = f"{operand} == {cond}" - else: - cond_expr = _normalize_sql_filter_operators(cond) - result = f"iif({cond_expr}, {then_val}, {result})" - return result, i - - -# Tokens of the call-shape rewriters (``extract``, ``count(distinct …)``): literals -# whole, identifiers Unicode-aware, so keyword text inside either never matches. -_CALL_TOKEN_RE = re.compile( - r"(?P'(?:\\.|[^'\\])*'|\"(?:\\.|[^\"\\])*\")" - r"|(?P[^\W\d]\w*)" - r"|(?P\()|(?P\))|(?P,)" - r"|(?P==|!=|<=|>=|=)" - r"|(?P\S)", - re.DOTALL, -) -_Token = Tuple[str, str, int, int] -_DISTINCT_ONLY = "DISTINCT is supported only as count(distinct x), i.e. count_distinct(x)" - - -def _call_tokens(text: str) -> List[_Token]: - return [(m.lastgroup or "", m.group(), m.start(), m.end()) for m in _CALL_TOKEN_RE.finditer(text)] - - -def _is_word(text: str, toks: List[_Token], i: int, word: str) -> bool: - """Whether ``toks[i]`` is the bare ASCII keyword ``word`` (any case), not part of a reference.""" - kind, val, start, end = toks[i] - return ( - kind == "id" and val.isascii() and val.lower() == word - and not _is_ident_adjacent(text, start - 1) and not _continues_ref(text=text, pos=end) - and not (i > 0 and toks[i - 1][1] == ".") - ) - - -def _matching_rp(toks: List[_Token], lp: int) -> Optional[int]: - depth = 0 - for j in range(lp, len(toks)): - if toks[j][0] == "lp": - depth += 1 - elif toks[j][0] == "rp": - depth -= 1 - if depth == 0: - return j - return None - - -def _top_level(toks: List[_Token], lp: int, rp: int) -> Iterator[int]: - """Indices of the tokens directly inside the parens at ``lp`` / ``rp``.""" - depth = 0 - for j in range(lp + 1, rp): - kind = toks[j][0] - if kind == "lp": - depth += 1 - elif kind == "rp": - depth -= 1 - elif depth == 0: - yield j - - -def _extract_call(text: str, toks: List[_Token], i: int) -> Optional[Tuple[int, int]]: - """``(rp, from)`` token indices when ``toks[i]`` opens ``extract(… from …)``.""" - if not (_is_word(text, toks, i, "extract") and i + 1 < len(toks) and toks[i + 1][0] == "lp"): - return None - rp = _matching_rp(toks, i + 1) - if rp is None: - return None - frm = next((j for j in _top_level(toks, i + 1, rp) if _is_word(text, toks, j, "from")), None) - return None if frm is None else (rp, frm) - - -def _rewrite_extract(text: str, *, original: str) -> str: - """``extract( from x)`` → ``date_part('', x)``.""" - if "extract" not in text.lower(): - return text - toks = _call_tokens(text) - out: List[str] = [] - last = i = 0 - while i < len(toks): - call = _extract_call(text, toks, i) - if call is None: - i += 1 - continue - rp, frm = call - part = toks[i + 2:frm] - operand = text[toks[frm][3]:toks[rp][2]].strip() - if len(part) != 1 or part[0][0] not in ("id", "str") or not operand: - raise ValueError( - f"Invalid Mode-B expression {original!r}: write extract( from ), " - f"e.g. extract(day from created_at)." - ) - word = part[0][1] if part[0][0] == "str" else f"'{part[0][1]}'" - out += [text[last:toks[i][2]], f"date_part({word}, {_rewrite_extract(operand, original=original)})"] - last, i = toks[rp][3], rp + 1 - out.append(text[last:]) - return "".join(out) - - -def _single_distinct_operand(toks: List[_Token], i: int, rp: int) -> bool: - """Whether ``distinct`` at ``toks[i]`` is followed by one non-star operand and only keyword arguments.""" - commas = [j for j in _top_level(toks, i - 1, rp) if toks[j][0] == "comma"] - operand = toks[i + 1:commas[0] if commas else rp] - if not operand or [t[1] for t in operand] == ["*"]: - return False - return all(toks[c + 1][0] == "id" and toks[c + 2][1] == "=" for c in commas) - - -def _in_call_top_level(toks: List[_Token], i: int) -> bool: - """Whether ``toks[i]`` sits directly inside a call's parentheses.""" - depth = 0 - for j in range(i - 1, -1, -1): - kind = toks[j][0] - if kind == "rp": - depth += 1 - elif kind == "lp": - if depth == 0: - return j > 0 and toks[j - 1][0] == "id" - depth -= 1 - return False - - -def _is_later_distinct_arg(text: str, toks: List[_Token], i: int) -> bool: - """Whether ``toks[i]`` is a ``distinct`` keyword opening a non-first call argument or a keyword value.""" - opens_arg = 0 < i < len(toks) - 1 and (toks[i - 1][0] == "comma" or toks[i - 1][1] == "=") - if not (opens_arg and _is_word(text, toks, i, "distinct")): - return False - kind, val = toks[i + 1][:2] - return (kind in ("id", "str", "lp") or val.isdigit() or val == "*") and _in_call_top_level(toks, i) - - -def _rewrite_count_distinct(text: str, *, original: str) -> str: - """``count(distinct x, …)`` → ``count_distinct(x, …)``; any other ``distinct`` keyword in a call raises.""" - if "distinct" not in text.lower(): - return text - toks = _call_tokens(text) - edits: List[Tuple[int, int, str]] = [] - for i in range(2, len(toks)): - if _is_later_distinct_arg(text, toks, i): - raise ValueError(f"Invalid Mode-B expression {original!r}: {_DISTINCT_ONLY}.") - if not (_is_word(text, toks, i, "distinct") and toks[i - 1][0] == "lp" and toks[i - 2][0] == "id"): - continue - rp = _matching_rp(toks, i - 1) - if rp is None: - continue - if not (_is_word(text, toks, i - 2, "count") and _single_distinct_operand(toks, i, rp)): - raise ValueError(f"Invalid Mode-B expression {original!r}: {_DISTINCT_ONLY}.") - edits += [(toks[i - 2][2], toks[i - 2][3], "count_distinct"), (toks[i][2], toks[i][3], "")] - for start, end, repl in reversed(edits): - text = text[:start] + repl + text[end:] - return text - - -_COLON_AGG_RE = re.compile( - r"(\*|[a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*(?:\.\*)?)" # source: * / ident / dotted - r":" - r"([a-zA-Z_]\w*)" - # No (args) consumption — Python's AST handles that. -) - -_FILTER_KEYWORDS = frozenset({"and", "or", "not", "in", "is"}) -_SCAN_TOKEN_RE = re.compile( - r"(?P\s+)" - r"|(?P'(?:\\.|[^'\\])*'|\"(?:\\.|[^\"\\])*\")" - r"|(?P[A-Za-z_]\w*)" - r"|(?P\d+(?:\.\d*)?|\.\d+)" - r"|(?P==|<=|>=|!=)" - r"|(?P=)" - r"|(?P\()" - r"|(?P\))" - r"|(?P\[)" - r"|(?P\])" - r"|(?P,)" - r"|(?P.)", - re.DOTALL, -) - - -_BIN_OP_MAP: Dict[type, str] = { - ast.Add: "+", ast.Sub: "-", ast.Mult: "*", ast.Div: "/", - ast.Mod: "%", ast.Pow: "**", ast.FloorDiv: "//", -} -_CMP_OP_MAP: Dict[type, str] = { - ast.Eq: "==", ast.NotEq: "!=", - ast.Lt: "<", ast.LtE: "<=", - ast.Gt: ">", ast.GtE: ">=", - # SQL ``IS [NOT] NULL`` lowers to ``is [not] None``; rendered back as - # ``IS [NOT] NULL`` by the SQL generator. - ast.Is: "is", ast.IsNot: "is not", - # ``IN`` / ``NOT IN`` with a literal-tuple RHS (shape enforced in - # the ``ast.Compare`` branch). - ast.In: "in", ast.NotIn: "not in", -} - - -# --------------------------------------------------------------------------- -# Public entry point -# --------------------------------------------------------------------------- - - -def parse_expr(text: str) -> ParsedExpr: - """Parse a Mode-B expression string into a ``ParsedExpr``. - - ``__`` in identifiers is legal; only the ``__slayer_`` prefix is - reserved. Raises ``ValueError`` (empty/syntax/unsupported node/chained - comparison/reserved prefix), ``UnknownFunctionError``, or - ``IllegalWindowInFilterError`` (raw ``OVER(...)``). - """ - if not text or not text.strip(): - raise ValueError("Empty Mode-B expression.") - - _reject_reserved_expr_token(text) - - # Scan for raw ``OVER(`` after blanking string literals so a quoted value - # (``status == 'OVER('``) isn't mistaken for window usage. Skip an ``OVER`` - # that only continues a reference — a Unicode-fused (``℘OVER(``) or dotted - # (``a.OVER(``) name the leading ``\b`` splits. - blanked = _PY_STRING_LITERAL_RE.sub("", text) - if any( - not _continues_ref(text=blanked, pos=m.start() - 1) - for m in _OVER_RE.finditer(blanked) - ): - raise IllegalWindowInFilterError( - filter_expr=text, - source="raw OVER(...) is not allowed in Mode-B DSL", - suggestion=( - "use a transform instead (rank, percent_rank, dense_rank, " - "ntile, cumsum, lag, lead, time_shift, …)." - ), - ) - - sql_spelled = _rewrite_count_distinct(_rewrite_extract(text, original=text), original=text) - preprocessed, agg_map = _preprocess_colons(_rewrite_case_when(sql_spelled)) - # AFTER the legacy colon pass — ``customers.*:count`` must become a - # placeholder first, so the only ``*`` left in call-first-arg position is - # the functional spelling (``count(*)`` / ``count(customers.*)``). - preprocessed = _preprocess_star_args(preprocessed) - - try: - py_ast = ast.parse(preprocessed, mode="eval").body - except SyntaxError as e: - raise ValueError( - f"Invalid Mode-B expression {text!r}: {e}" - ) - - parsed = _convert(py_ast, agg_map=agg_map, original=text) - if any(_is_interval(n) for n in _walk_parsed(parsed)): - raise ValueError( - f"Invalid Mode-B expression {text!r}: interval(n, unit) is only valid as " - f"`x + interval(n, unit)`, `interval(n, unit) + x` or `x - interval(n, unit)`; " - f"otherwise use date_add(x, n, unit)." - ) - return parsed - - -def parse_filter_expr(text: str) -> ParsedExpr: - """Parse a Mode-B *filter* string, accepting SQL operator spellings. - - Filters historically accepted SQL-style operators (``=``, ``<>``, ``NULL``, - and the keyword forms ``AND`` / ``OR`` / ``NOT`` / ``IS`` / ``IN``) - alongside the Python spellings. This wrapper normalizes those to their - Python equivalents (string-literal-aware, so quoted contents are - untouched) and then delegates to :func:`parse_expr`. Measures / order use - ``parse_expr`` directly — only filters get the SQL-operator leniency. - """ - return parse_expr(_normalize_sql_filter_operators(text)) - - -# SQL keyword rewrites as explicit ASCII character classes — ``re.IGNORECASE`` -# folds Unicode spoofs (``ıs`` → ``is``) into keywords. -_SQL_NULL_RE = re.compile(r"\b[nN][uU][lL][lL]\b") -_SQL_KEYWORD_RES: Tuple[Tuple[re.Pattern, str], ...] = tuple( - (re.compile(r"\b" + "".join(f"[{c}{c.upper()}]" for c in kw) + r"\b"), kw) - for kw in ("is", "not", "and", "or", "in") -) - - -def _normalize_sql_filter_operators(text: str) -> str: - """Rewrite SQL operator spellings to Python ones outside string literals. - - ``NULL`` → ``None``; ``IS`` / ``NOT`` / ``AND`` / ``OR`` / ``IN`` → - lowercase; standalone ``=`` → ``==``; ``<>`` → ``!=``; ``col [NOT] LIKE - 'p%'`` → ``[not ]like(col, 'p%')``. - """ - # LIKE first, on the whole string (its pattern is a quoted literal). The - # rest run per non-literal part (escape-aware split) so a keyword inside a - # quoted value isn't rewritten. - text = _rewrite_sql_like(text) - parts = _PY_STRING_LITERAL_RE.split(text) - literals = _PY_STRING_LITERAL_RE.findall(text) - result: List[str] = [] - for i, part in enumerate(parts): - part = _sub_keyword_isolated(_SQL_NULL_RE, "None", part) - for kw_re, kw in _SQL_KEYWORD_RES: - part = _sub_keyword_isolated(kw_re, kw, part) - part = part.replace("<>", "!=") - # SQL ``||`` → Python ``|`` (BitOr), desugared to ``concat`` in - # ``_convert``; same precedence relative to comparisons. - part = part.replace("||", "|") - result.append(part) - if i < len(literals): - result.append(literals[i]) - # ``=`` → ``==`` runs last, on the rejoined string, via a paren-aware scanner - # that leaves kwarg ``=`` inside non-scalar calls alone. - return _rewrite_comparison_equals("".join(result)) - - -def _classify_paren( - hist: List[Tuple[str, str]], -) -> Tuple[bool, Optional[str]]: - """Classify an open ``(`` as a CALL or GROUPING paren. - - A call paren follows a bare identifier or a ``)`` / ``]``; lowercase keywords - (``and`` / ``not`` / ``in`` …) do not open one. After a legacy colon - (``revenue:first(...)``) the callee is dropped to ``None`` so - :func:`_is_kwarg_equals` treats it as an aggregation, not a transform. - """ - prev_kind = hist[-1][0] if hist else None - prev_text = hist[-1][1] if hist else "" - is_call = prev_kind == "NAME" or prev_text in (")", "]") - callee = hist[-1][1] if (is_call and prev_kind == "NAME") else None - if callee is not None and len(hist) >= 2 and hist[-2] == ("OTHER", ":"): - callee = None - return is_call, callee - - -def _is_kwarg_equals( - stack: List[Tuple[bool, Optional[str]]], - hist: List[Tuple[str, str]], -) -> bool: - """Whether a lone ``=`` is a Python keyword-argument separator. - - Scalars never take kwargs. Transforms take the value first, so a kwarg only - follows a ``,`` (keeping ``consecutive_periods(status = 'paid')`` a - predicate). Aggregations/unknowns may take a kwarg first, so ``=`` follows - ``(`` or ``,``. - """ - top = stack[-1] if stack else None - if top is None or not top[0]: - return False - callee = top[1] - if callee is not None and callee.lower() in SCALAR_FUNCTIONS: - return False - prev_kind = hist[-1][0] if hist else None - if prev_kind != "NAME": - return False - prev_prev_kind = hist[-2][0] if len(hist) >= 2 else None - if callee is not None and callee in ALL_TRANSFORMS: - return prev_prev_kind == "COMMA" - return prev_prev_kind in ("LPAREN", "COMMA") - - -def _push_hist(hist: List[Tuple[str, str]], kind: str, text: str) -> None: - """Append ``(kind, text)`` and trim ``hist`` to the last 2 entries.""" - hist.append((kind, text)) - if len(hist) > 2: - del hist[0] - - -def _handle_pass_through( - m: "re.Match[str]", - out: List[str], - stack: List[Tuple[bool, Optional[str]]], - hist: List[Tuple[str, str]], -) -> None: - """Whitespace and string literals: emit verbatim, don't touch hist.""" - out.append(m.group(0)) - - -def _handle_ident( - m: "re.Match[str]", - out: List[str], - stack: List[Tuple[bool, Optional[str]]], - hist: List[Tuple[str, str]], -) -> None: - ident = m.group(0) - out.append(ident) - _push_hist(hist, "KW" if ident in _FILTER_KEYWORDS else "NAME", ident) - - -def _handle_other( - m: "re.Match[str]", - out: List[str], - stack: List[Tuple[bool, Optional[str]]], - hist: List[Tuple[str, str]], -) -> None: - """Numbers, compound ops (``==``/``<=``/``>=``/``!=``), brackets, and - any single char not otherwise classified (``:``, ``.``, ``+``, ``-``, - ``*``, ``/``, ``%``, ``<``, ``>``, ``!``, ``|``, ``{``, ``}``, ...).""" - text = m.group(0) - out.append(text) - _push_hist(hist, "OTHER", text) - - -def _handle_comma( - m: "re.Match[str]", - out: List[str], - stack: List[Tuple[bool, Optional[str]]], - hist: List[Tuple[str, str]], -) -> None: - out.append(",") - _push_hist(hist, "COMMA", ",") - - -def _handle_lparen( - m: "re.Match[str]", - out: List[str], - stack: List[Tuple[bool, Optional[str]]], - hist: List[Tuple[str, str]], -) -> None: - stack.append(_classify_paren(hist)) - out.append("(") - _push_hist(hist, "LPAREN", "(") - - -def _handle_rparen( - m: "re.Match[str]", - out: List[str], - stack: List[Tuple[bool, Optional[str]]], - hist: List[Tuple[str, str]], -) -> None: - if stack: - stack.pop() - out.append(")") - _push_hist(hist, "OTHER", ")") - - -def _handle_op_eq( - m: "re.Match[str]", - out: List[str], - stack: List[Tuple[bool, Optional[str]]], - hist: List[Tuple[str, str]], -) -> None: - out.append("=" if _is_kwarg_equals(stack, hist) else "==") - _push_hist(hist, "OTHER", "=") - - -_HANDLERS: Dict[str, Any] = { - "ws": _handle_pass_through, - "string": _handle_pass_through, - "ident": _handle_ident, - "number": _handle_other, - "op_eq2": _handle_other, - "op_eq": _handle_op_eq, - "lparen": _handle_lparen, - "rparen": _handle_rparen, - "lbrack": _handle_other, - "rbrack": _handle_other, - "comma": _handle_comma, - "other": _handle_other, -} - - -def _rewrite_comparison_equals(text: str) -> str: - """Rewrite SQL ``=`` to Python ``==`` except where it is a kwarg separator. - - A lone ``=`` is kept (kwarg) inside a non-scalar CALL paren when it follows - ``ident`` after ``(`` or ``,`` — Python's kwarg grammar (see - :func:`_is_kwarg_equals`); inside scalars a ``=`` is always the user's - comparison. Compound ops and string literals pass through untouched. - Token dispatch is keyed by the scan regex's ``lastgroup``. - """ - out: List[str] = [] - # Each frame: (is_call, callee) — callee is the ident before ``(``, else None. - stack: List[Tuple[bool, Optional[str]]] = [] - # Last 2 significant tokens (NAME / KW / LPAREN / COMMA / OTHER; no strings). - hist: List[Tuple[str, str]] = [] - for m in _SCAN_TOKEN_RE.finditer(text): - _HANDLERS[m.lastgroup](m, out, stack, hist) - return "".join(out) - - -# --------------------------------------------------------------------------- -# Reference walk (best-effort textual extraction) -# --------------------------------------------------------------------------- - - -def walk_parsed_refs( - parsed: ParsedExpr, -) -> Iterator[Ref | DottedRef | AggCall]: - """Yield the reference-bearing leaves (``Ref`` / ``DottedRef`` / ``AggCall``) - of a tree — scope-free name extraction for schema-drift / memory tagging. - - Descent (matches the binder's walk): ``AggCall`` yielded whole (args/kwargs - opaque, so ``weighted_avg(weight=quantity)`` surfaces ``price`` not - ``quantity``); ``TransformCall`` descends ``input`` only; ``ScalarCall`` / - arithmetic / comparison / boolean descend operands; literals yield nothing. - """ - if isinstance(parsed, (Ref, DottedRef, AggCall)): - yield parsed - return - if isinstance(parsed, TransformCall): - yield from walk_parsed_refs(parsed.input) - return - if isinstance(parsed, ScalarCall): - for a in parsed.args: - yield from walk_parsed_refs(a) - return - if isinstance(parsed, Arith): - yield from walk_parsed_refs(parsed.left) - yield from walk_parsed_refs(parsed.right) - return - if isinstance(parsed, Cmp): - yield from walk_parsed_refs(parsed.left) - yield from walk_parsed_refs(parsed.right) - return - if isinstance(parsed, UnaryOp): - yield from walk_parsed_refs(parsed.operand) - return - if isinstance(parsed, BoolOp): - for op in parsed.operands: - yield from walk_parsed_refs(op) - # Literal / Placeholder / StarSource / TupleLit → no references (TupleLit - # holds only Literals by construction). - - -# --------------------------------------------------------------------------- -# Internals -# --------------------------------------------------------------------------- - - -def _reject_reserved_expr_token(text: str) -> None: - """Reject the reserved ``__slayer_`` token in RAW Mode-B input (P3). - - Runs BEFORE ``_preprocess_colons`` mints its own ``__slayer_agg_N__`` - placeholders, so a legacy colon-spelling ``revenue:sum`` is unaffected while a - literal ``__slayer_agg_0__`` spoof is rejected [C3]. String literals are - blanked first (Python syntax, so escapes count) so quoted data is never - mistaken for an identifier. Matched at an identifier boundary so a legal - embedded name like ``foo__slayer_bar`` stays referenceable. - """ - if _RESERVED_EXPR_PREFIX_RE.search(_PY_STRING_LITERAL_RE.sub("", text)): - raise ValueError( - f"Mode-B expression {text!r} uses the reserved " - f"{_RESERVED_EXPR_PREFIX!r} identifier prefix, which is reserved " - f"for SLayer-internal placeholders." - ) - - -def _preprocess_colons( - text: str, -) -> Tuple[str, Dict[int, Tuple[Ref | DottedRef | StarSource, str]]]: - """Replace the legacy colon spelling ``:`` with placeholder identifiers. - - Captures source kind + agg name. Any trailing ``(args)`` is left in - place so Python's AST parses it naturally as a Call. String literal - spans are skipped — the literal text is user data, not DSL syntax. - """ - agg_map: Dict[int, Tuple[Ref | DottedRef | StarSource, str]] = {} - counter = [0] - literal_spans = [ - # Escape-aware, so escaped quotes don't leak colon rewrites into the string body. - (m.start(), m.end()) for m in _PY_STRING_LITERAL_RE.finditer(text) - ] - - def _in_literal(pos: int) -> bool: - return any(s <= pos < e for s, e in literal_spans) - - def _replace(match: re.Match) -> str: - if _in_literal(match.start()): - return match.group(0) - source_str = match.group(1) - agg_name = match.group(2) - source: Ref | DottedRef | StarSource - if source_str == "*": - source = StarSource() - elif "." in source_str: - source = DottedRef(parts=tuple(source_str.split("."))) - else: - source = Ref(name=source_str) - idx = counter[0] - counter[0] += 1 - agg_map[idx] = (source, agg_name) - return f"{_PLACEHOLDER_PREFIX}{idx}__" - - return _COLON_AGG_RE.sub(_replace, text), agg_map - - -def _preprocess_star_args(text: str) -> str: # NOSONAR(S3776) — one token-scan with two star shapes (bare ``(*``, dotted ``path.*``); the prev/next-significant-token checks ARE the grammar being recognised, and splitting them hides the call-first-argument contract. - """Replace a call's first-argument ``*`` / ``path.*`` with the reserved - star token (``count(*)`` → ``count(__slayer_star__)``) so Python's AST can - parse it. Token-aware, string-literal-safe; multiplication is untouched - (a multiplying ``*`` never directly follows a call's ``(`` or a ``.``). - """ - if "*" not in text: - return text - toks = [(m.lastgroup, m.group(0)) for m in _SCAN_TOKEN_RE.finditer(text)] - n = len(toks) - - def _prev(i: int) -> int: - j = i - 1 - while j >= 0 and toks[j][0] == "ws": - j -= 1 - return j - - def _next(i: int) -> int: - j = i + 1 - while j < n and toks[j][0] == "ws": - j += 1 - return j - - def _is_call_lparen(i: int) -> bool: - p = _prev(i) - return p >= 0 and toks[p][0] == "ident" - - def _dotted_star_opens_call(dot_idx: int) -> bool: - # Walk ``ident(.ident)*`` back from the ``.`` before the star; the - # path must open a call's first argument (directly after ``ident(``). - j = dot_idx - while True: - q = _prev(j) - if q < 0 or toks[q][0] != "ident": - return False - r = _prev(q) - if r >= 0 and toks[r] == ("other", "."): - j = r - continue - return r >= 0 and toks[r][0] == "lparen" and _is_call_lparen(r) - - out: List[str] = [] - for i, (kind, val) in enumerate(toks): - if kind == "other" and val == "*": - nx = _next(i) - closes_arg = nx < n and toks[nx][1] in (",", ")") - p = _prev(i) - if closes_arg and p >= 0: - if toks[p][0] == "lparen" and _is_call_lparen(p): - out.append(_STAR_ARG_TOKEN) - continue - if toks[p] == ("other", ".") and _dotted_star_opens_call(p): - out.append(_STAR_ARG_TOKEN) - continue - out.append(val) - return "".join(out) - - -def _convert(node: ast.AST, *, agg_map: Dict, original: str) -> ParsedExpr: # NOSONAR(S3776) — one-pass dispatch over ast node kinds (Constant/Name/Compare/BinOp/UnaryOp/BoolOp/Call/Attribute…) producing typed ParsedExpr; the branches are flat and short, and splitting hides the exhaustive ast-kind coverage one read scans for. Surfaces ParsedExpr.kind contract directly. - if isinstance(node, ast.Constant): - return _convert_constant(node, original=original) - - if isinstance(node, ast.Name): - m = _PLACEHOLDER_RE.match(node.id) - if m: - idx = int(m.group(1)) - source, agg = agg_map[idx] - return AggCall(source=source, agg=agg) - if node.id == _STAR_ARG_TOKEN: - return StarSource() - # SQL-cased boolean / null literals: Python's ast only treats True/False/None - # as constants, so `true`/`FALSE`/`NULL`/... arrive as names. All are - # reserved words in every target dialect, never columns. - if node.id.lower() in ("true", "false"): - return Literal(value=node.id.lower() == "true") - if node.id.lower() == "null": - return Literal(value=None) - return Ref(name=node.id) - - if isinstance(node, ast.Attribute): - parts = _flatten_attribute(node, original=original) - # ``count(customers.*)`` — the star pre-pass turned the trailing ``*`` - # into the star token; restore it so the DottedRef matches the legacy colon form. - parts = ["*" if p == _STAR_ARG_TOKEN else p for p in parts] - return DottedRef(parts=tuple(parts)) - - if isinstance(node, ast.Call): - return _convert_call(node, agg_map=agg_map, original=original) - - if isinstance(node, ast.BinOp): - op_type = type(node.op) - if op_type is ast.BitOr: - # SQL ``||`` (normalized to ``|``) desugars to the ``concat`` scalar. - return ScalarCall( - name="concat", - args=( - _convert(node.left, agg_map=agg_map, original=original), - _convert(node.right, agg_map=agg_map, original=original), - ), - ) - if op_type not in _BIN_OP_MAP: - raise ValueError( - f"Invalid Mode-B expression {original!r}: unsupported " - f"binary operator {op_type.__name__}." - ) - left = _convert(node.left, agg_map=agg_map, original=original) - right = _convert(node.right, agg_map=agg_map, original=original) - return _interval_as_date_add(op=_BIN_OP_MAP[op_type], left=left, right=right) or Arith( - op=_BIN_OP_MAP[op_type], left=left, right=right, - ) - - if isinstance(node, ast.UnaryOp): - op_type = type(node.op) - if op_type is ast.USub: - return UnaryOp( - op="-", - operand=_convert(node.operand, agg_map=agg_map, original=original), - ) - if op_type is ast.UAdd: - # `+x` is a no-op; collapse to the operand directly. - return _convert(node.operand, agg_map=agg_map, original=original) - if op_type is ast.Not: - return UnaryOp( - op="not", - operand=_convert(node.operand, agg_map=agg_map, original=original), - ) - raise ValueError( - f"Invalid Mode-B expression {original!r}: unsupported unary " - f"operator {op_type.__name__}." - ) - - if isinstance(node, ast.Compare): - if len(node.ops) != 1 or len(node.comparators) != 1: - raise ValueError( - f"Invalid Mode-B expression {original!r}: chained " - f"comparisons are not supported. Each Cmp must be a " - f"single comparison; split (e.g.) `1 < x < 10` into " - f"`1 < x and x < 10`." - ) - op_type = type(node.ops[0]) - if op_type not in _CMP_OP_MAP: - raise ValueError( - f"Invalid Mode-B expression {original!r}: unsupported " - f"comparison operator {op_type.__name__}." - ) - # ``IN`` / ``NOT IN`` carry a literal-only tuple RHS, or a single string - # (a time-point membership); empty and non-literal RHS are rejected. - if op_type in (ast.In, ast.NotIn): - rhs_node = node.comparators[0] - if isinstance(rhs_node, ast.Constant) and isinstance(rhs_node.value, str): - return Cmp( - op=_CMP_OP_MAP[op_type], - left=_convert(node.left, agg_map=agg_map, original=original), - right=Literal(value=rhs_node.value), - ) - placeholder = _placeholder_of(rhs_node) - if placeholder is not None: - return Cmp( - op=_CMP_OP_MAP[op_type], - left=_convert(node.left, agg_map=agg_map, original=original), - right=placeholder, - ) - if not isinstance(rhs_node, (ast.Tuple, ast.List)): - raise ValueError( - f"Invalid Mode-B expression {original!r}: the right-" - f"hand side of ``in`` / ``not in`` must be a tuple/" - f"list literal (e.g. ``status in ('a', 'b')``) or a " - f"time-point string (``created_at in '2025-Q1'``); got " - f"{type(rhs_node).__name__}." - ) - if not rhs_node.elts: - raise ValueError( - f"Invalid Mode-B expression {original!r}: empty " - f"tuple is not allowed on the right-hand side of " - f"``in`` / ``not in`` (dialect-dependent SQL); use " - f"a non-empty literal tuple." - ) - elements: List[Literal] = [] - for elt in rhs_node.elts: - converted = _convert_in_rhs_element( - elt, agg_map=agg_map, original=original, - ) - elements.append(converted) - return Cmp( - op=_CMP_OP_MAP[op_type], - left=_convert(node.left, agg_map=agg_map, original=original), - right=TupleLit(elements=tuple(elements)), - ) - return Cmp( - op=_CMP_OP_MAP[op_type], - left=_convert(node.left, agg_map=agg_map, original=original), - right=_convert(node.comparators[0], agg_map=agg_map, original=original), - ) - - if isinstance(node, ast.BoolOp): - op_str = "and" if isinstance(node.op, ast.And) else "or" - operands = tuple( - _convert(v, agg_map=agg_map, original=original) for v in node.values - ) - return BoolOp(op=op_str, operands=operands) - - placeholder = _placeholder_of(node) - if placeholder is not None: - return placeholder - - if isinstance(node, ast.IfExp): - raise ValueError( - f"Invalid Mode-B expression {original!r}: the Python conditional " - f"'x if cond else y' is not supported. Use SQL " - f"'CASE WHEN cond THEN x ELSE y END' or iif(cond, x, y)." - ) - - raise ValueError( - f"Invalid Mode-B expression {original!r}: unsupported AST node " - f"{type(node).__name__}." - ) - - -_PLACEHOLDER_NAME_RE = re.compile(r"[a-zA-Z_]\w*", re.ASCII) - - -def _placeholder_of(node: ast.AST) -> Optional[Placeholder]: - """``Placeholder`` for a set literal holding exactly one bare variable name, else ``None``.""" - if not (isinstance(node, ast.Set) and len(node.elts) == 1 and isinstance(node.elts[0], ast.Name)): - return None - name = node.elts[0].id - if name == _STAR_ARG_TOKEN or _PLACEHOLDER_RE.match(name) or not _PLACEHOLDER_NAME_RE.fullmatch(name): - return None - return Placeholder(name=name) - - -def placeholder_names(parsed: Any) -> List[str]: - """Names of every ``Placeholder`` in a parsed tree, in walk order.""" - return [n.name for n in _walk_parsed(parsed) if isinstance(n, Placeholder)] - - -def _convert_in_rhs_element( - node: ast.AST, *, agg_map: Dict, original: str, -) -> Literal: - """Convert one element of an ``IN`` / ``NOT IN`` literal-tuple RHS. - - The Python parser emits a negative numeric literal as - ``UnaryOp(USub, Constant(int|float))`` rather than a bare - ``Constant`` with a negative value, so a strict ``isinstance(_, - Literal)`` check against ``_convert``'s output would reject - ``amount in (-1, -2)``. This helper collapses the sign onto the - inner numeric before the literal check (Codex review). Boolean and - string literals are unaffected. - """ - if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)): - inner = node.operand - if ( - isinstance(inner, ast.Constant) - and isinstance(inner.value, (int, float)) - and not isinstance(inner.value, bool) - ): - signed = -inner.value if isinstance(node.op, ast.USub) else inner.value - if isinstance(signed, int): - return Literal(value=Decimal(signed)) - return Literal(value=Decimal(str(signed))) - converted = _convert(node, agg_map=agg_map, original=original) - if not isinstance(converted, Literal): - raise ValueError( - f"Invalid Mode-B expression {original!r}: every element on " - f"the right-hand side of ``in`` / ``not in`` must be a " - f"literal (string, number, or boolean); got " - f"{type(converted).__name__}." - ) - return converted - - -def _convert_constant(node: ast.Constant, *, original: str) -> Literal: - val = node.value - if isinstance(val, bool): - return Literal(value=val) - if val is None: - return Literal(value=None) - if isinstance(val, int): - return Literal(value=Decimal(val)) - if isinstance(val, float): - return Literal(value=Decimal(str(val))) - if isinstance(val, str): - return Literal(value=val) - raise ValueError( - f"Invalid Mode-B expression {original!r}: unsupported literal " - f"type {type(val).__name__}." - ) - - -def _flatten_attribute( - node: ast.Attribute, *, original: str, -) -> List[str]: - parts: List[str] = [node.attr] - cur: ast.AST = node.value - while isinstance(cur, ast.Attribute): - parts.append(cur.attr) - cur = cur.value - if isinstance(cur, ast.Name): - parts.append(cur.id) - else: - raise ValueError( - f"Invalid Mode-B expression {original!r}: unsupported " - f"attribute base {type(cur).__name__}." - ) - return list(reversed(parts)) - - -def _convert_kwarg_value(node: ast.AST, *, agg_map: Dict, original: str): - """Convert a call keyword-argument value. - - List / tuple values (e.g. ``partition_by=[region, channel]`` for the - rank family) convert to a tuple of converted elements so the parser - accepts the documented multi-column transform-kwarg grammar instead of - raising on the bare ``ast.List`` node; scalar values convert normally. - """ - if isinstance(node, (ast.List, ast.Tuple)): - return tuple( - _convert(e, agg_map=agg_map, original=original) for e in node.elts - ) - return _convert(node, agg_map=agg_map, original=original) - - -# The node kinds an ``AggCall.source`` may take: a column, star, or an aggregation-free -# scalar expression, predicates included (a bare TupleLit is not a value). -_AGG_SOURCE_KINDS = (Ref, DottedRef, StarSource, Literal, Placeholder, ScalarCall, Arith, UnaryOp, Cmp, BoolOp) - - -def _contains_agg_or_transform(node: Any) -> bool: - """Whether a parsed subtree contains any AggCall / TransformCall.""" - if isinstance(node, (AggCall, TransformCall)): - return True - if isinstance(node, ScalarCall): - return any(_contains_agg_or_transform(a) for a in node.args) - if isinstance(node, (Arith, Cmp)): - return _contains_agg_or_transform(node.left) or _contains_agg_or_transform( - node.right - ) - if isinstance(node, UnaryOp): - return _contains_agg_or_transform(node.operand) - if isinstance(node, BoolOp): - return any(_contains_agg_or_transform(o) for o in node.operands) - return False - - -def _source_leaves(node: Any): - """Composition leaves of an agg source, treating an inner AggCall / - TransformCall as opaque (not descended — its own source was already - validated when it was converted).""" - if isinstance(node, ScalarCall): - for a in node.args: - yield from _source_leaves(a) - elif isinstance(node, (Arith, Cmp)): - yield from _source_leaves(node.left) - yield from _source_leaves(node.right) - elif isinstance(node, UnaryOp): - yield from _source_leaves(node.operand) - elif isinstance(node, BoolOp): - for o in node.operands: - yield from _source_leaves(o) - else: - yield node - - -def _validated_agg_source(source: Any, *, func_name: str, original: str) -> Any: - """Validate a functional aggregation's first argument as its source. - - A source resolving to attached values — nested AggCalls and/or grained - transforms, alone or composed through arithmetic / scalar calls — is a - (second-order) re-aggregation; a source mixing row-level - references with attached values is a row-grain aggregation; both - are accepted.""" - leaves = list(_source_leaves(source)) - if any(isinstance(leaf, (AggCall, TransformCall)) for leaf in leaves): - return source - if not isinstance(source, _AGG_SOURCE_KINDS): - raise ValueError( - f"Invalid Mode-B expression {original!r}: {func_name!r} cannot " - f"aggregate a {type(source).__name__}; the aggregated expression " - f"must be built from columns, literals, arithmetic, comparisons, " - f"and scalar functions." - ) - return source - - -def _reject_bare_star_args( - args: Tuple[Any, ...], - kwargs: Tuple[Tuple[str, Any], ...], - *, - func_name: str, - original: str, -) -> None: - """``*`` is only an aggregation source — reject it as a plain call arg.""" - values = list(args) + [v for _, v in kwargs] - if any(isinstance(v, StarSource) for v in values): - raise ValueError( - f"Invalid Mode-B expression {original!r}: '*' is only valid as " - f"an aggregation source (e.g. count(*)), not as an argument to " - f"{func_name!r}." - ) - - -def _reject_repeated_keywords(node: ast.Call, *, call: str, original: str) -> None: - seen: set = set() - for kw in node.keywords: - if kw.arg in seen: - raise ValueError( - f"Invalid Mode-B expression {original!r}: {call}() got keyword " - f"argument {kw.arg!r} more than once." - ) - seen.add(kw.arg) - - -def _call_arguments( - node: ast.Call, *, call: str, agg_map: Dict, original: str, -) -> Tuple[Tuple[Any, ...], Tuple[Tuple[str, Any], ...]]: - args = tuple( - _convert(a, agg_map=agg_map, original=original) for a in node.args - ) - # Reject ``**kwargs`` dictionary unpacking (``kw.arg is None``) rather - # than silently dropping it (CR) — a dropped ``**`` would change call - # semantics without warning. - if any(kw.arg is None for kw in node.keywords): - raise ValueError( - f"Invalid Mode-B expression {original!r}: dictionary unpacking " - f"(**kwargs) is not supported in calls." - ) - _reject_repeated_keywords(node, call=call, original=original) - kwargs = tuple( - (kw.arg, _convert_kwarg_value(kw.value, agg_map=agg_map, original=original)) - for kw in node.keywords - if kw.arg is not None # guarded above; narrows kw.arg to str - ) - return args, kwargs - - -def _convert_call( - node: ast.Call, *, agg_map: Dict, original: str, -) -> ParsedExpr: - if isinstance(node.func, ast.Attribute): - return _convert_method_call(node, func=node.func, agg_map=agg_map, original=original) - if not isinstance(node.func, ast.Name): - raise ValueError( - f"Invalid Mode-B expression {original!r}: function calls " - f"with non-name callee are not supported." - ) - func_name = node.func.id - m = _PLACEHOLDER_RE.match(func_name) - args, kwargs = _call_arguments( - node, call=agg_map[int(m.group(1))][1] if m else func_name, agg_map=agg_map, original=original, - ) - # Legacy colon-spelling placeholder? - if m: - source, agg = agg_map[int(m.group(1))] - return AggCall(source=source, agg=agg, args=args, kwargs=kwargs) - return _dispatch_call(func_name, args=args, kwargs=kwargs, original=original) - - -def _onto_path(node: Any, *, path: Tuple[str, ...]) -> Any: - """``node`` with every column leaf (and ``*``) read under the join ``path``.""" - if isinstance(node, Ref): - return DottedRef(parts=(*path, node.name)) - if isinstance(node, DottedRef): - return DottedRef(parts=(*path, *node.parts)) - if isinstance(node, StarSource): - return DottedRef(parts=(*path, "*")) - if isinstance(node, ScalarCall): - return node.model_copy(update={"args": tuple(_onto_path(a, path=path) for a in node.args)}) - if isinstance(node, (Arith, Cmp)): - return node.model_copy(update={"left": _onto_path(node.left, path=path), "right": _onto_path(node.right, path=path)}) - if isinstance(node, UnaryOp): - return node.model_copy(update={"operand": _onto_path(node.operand, path=path)}) - if isinstance(node, BoolOp): - return node.model_copy(update={"operands": tuple(_onto_path(o, path=path) for o in node.operands)}) - return node - - -def _is_literal_arg(node: Any) -> bool: - if isinstance(node, UnaryOp) and node.op == "-": - return isinstance(node.operand, Literal) - return isinstance(node, (Literal, Placeholder)) - - -def _method_call_refusal(name: str, *, args: Tuple[Any, ...], kwargs: Tuple[Tuple[str, Any], ...]) -> Optional[str]: - """Why ``.(args)`` has no single functional reading, else ``None``.""" - lowered = name.lower() - transform_only = name in ALL_TRANSFORMS and normalize_aggregation_name(name) not in BUILTIN_AGGREGATIONS - if lowered in SCALAR_FUNCTIONS or lowered in DATE_PART_ALIASES or transform_only: - return f"calls {name!r}, which is not an aggregation" - if not args: - return "needs the aggregated value as its first argument" - if any(isinstance(leaf, (DottedRef, AggCall, TransformCall)) for leaf in _source_leaves(args[0])): - return "may read only bare column names of that path" - if any(k == "partition_by" for k, _ in kwargs): - return "cannot take partition_by=" - if not all(_is_literal_arg(v) for v in (*args[1:], *(v for _, v in kwargs))): - return "takes only literal further arguments" - return None - - -def _convert_method_call(node: ast.Call, *, func: ast.Attribute, agg_map: Dict, original: str) -> ParsedExpr: - """``.(, )`` as its functional twin ``(., …)``.""" - *path_parts, name = _flatten_attribute(func, original=original) - path = tuple(path_parts) - args, kwargs = _call_arguments(node, call=name, agg_map=agg_map, original=original) - twin_args = (_onto_path(args[0], path=path), *args[1:]) if args else () - reason = _method_call_refusal(name, args=args, kwargs=kwargs) - if reason is not None: - raise ValueError( - f"Invalid Mode-B expression {original!r}: the method call {'.'.join(path)}.{name}(...) " - f"{reason}; write the functional form, e.g. {name}{_canonical_call_params(twin_args, kwargs)}." - ) - return _dispatch_call(name, args=twin_args, kwargs=kwargs, original=original) - - -def _dispatch_call( # NOSONAR(S3776) — the one call-dispatch ladder (builtin functional aggregation → transform → scalar → date-part alias → unknown-name AggCall deferral); each rung IS the documented dispatch order and splitting them hides it. first/last carry no grain decision here — they dispatch at bind. - func_name: str, *, args: Tuple[Any, ...], kwargs: Tuple[Tuple[str, Any], ...], original: str, -) -> ParsedExpr: - # Functional builtin aggregation? Matched via alias/case healing, exactly - # like legacy colon names heal at binding; ``agg`` stores the RAW token so both - # spellings collapse to the identical AggCall. ``first``/``last`` dispatch at - # bind by the operand's type. - if normalize_aggregation_name(func_name) in BUILTIN_AGGREGATIONS: - if not args: - raise ValueError( - f"Invalid Mode-B expression {original!r}: aggregation " - f"{func_name!r} requires a value argument, e.g. " - f"{func_name}(column)." - ) - source = _validated_agg_source( - args[0], func_name=func_name, original=original, - ) - return AggCall(source=source, agg=func_name, args=args[1:], kwargs=kwargs) - - # Transform? - if func_name in ALL_TRANSFORMS: - if not args: - raise ValueError( - f"Invalid Mode-B expression {original!r}: transform " - f"{func_name!r} requires at least one positional argument " - f"(the value to transform)." - ) - _reject_bare_star_args(args, kwargs, func_name=func_name, original=original) - return TransformCall( - op=func_name, - input=args[0], - args=args[1:], - kwargs=kwargs, - ) - - # Scalar function? Case-insensitive match; normalised to lower case into - # ``ScalarCall`` so ``COALESCE`` / ``coalesce`` intern to one key. - if func_name.lower() in SCALAR_FUNCTIONS: - if kwargs: - raise ValueError( - f"Invalid Mode-B expression {original!r}: scalar function " - f"{func_name!r} does not accept keyword arguments. Pass " - f"values positionally." - ) - _reject_bare_star_args(args, kwargs, func_name=func_name, original=original) - name = func_name.lower() - return ScalarCall(name=name, args=_date_args(name=name, args=args, original=original)) - - part = DATE_PART_ALIASES.get(func_name.lower()) - if part is not None: - if kwargs or len(args) != 1: - raise ValueError( - f"Invalid Mode-B expression {original!r}: date-part function {func_name!r} " - f"takes 1 argument; got {len(args) + len(kwargs)}." - ) - _reject_bare_star_args(args, kwargs, func_name=func_name, original=original) - return ScalarCall(name="date_part", args=(Literal(value=part.value), args[0])) - - # Unknown name with an aggregatable first arg → AggCall candidate, validated - # at binding. A custom aggregation over an - # attached source is a re-aggregation, validated like any other. - if args and isinstance(args[0], _AGG_SOURCE_KINDS) and not _contains_agg_or_transform(args[0]): - return AggCall(source=args[0], agg=func_name, args=args[1:], kwargs=kwargs) - if args and any(isinstance(leaf, AggCall) for leaf in _source_leaves(args[0])): - source = _validated_agg_source(args[0], func_name=func_name, original=original) - return AggCall(source=source, agg=func_name, args=args[1:], kwargs=kwargs) - - if func_name.lower() in GRANULARITY_NAMES: - raise GranularityCallError.wrong_shape(original) - raise UnknownFunctionError( - name=func_name, - location=original, - suggestion=ambiguous_weekday_message(func_name) if func_name.lower() in AMBIGUOUS_WEEKDAY_NAMES else ( - f"Mode-B accepts the closed scalar allowlist " - f"({sorted(SCALAR_FUNCTIONS)}), transforms " - f"({sorted(ALL_TRANSFORMS)}), and aggregations " - f"(`sum(revenue)`); custom " - f"aggregation names must be defined on the model." - ), - ) - - -# Position of the count argument of the date-offset calls. -_COUNT_ARG = {"date_add": 1, "interval": 0} - - -def _date_args(*, name: str, args: Tuple[Any, ...], original: str) -> Tuple[Any, ...]: - """Validate and lower-case a date scalar's unit word; fold a negated numeric count literal.""" - out = list(args) - unit = DATE_UNIT_ARGS.get(name) - if unit is not None and unit[0] < len(out): - pos, vocab = unit - words = tuple(v.value for v in vocab) - out[pos] = Literal(value=_unit_word(out[pos], name=name, words=words, original=original)) - count_pos = _COUNT_ARG.get(name) - if count_pos is not None and count_pos < len(out): - out[count_pos] = _folded(out[count_pos]) - return tuple(out) - - -def _unit_word(arg: Any, *, name: str, words: Tuple[str, ...], original: str) -> str: - """The canonical lower-case unit (or ``date_part`` part) word of ``arg``, one of ``words``.""" - accepted = ", ".join(words) - what = "part" if name == "date_part" else "unit" - if not (isinstance(arg, Literal) and isinstance(arg.value, str)): - raise ValueError( - f"Invalid Mode-B expression {original!r}: the {what} of {name}() must be a " - f"string literal, one of: {accepted}." - ) - word = arg.value.lower() - if name == "date_part": - if word in AMBIGUOUS_WEEKDAY_NAMES: - raise ValueError(f"Invalid Mode-B expression {original!r}: {ambiguous_weekday_message(arg.value)}") - word = DATE_PART_ALIASES[word].value if word in DATE_PART_ALIASES else word - if word not in words: - raise ValueError( - f"Invalid Mode-B expression {original!r}: unknown {name}() {what} " - f"{arg.value!r}; accepted: {accepted}." - ) - return word - - -def _folded(node: Any) -> Any: - """``-`` as one negative literal, so both count spellings intern alike.""" - if ( - isinstance(node, UnaryOp) and node.op == "-" - and isinstance(node.operand, Literal) and isinstance(node.operand.value, Decimal) - ): - return Literal(value=-node.operand.value) - return node - - -def _is_interval(node: Any) -> bool: - return isinstance(node, ScalarCall) and node.name == "interval" - - -def _interval_as_date_add(*, op: str, left: Any, right: Any) -> Optional[ScalarCall]: - """``x + interval(n, u)`` / ``interval(n, u) + x`` / ``x - interval(n, u)`` as ``date_add``.""" - if _is_interval(left) == _is_interval(right) or op not in ("+", "-"): - return None - if op == "-" and _is_interval(left): - return None - interval, base = (right, left) if _is_interval(right) else (left, right) - if len(interval.args) != 2: - raise ValueError(check_scalar_arity(name="interval", argc=len(interval.args))) - count, unit = interval.args - if op == "-": - count = _folded(UnaryOp(op="-", operand=count)) - return ScalarCall(name="date_add", args=(base, count, unit)) - - -def _walk_parsed(node: Any) -> Iterator[Any]: - """Every node of a parsed tree, including call args, kwargs and agg sources.""" - if isinstance(node, tuple): - for item in node: - yield from _walk_parsed(item) - return - if not isinstance(node, _BaseNode): - return - yield node - for field in type(node).model_fields: - yield from _walk_parsed(getattr(node, field)) - - -# --------------------------------------------------------------------------- -# Canonical text rendering + entity-ref splitting -# --------------------------------------------------------------------------- - - -def _canonical_call_params( - args: Tuple[Any, ...], kwargs: Tuple[Tuple[str, Any], ...], *, bare_direction: bool = False, -) -> str: - parts = [canonical_measure_text(a) for a in args] - parts += [_canonical_kwarg(k, v, bare_direction=bare_direction) for k, v in kwargs] - return f"({', '.join(parts)})" if parts else "" - - -def _canonical_kwarg(name: str, value: Any, *, bare_direction: bool) -> str: - """``name=value``; a transform's recognised ``direction`` renders as its bare normalised value.""" - if bare_direction and name == "direction" and isinstance(value, Literal): - direction = normalize_direction(value.value) - if direction is not None: - return direction - return f"{name}={_canonical_kwarg_text(value)}" - - -def _canonical_kwarg_text(value: Any) -> str: - if isinstance(value, tuple): - return f"[{', '.join(canonical_measure_text(v) for v in value)}]" - return canonical_measure_text(value) - - -def canonical_measure_text(parsed: Any) -> str: # NOSONAR(S3776) — flat per-node-kind rendering table; each branch is one spelling rule. - """Render a ``ParsedExpr`` back to canonical legacy colon-spelling text. - - Used for alias derivation so the functional and legacy colon spellings of one - formula sanitize to the SAME public name. Deterministic, not a - verbatim round-trip: grouping parens are dropped and spacing normalised. - """ - if isinstance(parsed, Ref): - return parsed.name - if isinstance(parsed, DottedRef): - return ".".join(parsed.parts) - if isinstance(parsed, StarSource): - return "*" - if isinstance(parsed, Literal): - if isinstance(parsed.value, str): - return f"'{parsed.value}'" - return str(parsed.value) - if isinstance(parsed, Placeholder): - return f"{{{parsed.name}}}" - if isinstance(parsed, TupleLit): - return f"({', '.join(canonical_measure_text(e) for e in parsed.elements)})" - if isinstance(parsed, AggCall): - source = canonical_measure_text(parsed.source) - if isinstance(parsed.source, (Ref, DottedRef, StarSource)): - return f"{source}:{parsed.agg}{_canonical_call_params(parsed.args, parsed.kwargs)}" - # Expression source (``sum(amount - cost)``): render functionally so the - # text can't collide with a distinct parse tree like ``amount - cost:sum``. - parts = [source] - parts += [canonical_measure_text(a) for a in parsed.args] - parts += [f"{k}={_canonical_kwarg_text(v)}" for k, v in parsed.kwargs] - return f"{parsed.agg}({', '.join(parts)})" - if isinstance(parsed, TransformCall): - inner = canonical_measure_text(parsed.input) - params = _canonical_call_params(parsed.args, parsed.kwargs, bare_direction=True) - return f"{parsed.op}({inner}{', ' + params[1:-1] if params else ''})" - if isinstance(parsed, ScalarCall): - return f"{parsed.name}({', '.join(canonical_measure_text(a) for a in parsed.args)})" - if isinstance(parsed, (Arith, Cmp)): - left = canonical_measure_text(parsed.left) - right = canonical_measure_text(parsed.right) - return f"{left} {parsed.op} {right}" - if isinstance(parsed, UnaryOp): - operand = canonical_measure_text(parsed.operand) - return f"not {operand}" if parsed.op == "not" else f"{parsed.op}{operand}" - if isinstance(parsed, BoolOp): - return f" {parsed.op} ".join( - canonical_measure_text(o) for o in parsed.operands - ) - return str(parsed) - - -def _functional_suffix_text(raw: str, *, agg: str) -> str: - """Rebuild the ``agg`` / ``agg(rest-args)`` suffix of a functional entity - ref, preserving any extra-arg text verbatim (``last(balance, updated_at)`` - → ``last(updated_at)``).""" - open_idx = raw.index("(") - depth = 0 - split_idx = None - for i in range(open_idx, len(raw)): - ch = raw[i] - if ch == "(": - depth += 1 - elif ch == ")": - depth -= 1 - if depth == 0: - split_idx = i - break - elif ch == "," and depth == 1: - split_idx = i - break - if split_idx is None or raw[split_idx] == ")": - return agg - rest = raw[split_idx + 1:].strip() - if rest.endswith(")"): - rest = rest[:-1].strip() - return f"{agg}({rest})" - - -def split_entity_agg_ref(raw: str) -> Tuple[str, Optional[str]]: - """``(prefix, agg_suffix)`` of a single aggregated-column entity - reference: ``sum(orders.amount)`` and the legacy ``orders.amount:sum`` split - identically. - - Legacy colon and call-free text splits exactly like - :func:`slayer.core.refs.split_agg_suffix`. Functional text must parse to - an ``AggCall`` over a pure column / star source; multi-column expression - text (``sum(a - b)``) raises ``ValueError`` — an expression is not an - entity. - """ - prefix, suffix = split_agg_suffix(raw) - if suffix is not None or "(" not in raw: - return prefix, suffix - parsed = parse_expr(raw) - if not isinstance(parsed, AggCall) or not isinstance( - parsed.source, (Ref, DottedRef, StarSource) - ): - raise ValueError( - f"{raw!r} is not a single aggregated column reference; only " - f"the `agg(column)` form names an entity." - ) - source = parsed.source - if isinstance(source, StarSource): - prefix = "*" - elif isinstance(source, Ref): - prefix = source.name - else: - prefix = ".".join(source.parts) - return prefix, _functional_suffix_text(raw, agg=parsed.agg) diff --git a/slayer/engine/unrelated.py b/slayer/engine/unrelated.py index b8360ee22..4dc89d284 100644 --- a/slayer/engine/unrelated.py +++ b/slayer/engine/unrelated.py @@ -21,7 +21,7 @@ from slayer.core.query import ComputedDimension, SlayerQuery, render_probe_text from slayer.core.time_spine import is_spine from slayer.engine.reference_closure import key_closure -from slayer.engine.syntax import DottedRef, parse_expr, parse_filter_expr +from slayer.core.syntax import DottedRef, parse_expr from slayer.ir.source_bundle import ResolvedSourceBundle __all__ = ["ProductSides", "first_hop_names", "stamp_unrelated", "unrelated_targets"] @@ -43,7 +43,7 @@ def _dotted_heads(node: Any) -> Iterator[str]: def _parsed_heads(text: str, *, as_filter: bool) -> Iterator[str]: try: - parsed = parse_filter_expr(render_probe_text(text)) if as_filter else parse_expr(text) + parsed = parse_expr(render_probe_text(text) if as_filter else text) except Exception: # noqa: BLE001 — the binder reports a malformed expression return yield from _dotted_heads(parsed) diff --git a/slayer/ir/bound.py b/slayer/ir/bound.py index 573ca3699..2e395a8ba 100644 --- a/slayer/ir/bound.py +++ b/slayer/ir/bound.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Optional, Tuple +from typing import Literal, Optional, Tuple from pydantic import BaseModel, ConfigDict, Field @@ -92,6 +92,7 @@ class OrderSpec(BaseModel): bound: BoundExpr direction: str = "asc" + nulls: Literal["default", "first", "last"] = "default" class GranularityConflict(BaseModel): diff --git a/slayer/ir/planned.py b/slayer/ir/planned.py index 7ec6b5874..e9b3d88ec 100644 --- a/slayer/ir/planned.py +++ b/slayer/ir/planned.py @@ -20,6 +20,7 @@ AggregateKey, LiteralKey, Phase, + CastKey, ScalarCallKey, TransformKey, ValueKey, @@ -775,6 +776,16 @@ def plan_value_keys(planned) -> Iterator[ValueKey]: yield from _walk_plan_keys(planned, set()) +def plans_try_cast(planned_list) -> bool: + """Whether any plan renders a ``TRY_CAST``.""" + return any( + isinstance(k, CastKey) and k.try_ + for planned in planned_list + for root in plan_value_keys(planned) + for k in walk_value_keys(root) + ) + + def plans_read_clock(planned_list) -> bool: """Whether any plan evaluates ``now()`` / ``current_date()``.""" return any( diff --git a/slayer/ir/terms.py b/slayer/ir/terms.py index 65ef1d5c6..33e1b2973 100644 --- a/slayer/ir/terms.py +++ b/slayer/ir/terms.py @@ -13,8 +13,8 @@ from pydantic import BaseModel, model_validator -from slayer.core.formula import TIME_TRANSFORMS from slayer.core.keys import AggregateKey, Grain, TransformKey, ValueKey +import slayer.core.enums @runtime_checkable @@ -84,7 +84,7 @@ class Transform(_Term, frozen=True): @model_validator(mode="after") def _require_time_axis(self) -> "Transform": - if self.recipe.op in TIME_TRANSFORMS and self.recipe.time_key is None: + if self.recipe.op in slayer.core.enums.TIME_TRANSFORMS and self.recipe.time_key is None: raise ValueError( f"Transform {self.recipe.op!r} requires a time axis " f"(recipe.time_key is unset).", diff --git a/slayer/ir/variables.py b/slayer/ir/variables.py index e76d189de..3a2e11c4c 100644 --- a/slayer/ir/variables.py +++ b/slayer/ir/variables.py @@ -2,18 +2,20 @@ - :func:`merge_query_variables` collapses the four variable layers into the effective dict (runtime > stage > outer > model_defaults). -- :func:`apply_variables_to_query` substitutes ``{var}`` (Mode-B regime) into a - query's filters, measure formulas, computed-dimension expressions, order - expressions and ``date_range`` bounds. +- :func:`apply_variables_to_query` binds ``{var}`` as literals into a query's + filters, measure formulas, computed-dimension expressions and order + expressions, and substitutes its ``date_range`` bounds. - :func:`substitute_model_sql_surfaces` substitutes a model's four Mode-A - surfaces (SQL regime) and its saved measure formulas (Mode-B regime). + surfaces (SQL regime) and binds its saved measure formulas. """ from __future__ import annotations +import re from typing import Any, Dict, Optional from slayer.core.models import ModelMeasure, SlayerModel +from slayer.core.syntax import bind_variables from slayer.core.query import ( ComputedDimension, OrderItem, @@ -50,34 +52,53 @@ def merge_query_variables( } -def _python_sub(text: str, variables: Dict[str, Any]) -> str: - # Mode-B takes the python regime: SQL quote-doubling would concatenate adjacent literals. - return substitute_variables(filter_str=text, variables=variables, escape="python") - - def _substituted_measure(m: ModelMeasure, variables: Dict[str, Any]) -> ModelMeasure: - text = _python_sub(m.formula, variables) + text = bind_variables(m.formula, variables) return m if text == m.formula else m.substituted(text) def _substituted_dimension(d: Any, variables: Dict[str, Any]) -> Any: if not isinstance(d, ComputedDimension): return d - text = _python_sub(d.expression, variables) - return d if text == d.expression else d.substituted(text) + text = bind_variables(d.expression, variables) + if text == d.expression: + return d + bound = d.model_copy(update={"expression": text}) + bound._template = d.template + return bound def _substituted_order(o: OrderItem, variables: Dict[str, Any]) -> OrderItem: if not o.raw_formula: return o - text = _python_sub(o.raw_formula, variables) + text = bind_variables(o.raw_formula, variables) return o if text == o.raw_formula else o.model_copy(update={"raw_formula": text}) +_BOUND_VARIABLE_RE = re.compile(r"\{([A-Za-z_]\w*)\}", re.ASCII) + + +def _date_range_bound(text: str, variables: Dict[str, Any]) -> str: + """A ``date_range`` bound with each ``{name}`` replaced by its value's text (a string or a whole number); + the result is then checked as a time point.""" + def _value(match: "re.Match[str]") -> str: + name = match.group(1) + if name not in variables: + raise ValueError( + f"Undefined variable '{name}' in date_range bound {text!r}. Available variables: {sorted(variables)}" + ) + value = variables[name] + if isinstance(value, bool) or not isinstance(value, (str, int)): + raise ValueError(f"Variable '{name}' in a date_range bound must be a string or an integer year.") + return str(value) + + return _BOUND_VARIABLE_RE.sub(_value, text) + + def _substituted_time_dimension(td: TimeDimension, variables: Dict[str, Any]) -> TimeDimension: if not td.date_range: return td - bounds = [None if b is None else _python_sub(b, variables) for b in td.date_range] + bounds = [None if b is None else _date_range_bound(b, variables) for b in td.date_range] if bounds == td.date_range: return td # Revalidated so a substituted bound is shape-checked like a literal one. @@ -99,7 +120,7 @@ def apply_variables_to_query( effective: Dict[str, Any] = dict(variables or {}) changed: Dict[str, Any] = {} if query.filters is not None: - changed["filters"] = [_python_sub(f, effective) for f in query.filters] + changed["filters"] = [bind_variables(f, effective) for f in query.filters] surfaces = ( ("measures", lambda m: _substituted_measure(m, effective)), ("dimensions", lambda d: _substituted_dimension(d, effective)), @@ -152,7 +173,7 @@ def substitute_model_sql_surfaces( *, model: SlayerModel, variables: Dict[str, Any], backslash_escapes: bool, ) -> SlayerModel: """Copy of ``model`` with ``{var}`` substituted into its four Mode-A surfaces (SQL regime) - and its saved measure formulas (python regime); no-op when unneeded.""" + and bound into its saved measure formulas; no-op when unneeded.""" if not variables and not model_needs_substitution_pass(model): return model variables = coerce_declared_list_variables( @@ -180,7 +201,7 @@ def _sub(text: str) -> str: update["filters"] = [_sub(f) for f in model.filters] if model.measures: update["measures"] = [ - m.model_copy(update={"formula": _python_sub(m.formula, variables)}) if _measure_needs_pass(m) else m + m.model_copy(update={"formula": bind_variables(m.formula, variables)}) if _measure_needs_pass(m) else m for m in model.measures ] return model.model_copy(update=update) diff --git a/slayer/memories/help_content/02_workflow.md b/slayer/memories/help_content/02_workflow.md index 2dca786b5..47dd2ea96 100644 --- a/slayer/memories/help_content/02_workflow.md +++ b/slayer/memories/help_content/02_workflow.md @@ -22,7 +22,7 @@ show" is the rows. ## Filter literals -Build every `==` / `in` / `like` condition on a text column from that column's +Build every `=` / `in` / `like` condition on a text column from that column's sampled values (inspect it with `compact=false`), never a guessed spelling; samples are a top-N snapshot, so verify a needed literal that is absent (e.g. a distinct-values query) rather than assume. diff --git a/slayer/memories/help_content/03_aggregations.md b/slayer/memories/help_content/03_aggregations.md index f38aaf0fb..9d9d7eabb 100644 --- a/slayer/memories/help_content/03_aggregations.md +++ b/slayer/memories/help_content/03_aggregations.md @@ -13,6 +13,8 @@ filters, order, and computed dimensions. `covar_pop(x, other=y)`, `first(x[, time_col])` / `last(x[, time_col])` (the earliest / latest record's value in each group), plus the model's custom aggregations (`inspect` the model to list them). +`name=value` is a keyword argument in an aggregation or transform but a comparison in a +scalar function (`iif(status = 'paid', …)`); write `==` for a comparison in a keyword slot. A sole primary-key column only takes `count`, `count_distinct`, `count_distinct_approx`, `min` and `max`; a column's `allowed_aggregations` may @@ -22,7 +24,7 @@ restrict it further. Expressions are not gated: `sum(price * quantity)` works. `sum(amount - cost)`, `count_distinct(upper(email))`, `percentile(price * qty, p=0.5)`. A comparison aggregates as 1 / 0: `sum(amount > 15)` counts the rows above 15. A -conditional aggregate uses `iif`: `sum(iif(status == 'paid', amount, 0))`; always give +conditional aggregate uses `iif`: `sum(iif(status = 'paid', amount, 0))`; always give it a `name`. ## Empty input diff --git a/slayer/memories/resolver.py b/slayer/memories/resolver.py index 61670d76f..a04869724 100644 --- a/slayer/memories/resolver.py +++ b/slayer/memories/resolver.py @@ -43,7 +43,7 @@ from slayer.core.models import SlayerModel from slayer.core.query import ComputedDimension, ModelExtension, SlayerQuery from slayer.engine.model_reads import FieldKind, query_field_texts -from slayer.engine.syntax import ( +from slayer.core.syntax import ( AggCall, DottedRef, ParsedExpr, diff --git a/slayer/osi/converter.py b/slayer/osi/converter.py index 01d05e0cb..0faa3efd4 100644 --- a/slayer/osi/converter.py +++ b/slayer/osi/converter.py @@ -20,7 +20,7 @@ from slayer.core.enums import DataType, JoinCardinality from slayer.core.errors import AmbiguousJoinPathError -from slayer.core.formula import parse_formula +from slayer.core.syntax import parse_expr from slayer.core.join_walker import resolve_hop from slayer.core.models import Column, ModelJoin, ModelMeasure, SlayerModel, join_key_error from slayer.core.refs import IDENTIFIER_RE as _IDENTIFIER_RE @@ -795,7 +795,7 @@ def _build_measure(self, metric: OSIMetric, sm_model_names: list[str]) -> None: return try: - parse_formula(result.formula) + parse_expr(result.formula) except Exception as exc: # noqa: BLE001 — reject un-parseable emission self._unconv( f"Metric {metric.name!r}: emitted formula {result.formula!r} is " diff --git a/slayer/sql/client.py b/slayer/sql/client.py index a9d73c8c7..48a379512 100644 --- a/slayer/sql/client.py +++ b/slayer/sql/client.py @@ -389,7 +389,7 @@ def _statement_timeout( conn, *, engine: Any, dialect: SqlDialect, datasource_name: str, timeout_seconds: int, ) -> Iterator[list[StatementTimeoutSkippedWarning]]: """Put the dialect's timeout on ``conn`` for the block; yields the skips to report.""" - if not dialect.timeout_permitted(_server_profile(conn, engine=engine, dialect=dialect)): + if dialect.timeout_reads_profile and not dialect.timeout_permitted(_server_profile(conn, engine=engine, dialect=dialect)): yield [_skip(datasource_name, timeout_seconds, "readonly_user")] return dbapi_connection = conn.connection.dbapi_connection @@ -415,7 +415,9 @@ async def _statement_timeout_async( conn, *, engine: Any, dialect: SqlDialect, datasource_name: str, timeout_seconds: int, ) -> AsyncIterator[list[StatementTimeoutSkippedWarning]]: """Async sibling of ``_statement_timeout``.""" - if not dialect.timeout_permitted(await _server_profile_async(conn, engine=engine, dialect=dialect)): + if dialect.timeout_reads_profile and not dialect.timeout_permitted( + await _server_profile_async(conn, engine=engine, dialect=dialect), + ): yield [_skip(datasource_name, timeout_seconds, "readonly_user")] return dbapi_connection = (await conn.get_raw_connection()).dbapi_connection diff --git a/slayer/sql/dialects/_tier2.py b/slayer/sql/dialects/_tier2.py index 19cc0ef43..b8ad74246 100644 --- a/slayer/sql/dialects/_tier2.py +++ b/slayer/sql/dialects/_tier2.py @@ -12,7 +12,10 @@ from __future__ import annotations +from typing import ClassVar + from sqlglot import exp +from sqlglot.expressions.datatypes import DType from slayer.sql.dialects.base import SqlDialect from slayer.sql.dialects.trino import PrestoFamilyDialect @@ -20,6 +23,8 @@ class RedshiftDialect(SqlDialect): sqlglot_name: str = "redshift" + ilike_native: ClassVar[bool] = True + vendor_cast_types: ClassVar[frozenset[DType]] = frozenset({exp.DataType.Type.MONEY}) ds_type_aliases: frozenset[str] = frozenset({"redshift"}) explain_prefix: str | None = "EXPLAIN" explain_postfix: str = "" @@ -32,7 +37,7 @@ def build_null_safe_eq( self, left: exp.Expression, right: exp.Expression, ) -> exp.Expression: """Redshift (Postgres 8.0.2 fork) has no ``IS NOT DISTINCT - FROM`` — emit the expanded ``a = b OR (a IS NULL AND b IS NULL)``.""" + FROM`` — emit the expanded two-valued form.""" return self._expanded_null_safe_eq(left, right) @@ -44,6 +49,7 @@ class PrestoDialect(PrestoFamilyDialect): class DatabricksDialect(SqlDialect): sqlglot_name: str = "databricks" + ilike_native: ClassVar[bool] = True ds_type_aliases: frozenset[str] = frozenset({"databricks"}) explain_prefix: str | None = "EXPLAIN EXTENDED" explain_postfix: str = "" @@ -55,6 +61,7 @@ class DatabricksDialect(SqlDialect): class SparkDialect(SqlDialect): sqlglot_name: str = "spark" + ilike_native: ClassVar[bool] = True ds_type_aliases: frozenset[str] = frozenset({"spark"}) explain_prefix: str | None = "EXPLAIN EXTENDED" explain_postfix: str = "" @@ -81,6 +88,5 @@ class OracleDialect(SqlDialect): def build_null_safe_eq( self, left: exp.Expression, right: exp.Expression, ) -> exp.Expression: - """Oracle has no ``IS NOT DISTINCT FROM`` — emit the expanded - ``a = b OR (a IS NULL AND b IS NULL)``.""" + """Oracle has no ``IS NOT DISTINCT FROM`` — emit the expanded two-valued form.""" return self._expanded_null_safe_eq(left, right) diff --git a/slayer/sql/dialects/base.py b/slayer/sql/dialects/base.py index bdb0837de..e91175641 100644 --- a/slayer/sql/dialects/base.py +++ b/slayer/sql/dialects/base.py @@ -13,8 +13,10 @@ from sqlalchemy.engine.url import make_url from sqlalchemy.exc import ArgumentError from sqlglot import exp +from sqlglot.expressions.datatypes import DType from sqlglot.expressions.core import Expression from sqlglot.dialects.dialect import Dialect as _SqlglotDialect +from sqlglot.errors import SqlglotError from slayer.core.enums import SUB_DAY_GRANULARITIES, DataType, DatePart, TimeGranularity from slayer.core.errors import IdentifierCollisionError, IdentifierLengthError @@ -104,6 +106,39 @@ def iso_text(value: date) -> str: } +#: The validity grammar where a database cannot check a cast itself: a signed decimal with an optional +#: fraction and exponent, surrounding spaces ignored (POSIX classes, so no backslash meets a string literal). +NUMBER_TEXT_PATTERN = r"^[[:space:]]*[+-]?([0-9]+[.]?[0-9]*|[.][0-9]+)([eE][+-]?[0-9]+)?[[:space:]]*$" +TRUE_WORDS = ("true", "t", "yes", "1") +FALSE_WORDS = ("false", "f", "no", "0") + + +def as_text(value: Expression) -> Expression: + """``CAST(value AS TEXT)`` — the input a validity check reads.""" + return exp.Cast(this=value.copy(), to=exp.DataType.build("TEXT")) + + +def cast_target(type_text: str, *, dialect: "SqlDialect") -> exp.DataType: + """The author's cast type as ``dialect``'s ``DataType``; ``ValueError`` when it cannot express it.""" + sqlglot_name = dialect.sqlglot_name + built: Optional[exp.DataType] = None + for reader in (sqlglot_name, "postgres"): + try: + built = exp.DataType.build(type_text, dialect=reader) + break + except (SqlglotError, ValueError): + continue + if built is None or built.this == exp.DataType.Type.USERDEFINED or not dialect.cast_type_supported(built): + raise ValueError(f"CAST to {type_text} cannot be expressed on {sqlglot_name}.") + return built + + +#: Types only some databases have; a dialect without one rejects a CAST to it at plan time. +VENDOR_ONLY_CAST_TYPES: frozenset[DType] = frozenset({ + exp.DataType.Type.MONEY, exp.DataType.Type.SMALLMONEY, +}) + + def temporal_type_of(value: date) -> DataType: return DataType.TIMESTAMP if isinstance(value, datetime) else DataType.DATE @@ -287,6 +322,14 @@ class SqlDialect(BaseModel): # Correlated subqueries need a server setting this user may be unable to set. correlated_subqueries_gated: bool = False + # Whether setting the statement timeout depends on the server profile (probed only then). + timeout_reads_profile: bool = False + + # The server's profile, when SQL generation needs it (Postgres TRY_CAST); unknown otherwise. + server_profile: ServerProfile = ServerProfile() + # Whether rendering a TRY_CAST reads ``server_profile``. + try_cast_reads_profile: bool = False + # An ``IN`` subquery and its joins run once on the initiator (``GLOBAL``): exact over shards. global_in_subqueries: bool = False @@ -329,6 +372,25 @@ def declared_cast_type(self, dt: Optional[DataType]) -> Optional[DataType]: """The CAST target for a value of type ``dt`` (``None`` skips the cast).""" return dt + # Vendor-only types sqlglot parses for every dialect; a dialect lists the ones it has. + vendor_cast_types: ClassVar[frozenset[DType]] = frozenset() + + def cast_type_supported(self, data_type: exp.DataType) -> bool: + """Whether this dialect can express a CAST to ``data_type``.""" + return data_type.this not in VENDOR_ONLY_CAST_TYPES or data_type.this in self.vendor_cast_types + + def build_try_cast(self, value: Expression, to: exp.DataType) -> Expression: + """``TRY_CAST``: NULL for an input the cast cannot convert; the input is read as text first.""" + return exp.TryCast(this=as_text(value), to=to) + + # A native case-insensitive ``ILIKE``; elsewhere it is ``LOWER(x) LIKE LOWER(p)``. + ilike_native: ClassVar[bool] = False + + def build_ilike(self, value: Expression, pattern: Expression) -> Expression: + if self.ilike_native: + return exp.ILike(this=value, expression=pattern) + return exp.Like(this=exp.Lower(this=value), expression=exp.Lower(this=pattern)) + def build_null_safe_eq( self, left: Expression, right: Expression, ) -> Expression: @@ -350,6 +412,18 @@ def build_ordered( kwargs["nulls_first"] = False return exp.Ordered(**kwargs) + def build_order_terms( + self, + order_col: Expression, + *, + descending: bool, + nulls: Literal["default", "first", "last"] = "default", + resolved: Optional[Expression] = None, # NOSONAR(S1172) — read by the T-SQL emulation + ) -> list[exp.Ordered]: + """The ``ORDER BY`` terms of one entry; ``resolved`` is the expression ``order_col`` names, when it is + an alias of the SELECT being ordered.""" + return [self.build_ordered(order_col, descending=descending, nulls=nulls)] + def native_nulls_first(self, *, descending: bool) -> bool: """Where NULLs natively sort for ``descending``, read from sqlglot's emitter.""" ordering = getattr( @@ -372,13 +446,17 @@ def native_nulls_first(self, *, descending: bool) -> bool: def _expanded_null_safe_eq( left: Expression, right: Expression, ) -> Expression: - """``left = right OR (left IS NULL AND right IS NULL)``.""" - eq = exp.EQ(this=left.copy(), expression=right.copy()) - both_null = exp.And( - this=exp.Is(this=left.copy(), expression=exp.Null()), - expression=exp.Is(this=right.copy(), expression=exp.Null()), + """Two-valued ``(l = r AND l IS NOT NULL AND r IS NOT NULL) OR (l IS NULL AND r IS NULL)``, safe to negate.""" + def is_null(side: Expression) -> Expression: + return exp.Is(this=side.copy(), expression=exp.Null()) + + both_known = exp.and_( + exp.EQ(this=left.copy(), expression=right.copy()), + exp.Not(this=is_null(left)), + exp.Not(this=is_null(right)), ) - return exp.paren(exp.Or(this=eq, expression=exp.paren(both_null))) + both_null = exp.And(this=is_null(left), expression=is_null(right)) + return exp.paren(exp.Or(this=exp.paren(both_known), expression=exp.paren(both_null))) def build_date_trunc( self, diff --git a/slayer/sql/dialects/clickhouse.py b/slayer/sql/dialects/clickhouse.py index 8e9308ebf..fccfa8b16 100644 --- a/slayer/sql/dialects/clickhouse.py +++ b/slayer/sql/dialects/clickhouse.py @@ -10,13 +10,13 @@ import re from collections.abc import Sequence from datetime import date -from typing import Any +from typing import Any, ClassVar from sqlglot import exp from sqlglot.expressions.core import Expression from slayer.core.enums import SUB_DAY_GRANULARITIES, DataType, DatePart, TimeGranularity -from slayer.sql.dialects.base import ServerProfile, SqlDialect, iso_text +from slayer.sql.dialects.base import ServerProfile, SqlDialect, as_text, iso_text _CH_CORRELATED_SETTING = "allow_experimental_correlated_subqueries" _CORRELATED_MIN_VERSION = (25, 4) @@ -106,6 +106,7 @@ def _settings_holder(ast: Expression) -> Expression: class ClickhouseDialect(SqlDialect): sqlglot_name: str = "clickhouse" + ilike_native: ClassVar[bool] = True ds_type_aliases: frozenset[str] = frozenset({"clickhouse"}) explain_prefix: str | None = "EXPLAIN" explain_postfix: str = "" @@ -114,6 +115,7 @@ class ClickhouseDialect(SqlDialect): max_identifier_bytes: int | None = None # unbounded approx_count_distinct_native: bool = True correlated_subqueries_gated: bool = True + timeout_reads_profile: bool = True global_in_subqueries: bool = True url_scheme: str | None = "clickhouse+http" install_extra: str | None = "clickhouse" @@ -132,6 +134,12 @@ def restore_connection_timeout(self, dbapi_connection: Any, prior: object) -> No else: settings[_TIMEOUT_SETTING] = prior + def build_try_cast(self, value: Expression, to: exp.DataType) -> Expression: + spelled = to.sql(dialect=self.sqlglot_name) + if spelled.startswith("Nullable(") and spelled.endswith(")"): + spelled = spelled[len("Nullable("):-1] + return exp.Anonymous(this="accurateCastOrNull", expressions=[as_text(value), exp.Literal.string(spelled)]) + def server_profile_sql(self) -> str | None: return "SELECT version(), getSetting('readonly')" diff --git a/slayer/sql/dialects/duckdb.py b/slayer/sql/dialects/duckdb.py index eaf061cee..5c310f2c4 100644 --- a/slayer/sql/dialects/duckdb.py +++ b/slayer/sql/dialects/duckdb.py @@ -7,6 +7,8 @@ from __future__ import annotations +from typing import ClassVar + from sqlglot import exp from slayer.sql.dialects.base import SqlDialect @@ -14,6 +16,7 @@ class DuckdbDialect(SqlDialect): sqlglot_name: str = "duckdb" + ilike_native: ClassVar[bool] = True ds_type_aliases: frozenset[str] = frozenset({"duckdb"}) explain_prefix: str | None = "EXPLAIN ANALYZE" explain_postfix: str = "" diff --git a/slayer/sql/dialects/mysql.py b/slayer/sql/dialects/mysql.py index 8fd89cf85..ebc9eee38 100644 --- a/slayer/sql/dialects/mysql.py +++ b/slayer/sql/dialects/mysql.py @@ -16,7 +16,11 @@ from slayer.core.enums import DataType, DatePart, TimeGranularity from slayer.sql.dialects.base import ( + FALSE_WORDS, + NUMBER_TEXT_PATTERN, + TRUE_WORDS, SqlDialect, + as_text, StatAgg1Name, StatAgg2Name, _build_covar_decomposition, @@ -38,6 +42,21 @@ class MysqlDialect(SqlDialect): sqlglot_name: str = "mysql" ds_type_aliases: frozenset[str] = frozenset({"mysql"}) + + def build_try_cast(self, value: Expression, to: exp.DataType) -> Expression: + """Numeric and boolean targets are guarded by the validity grammar; MySQL's date casts already yield NULL.""" + text = as_text(value) + if to.is_type(*exp.DataType.NUMERIC_TYPES): + guard = exp.RegexpLike(this=text, expression=exp.Literal.string(NUMBER_TEXT_PATTERN)) + return exp.Case(ifs=[exp.If(this=guard, true=exp.Cast(this=value.copy(), to=to))]) + if to.is_type(exp.DataType.Type.BOOLEAN): + word = exp.Lower(this=exp.Trim(this=text)) + return exp.Case(ifs=[ + exp.If(this=exp.In(this=word, expressions=[exp.Literal.string(w) for w in TRUE_WORDS]), true=exp.true()), + exp.If(this=exp.In(this=word.copy(), expressions=[exp.Literal.string(w) for w in FALSE_WORDS]), + true=exp.false()), + ]) + return exp.Cast(this=value, to=to) explain_prefix: str | None = "EXPLAIN FORMAT=JSON" explain_postfix: str = "" log10_native: bool = True diff --git a/slayer/sql/dialects/postgres.py b/slayer/sql/dialects/postgres.py index 6844a98df..9d71ee490 100644 --- a/slayer/sql/dialects/postgres.py +++ b/slayer/sql/dialects/postgres.py @@ -2,10 +2,17 @@ from __future__ import annotations +from collections.abc import Sequence +from typing import Any, ClassVar from sqlglot import exp +from sqlglot.expressions.datatypes import DType +from sqlglot.expressions.core import Expression -from slayer.sql.dialects.base import SqlDialect +from slayer.sql.dialects.base import ServerProfile, SqlDialect, as_text + +#: The first Postgres major version with ``pg_input_is_valid``. +_PG_INPUT_IS_VALID_SINCE = (16, 0) def _cast_round_arg_to_numeric(node: exp.Expression) -> exp.Expression: @@ -37,6 +44,8 @@ def _cast_round_arg_to_numeric(node: exp.Expression) -> exp.Expression: class PostgresDialect(SqlDialect): sqlglot_name: str = "postgres" ds_type_aliases: frozenset[str] = frozenset({"postgres", "postgresql"}) + vendor_cast_types: ClassVar[frozenset[DType]] = frozenset({exp.DataType.Type.MONEY}) + ilike_native: ClassVar[bool] = True explain_prefix: str | None = "EXPLAIN ANALYZE" explain_postfix: str = "" log10_native: bool = True @@ -44,10 +53,34 @@ class PostgresDialect(SqlDialect): # NAMEDATALEN: 63 usable bytes; over-length names are SILENTLY truncated. max_identifier_bytes: int | None = 63 statement_timeout_best_effort: bool = True + try_cast_reads_profile: bool = True url_scheme: str | None = "postgresql+psycopg2" async_driver: str | None = "asyncpg" install_extra: str | None = "postgres" + def server_profile_sql(self) -> str | None: + return "SELECT current_setting('server_version_num')" + + def parse_server_profile( + self, *, base_row: Sequence[Any] | None, correlated_row: Sequence[Any] | None = None, + ) -> ServerProfile: + number = base_row[0] if base_row else None + try: + version_num = int(number) if number is not None else None + except (TypeError, ValueError): + version_num = None + return ServerProfile(version=None if version_num is None else (version_num // 10000, version_num % 10000)) + + def build_try_cast(self, value: Expression, to: exp.DataType) -> Expression: + """Postgres 16+ checks the input with ``pg_input_is_valid``; an older or unknown server gets a plain cast.""" + version = self.server_profile.version + if version is None or version < _PG_INPUT_IS_VALID_SINCE: + return exp.Cast(this=value, to=to) + check = exp.Anonymous(this="pg_input_is_valid", expressions=[ + as_text(value), exp.Literal.string(to.sql(dialect=self.sqlglot_name)), + ]) + return exp.Case(ifs=[exp.If(this=check, true=exp.Cast(this=value.copy(), to=to))]) + def build_integer_sequence(self, *, size: int) -> exp.Select: return exp.select(exp.column("i")).from_(exp.Table( this=exp.Anonymous(this="GENERATE_SERIES", expressions=[ diff --git a/slayer/sql/dialects/snowflake.py b/slayer/sql/dialects/snowflake.py index 77fd8e171..6cb0fa820 100644 --- a/slayer/sql/dialects/snowflake.py +++ b/slayer/sql/dialects/snowflake.py @@ -100,6 +100,7 @@ class SnowflakeDialect(SqlDialect): """Snowflake dialect.""" sqlglot_name: str = "snowflake" + ilike_native: ClassVar[bool] = True ds_type_aliases: frozenset[str] = frozenset({"snowflake"}) explain_prefix: str | None = "EXPLAIN USING JSON" explain_postfix: str = "" diff --git a/slayer/sql/dialects/sqlite.py b/slayer/sql/dialects/sqlite.py index 444532b0c..c63872cff 100644 --- a/slayer/sql/dialects/sqlite.py +++ b/slayer/sql/dialects/sqlite.py @@ -6,14 +6,27 @@ import re from calendar import monthrange from datetime import date, datetime, timedelta +from decimal import Decimal from sqlglot import exp +from sqlglot.expressions.datatypes import DType from sqlglot.expressions.core import Expression -from typing import Optional +from typing import ClassVar, Optional from slayer.core.enums import DataType, DatePart, TimeGranularity -from slayer.sql.dialects.base import COMPARISON_NODES, SqlDialect, TemporalComparisonOp, iso_text +from slayer.core.keys import cast_data_type +from slayer.core.time_points import parse_temporal_value +from slayer.sql.dialects.base import ( + COMPARISON_NODES, + FALSE_WORDS, + TRUE_WORDS, + VENDOR_ONLY_CAST_TYPES, + SqlDialect, + TemporalComparisonOp, + as_text, + iso_text, +) def rewrite_sqlite_json_extract(node: Expression) -> Expression: @@ -345,6 +358,56 @@ def _slayer_date_add(ts, n, unit): return moved.isoformat(sep=" ") +_NUMBER_TEXT_RE = re.compile(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?") + + +_INT64_BOUND = 2 ** 63 + + +def _try_number(text: str, kind: DataType): + stripped = text.strip() + if not _NUMBER_TEXT_RE.fullmatch(stripped): + return None + if kind is DataType.DOUBLE: + number = float(stripped) + return number if math.isfinite(number) else None + exact = Decimal(stripped) + return int(exact) if -_INT64_BOUND <= exact < _INT64_BOUND else None + + +def _try_boolean(text: str): + word = text.strip().lower() + if word in TRUE_WORDS: + return 1 + if word in FALSE_WORDS: + return 0 + return None + + +def _try_temporal(text: str, kind: DataType): + moment = parse_temporal_value(text.strip()) + if moment is None: + return None + if kind is DataType.DATE: + return (moment.date() if isinstance(moment, datetime) else moment).isoformat() + stamp = moment if isinstance(moment, datetime) else datetime(moment.year, moment.month, moment.day) + return stamp.isoformat(sep=" ") + + +def _slayer_try_cast(text, target): + """``TRY_CAST(text AS target)``: the converted value, or NULL where the cast cannot convert ``text``.""" + if text is None: + return None + kind = cast_data_type(target) + if kind in (DataType.INT, DataType.DOUBLE): + return _try_number(text, kind) + if kind is DataType.BOOLEAN: + return _try_boolean(text) + if kind in (DataType.DATE, DataType.TIMESTAMP): + return _try_temporal(text, kind) + return text + + def register_sqlite_udfs(dbapi_connection) -> None: """Register all SLayer SQLite UDFs on a fresh DBAPI connection (idempotent).""" dbapi_connection.create_function("ln", 1, _ln) @@ -357,6 +420,7 @@ def register_sqlite_udfs(dbapi_connection) -> None: dbapi_connection.create_function("pow", 2, _pow) dbapi_connection.create_function("power", 2, _pow) dbapi_connection.create_function("slayer_date_add", 3, _slayer_date_add) + dbapi_connection.create_function("slayer_try_cast", 2, _slayer_try_cast) dbapi_connection.create_aggregate("median", 1, _MedianAgg) dbapi_connection.create_aggregate("percentile_cont", 2, _PercentileContAgg) @@ -376,6 +440,8 @@ def register_sqlite_udfs(dbapi_connection) -> None: class SqliteDialect(SqlDialect): sqlglot_name: str = "sqlite" + # Any type name is a column affinity on SQLite. + vendor_cast_types: ClassVar[frozenset[DType]] = VENDOR_ONLY_CAST_TYPES ds_type_aliases: frozenset[str] = frozenset({"sqlite"}) explain_prefix: str | None = "EXPLAIN QUERY PLAN" explain_postfix: str = "" @@ -392,6 +458,11 @@ def declared_cast_type(self, dt: Optional[DataType]) -> Optional[DataType]: return None return dt + def build_try_cast(self, value: Expression, to: exp.DataType) -> Expression: + return exp.Anonymous(this="slayer_try_cast", expressions=[ + as_text(value), exp.Literal.string(to.sql(dialect="postgres")), + ]) + def build_null_safe_eq( self, left: Expression, right: Expression, ) -> Expression: diff --git a/slayer/sql/dialects/tsql.py b/slayer/sql/dialects/tsql.py index da39729f4..c7e5a7928 100644 --- a/slayer/sql/dialects/tsql.py +++ b/slayer/sql/dialects/tsql.py @@ -7,15 +7,17 @@ from __future__ import annotations import re -from typing import ClassVar, Literal +from typing import ClassVar, Literal, Optional from sqlglot import exp +from sqlglot.expressions.datatypes import DType from sqlglot.expressions.core import Expr, Expression from slayer.core.enums import SUB_DAY_GRANULARITIES, DataType, DatePart, TimeGranularity from slayer.sql.dialects.base import ( DottedAliasManglingMixin, SqlDialect, + operand_copy, StatAgg1Name, StatAgg2Name, _build_covar_decomposition, @@ -96,6 +98,9 @@ def _bit_value(predicate: Expr) -> Expression: class TsqlDialect(DottedAliasManglingMixin, SqlDialect): sqlglot_name: str = "tsql" ds_type_aliases: frozenset[str] = frozenset({"mssql", "sqlserver", "tsql"}) + vendor_cast_types: ClassVar[frozenset[DType]] = frozenset({ + exp.DataType.Type.MONEY, exp.DataType.Type.SMALLMONEY, + }) explain_prefix: str | None = "SET SHOWPLAN_ALL ON;" explain_postfix: str = "; SET SHOWPLAN_ALL OFF" log10_native: bool = True @@ -135,6 +140,30 @@ def build_ordered( order_col, descending=descending, nulls=nulls, ) + def build_order_terms( + self, + order_col: Expression, + *, + descending: bool, + nulls: Literal["default", "first", "last"] = "default", + resolved: Optional[Expression] = None, + ) -> list[exp.Ordered]: + """An explicit NULL placement is a leading ``CASE WHEN IS NULL`` key over the resolved + expression (an alias is not visible inside an ORDER BY expression), then the native term.""" + if nulls == "default": + return [self.build_ordered(order_col, descending=descending)] + null_rank = exp.Case( + ifs=[exp.If( + this=exp.Is(this=operand_copy(resolved if resolved is not None else order_col), expression=exp.Null()), + true=exp.Literal.number(0 if nulls == "first" else 1), + )], + default=exp.Literal.number(1 if nulls == "first" else 0), + ) + return [ + self.build_ordered(null_rank, descending=False), + self.build_ordered(order_col, descending=descending), + ] + def build_date_trunc( self, col_expr: Expression, diff --git a/slayer/sql/generator.py b/slayer/sql/generator.py index cd9941889..589315316 100644 --- a/slayer/sql/generator.py +++ b/slayer/sql/generator.py @@ -40,7 +40,7 @@ from slayer.core.enums import RANK_FAMILY_TRANSFORMS from slayer.core.granularity import CustomGranularity, Granularity, granularity_parts from slayer.core.time_spine import TIME_SPINE_COLUMN, is_spine -from slayer.core.keys import BOOL_CONNECTIVE_OPS, KIND_POLICY, REGROUP_LEAF_PREFIX, SLOT_COMPOSITE_KINDS, TEMPORAL_TYPES, VALUE_KEY_TYPES, AggregateKey, ArithmeticKey, ColumnKey, ColumnSqlKey, ColumnTypeFn, InKey, LiteralKey, Phase, ScalarCallKey, SqlFragmentKey, StarKey, TimePointCmpKey, TimeTruncKey, TransformKey, aggregation_source_type, column_path, date_add_type, is_boolean_shaped, parameter_row_leaves, shift_offset_of, source_anchor_path, substitute_value_keys, temporal_type, walk_value_keys +from slayer.core.keys import BOOL_CONNECTIVE_OPS, KIND_POLICY, REGROUP_LEAF_PREFIX, SLOT_COMPOSITE_KINDS, TEMPORAL_TYPES, VALUE_KEY_TYPES, AggregateKey, ArithmeticKey, CastKey, ColumnKey, ColumnSqlKey, ColumnTypeFn, InKey, LiteralKey, Phase, ScalarCallKey, SqlFragmentKey, StarKey, TimePointCmpKey, TimeTruncKey, TransformKey, aggregation_source_type, column_path, date_add_type, is_boolean_shaped, parameter_row_leaves, shift_offset_of, source_anchor_path, substitute_value_keys, temporal_type, walk_value_keys from slayer.core.join_walker import ( aggregation_owner, follow, @@ -50,7 +50,7 @@ resolved_terminal, ) from slayer.core.models import VALUE_PLACEHOLDER, aggregation_definition, rendered_formula, reserved_value_param_message -from slayer.core.refs import EXPRESSION_SOURCE_KINDS as _EXPRESSION_SOURCE_KINDS +from slayer.core.refs import EXPRESSION_SOURCE_KINDS as _EXPRESSION_SOURCE_KINDS, key_display from slayer.core.window_duration import WINDOW_UNIT_GRANULARITY from slayer.sql.column_expansion import ( is_trivial_base, @@ -599,7 +599,7 @@ def _conjunction(parts: Sequence[Expression]) -> Optional[Expression]: # pyrigh return out # Parameter values rendered through a scope; scalars render as literals, markers never. -_ROW_PARAM_KINDS = (ColumnKey, ColumnSqlKey, SqlFragmentKey) +_ROW_PARAM_KINDS = (ColumnKey, ColumnSqlKey, SqlFragmentKey, ScalarCallKey) # Shift units whose whole-unit offsets map each bucket start onto another, making the outer re-trunc a per-row no-op. _BUCKET_ALIGNED_SHIFT_UNITS: dict[str, frozenset[str]] = { @@ -669,7 +669,7 @@ def _is_host_grain(key) -> bool: # Real ValueKey args (a ScalarCallKey / iif may also carry raw scalar literals). _COMPOUND_VALUE_KEYS = ( ColumnKey, ColumnSqlKey, TimeTruncKey, StarKey, - AggregateKey, TransformKey, ArithmeticKey, ScalarCallKey, InKey, + AggregateKey, TransformKey, ArithmeticKey, ScalarCallKey, InKey, CastKey, ) @@ -706,6 +706,8 @@ def _walk_cp_predicate(*, op: str, key, connective_operand: bool) -> None: _walk_cp_predicate(op=op, key=a, connective_operand=False) elif isinstance(key, InKey): _walk_cp_predicate(op=op, key=key.column, connective_operand=False) + elif isinstance(key, CastKey): + _walk_cp_predicate(op=op, key=key.source, connective_operand=False) def _effective_src_filters(*, lowered_filters, plan) -> list: @@ -1990,6 +1992,8 @@ def _walk(key, fn) -> None: elif isinstance(key, ScalarCallKey): for a in key.args: _walk(a, fn) + elif isinstance(key, CastKey): + _walk(key.source, fn) def _for_each_local_agg(fn) -> None: for sid in base_render_order: @@ -2021,9 +2025,14 @@ def _resolve_first_last_time_arg(key) -> None: return scope.resolve(arg) + def _resolve_filter(key) -> None: + if key.filter is not None: + scope.resolve(key.filter) + _for_each_local_agg(_resolve_source) _for_each_local_agg(_resolve_kwargs) _for_each_local_agg(_resolve_first_last_time_arg) + _for_each_local_agg(_resolve_filter) return resolved def _throwaway_frame( @@ -2234,6 +2243,10 @@ def _defer_composite(sid: str, slot, full_alias: str) -> None: select_columns.append(dim_expr.copy().as_(full_alias)) group_by_keys.setdefault(sid, dim_expr) _record_alias(sid, full_alias) + elif isinstance(key, LiteralKey) and slot.is_dimension: + # A constant needs no GROUP BY entry (a constant one is refused or read as a position). + select_columns.append(_literal(key.value, dialect=self._dialect).as_(full_alias)) + _record_alias(sid, full_alias) elif isinstance(key, SLOT_COMPOSITE_KINDS) or ( isinstance(key, LiteralKey) and not slot.is_dimension ): @@ -2866,13 +2879,19 @@ def _frame(*, attached=None) -> ScopeFrame: ), )) - value_ref = ranked_scope.materialize_for( - self._ranked_value_expr( - key=local_key, root_model=root_model, - root_relation=root_relation, bundle=bundle, scope=ranked_scope, - ), - consumer=cte_scope, + value_expr = self._ranked_value_expr( + key=local_key, root_model=root_model, + root_relation=root_relation, bundle=bundle, scope=ranked_scope, ) + # A FILTER ranks the passing rows first and picks only a passing row's value. + passing_first: List[exp.Ordered] = [] + if local_key.filter is not None: + value_expr = self._passing_value(local_key.filter, value=value_expr, scope=ranked_scope) + passing_first.append(exp.Ordered(this=exp.Case( + ifs=[exp.If(this=value_expr.args["ifs"][0].this.copy(), true=exp.Literal.number(0))], + default=exp.Literal.number(1), + ))) + value_ref = ranked_scope.materialize_for(value_expr, consumer=cte_scope) ranking_time = self._ranked_scope_expr( key=plan.ranking_time_key, root_model=root_model, root_relation=root_relation, bundle=bundle, scope=ranked_scope, @@ -2905,6 +2924,7 @@ def _frame(*, attached=None) -> ScopeFrame: descending=plan.agg == "last", ), ), + leading=passing_first, )) inner = inner.from_(from_expr) inner = _apply_joins(select=inner, joins=joins) @@ -3108,6 +3128,11 @@ def _render_association_producer_body( # NOSONAR(S3776) — one cohesive two-le # the input is root-determined, constant per entity); ``count(*)`` keeps no # value column — level 2 counts the entity rows. is_star = isinstance(agg_slot.key.source, StarKey) + if agg_slot.key.filter is not None: + raise RuntimeError( + f"{key_display(agg_slot.key)}: a key-carried FILTER reached an association producer; binding " + f"carries one only on a local star count or a ranked pick.", + ) picked_alias = "_v" # Parameters the grain determines are picked once per cell as # _p (below) and read by level 2 as _base._p; the level-1 value pick @@ -3868,8 +3893,7 @@ def _render_outer_composite(cslot) -> Expression: else grain_alias_column(alias=_full_alias, table="_base") ) order_terms = [ - resolve_order_term(entry=entry, env=order_env) - for entry in lowered.order + term for entry in lowered.order for term in resolve_order_term(entry=entry, env=order_env) ] if order_terms: combined_statement.set("order", exp.Order(expressions=order_terms)) @@ -4738,8 +4762,8 @@ def _planned_order_terms( dialect=self._dialect, ) return [ - resolve_order_term(entry=entry, env=env) - for entry in _lower_positions(planned_query).order + term for entry in _lower_positions(planned_query).order + for term in resolve_order_term(entry=entry, env=env) ] def _series_shift_cte( @@ -5525,6 +5549,8 @@ def _walk(k) -> None: _walk(a) elif isinstance(k, InKey): _walk(k.column) + elif isinstance(k, CastKey): + _walk(k.source) _walk(key) return out @@ -5665,21 +5691,24 @@ def _build_agg_render_spec_from_planned( # NOSONAR(S3776) — sequential isinst f"'count(*)' takes no args or kwargs; got " f"args={key.args!r}, kwargs={key.kwargs!r}." ) + star_scope = scope or self._throwaway_frame( + model=source_model, relation=source_relation, + bundle=bundle or ResolvedSourceBundle(dialect=self.dialect, source_model=source_model), + ) return AggRenderSpec( name="", sql=None, + # A key-carried FILTER counts the passing rows. + value=None if key.filter is None else self._passing_value( + key.filter, value=exp.Literal.number(1), scope=star_scope, + ), aggregation=key.agg, alias=full_alias, model_name=source_relation, type=slot_type, formula=formula, agg_kwargs=self._agg_param_exprs( - key=key, formula=formula, resolved=resolved_agg_kwargs, - scope=scope or self._throwaway_frame( - model=source_model, relation=source_relation, - bundle=bundle or ResolvedSourceBundle( - dialect=self.dialect, source_model=source_model), - ), + key=key, formula=formula, resolved=resolved_agg_kwargs, scope=star_scope, ), ) param_scope = scope or self._throwaway_frame( @@ -5798,6 +5827,11 @@ def _build_agg_render_spec_from_planned( # NOSONAR(S3776) — sequential isinst f"AggregateKey source {type(source).__name__} not supported.", ) + def _passing_value(self, condition, *, value: Expression, scope: ScopeFrame) -> exp.Case: + """``CASE WHEN THEN END``: ``value`` on the rows a FILTER keeps.""" + passing = render_value_key(key=condition, ctx=RenderContext(scope=scope, dialect=self._dialect)) + return exp.Case(ifs=[exp.If(this=passing, true=value)]) + def _semi_join_exists_conditions( self, *, planned_query, source_model, source_relation: str, bundle, ) -> "List[Expression]": @@ -6125,6 +6159,7 @@ def _host_order_terms( source_model=None, bundle=None, aliases_by_slot_id: Optional[Dict[str, List[str]]] = None, + select: Optional[exp.Select] = None, ) -> List[exp.Ordered]: """ORDER BY terms resolved against a base SELECT with no CTE chain.""" scoped_order = _lower_positions(planned_query).order @@ -6136,7 +6171,11 @@ def _host_order_terms( bundle=bundle, aliases_by_slot_id=aliases_by_slot_id, ) - return [resolve_order_term(entry=entry, env=env) for entry in scoped_order] + if select is not None: + env.projection = { + a.alias: a.this for a in select.expressions if isinstance(a, exp.Alias) and a.alias + } + return [term for entry in scoped_order for term in resolve_order_term(entry=entry, env=env)] def _apply_planned_order_limit( self, @@ -6159,6 +6198,7 @@ def _apply_planned_order_limit( source_model=source_model, bundle=bundle, aliases_by_slot_id=aliases_by_slot_id, + select=select, ), planned_query=planned_query, ) @@ -6250,6 +6290,18 @@ def _host_base_order_ref( # NOSONAR(S3776) — per-key-kind resolution of ONE h if expr is not None: return expr + # A hidden row-level expression in an ungrouped query renders in place over the host's + # rows; its joins were collected with the order slots. + if ( + source_model is not None and bundle is not None and slot.phase == Phase.ROW + and isinstance(key, SLOT_COMPOSITE_KINDS) + ): + frame = self._scope_frame( + model=source_model, relation=source_relation, bundle=bundle, + allocator=self._gen_allocator or self._new_allocator(), + ) + return render_value_key(key=key, ctx=RenderContext(scope=frame, dialect=self._dialect)) + raise NotImplementedError( f"ORDER BY references a hidden slot (id={slot.id!r}, key=" f"{type(slot.key).__name__}) that was not resolved at plan " @@ -6363,11 +6415,15 @@ def read_models(statement: Expression) -> Set[str]: } +def _as_dialect(dialect: "str | SqlDialect") -> SqlDialect: + return dialect if isinstance(dialect, SqlDialect) else get_dialect(dialect) + + def generate_planned_stages( planned_queries, *, bundle, - dialect: str = "postgres", + dialect: "str | SqlDialect" = "postgres", projection_aliases: "Sequence[str]" = (), kept_stages: "Optional[Set[str]]" = None, read_set: "Optional[Set[str]]" = None, @@ -6385,10 +6441,10 @@ def generate_planned_stages( read_set.update(read_models(statement)) spine_sizes = [p.spine.size for p in planned_queries if p.spine is not None] if spine_sizes: - get_dialect(dialect).attach_sequence_setting(statement, size=max(spine_sizes)) + _as_dialect(dialect).attach_sequence_setting(statement, size=max(spine_sizes)) # Semi-join pushdown emits a correlated EXISTS; attached unconditionally so SQL never depends on server state. if any(plan_has_semi_join_filters(p) for p in emitted_plans(planned_queries, kept_stages=kept)): - get_dialect(dialect).attach_correlated_setting(statement) + _as_dialect(dialect).attach_correlated_setting(statement) # Length-fit over-limit projection aliases from the plan-derived canonical keys, not parsed off the SQL — # BigQuery can't parse a backticked dotted alias. return _finish_statement( @@ -6400,7 +6456,7 @@ def generate_planned_stages( def _build_planned_stages_ast( - planned_queries, *, bundle, dialect: str, kept_stages: "Optional[Set[str]]" = None, + planned_queries, *, bundle, dialect: "str | SqlDialect", kept_stages: "Optional[Set[str]]" = None, ) -> exp.Select: """Compose a multi-stage DAG (``plan_stages`` output) as one statement AST.""" if not planned_queries: @@ -6467,7 +6523,7 @@ def _build_planned_stages_ast( source_relation=planned.source_relation, inner=body, expected_columns=[c.name for c in planned.stage_schema.columns], - dialect=dialect, + dialect=_as_dialect(dialect).sqlglot_name, ), depends_on=_merged_deps( [h.name for h in hoisted], @@ -6500,14 +6556,14 @@ def _build_planned_stages_ast( def _fit_overlimit_identifiers( - tree, *, dialect: str, exempt: AbstractSet[str], stage_columns: AbstractSet[str], + tree, *, dialect: "str | SqlDialect", exempt: AbstractSet[str], stage_columns: AbstractSet[str], ) -> None: """Length-fit over-limit SLayer-minted identifiers in place the way the emission pass fits quoted ones: every unquoted one (stage identities, stage column references) and the stage column names wherever they occur, so a definition and its references agree; user-authored ``exempt`` names pass unless they name a stage column.""" - d = get_dialect(dialect) + d = _as_dialect(dialect) if d.max_identifier_bytes is None: return for ident in tree.find_all(exp.Identifier): diff --git a/slayer/sql/render/joins.py b/slayer/sql/render/joins.py index 848ff6084..7cd616330 100644 --- a/slayer/sql/render/joins.py +++ b/slayer/sql/render/joins.py @@ -11,7 +11,7 @@ outer-joined column, a bucket with no rows. Plain ``=`` never matches NULL against NULL, so the group silently loses its aggregate instead of receiving it. The dialect strategy owns the spelling (``IS NOT DISTINCT FROM``, MySQL -``<=>``, SQLite ``IS``, or the expanded ``a = b OR (a IS NULL AND b IS NULL)`` +``<=>``, SQLite ``IS``, or an expanded two-valued ``=`` / ``IS NULL`` disjunction where there is no native operator). **Built as AST, never re-parsed.** The superseded helper rendered both sides to diff --git a/slayer/sql/render/order_terms.py b/slayer/sql/render/order_terms.py index 7fbdb826e..ff5d1c06c 100644 --- a/slayer/sql/render/order_terms.py +++ b/slayer/sql/render/order_terms.py @@ -29,10 +29,11 @@ from __future__ import annotations from enum import Enum -from typing import Dict, Literal +from typing import Dict, List, Literal from pydantic import BaseModel, ConfigDict, Field from sqlglot import exp +from sqlglot.expressions.core import Expression from slayer.ir.planned import SlotId from slayer.sql.dialects.base import SqlDialect @@ -106,6 +107,8 @@ class OrderEnv(BaseModel): ranked_cte: Dict[str, exp.Expression] = Field(default_factory=dict) transform_step: Dict[str, exp.Expression] = Field(default_factory=dict) outer_composite: Dict[str, exp.Expression] = Field(default_factory=dict) + #: The ordered SELECT's own projection (alias → expression), where the ORDER BY attaches to it. + projection: Dict[str, Expression] = Field(default_factory=dict) #: Owns the null-ordering spelling (P-H). Required so no render site can #: silently fall back to a default dialect's null ordering. dialect: SqlDialect @@ -139,8 +142,8 @@ def uniform( ) -def resolve_order_term(*, entry: ScopedOrder, env: OrderEnv) -> exp.Ordered: - """One scoped order entry → one ``ORDER BY`` term. +def resolve_order_term(*, entry: ScopedOrder, env: OrderEnv) -> List[exp.Ordered]: + """One scoped order entry → its ``ORDER BY`` terms. Raises :class:`OrderSlotNotMaterialisedError` when the entry's scope did not materialise the slot. Returning an unsorted result instead is the one @@ -154,6 +157,8 @@ def resolve_order_term(*, entry: ScopedOrder, env: OrderEnv) -> exp.Ordered: f"{entry.scope.value} scope did not materialise " f"(it carries {sorted(refs)}).", ) - return env.dialect.build_ordered( + resolved = env.projection.get(ref.name) if isinstance(ref, exp.Column) and not ref.table else None + return env.dialect.build_order_terms( ref.copy(), descending=entry.direction == "desc", nulls=entry.nulls, + resolved=None if resolved is None else resolved.copy(), ) diff --git a/slayer/sql/render/ranked.py b/slayer/sql/render/ranked.py index 01e453024..5feec7494 100644 --- a/slayer/sql/render/ranked.py +++ b/slayer/sql/render/ranked.py @@ -82,6 +82,7 @@ def build_rank_column( *, partition_by: Sequence[Expression], ranking_time: exp.Ordered, + leading: Sequence[exp.Ordered] = (), ) -> Expression: """``ROW_NUMBER() OVER (PARTITION BY ORDER BY