From 35c6ef4420c77c97ec463fd6563abfc300a6f4a9 Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 3 Sep 2026 08:16:31 -0400 Subject: [PATCH 1/5] feat: Rebuild scalar expression projections on the IEJoin path A column-to-column INTERSECTS join whose SELECT list held a scalar expression over qualified columns declined to the naive predicate, even though the join itself was an INNER pure-inequality join the dialect already routes through IE_JOIN. The projection shape alone cost the query its plan: a hash join on chrom with the position inequalities as a residual filter, quadratic when the chromosome key has low cardinality. That gate answers whether the projection rebuild can express a shape, and it was also deciding which join plan the query gets, though the projection takes no part in join planning. The rebuild needed nothing new. A plain aggregate already projects its argument columns inward as __giql_p inner columns and re-emits the call over them on the outer wrapper SELECT; the same two steps applied to the leaves of an expression tree serve GREATEST, LEAST, arithmetic, CASE, casts and concatenation alike, because the wrapper relation carries exactly the leaf values the per-chromosome join produced. An unaliased expression is named __giql_expr_, as an unaliased aggregate is named __giql_agg_, since DuckDB's automatic name is derived from text the rewrite has changed. This costs two of the twelve bedtools intersect modes in the benchmark suite: the default clipped overlap span and -wo, both quadratic today. The decline from #205 is narrowed rather than reverted. A window aggregate, a FILTER clause, a scalar subquery, an aggregate nested in an expression, a bare literal and a star anywhere in the tree still fall back to the naive plan, so dialect="duckdb" stays consistent with every other backend. An out-of-scope leaf still raises: unqualified, unknown table, or a right-side reference under SEMI or ANTI, which now reaches the same diagnostic from anywhere in the tree rather than only at its root. The outer-join decomposition stays column-only, since its unmatched half would have to evaluate the expression over NULL-filled leaves from the other side. --- src/giql/expanders/intersects_duckdb.py | 754 ++++++++++++++++-------- src/giql/transpile.py | 6 + 2 files changed, 525 insertions(+), 235 deletions(-) diff --git a/src/giql/expanders/intersects_duckdb.py b/src/giql/expanders/intersects_duckdb.py index 5c70069..2ed56b4 100644 --- a/src/giql/expanders/intersects_duckdb.py +++ b/src/giql/expanders/intersects_duckdb.py @@ -32,12 +32,16 @@ from typing import Literal from sqlglot import exp +from sqlglot import parse_one +from sqlglot.errors import ParseError +from sqlglot.helper import find_new_name from giql.canonical import canonical_end from giql.canonical import canonical_start from giql.constants import DEFAULT_CHROM_COL from giql.constants import DEFAULT_END_COL from giql.constants import DEFAULT_START_COL +from giql.dialect import GIQLDialect from giql.expander import REGISTRY from giql.expander import ExpansionContext from giql.expander import register @@ -54,6 +58,41 @@ from giql.targets import DuckDBTarget from giql.targets import GenericTarget +#: Node classes that disqualify a SELECT-list expression from the IEJoin's +#: projection rebuild, grouped by the property each one violates (#109). Stated +#: as classes rather than as a property test because each group *is* the +#: property for the node kinds sqlglot models; the one property no class list +#: can predict -- that the DuckDB generator reproduces what the GIQL parser +#: read -- is verified instead, by :func:`_renders_faithfully_for_duckdb`. +_NON_REBUILDABLE_NODES = ( + # Not row-wise. The rebuild evaluates the tree once per wrapper row, so a + # node that aggregates across rows or multiplies them cannot survive it. A + # ``FILTER`` clause is reached through its aggregate, not listed separately. + exp.AggFunc, + exp.Window, + exp.Unnest, + exp.Explode, + # Aggregates sqlglot models outside ``AggFunc``. Together with ``AggFunc`` + # and ``Anonymous`` below these cover every aggregate DuckDB exposes, which + # ``TestDuckDBAggregateCoverage`` pins against ``duckdb_functions()``. + exp.List, + exp.RowNumber, + # Introduces a name-binding scope. :meth:`~IntersectsDuckDBIEJoinTransformer + # ._rewrite_refs_for_wrapper_relation` is not scope-aware, and swapping the + # join for the wrapper relation changes which aliases are in scope, so a + # lambda parameter sharing a join alias's name would bind differently. + exp.Select, + exp.Subquery, + exp.Lambda, + # Not enumerable at transpile time: the dialect has no live schema. Covers + # ``a.*`` and ``COLUMNS(*)`` alike, since both carry a ``Star``. + exp.Star, + # Unrecognized function. sqlglot cannot tell us whether it aggregates, + # multiplies rows, or binds names, so the module's decline-rather-than-guess + # posture applies. + exp.Anonymous, +) + #: Which side of an outer join keeps its unmatched rows. Narrowed by explicit #: per-branch assignment rather than a cast: nothing type-checks this package, so #: a cast asserts without verifying, and the value decides whether the emission @@ -78,9 +117,10 @@ class _UnqualifiedProjectionError(Exception): boundary. Shapes the naive plan *accepts* but the IEJoin cannot rebuild do NOT raise - — star projections (#202), and expressions / window aggregates / ``FILTER`` - clauses / scalar subqueries / aggregate-nested stars (#204, #205) — they - signal :class:`_DeclineIEJoin` and fall back to the naive plan instead. + — star projections (#202) and every shape :data:`_NON_REBUILDABLE_NODES` + names (#204, #205, #109) — they signal :class:`_DeclineIEJoin` and fall + back to the naive plan instead. A scalar expression over qualified columns + is rebuilt on the IEJoin path (#109). """ @@ -89,9 +129,10 @@ class _DeclineIEJoin(Exception): Distinct from :class:`_UnqualifiedProjectionError`: this is not a user error but a shape the naive-predicate plan compiles for free while the - IEJoin projection rebuild cannot express it (an expression, a window - aggregate, a ``FILTER`` clause, a scalar subquery, an aggregate nested in - an expression, or a star nested in an aggregate argument — #204, #205). + IEJoin projection rebuild cannot express it — see + :data:`_NON_REBUILDABLE_NODES` for the rule and + :func:`_classify_projection` for the verdicts (#204, #205; a scalar + expression over qualified columns is rebuilt instead, #109). Caught in :meth:`IntersectsDuckDBIEJoinTransformer.transform_to_parts`, which returns ``None`` so the generic naive-predicate plan handles the query, keeping ``dialect="duckdb"`` consistent with every other backend. @@ -247,68 +288,90 @@ def _has_star_projection(query: exp.Select) -> bool: return False -def _projection_declines_to_naive( +def _renders_faithfully_for_duckdb(node: exp.Expression) -> bool: + """Return True if the DuckDB generator reproduces what the GIQL parser read. + + The query arrives parsed with :class:`~giql.dialect.GIQLDialect`, but the + rebuilt projection is re-emitted with the DuckDB generator, so any + read/write divergence between the two becomes a silent miscompile rather + than an error (#109). A literal subscript is the reachable instance: + ``GIQLDialect`` derives from sqlglot's base dialect and inherits + ``INDEX_OFFSET`` 0, DuckDB uses 1, so ``a.arr[1]`` re-emits as + ``a.arr[2]`` and silently returns the next element. + + Rather than enumerate the divergences -- an open set that grows with every + sqlglot release, and the reason #109's first cut admitted this bug -- verify + the round trip: emit with DuckDB, re-read with the dialect the query was + parsed with, and require the tree to come back unchanged. Note this is not + the same as rendering the rebuild with ``GIQLDialect`` instead, which would + trade the wrong answer for a broken query: that generator emits + ``STRUCT(x AS k)`` and ``ARRAY(a, b)`` for DuckDB's ``{'k': x}`` and + ``[a, b]``, neither of which DuckDB can parse. + """ + try: + reparsed = parse_one(node.sql(dialect="duckdb"), dialect=GIQLDialect) + except ParseError: + return False + return reparsed == node + + +def _is_rebuildable_scalar(target: exp.Expression) -> bool: + """Return True if *target* is a scalar expression the IEJoin can rebuild (#109). + + The rebuild projects each leaf column inward as a ``__giql_p`` inner + column and re-emits the expression tree on the outer wrapper ``SELECT`` + over those -- the same mechanism :meth:`~IntersectsDuckDBIEJoinTransformer + ._resolve_projections` applies to a plain qualified column, applied to + every leaf of the tree. + + That is sound exactly when four properties hold, and this function tests + all four rather than predicting them from a list of shapes: + + 1. the tree contains no node in :data:`_NON_REBUILDABLE_NODES`, which is + grouped by the property each class violates -- not row-wise, models an + aggregate outside ``AggFunc``, binds names, or is unenumerable / + unrecognized; + 2. at least one leaf is a qualified column, so there is something to + attribute to a join side (a leafless tree such as ``100`` declines, and + one declining SELECT item declines the whole query); + 3. the DuckDB generator reproduces the tree faithfully -- see + :func:`_renders_faithfully_for_duckdb`. + + Callers have already verified that every own-scope leaf is qualified with + one of the join's aliases, which is the fourth property and the one this + function does not re-derive. + """ + if target.find(*_NON_REBUILDABLE_NODES) is not None: + return False + if not any(col.table for col in target.find_all(exp.Column)): + return False + return _renders_faithfully_for_duckdb(target) + + +def _first_out_of_scope_leaf( target: exp.Expression, l_alias: str, r_alias: str, is_left_only: bool, -) -> bool: - """Return True if *target* is a projection the IEJoin should decline to naive. - - The IEJoin projection rebuild only handles a qualified column (``a.col``) - or a plain aggregate over qualified columns (``SUM(a.score)`` / ``COUNT(*)``); - it emits explicitly-named ``__giql_p`` inner columns and reconstructs the - outer SELECT from them. Any other shape the naive-predicate plan compiles - for free — an expression (``a.start + 1``), a window aggregate, a ``FILTER`` - clause, a scalar subquery, an aggregate nested in an expression - (``COUNT(*) * 2``), or a star nested in an aggregate argument - (``COUNT(a.*)`` / ``MIN(COLUMNS(*))``) — should decline so ``dialect="duckdb"`` - stays consistent with every other backend instead of hard-erroring or, for - the aggregate-nested star, miscompiling (#204, #205). - - A projection that references a column **in its own scope** the rebuild - cannot attribute to a join side (unqualified, an unknown table, or the - right side under a SEMI / ANTI join) does NOT decline — the dispatch raises - a clean :class:`_UnqualifiedProjectionError` instead. The unknown-table and - right-side cases are rejected by the naive plan too; a bare unqualified - column may be naive-valid when unambiguous, but the dialect has no live - schema to pick a side, so a transpile-time error is more actionable than - guessing. Columns inside a nested subquery resolve against that subquery, - not the join, so they are ignored by this test. +) -> exp.Column | None: + """Return *target*'s first own-scope leaf the rebuild cannot attribute to a side. + + Own-scope means the leaf resolves against the join rather than against a + subquery nested inside *target*: a nested subquery's columns resolve in its + own scope, so they are skipped. A qualified star (``b.*``) is skipped too -- + it is declinable rather than a plain column reference. + + Returns the offending :class:`~sqlglot.expressions.Column` so the caller can + diagnose it precisely, or ``None`` when every leaf is side-attributable. """ - # A qualified, non-star column is rebuilt directly — never declines here. - if ( - isinstance(target, exp.Column) - and target.table - and not isinstance(target.this, exp.Star) - ): - return False - # An aggregate is decided here in full (it never falls through to the - # column scan below — an unqualified aggregate argument like ``SUM(score)`` - # is diagnosed in ``render_aggregate``, not treated as a decline). A star - # nested in an aggregate argument (``COUNT(a.*)`` / ``MIN(COLUMNS(*))`` / - # ``COUNT(a.* EXCLUDE (name))``) cannot be enumerated, so it declines - # regardless of any EXCLUDE / REPLACE modifier columns (#204); a plain - # aggregate over qualified columns (``COUNT(*)`` / ``SUM(a.score)`` / - # ``COUNT(DISTINCT a.col)``) is rebuilt directly. - if isinstance(target, exp.AggFunc): - has_star_arg = any( - isinstance(col.this, exp.Star) for col in target.find_all(exp.Column) - ) - return has_star_arg or target.find(exp.Columns) is not None - # Unsupported by the rebuild. Decline to naive unless an own-scope column - # is out of scope (a user error the naive plan rejects too — let the - # dispatch raise instead). for col in target.find_all(exp.Column): if isinstance(col.this, exp.Star): - continue # qualified star — declinable, not a plain column ref - # Skip a column inside a nested subquery within *target* (it resolves - # against that subquery, not the join). *target* is an ancestor of - # *col*, so walking up from ``col.parent`` reaches it; a Select / - # Subquery strictly between them is a nested scope. When ``col`` IS - # ``target`` (a bare column projection), there is no interior to walk. - in_subquery = False + continue + # *target* is an ancestor of *col*, so walking up from ``col.parent`` + # reaches it; a Select / Subquery strictly between them is a nested + # scope. When ``col`` IS ``target`` there is no interior to walk. node = col.parent if col is not target else target + in_subquery = False while node is not None and node is not target: if isinstance(node, (exp.Select, exp.Subquery)): in_subquery = True @@ -318,8 +381,106 @@ def _projection_declines_to_naive( continue tbl = _fold_identifier(col.table) if col.table else "" if tbl not in (l_alias, r_alias) or (is_left_only and tbl == r_alias): - return False - return True + return col + if col.args.get("db") or col.args.get("catalog"): + return col + return None + + +@dataclass(frozen=True) +class _ProjectionPlan: + """How one SELECT item will be resolved, decided once (#109). + + :meth:`~IntersectsDuckDBIEJoinTransformer._resolve_projections` classifies + every SELECT item up front and then dispatches on :attr:`kind`, so the + question "are this tree's leaves attributable to a join side?" is answered + in exactly one place. Before #109 it was answered by three independently + written formulations, and the two that ran later silently depended on the + first having run. + """ + + #: The projection with any user alias unwrapped -- what gets rebuilt. + target: exp.Expression + #: The user's ``AS`` name, or ``None`` for an unaliased item. + user_alias: str | None + #: ``"column"`` a plain qualified column, ``"aggregate"`` a plain aggregate + #: over qualified columns, ``"scalar"`` a rebuildable expression tree, + #: ``"decline"`` a naive-valid shape the rebuild cannot express (#204, + #: #205), ``"raise"`` a user error the naive plan rejects too. + kind: Literal["column", "aggregate", "scalar", "decline", "raise"] + #: For ``"raise"``, the leaf that forced it -- carried so the diagnosis can + #: name the offending column rather than echo the whole projection. + offending: exp.Column | None = None + + +def _classify_projection( + sel: exp.Expression, + l_alias: str, + r_alias: str, + is_left_only: bool, +) -> _ProjectionPlan: + """Decide how the IEJoin will resolve one SELECT item. + + The IEJoin projection rebuild handles a qualified column (``a.col``), a + plain aggregate over qualified columns (``SUM(a.score)`` / ``COUNT(*)``), + and a scalar expression whose leaves are qualified columns + (``GREATEST(a.start, b.start)`` / ``a.start + 1`` -- #109); it emits + explicitly-named ``__giql_p`` inner columns for every leaf and + reconstructs the outer SELECT from them. + + The three verdicts that are not a rebuild: + + - ``"decline"`` -- a shape the naive-predicate plan compiles for free while + the rebuild cannot express it: a window aggregate, a ``FILTER`` clause, a + scalar subquery, an aggregate nested in an expression (``COUNT(*) * 2``), + a bare literal, a subscript, a lambda, an unrecognized function, or a + star nested in an aggregate argument (``COUNT(a.*)`` / ``MIN(COLUMNS(*))``). + Declining keeps ``dialect="duckdb"`` consistent with every other backend + instead of hard-erroring or, for the aggregate-nested star, miscompiling + (#204, #205). :func:`_is_rebuildable_scalar` holds the exact rule. + - ``"raise"`` -- a projection referencing a column **in its own scope** the + rebuild cannot attribute to a join side (unqualified, an unknown table, a + catalog/schema qualifier, or the right side under a SEMI / ANTI join). + The unknown-table, catalog-qualified and right-side cases are rejected by + the naive plan too; a bare unqualified column may be naive-valid when + unambiguous, but the dialect has no live schema to pick a side, so a + transpile-time error is more actionable than guessing. + - an aggregate is decided in full here and never falls through to the + expression rule: an unqualified aggregate argument (``SUM(score)``) is a + ``"raise"``, while a star nested in an aggregate argument cannot be + enumerated and is a ``"decline"`` regardless of any ``EXCLUDE`` / + ``REPLACE`` modifier columns (#204). + """ + target = _unwrap_projection_target(sel) + user_alias = sel.alias if isinstance(sel, exp.Alias) else None + + def plan( + kind: Literal["column", "aggregate", "scalar", "decline", "raise"], + offending: exp.Column | None = None, + ) -> _ProjectionPlan: + return _ProjectionPlan(target, user_alias, kind, offending) + + if isinstance(target, exp.Column) and not isinstance(target.this, exp.Star): + offending = _first_out_of_scope_leaf(target, l_alias, r_alias, is_left_only) + return plan("raise", offending) if offending else plan("column") + + if isinstance(target, exp.AggFunc): + has_star_arg = any( + isinstance(col.this, exp.Star) for col in target.find_all(exp.Column) + ) + if has_star_arg or target.find(exp.Columns) is not None: + return plan("decline") + offending = _first_out_of_scope_leaf(target, l_alias, r_alias, is_left_only) + return plan("raise", offending) if offending else plan("aggregate") + + # Anything else is an expression tree. An own-scope leaf the rebuild cannot + # attribute to a side is a user error the naive plan rejects too, so it + # raises rather than declining -- checked before the rebuildability rule so + # a malformed leaf is never masked by an unsupported wrapper around it. + offending = _first_out_of_scope_leaf(target, l_alias, r_alias, is_left_only) + if offending is not None: + return plan("raise", offending) + return plan("scalar") if _is_rebuildable_scalar(target) else plan("decline") def _strip_intersects( @@ -727,8 +888,11 @@ def _match_outer_join_decomposition(query: exp.Select) -> "_OuterJoinShape | Non transpile of each half would apply them per branch instead. - A projection that is not a side-attributable qualified column (a star, an expression, an aggregate) cannot be split into "survives" and "becomes - NULL", which is exactly what the unmatched half needs. This is also why - the ``bedtools -wao`` ``CASE`` projection still declines until #109 lands. + NULL", which is exactly what the unmatched half needs. The INNER / SEMI / + ANTI path rebuilds a scalar expression over its leaf columns (#109), but + the unmatched half would additionally have to evaluate that expression + over NULL-filled other-side leaves, which this rewrite does not yet do — + so the ``bedtools -wao`` ``CASE`` projection still declines here. """ # Whitelist, not a blocklist: the rewrite re-emits the query as a UNION ALL # of two independently transpiled halves, so ANY top-level clause it does @@ -846,15 +1010,21 @@ class IntersectsDuckDBIEJoinTransformer: end / strand), not arbitrary user columns — so declining keeps the projection identical across backends rather than silently narrowing it (#202). + - A scalar expression over qualified columns (``GREATEST(a.start, + b.start)``, ``LEAST(a."end", b."end") - GREATEST(a.start, b.start)``, + ``a.start + 1``) keeps the IEJoin plan: its leaf columns are projected + inward as ``__giql_p`` inner columns and the expression is rebuilt + on the outer wrapper SELECT over them (#109). An unaliased expression + is named ``__giql_expr_``, like ``__giql_agg_`` for an unaliased + aggregate. - A projection the rebuild cannot express but the naive plan handles — - an expression (``a.start + 1``), a window aggregate, a ``FILTER`` - clause, a scalar subquery, an aggregate nested in an expression - (``COUNT(*) * 2``), or a star nested in an aggregate argument - (``COUNT(a.*)`` / ``MIN(COLUMNS(*))``) — also declines to the naive - plan, keeping ``dialect="duckdb"`` consistent with every backend - rather than hard-erroring or miscompiling (#204, #205). Only a - genuine out-of-scope column reference (unqualified / unknown-table / - right-side) still raises. + the shapes :data:`_NON_REBUILDABLE_NODES` names, plus a bare literal + and a star nested in an aggregate argument (``COUNT(a.*)`` / + ``MIN(COLUMNS(*))``) — declines to the naive plan, keeping + ``dialect="duckdb"`` consistent with every backend rather than + hard-erroring or miscompiling (#204, #205). Only a genuine + out-of-scope column reference (unqualified / unknown-table / + catalog-qualified / right-side) still raises. - ``SEMI JOIN`` and ``ANTI JOIN`` outputs are left-side only. Any ``b.col`` reference in the outer SELECT, aggregate arguments, GROUP BY, HAVING, or ORDER BY raises :class:`_UnqualifiedProjectionError` @@ -920,16 +1090,15 @@ def transform_to_parts(self, query: exp.Expression) -> tuple[str, str] | None: query(getvariable(...))``, so any top-level clause :meth:`_build_sql` does not read would be silently dropped. The check below is a whitelist: engage the dialect path only for a query that is - exactly ``SELECT FROM - [JOIN ] {ON|WHERE} - `` with no other top-level clause. - Star projections (schema-less enumeration would narrow them, #202), - projections the rebuild cannot express but the naive plan handles — - expressions / window aggregates / FILTER clauses / scalar subqueries / - aggregate-nested stars (#204, #205) — and a ``SEMI`` / ``ANTI`` join - whose ``INTERSECTS`` sits out of scope in the ``WHERE`` (#201) also - decline. Everything else returns ``None`` so the generic - naive-predicate plan handles it. + exactly ``SELECT FROM [JOIN ] {ON|WHERE} `` with no other + top-level clause. Star projections (schema-less enumeration would + narrow them, #202), projections the rebuild cannot express but the + naive plan handles (:func:`_classify_projection` decides; #204, #205) + — and a ``SEMI`` / ``ANTI`` join whose ``INTERSECTS`` + sits out of scope in the ``WHERE`` (#201) also decline. Everything + else returns ``None`` so the generic naive-predicate plan handles it. """ if not isinstance(query, exp.Select): return None @@ -1091,9 +1260,9 @@ def transform_to_parts(self, query: exp.Expression) -> tuple[str, str] | None: try: return self._build_sql(query, intersects, sides, the_join) except _DeclineIEJoin: - # A projection the naive plan handles but the IEJoin cannot rebuild - # (expression / window / FILTER / scalar subquery / aggregate-nested - # star) — fall back to the naive-predicate plan (#204, #205). + # A projection the naive plan handles but the IEJoin cannot + # rebuild (:func:`_classify_projection` decided "decline") — fall + # back to the naive-predicate plan (#204, #205). return None except _UnqualifiedProjectionError as exc: raise ValueError(str(exc)) from exc @@ -1156,50 +1325,90 @@ def _classify_extras( return "inline" @staticmethod - def _validate_extra_qualifiers( - extras: list[exp.Expression], + def _raise_unattributable_column( + col: exp.Column, sides: "_IEJoinSides", + context: str, + *, + reject_right_side: bool = False, ) -> None: - """Raise for unqualified, unknown-aliased, or schema-qualified columns. - - Surfaces user mistakes at transpile time rather than deferring to a - downstream binder error. Three failure modes raise: - - - **Unqualified column:** no ``.table`` qualifier. - - **Unknown alias:** ``.table`` doesn't match either side of the join. + """Raise naming *col* as the reason the rebuild cannot attribute it. + + The single diagnosis site for a column the dialect cannot resolve, + shared by the extras-predicate validator and the SELECT-list + classifier so both name the offending leaf rather than echoing the + whole predicate or projection around it. *context* is interpolated + after ``dialect='duckdb' `` and says what was being resolved. + + Four failure modes raise, in this order: + + - **Unqualified column:** no ``.table`` qualifier. The dialect has no + live schema, so it cannot pick a side even when the naive plan + could. + - **Unknown alias:** ``.table`` matches neither side of the join. The + naive plan rejects this too. + - **Right side under a left-only join:** a SEMI / ANTI join's outer + SELECT resolves only left-side columns, so ``b.col`` is never valid + there and the generic "qualify the column" guidance would steer the + user toward another invalid form. Applies only when + *reject_right_side* is set: an extras predicate inlines into the + per-chromosome ``ON`` and may legitimately reference either side. - **Catalog/schema-qualified column:** ``mycat.myschema.a.score`` would silently mis-bind once the alias-rewriter remaps ``a`` to the inner subquery's hardcoded ``a`` alias (the ``mycat.myschema`` qualifier survives the rewrite and addresses a different relation than intended). """ - valid_aliases = {sides.left_user_alias, sides.right_user_alias} + lead = f"dialect='duckdb' {context}" + col_table = _fold_identifier(col.table) if col.table else "" + if not col_table: + raise _UnqualifiedProjectionError( + f"{lead}: column {col.sql()!r} must be qualified with " + f"{sides.left_user_alias!r} or {sides.right_user_alias!r}; " + f"use the {sides.left_user_alias}.col or " + f"{sides.right_user_alias}.col form." + ) + if col_table not in (sides.left_user_alias, sides.right_user_alias): + raise _UnqualifiedProjectionError( + f"{lead}: column {col.sql()!r} references unknown table " + f"qualifier {col.table!r}; expected " + f"{sides.left_user_alias!r} or {sides.right_user_alias!r}." + ) + if reject_right_side and col_table == sides.right_user_alias: + raise _UnqualifiedProjectionError( + f"dialect='duckdb' with kind={sides.kind!r} only projects " + f"left-side columns; got {col.sql()!r} which references the " + f"right side ({sides.right_user_alias!r})." + ) + if col.args.get("db") or col.args.get("catalog"): + raise _UnqualifiedProjectionError( + f"{lead}: column {col.sql()!r} carries a catalog/schema " + f"qualifier; use the alias-only form " + f"({sides.left_user_alias!r}. or " + f"{sides.right_user_alias!r}.)." + ) + + @classmethod + def _validate_extra_qualifiers( + cls, + extras: list[exp.Expression], + sides: "_IEJoinSides", + ) -> None: + """Raise for unqualified, unknown-aliased, or schema-qualified columns. + + Surfaces user mistakes at transpile time rather than deferring to a + downstream binder error. :meth:`_raise_unattributable_column` holds the + diagnoses; a right-side reference is *not* one of them here, because an + extras predicate inlines into the per-chromosome ``ON`` clause where + both sides are in scope. + """ for predicate in extras: for col in predicate.find_all(exp.Column): - col_table = _fold_identifier(col.table) if col.table else "" - if not col_table: - raise _UnqualifiedProjectionError( - f"dialect='duckdb' cannot inline extra predicate " - f"{predicate.sql()!r}: column {col.sql()!r} must be " - f"qualified with {sides.left_user_alias!r} or " - f"{sides.right_user_alias!r}." - ) - if col_table not in valid_aliases: - raise _UnqualifiedProjectionError( - f"dialect='duckdb' cannot inline extra predicate " - f"{predicate.sql()!r}: column references unknown " - f"table qualifier {col.table!r}; expected " - f"{sides.left_user_alias!r} or " - f"{sides.right_user_alias!r}." - ) - if col.args.get("db") or col.args.get("catalog"): - raise _UnqualifiedProjectionError( - f"dialect='duckdb' cannot inline extra predicate " - f"{predicate.sql()!r}: column {col.sql()!r} carries " - f"a catalog/schema qualifier; use the alias-only " - f"form ({sides.left_user_alias!r}. or " - f"{sides.right_user_alias!r}.)." - ) + cls._raise_unattributable_column( + col, + sides, + f"cannot inline extra predicate {predicate.sql()!r}", + ) @staticmethod def _rewrite_refs_for_per_chrom_subquery( @@ -1265,12 +1474,23 @@ def _rewrite_refs_for_wrapper_relation( Scope caveat: this walker is not scope-aware — it rewrites every :class:`~sqlglot.expressions.Column` whose ``.table`` matches either - side's alias anywhere inside ``node``, including correlated or - non-correlated subqueries that may rebind those aliases to a - different table. The dispatcher gates already reject any subquery - in GROUP BY / HAVING / ORDER BY / SELECT list so the unsafe shape - can't reach here today. A future relaxation of those gates must - add a scope-aware walker here. + side's alias anywhere inside ``node``. The invariant callers must + uphold is therefore that **no node which binds names may reach it**, + because the rewrite swaps the join for the wrapper relation and so + changes which aliases are in scope. Two ways that breaks: a subquery + rebinding an alias to a different table, and a lambda parameter + sharing a join alias's name, where ``list_transform(b.arr, a -> a.x)`` + binds ``a.x`` to the lambda's struct field once the relation no longer + carries alias ``a`` — same SQL text, different answer, no error. + + Two gates uphold this, and both must keep doing so: + :meth:`_resolve_projections`' clause dispatch rejects a subquery in + GROUP BY / HAVING / ORDER BY, and :func:`_is_rebuildable_scalar` + rejects :class:`~sqlglot.expressions.Select`, + :class:`~sqlglot.expressions.Subquery` and + :class:`~sqlglot.expressions.Lambda` in a SELECT-list expression tree + (see :data:`_NON_REBUILDABLE_NODES`). Relaxing either one requires a + scope-aware walker here first. """ valid_aliases = (sides.left_user_alias, sides.right_user_alias) @@ -1280,7 +1500,7 @@ def replace(n: exp.Expression) -> exp.Expression: col_table = _fold_identifier(n.table) if n.table else "" if col_table not in valid_aliases: return n - key = (col_table, n.name) + key = (col_table, _fold_identifier(n.name)) if key not in projection_map: raise _UnqualifiedProjectionError( f"dialect='duckdb' cannot resolve {n.sql()!r} in " @@ -1596,11 +1816,12 @@ def _build_sql( dialect cannot inline, or a single-column USING references a column that is not both tables' ``chrom_col``). May propagate :class:`_DeclineIEJoin` from the :meth:`_resolve_projections` projection - pre-scan (a naive-valid projection the rebuild cannot express — #204, - #205); :meth:`transform_to_parts` catches it and declines to the naive - plan. Raises :class:`_UnqualifiedProjectionError` when a user-mistake - condition fires; callers translating to the public surface wrap the - raise to :class:`ValueError`. + pre-scan (a naive-valid projection the rebuild cannot express — a + window / FILTER / subquery / aggregate-nested wrapper or a bare + literal, #204, #205); :meth:`transform_to_parts` catches it and + declines to the naive plan. Raises :class:`_UnqualifiedProjectionError` + when a user-mistake condition fires; callers translating to the public + surface wrap the raise to :class:`ValueError`. """ on_residuals, where_residuals = self._extract_extra_predicates(query, intersects) all_residuals = on_residuals + where_residuals @@ -1907,11 +2128,13 @@ def _resolve_projections( list — every column the query touches in any clause, projected once each under a unique ``__giql_p`` alias. * ``outer_projections`` is the outer SELECT's projection list, - rendered using the inner aliases (qualified columns and plain - aggregate calls). Star projections never reach here — they decline + rendered using the inner aliases (qualified columns, plain + aggregate calls, and scalar expressions rebuilt over their leaf + columns — #109). Star projections never reach here — they decline upstream (#202) — nor do the naive-valid shapes the projection - pre-scan declines (expressions, window aggregates, FILTER clauses, - scalar subqueries, aggregate-nested stars — #204, #205). + pre-scan declines (window aggregates, FILTER clauses, scalar + the shapes :data:`_NON_REBUILDABLE_NODES` names, plus bare literals + and aggregate-nested stars — #204, #205). * ``projection_map`` binds each ``(normalized_alias, column_name)`` referenced anywhere in the query to its inner alias name so the wrapper-relation rewriter and aggregate-argument rewriting can @@ -1928,33 +2151,51 @@ def _resolve_projections( l_alias = sides.left_user_alias r_alias = sides.right_user_alias - # Decline (fall back to the naive-predicate plan) for any projection the - # IEJoin cannot rebuild but the naive plan handles — an expression, a - # window aggregate, a FILTER clause, a scalar subquery, an aggregate - # nested in an expression, or a star nested in an aggregate argument - # (#204, #205). Run before any allocation so an aggregate-nested star - # (COUNT(a.*)) declines rather than allocating a bogus ``a."*"`` column. - # A projection with an out-of-scope column reference is a user error the - # naive plan also rejects, so it is left to the dispatch below to raise. - for sel in query.expressions: - target = _unwrap_projection_target(sel) - if _projection_declines_to_naive( - target, l_alias, r_alias, sides.is_left_only_join - ): + # Classify every SELECT item once, before any allocation, so that an + # aggregate-nested star (COUNT(a.*)) declines rather than allocating a + # bogus ``a."*"`` column, and so the loops below can switch on the + # recorded verdict instead of re-deriving it (#109). + plans = [ + _classify_projection(sel, l_alias, r_alias, sides.is_left_only_join) + for sel in query.expressions + ] + + # One declining SELECT item declines the whole query: the naive plan + # compiles the projection for free (#204, #205). + for plan in plans: + if plan.kind == "decline": raise _DeclineIEJoin( - f"dialect='duckdb' cannot rebuild projection {target.sql()!r}; " - "deferring to the naive-predicate plan." + f"dialect='duckdb' cannot rebuild projection " + f"{plan.target.sql()!r}; deferring to the naive-predicate " + "plan." + ) + + # A user error the naive plan rejects too. Diagnosed against the leaf + # that caused it rather than the projection around it, so + # ``GREATEST(a.start, c.start)`` names ``c.start`` instead of + # prescribing that the user qualify a column they already qualified. + for plan in plans: + if plan.kind == "raise": + assert plan.offending is not None # set by _classify_projection + self._raise_unattributable_column( + plan.offending, + sides, + "requires qualified projections; cannot rebuild " + f"{plan.target.sql()!r}", + reject_right_side=sides.is_left_only_join, ) inner_projections: list[str] = [] projection_map: dict[tuple[str, str], str] = {} - def reject_right_side_if_left_only(tbl_normalized: str, source_sql: str) -> None: + def reject_right_side_if_left_only( + tbl_normalized: str, source: exp.Expression + ) -> None: """Raise when a SEMI / ANTI join projects from the right side.""" if sides.is_left_only_join and tbl_normalized == r_alias: raise _UnqualifiedProjectionError( f"dialect='duckdb' with kind={sides.kind!r} only " - f"projects left-side columns; got {source_sql!r} " + f"projects left-side columns; got {source.sql()!r} " f"which references the right side " f"({sides.right_user_alias!r})." ) @@ -1962,17 +2203,30 @@ def reject_right_side_if_left_only(tbl_normalized: str, source_sql: str) -> None def allocate(tbl: str, col: str) -> str: """Return the inner alias for ``(tbl, col)``, allocating if new. + The key folds both halves, because DuckDB resolves identifiers + case-insensitively: ``a.START`` and ``a.start`` name one physical + column and must share one ``__giql_p``, or the duplicate is + carried per row through every branch of the per-chromosome union. + The emitted inner projection keeps the spelling the user wrote, + which DuckDB resolves to the same column. + Allocation order is deterministic per query but depends on (a) the fixed pre-walk order in :meth:`_resolve_projections` — modifier columns first (``("group", "having", "order")``), - then aggregate-argument columns, then the SELECT list — and + then WHERE residuals, then the SELECT list — and (b) sqlglot's ``find_all`` traversal order within each sub-clause. Tests therefore should NOT couple to a specific ``__giql_p`` ↔ source-column mapping (e.g. ``assert projection_map[(a, start)] == "__giql_p3"``). + + The *outer* output name still follows the user's spelling, so + ``SELECT a.START, a.start`` reports ``['START', 'start']`` where + the naive plan reports ``['start', 'start']`` — a pre-existing, + cosmetic, case-only asymmetry this folding does not close, because + recovering the physical spelling would need a live schema. """ tbl = _fold_identifier(tbl) - key = (tbl, col) + key = (tbl, _fold_identifier(col)) existing = projection_map.get(key) if existing is not None: return existing @@ -1990,29 +2244,6 @@ def allocate(tbl: str, col: str) -> str: projection_map[key] = alias return alias - def render_aggregate(agg_node: exp.Expression, user_alias: str | None) -> None: - """Validate aggregate-argument qualifiers and emit the outer projection.""" - for col in agg_node.find_all(exp.Column): - col_table = _fold_identifier(col.table) if col.table else "" - if col_table not in (l_alias, r_alias): - raise _UnqualifiedProjectionError( - "dialect='duckdb' requires qualified aggregate " - f"arguments; got {agg_node.sql()!r}. Use a " - "qualified column reference like " - f"'{type(agg_node).__name__.upper()}(a.col)' or " - f"'{type(agg_node).__name__.upper()}(b.col)'." - ) - reject_right_side_if_left_only(col_table, agg_node.sql()) - rewritten_agg = self._rewrite_refs_for_wrapper_relation( - agg_node, projection_map, sides, "aggregate argument" - ) - outer_name = ( - user_alias - if user_alias is not None - else f"__giql_agg_{len(outer_projections)}" - ) - outer_projections.append(f"{rewritten_agg} AS {q(outer_name)}") - # Pre-allocate inner aliases for every column referenced in # GROUP BY / HAVING / ORDER BY so the wrapper relation exposes # them even if the user didn't put them in the SELECT list. @@ -2023,7 +2254,7 @@ def render_aggregate(agg_node: exp.Expression, user_alias: str | None) -> None: for col in modifier_node.find_all(exp.Column): col_table = _fold_identifier(col.table) if col.table else "" if col_table in (l_alias, r_alias): - reject_right_side_if_left_only(col_table, col.sql()) + reject_right_side_if_left_only(col_table, col) allocate(col_table, col.name) # Pre-allocate inner aliases for columns referenced in the WHERE @@ -2035,72 +2266,125 @@ def render_aggregate(agg_node: exp.Expression, user_alias: str | None) -> None: for col in residual.find_all(exp.Column): col_table = _fold_identifier(col.table) if col.table else "" if col_table in (l_alias, r_alias): - reject_right_side_if_left_only(col_table, col.sql()) + reject_right_side_if_left_only(col_table, col) allocate(col_table, col.name) - # Pre-allocate inner aliases for columns referenced inside any - # aggregate function in the SELECT list (we'll rewrite the - # aggregate's argument to use the inner alias when rendering the - # outer projection below). - for sel in query.expressions: - target = _unwrap_projection_target(sel) - if isinstance(target, exp.AggFunc): - for col in target.find_all(exp.Column): - col_table = _fold_identifier(col.table) if col.table else "" - if col_table in (l_alias, r_alias): - reject_right_side_if_left_only(col_table, col.sql()) - allocate(col_table, col.name) + # Pre-allocate inner aliases for every leaf column referenced inside a + # non-column SELECT item — an aggregate's argument or a scalar + # expression's leaves (#109) — so the wrapper relation exposes them; + # the outer projection below is rendered with those leaves rewritten to + # the inner aliases. A right-side leaf under SEMI / ANTI is rejected + # here with the dedicated left-side-only message. + for plan in plans: + if plan.kind == "column": + continue + for col in plan.target.find_all(exp.Column): + col_table = _fold_identifier(col.table) if col.table else "" + if col_table in (l_alias, r_alias): + reject_right_side_if_left_only(col_table, col) + allocate(col_table, col.name) outer_projections: list[str] = [] + # Output names the user actually wrote, case-folded. A synthesized name + # is chosen around these so it never shadows one of them (#109): DuckDB + # accepts duplicate result-column names silently, which would make + # ``fetchdf()`` and any name-keyed access collapse or shadow a column. + # Two same-named *user* columns (``SELECT a.start, b.start``) are left + # alone — the naive plan reports ``['start', 'start']`` there too, and + # the dialect must not be stricter than the plan it accelerates. + taken_names = { + _fold_identifier( + plan.user_alias + if plan.user_alias is not None + else plan.target.name + if plan.kind == "column" + else "" + ) + for plan in plans + } - {""} + + def render_rewritten( + node: exp.Expression, + user_alias: str | None, + *, + name_prefix: str, + clause_label: str, + ) -> None: + """Emit one outer projection rebuilt over the wrapper's inner aliases. + + Shared by the aggregate and scalar-expression paths, which differ + only in the synthesized name's prefix and the rewriter's diagnostic + label. Every leaf was pre-allocated above, so the rewriter finds + each one in ``projection_map``. + + An unaliased item gets a synthesized output name because DuckDB's + automatic name would be derived from the rewritten text, which + names ``__giql_p`` inner columns the caller never wrote. + + The *leaves* rather than the computed value go inward on purpose. + Computing the expression inside the per-chromosome subquery would + narrow the wrapper relation, but the session-variable name is a + content digest of that subquery's SQL, so the expression's text + would fold into the digest and ``a.start + 1``, ``a.start + 2``, … + would each mint a distinct ``SET VARIABLE`` on a long-lived + connection — trading the bounded variable set this module + guarantees for an unbounded one. Leaf dedup also means the shapes + #109 exists for pay nothing: the bedtools clipped-span and ``-wo`` + projections already carry their leaves as plain columns. + """ + rewritten = self._rewrite_refs_for_wrapper_relation( + node, projection_map, sides, clause_label + ) + if user_alias is not None: + outer_name = user_alias + else: + outer_name = find_new_name( + taken_names, f"{name_prefix}{len(outer_projections)}" + ) + taken_names.add(_fold_identifier(outer_name)) + outer_projections.append(f"{rewritten} AS {q(outer_name)}") - for sel in query.expressions: - target = _unwrap_projection_target(sel) - user_alias = sel.alias if isinstance(sel, exp.Alias) else None + def render_aggregate(agg_node: exp.Expression, user_alias: str | None) -> None: + """Validate aggregate-argument qualifiers, then emit the projection.""" + for col in agg_node.find_all(exp.Column): + col_table = _fold_identifier(col.table) if col.table else "" + if col_table not in (l_alias, r_alias): + raise _UnqualifiedProjectionError( + "dialect='duckdb' requires qualified aggregate " + f"arguments; got {agg_node.sql()!r}. Use a " + "qualified column reference like " + f"'{type(agg_node).__name__.upper()}(a.col)' or " + f"'{type(agg_node).__name__.upper()}(b.col)'." + ) + reject_right_side_if_left_only(col_table, agg_node) + render_rewritten( + agg_node, + user_alias, + name_prefix="__giql_agg_", + clause_label="aggregate argument", + ) + for plan in plans: # Star projections (bare ``*`` and ``a.*`` / ``b.*``) never reach - # here — :func:`_has_star_projection` declines them upstream (#202) — - # and neither do the naive-valid shapes the projection pre-scan - # above declined (expressions / window aggregates / FILTER clauses / - # scalar subqueries / aggregate-nested stars — #204, #205). - - if isinstance(target, exp.AggFunc): - render_aggregate(target, user_alias) - continue - - if isinstance(target, exp.Column) and target.table: - tbl = _fold_identifier(target.table) - reject_right_side_if_left_only(tbl, target.sql()) - col = target.name + # here — :func:`_has_star_projection` declines them upstream (#202) + # — and neither do the ``"decline"`` / ``"raise"`` verdicts, both + # of which left this method above. + if plan.kind == "aggregate": + render_aggregate(plan.target, plan.user_alias) + elif plan.kind == "column": + tbl = _fold_identifier(plan.target.table) + reject_right_side_if_left_only(tbl, plan.target) + col = plan.target.name alias = allocate(tbl, col) - outer_name = user_alias if user_alias is not None else col + outer_name = plan.user_alias if plan.user_alias is not None else col outer_projections.append(f"{alias} AS {q(outer_name)}") - continue - - # Everything reaching here is an unsupported wrapper (an - # expression, a window aggregate, a FILTER clause, or an aggregate - # nested in an expression) that the pre-scan did NOT decline because - # it references an out-of-scope column — an unqualified column, an - # unknown table, or the right side under a SEMI / ANTI join (e.g. - # ``SUM(score) OVER (...)``, ``score + 1``). A plain aggregate with - # an unqualified argument (``SUM(score)``) is diagnosed earlier in - # ``render_aggregate``. - # - # A right-side column wrapped in an unsupported shape under a - # left-only join gets the dedicated left-side-only message rather - # than the generic "qualify the column" one, which would steer the - # user toward another invalid form (``b.col`` is never valid here). - for col in target.find_all(exp.Column): - col_table = _fold_identifier(col.table) if col.table else "" - reject_right_side_if_left_only(col_table, target.sql()) - # Otherwise the fault is an unqualified / unknown-table column; the - # accurate guidance is to qualify it (once qualified, the wrapper - # itself declines to the naive plan — #204, #205 — rather than - # erroring). - raise _UnqualifiedProjectionError( - "dialect='duckdb' requires qualified projections; got " - f"{target.sql()!r}. Use a qualified column reference like " - "'a.col' or 'b.col [AS x]'." - ) + else: + render_rewritten( + plan.target, + plan.user_alias, + name_prefix="__giql_expr_", + clause_label="SELECT", + ) if not outer_projections: raise _UnqualifiedProjectionError( diff --git a/src/giql/transpile.py b/src/giql/transpile.py index 40b80ee..e3e2ab9 100644 --- a/src/giql/transpile.py +++ b/src/giql/transpile.py @@ -80,6 +80,12 @@ def transpile( declines silently to the naive predicate for every projection it cannot split, so ``dialect="duckdb"`` never changes an outer join's result — only its plan. See the performance guide for the full enumeration. + This dialect also names SELECT-list items the other targets leave to + the engine: an unaliased scalar expression or aggregate is emitted as + ``__giql_expr_`` / ``__giql_agg_``, because the rebuilt text + DuckDB would derive a name from refers to inner columns the caller + never wrote. Alias such an item to pin its result-column name across + targets. The target's capabilities also choose the coordinate-canonicalization emit form for a non-canonically-encoded From f2b461ea45ea7ce9d0402e79b8b40c493a98b161 Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 3 Sep 2026 08:16:49 -0400 Subject: [PATCH 2/5] test: Cover scalar expression projections on the IEJoin path The four tests that pinned an expression projection as a decline now assert it engages, and a.start + 1 leaves the unsupported-projection parametrizations. What replaces it there is the boundary the rebuild actually draws: each carve-out nested one level inside an otherwise rebuildable tree, since a window aggregate or a subquery reached through a GREATEST must decline as surely as one at the root; literal-only trees, which have no leaf to attribute to a side; and an expression sharing a SELECT list with a star or a window aggregate, where the older declines still win. The deep-leaf cases matter for the same reason on the error side: an unqualified or right-side column must raise from anywhere in the tree, not only where it used to sit. The execution tests cover what the property strategy cannot reach, because it generates integer arithmetic only: string concatenation, a boolean comparison, a CASE over literals and a cast. Alongside them the compositions the rebuild has to survive, each verified against the naive plan in rows and result schema: ANTI with a WHERE residual layered on the wrapper, an implicit comma join, DISTINCT over a rebuilt value, GROUP BY on an expression rather than on its leaf, LIMIT and OFFSET behind an ORDER BY alias, USING, custom column names, one-based closed coordinates where the rebuilt expression sees raw values while the join sees canonical ones, and case-folded user aliases. Four invariants are pinned that no single-shape test reaches. Both left-only shapes still plan through IE_JOIN rather than the nested loop the correlated EXISTS form exists to avoid. The unpartitioned branch above the contig ceiling carries the same projection list as the per-chromosome one. The empty-schema fallback types an expression column from the source tables under WHERE FALSE. And two queries whose inner projections coincide share one content-addressed session variable without contaminating each other's results, which is what makes that sharing safe now that the outer SELECT can differ while the inner one does not. Two property tests join the existing one. The first draws expression trees over left-only leaves under SEMI and ANTI with a WHERE residual and an ORDER BY, the path where a right-side leaf must raise instead. The second checks the clipped-span and overlap-width shapes against a Python reference rather than against the naive plan, so the bedtools semantics are pinned by something other than the implementation they are compared with. --- tests/test_duckdb_iejoin.py | 2457 +++++++++++++++++++++++++++++------ 1 file changed, 2058 insertions(+), 399 deletions(-) diff --git a/tests/test_duckdb_iejoin.py b/tests/test_duckdb_iejoin.py index 4a4f9a4..022db7e 100644 --- a/tests/test_duckdb_iejoin.py +++ b/tests/test_duckdb_iejoin.py @@ -21,6 +21,14 @@ from hypothesis import settings # noqa: E402 from hypothesis import strategies as st # noqa: E402 +#: Leaf columns the random scalar-expression strategy draws from — both sides' +#: start and end, so a generated tree can mix join sides freely. +_SCALAR_LEAVES = ("a.start", 'a."end"', "b.start", 'b."end"') + +#: The left side's leaves only, for trees projected under SEMI / ANTI joins +#: whose output may not reference the right side. +_LEFT_ONLY_LEAVES = ("a.start", 'a."end"') + def _make_table(conn, name: str, rows: list[tuple]) -> None: conn.execute(f""" @@ -52,6 +60,24 @@ def _python_overlap(peaks: list[tuple], genes: list[tuple]) -> list[tuple]: return sorted(out) +def _python_clipped_overlap(peaks: list[tuple], genes: list[tuple]) -> list[tuple]: + """Reference ``bedtools intersect`` clipped span plus ``-wo`` overlap width. + + For every overlapping half-open pair returns ``(chrom, clipped start, + clipped end, width)`` — what the projection ``GREATEST(a.start, b.start)`` + / ``LEAST(a."end", b."end")`` / their difference computes. Sorted multiset, + for the same reason :func:`_python_overlap` returns one; composed on that + oracle, as :func:`_python_anti_overlap` composes on + :func:`_python_semi_overlap`, so the half-open overlap predicate is + defined in exactly one place and a correction cannot land in only one of + two reference implementations. + """ + return sorted( + (pc, max(ps, gs), min(pe, ge), min(pe, ge) - max(ps, gs)) + for pc, ps, pe, _gc, gs, ge in _python_overlap(peaks, genes) + ) + + def _python_semi_overlap(peaks: list[tuple], genes: list[tuple]) -> list[tuple]: """Reference SEMI-join overlap: distinct left rows with any match.""" matched: set[tuple] = set() @@ -98,6 +124,35 @@ def _make_interval_tables(conn, peak_rows: list[tuple], gene_rows: list[tuple]) conn.executemany(f"INSERT INTO {table} VALUES (?, ?, ?)", rows) +def _make_planner_scale_tables(conn, *, genes_where: str | None = None) -> None: + """Recreate 20k-row peaks/genes tables over four chromosomes. + + Sized so DuckDB's planner picks a range join rather than a nested loop, + which is what the ``IE_JOIN`` plan assertions read. The two strides (37 + and 41, the latter offset by 17) are coprime with the 100000 modulus, so + starts spread evenly and the interval sets overlap without either side + being a shifted copy of the other. *genes_where* restricts genes to a + subset of the chromosomes, leaving peaks with unmatched contigs. + """ + conn.execute("DROP TABLE IF EXISTS peaks") + conn.execute(""" + CREATE TABLE peaks AS + SELECT 'chr' || ((i % 4) + 1) AS chrom, + ((i * 37) % 100000)::INTEGER AS "start", + (((i * 37) % 100000) + 200)::INTEGER AS "end" + FROM range(20000) t(i) + """) + conn.execute("DROP TABLE IF EXISTS genes") + conn.execute(f""" + CREATE TABLE genes AS + SELECT 'chr' || ((i % 4) + 1) AS chrom, + ((i * 41 + 17) % 100000)::INTEGER AS "start", + (((i * 41 + 17) % 100000) + 200)::INTEGER AS "end" + FROM range(20000) t(i) + {f"WHERE {genes_where}" if genes_where else ""} + """) + + def _null_key(row: tuple) -> tuple: """Total sort key for rows that may contain NULLs, which are not comparable.""" return tuple((value is None, str(value)) for value in row) @@ -170,6 +225,67 @@ def _dynamic_sql_variables(sql: str) -> list[str]: return list(dict.fromkeys(re.findall(r"__giql_iejoin_[0-9a-f]+", sql))) +def _expression_trees(leaves: tuple[str, ...]) -> st.SearchStrategy[str]: + """Return a Hypothesis strategy for scalar expression trees over *leaves*. + + Leaves are the given columns plus small integer literals; a generated tree + is filtered to keep at least one column leaf, so a literal-only subtree + (which the dialect declines) can appear inside a tree but never be the + whole projection. Combines leaves through the operators the ``bedtools + intersect`` shapes use (``+`` / ``-`` / ``GREATEST`` / ``LEAST``) plus + ``ABS``, ``CASE`` and a ``CAST`` so the tree exercises function calls, + arithmetic, conditional and unary forms. Every operator stays + integer-valued over integer leaves, so the two plans agree on the result + column's type as well as its values. + + Nodes carry ``(sql, has_column)`` while the tree is built and the flag is + dropped at the end, so "this tree has a column leaf" is decided + structurally. Scanning the rendered text for a leaf substring instead + would hold only as long as no operator template can synthesize one. + """ + + def extend(children): + def combine(template): + return st.tuples(children, children).map( + lambda p: (template(p[0][0], p[1][0]), p[0][1] or p[1][1]) + ) + + return st.one_of( + combine(lambda x, y: f"({x} + {y})"), + combine(lambda x, y: f"({x} - {y})"), + combine(lambda x, y: f"GREATEST({x}, {y})"), + combine(lambda x, y: f"LEAST({x}, {y})"), + combine(lambda x, y: f"ABS({x} - {y})"), + combine(lambda x, y: f"CASE WHEN {x} < {y} THEN {x} ELSE {y} END"), + children.map(lambda c: (f"CAST({c[0]} AS BIGINT)", c[1])), + ) + + base = st.one_of( + st.sampled_from(leaves).map(lambda leaf: (leaf, True)), + st.integers(min_value=-5, max_value=5).map(lambda n: (str(n), False)), + ) + return ( + st.recursive(base, extend, max_leaves=6) + .filter(lambda tree: tree[1]) + .map(lambda tree: tree[0]) + ) + + +def _interval_lists( + chroms: tuple[str, ...] = ("chr1", "chr2"), max_size: int = 6 +) -> st.SearchStrategy[list[tuple]]: + """Return a strategy for (chrom, start, length) triples over *chroms*.""" + return st.lists( + st.tuples( + st.sampled_from(chroms), + st.integers(min_value=0, max_value=100), + st.integers(min_value=1, max_value=30), + ), + min_size=0, + max_size=max_size, + ) + + def _explain_dynamic_sql(conn, sql: str, half: int = 0) -> str: """Return the DuckDB EXPLAIN text of the per-chromosome dynamic SQL for *sql*. @@ -2278,8 +2394,8 @@ def test_transpile_should_raise_when_projection_references_unknown_table_alias( with pytest.raises(ValueError, match="[Uu]nknown.*qualifier|'c'"): transpile(query, tables=["peaks", "genes"], dialect="duckdb") - def test_transpile_should_decline_expression_form_projection_to_naive_plan(self): - """Test that an arithmetic projection expression declines to the naive plan. + def test_transpile_should_engage_iejoin_for_expression_form_projection(self): + """Test that an arithmetic projection expression keeps the IEJoin plan. Given: A SELECT list containing an expression over a qualified column @@ -2287,8 +2403,9 @@ def test_transpile_should_decline_expression_form_projection_to_naive_plan(self) When: ``transpile`` is called with ``dialect='duckdb'``. Then: - It should decline the IEJoin (no ``SET VARIABLE`` scaffolding) so - the naive-predicate plan evaluates the expression (#205). + It should engage the IEJoin path (``SET VARIABLE __giql_iejoin_`` + emitted) — the projection shape alone must not cost the join its + fast plan (#109). """ # Arrange query = """ @@ -2301,8 +2418,290 @@ def test_transpile_should_decline_expression_form_projection_to_naive_plan(self) sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") # Assert - assert "SET VARIABLE" not in sql.upper() - assert "getvariable" not in sql + assert "SET VARIABLE __giql_iejoin_" in sql + assert "getvariable" in sql + + def test_transpile_should_rebuild_both_side_expression_on_outer_select(self): + """Test that a clipped-span projection is rebuilt over the inner columns. + + Given: + The ``bedtools intersect`` clipped-span projection ``GREATEST(a.start, + b.start) AS start, LEAST(a."end", b."end") AS "end"`` over an + INTERSECTS INNER join. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should engage the IEJoin and re-emit each expression on the + outer SELECT over ``__giql_p`` inner columns under the user's + alias, with no raw ``a.``/``b.`` reference left on the outer SELECT + (#109). + """ + # Arrange + query = ( + "SELECT a.chrom, GREATEST(a.start, b.start) AS start, " + 'LEAST(a."end", b."end") AS "end" ' + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + outer_select = sql.split(";\n")[-1] + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert re.search( + r'GREATEST\(__giql_p\d+, __giql_p\d+\) AS "start"', outer_select + ) + assert re.search(r'LEAST\(__giql_p\d+, __giql_p\d+\) AS "end"', outer_select) + assert "a.start" not in outer_select + assert "b.start" not in outer_select + + def test_transpile_should_rebuild_mixed_side_expression_when_leaves_span_both_sides( + self, + ): + """Test that an expression mixing both sides projects every leaf inward. + + Given: + The ``bedtools intersect -wo`` overlap-width projection + ``LEAST(a."end", b."end") - GREATEST(a.start, b.start) AS ov``, whose + leaves come from both join sides. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should engage the IEJoin, project all four leaf columns into the + per-chromosome subquery, and rebuild the subtraction on the outer + SELECT under the ``ov`` alias (#109). + """ + # Arrange + query = ( + 'SELECT LEAST(a."end", b."end") - GREATEST(a.start, b.start) AS ov ' + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + setup, outer_select = sql.split(";\n") + + # Assert + assert "SET VARIABLE __giql_iejoin_" in setup + for leaf in ('a."start"', 'a."end"', 'b."start"', 'b."end"'): + assert re.search(rf"{re.escape(leaf)} AS __giql_p\d+", setup) + assert re.search( + r"LEAST\(__giql_p\d+, __giql_p\d+\) - GREATEST\(__giql_p\d+, __giql_p\d+\)" + r' AS "ov"', + outer_select, + ) + + def test_transpile_should_rebuild_single_side_expression(self): + """Test that a single-side expression is rebuilt over its one inner column. + + Given: + A ``a.start + 1 AS s`` projection over an INTERSECTS INNER join. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should engage the IEJoin and emit ``__giql_p + 1 AS "s"`` on + the outer SELECT (#109). + """ + # Arrange + query = ( + "SELECT a.start + 1 AS s " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + outer_select = sql.split(";\n")[-1] + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert re.search(r'__giql_p\d+ \+ 1 AS "s"', outer_select) + + def test_transpile_should_synthesize_output_name_when_expression_is_unaliased( + self, + ): + """Test that an unaliased expression projection gets a synthesized name. + + Given: + A SELECT list whose second item is the unaliased expression + ``a.start + 1`` over an INTERSECTS INNER join. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should engage the IEJoin and name the rebuilt expression + ``__giql_expr_1`` (position-indexed, like ``__giql_agg_`` for an + unaliased aggregate), since DuckDB's auto-name is derived from the + original expression text the rewrite has changed (#109). + """ + # Arrange + query = ( + "SELECT a.chrom, a.start + 1 " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + outer_select = sql.split(";\n")[-1] + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert re.search(r'__giql_p\d+ \+ 1 AS "__giql_expr_1"', outer_select) + + @pytest.mark.parametrize("kind", ["SEMI", "ANTI"]) + @pytest.mark.parametrize( + "projection", + [ + "GREATEST(a.start, b.start)", + "CASE WHEN a.start > 0 THEN b.start ELSE a.start END", + 'LEAST(a."end", GREATEST(a.start, b.start))', + "b.start + 1", + ], + ) + def test_transpile_should_raise_left_side_only_when_expression_references_right_side_under_left_only_join( # noqa: E501 + self, kind, projection + ): + """Test that a right-side leaf in an expression under SEMI/ANTI raises. + + Given: + An expression projection under a SEMI / ANTI join carrying a + ``b.start`` leaf — at the top level, inside a ``CASE`` branch, or + nested two calls deep — that is out of scope for the left-only + output. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should raise the dedicated left-side-only ``ValueError`` naming + the offending leaf rather than rebuilding or declining — the naive + plan rejects the reference too (#109). + """ + # Arrange + query = ( + f"SELECT {projection} FROM peaks a {kind} JOIN genes b " + "ON a.interval INTERSECTS b.interval" + ) + + # Act & assert + with pytest.raises(ValueError, match="left-side") as excinfo: + transpile(query, tables=["peaks", "genes"], dialect="duckdb") + assert "b.start" in str(excinfo.value) + + def test_transpile_should_allocate_one_inner_column_when_leaf_is_shared(self): + """Test that a leaf shared by a column and two expressions is projected once. + + Given: + A SELECT list projecting ``a.start`` plainly and twice more inside + ``a.start + 1`` expressions. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should project ``a."start"`` into the per-chromosome subquery + exactly once and rebuild all three outer items over that one inner + alias (#109). + """ + # Arrange + query = ( + "SELECT a.start, a.start + 1 AS s, a.start + 1 AS s2 " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + setup, outer_select = sql.split(";\n") + shared = re.search(r'(__giql_p\d+) AS "start"', outer_select) + + # Assert — the inner list is rendered once per emitted branch, so count + # distinct aliases rather than occurrences. + assert shared is not None + assert set(re.findall(r"__giql_p\d+", setup)) == {shared.group(1)} + assert f'{shared.group(1)} + 1 AS "s"' in outer_select + assert f'{shared.group(1)} + 1 AS "s2"' in outer_select + + def test_transpile_should_index_synthesized_names_by_position_across_kinds( + self, + ): + """Test that unaliased expressions and aggregates share one position counter. + + Given: + A SELECT list with an unaliased expression at position 0, a plain + column, an unaliased aggregate, and another unaliased expression. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should name them ``__giql_expr_0``, ``chrom``, ``__giql_agg_2`` + and ``__giql_expr_3`` — every synthesized name carries its SELECT + position, whichever scheme produced it (#109). + """ + # Arrange + query = ( + "SELECT a.start + 1, a.chrom, COUNT(*), b.start + 1 " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom, a.start, b.start" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + outer_select = sql.split(";\n")[-1] + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + for name in ("__giql_expr_0", "chrom", "__giql_agg_2", "__giql_expr_3"): + assert f'AS "{name}"' in outer_select + + @pytest.mark.parametrize("projection", ["c.start + 1", "GREATEST(a.start, c.start)"]) + def test_transpile_should_raise_when_expression_leaf_has_unknown_qualifier( + self, projection + ): + """Test that an unknown table qualifier inside an expression raises. + + Given: + An expression projection whose leaf is qualified with ``c``, an + alias absent from the FROM / JOIN clauses, at the top level or + nested inside a function call. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should raise ``ValueError`` naming the offending leaf rather than + rebuilding or declining — the naive plan rejects it too (#109). + """ + # Arrange + query = ( + f"SELECT {projection} FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act & assert + with pytest.raises(ValueError) as excinfo: + transpile(query, tables=["peaks", "genes"], dialect="duckdb") + assert "c.start" in str(excinfo.value) + + def test_transpile_should_engage_iejoin_when_literal_sits_beside_column_leaf( + self, + ): + """Test that a literal argument next to a column leaf is rebuildable. + + Given: + A ``COALESCE(a.start, 100) AS v`` projection — a literal inside an + expression that also has a qualified column leaf. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should engage the IEJoin and rebuild the call over the inner + alias — the bare-literal decline must not over-reach onto literal + arguments (#109). + """ + # Arrange + query = ( + "SELECT COALESCE(a.start, 100) AS v " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + outer_select = sql.split(";\n")[-1] + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert re.search(r'COALESCE\(__giql_p\d+, 100\) AS "v"', outer_select) def test_transpile_should_decline_star_with_unknown_qualifier_to_naive_plan( self, @@ -3703,18 +4102,7 @@ def test_count_overlaps_should_match_python_reference_for_random_inputs( # Arrange peak_rows = [(c, s, s + length) for (c, s, length) in peaks] gene_rows = [(c, s, s + length) for (c, s, length) in genes] - conn.execute("DROP TABLE IF EXISTS peaks") - conn.execute("DROP TABLE IF EXISTS genes") - conn.execute( - 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' - ) - if peak_rows: - conn.executemany("INSERT INTO peaks VALUES (?, ?, ?)", peak_rows) - conn.execute( - 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' - ) - if gene_rows: - conn.executemany("INSERT INTO genes VALUES (?, ?, ?)", gene_rows) + _make_interval_tables(conn, peak_rows, gene_rows) sql = transpile( 'SELECT a.chrom, a.start, a."end", COUNT(b.chrom) AS n ' "FROM peaks a LEFT JOIN genes b ON a.interval INTERSECTS b.interval " @@ -3769,6 +4157,38 @@ def test_count_overlaps_plan_should_avoid_nested_loop_when_executed_on_duckdb( # Assert assert "BLOCKWISE_NL_JOIN" not in plan + @pytest.mark.parametrize( + "group_by", ["a.chrom, a.start + 1", "a.chrom, s"], ids=["expression", "alias"] + ) + def test_transpile_should_decline_count_overlaps_when_key_is_expression( + self, group_by + ): + """Test that an expression group key declines the count fast path. + + Given: + A LEFT-join COUNT whose projected key ``a.start + 1 AS s`` is an + expression, grouped by the expression or by its alias. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should decline to the naive plan (no ``SET VARIABLE``) — the + count_overlaps matcher stays closed to expression keys even though + the INNER path now rebuilds expressions (#109). + """ + # Arrange + query = ( + "SELECT a.chrom, a.start + 1 AS s, COUNT(b.chrom) AS n " + "FROM peaks a LEFT JOIN genes b ON a.interval INTERSECTS b.interval " + f"GROUP BY {group_by}" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert "SET VARIABLE" not in sql.upper() + assert "__giql_counts" not in sql + class TestTranspileDuckDBIEJoinOuterJoinDecomposition: """Outer-join INTERSECTS as INNER pairs UNION ALL unmatched rows (#95).""" @@ -4024,23 +4444,39 @@ def test_transpile_should_decline_outer_join_when_full_outer(self): assert "SET VARIABLE __giql_iejoin_" not in sql assert 'a."chrom" = b."chrom"' in sql - def test_transpile_should_decline_outer_join_when_projection_is_expression(self): + @pytest.mark.parametrize( + "join, projection", + [ + ("LEFT JOIN", "CASE WHEN b.chrom IS NULL THEN 0 ELSE 1 END AS ov"), + ("LEFT JOIN", "a.start + 1 AS v"), + ("RIGHT JOIN", "GREATEST(a.start, b.start) AS v"), + ("FULL OUTER JOIN", "a.start + 1 AS v"), + ], + ids=["left_wao_case", "left_arithmetic", "right_greatest", "full_arithmetic"], + ) + def test_transpile_should_decline_outer_join_when_projection_is_expression( + self, join, projection + ): """Test that an expression projection under an outer join declines. Given: - A LEFT-join INTERSECTS whose SELECT list contains a ``CASE`` - expression (the ``bedtools -wao`` shape). + A LEFT / RIGHT / FULL OUTER-join INTERSECTS whose SELECT list + contains an expression — the ``bedtools -wao`` ``CASE`` shape or + plain arithmetic. When: ``transpile`` is called with ``dialect='duckdb'``. Then: It should decline to the naive-predicate plan — the unmatched half can only NULL-fill a side-attributable column, not an arbitrary - expression (this shape unblocks with #109). + expression. The INNER / SEMI / ANTI path rebuilds scalar + expressions (#109), but the decomposition stays column-only until + its unmatched half can evaluate an expression over NULL-filled + other-side leaves. """ # Arrange & act sql = transpile( - "SELECT a.chrom, CASE WHEN b.chrom IS NULL THEN 0 ELSE 1 END AS ov " - "FROM peaks a LEFT JOIN genes b ON a.interval INTERSECTS b.interval", + f"SELECT a.chrom, {projection} " + f"FROM peaks a {join} genes b ON a.interval INTERSECTS b.interval", tables=["peaks", "genes"], dialect="duckdb", ) @@ -4049,27 +4485,63 @@ def test_transpile_should_decline_outer_join_when_projection_is_expression(self) assert "SET VARIABLE __giql_iejoin_" not in sql assert 'a."chrom" = b."chrom"' in sql - def test_transpile_should_decline_outer_join_when_order_by(self): - """Test that a top-level ORDER BY under an outer join declines. + def test_query_should_match_naive_plan_when_declined_wao_expression_runs(self, conn): + """Test that the declined ``bedtools -wao`` shape executes like the naive plan. Given: - A LEFT-join INTERSECTS carrying a top-level ``ORDER BY``. + Peaks with one matched and one unmatched row, and a LEFT-join + INTERSECTS projecting the overlap width guarded by a ``CASE`` on + the right side's NULL. When: - ``transpile`` is called with ``dialect='duckdb'``. + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. Then: - It should decline to the naive-predicate plan, because the modifier - belongs on the union rather than on each half independently. + It should decline the decomposition and return the naive plan's + rows and schema — the width for the matched row and ``0`` for the + unmatched one. """ - # Arrange & act - sql = transpile( - "SELECT a.chrom, a.start FROM peaks a " - "LEFT JOIN genes b ON a.interval INTERSECTS b.interval " - "ORDER BY a.start", - tables=["peaks", "genes"], - dialect="duckdb", - ) - - # Assert + # Arrange + _make_table( + conn, + "peaks", + [("chr1", 100, 200, "p1", 1, "+"), ("chr1", 900, 950, "p2", 2, "+")], + ) + _make_table(conn, "genes", [("chr1", 150, 250, "g1", 1, "-")]) + query = ( + "SELECT a.chrom, a.start, CASE WHEN b.start IS NULL THEN 0 " + 'ELSE LEAST(a."end", b."end") - GREATEST(a.start, b.start) END AS ov ' + "FROM peaks a LEFT JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = _assert_agrees_with_naive(conn, query) + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + assert rows == [("chr1", 100, 50), ("chr1", 900, 0)] + + def test_transpile_should_decline_outer_join_when_order_by(self): + """Test that a top-level ORDER BY under an outer join declines. + + Given: + A LEFT-join INTERSECTS carrying a top-level ``ORDER BY``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should decline to the naive-predicate plan, because the modifier + belongs on the union rather than on each half independently. + """ + # Arrange & act + sql = transpile( + "SELECT a.chrom, a.start FROM peaks a " + "LEFT JOIN genes b ON a.interval INTERSECTS b.interval " + "ORDER BY a.start", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert assert "SET VARIABLE __giql_iejoin_" not in sql assert 'a."chrom" = b."chrom"' in sql @@ -4452,38 +4924,63 @@ def test_transform_to_parts_should_return_setup_and_select_separately(self): assert select.startswith("SELECT ") assert "SET VARIABLE" not in select - def test_transform_to_parts_should_not_mutate_its_query(self): + @pytest.mark.parametrize( + "query", + [ + "SELECT a.chrom, a.start FROM peaks a " + "LEFT JOIN genes b ON a.interval INTERSECTS b.interval", + "SELECT a.chrom, GREATEST(a.start, b.start) AS s, " + 'LEAST(a."end", b."end") AS e FROM peaks a ' + "JOIN genes b ON a.interval INTERSECTS b.interval", + 'SELECT a.chrom, a."end" - a.start AS len FROM peaks a ' + "SEMI JOIN genes b ON a.interval INTERSECTS b.interval", + "SELECT a.chrom, a.start + 1 AS s FROM peaks a " + "ANTI JOIN genes b ON a.interval INTERSECTS b.interval " + "WHERE a.score > 1", + "SELECT a.chrom, a.start + 1 AS s, COUNT(*) AS n FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom, a.start ORDER BY s", + ], + ids=[ + "left_outer_columns", + "inner_clipped_span", + "semi", + "anti_where", + "inner_group_order", + ], + ) + def test_transform_to_parts_should_not_mutate_its_query(self, query): """Test that the primitive leaves the AST it is handed unchanged. Given: - A parsed decomposed outer join. + A parsed decomposed outer join, or an INNER / SEMI / ANTI join + whose SELECT list carries a scalar expression, with or without + modifiers. When: transform_to_parts is called on it. Then: - The AST should be unchanged. + It should return the parts and leave the AST unchanged. The registry entry point passes the live statement AST that pass 3 goes on transforming, and the naive-predicate fallback rewrites that same node in place when this path declines. Every composed rewrite here now transpiles a copy rather than the caller's node, which is what lets this - hold without each of them defending it. + hold without each of them defending it — including the leaf + pre-allocation and expression rebuild the scalar cases exercise (#109). """ # Arrange tables = Tables() for name in ("peaks", "genes"): tables.register(name, Table(name)) transformer = IntersectsDuckDBIEJoinTransformer(tables) - ast = parse_one( - "SELECT a.chrom, a.start FROM peaks a " - "LEFT JOIN genes b ON a.interval INTERSECTS b.interval", - dialect=GIQLDialect, - ) + ast = parse_one(query, dialect=GIQLDialect) before = repr(ast) # Act - transformer.transform_to_parts(ast) + parts = transformer.transform_to_parts(ast) # Assert + assert parts is not None assert repr(ast) == before @pytest.mark.parametrize( @@ -4891,23 +5388,7 @@ def test_query_should_plan_both_decomposition_halves_through_ie_join(self, conn) report nothing at all about the unmatched half. """ # Arrange - conn.execute("DROP TABLE IF EXISTS peaks") - conn.execute(""" - CREATE TABLE peaks AS - SELECT 'chr' || ((i % 4) + 1) AS chrom, - ((i * 37) % 100000)::INTEGER AS "start", - (((i * 37) % 100000) + 200)::INTEGER AS "end" - FROM range(20000) t(i) - """) - conn.execute("DROP TABLE IF EXISTS genes") - conn.execute(""" - CREATE TABLE genes AS - SELECT 'chr' || ((i % 4) + 1) AS chrom, - ((i * 41 + 17) % 100000)::INTEGER AS "start", - (((i * 41 + 17) % 100000) + 200)::INTEGER AS "end" - FROM range(20000) t(i) - WHERE (i % 4) + 1 <= 3 - """) + _make_planner_scale_tables(conn, genes_where="(i % 4) + 1 <= 3") sql = transpile( "SELECT a.chrom, a.start, b.start AS bs FROM peaks a " "LEFT JOIN genes b ON a.interval INTERSECTS b.interval", @@ -5664,224 +6145,1133 @@ def test_query_should_propagate_user_alias_on_qualified_column(self, conn): assert column_names == ["s"] assert rows == [(100,)] - def test_query_should_treat_touching_half_open_intervals_as_non_overlapping( - self, conn - ): - """Test that half-open intervals that share an endpoint do not overlap. + def test_query_should_match_naive_plan_for_both_side_expression(self, peaks_genes): + """Test that the clipped-span projection runs on the IEJoin and matches naive. Given: - Two half-open tables where ``peaks[100, 200)`` ends exactly - where ``genes[200, 300)`` begins. + The ``peaks`` / ``genes`` fixture tables and the ``bedtools + intersect`` clipped-span projection ``GREATEST(a.start, b.start)`` / + ``LEAST(a."end", b."end")`` over an INTERSECTS INNER join. When: - The DuckDB-dialect SQL is executed. + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. Then: - It should return an empty result (touching half-open - intervals do not overlap). + It should engage the IEJoin and return the naive plan's rows and + schema — the clipped overlap span of each overlapping pair (#109). """ # Arrange - _make_table(conn, "peaks", [("chr1", 100, 200, "p", 0, "+")]) - _make_table(conn, "genes", [("chr1", 200, 300, "g", 0, "+")]) - sql = transpile( - """ - SELECT a.start AS a_s, b.start AS b_s - FROM peaks a - JOIN genes b ON a.interval INTERSECTS b.interval - """, - tables=[ - Table("peaks", coordinate_system="0based", interval_type="half_open"), - Table("genes", coordinate_system="0based", interval_type="half_open"), - ], - dialect="duckdb", + conn = peaks_genes + query = ( + "SELECT a.chrom, GREATEST(a.start, b.start) AS start, " + 'LEAST(a."end", b."end") AS "end" ' + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" ) # Act - rows = conn.execute(sql).fetchall() + sql = _assert_agrees_with_naive(conn, query) + rows = sorted(conn.execute(sql).fetchall()) # Assert - assert rows == [] + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [("chr1", 150, 200), ("chr1", 500, 600), ("chr2", 100, 150)] - def test_query_should_handle_mixed_coordinate_systems(self, conn): - """Test semantic equivalence across mixed coordinate systems. + def test_query_should_match_naive_plan_for_single_side_expression(self, peaks_genes): + """Test that a single-side expression runs on the IEJoin and matches naive. Given: - ``peaks`` declared as 1-based closed (``[100, 200]``) and - ``genes`` declared as 0-based half-open (``[99, 200)``) - describing the same genomic span. + The ``peaks`` / ``genes`` fixture tables and a ``a.start + 1 AS s`` + projection over an INTERSECTS INNER join. When: - The DuckDB-dialect SQL is executed. + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. Then: - It should report the pair as overlapping (both canonicalize - to the same 0-based half-open span). + It should engage the IEJoin and return the naive plan's rows and + schema — one shifted start per overlapping pair (#109). """ # Arrange - _make_table(conn, "peaks", [("chr1", 100, 200, "p", 0, "+")]) - _make_table(conn, "genes", [("chr1", 99, 200, "g", 0, "+")]) - sql = transpile( - """ - SELECT a.start AS a_s, b.start AS b_s - FROM peaks a - JOIN genes b ON a.interval INTERSECTS b.interval - """, - tables=[ - Table("peaks", coordinate_system="1based", interval_type="closed"), - Table("genes", coordinate_system="0based", interval_type="half_open"), - ], - dialect="duckdb", + conn = peaks_genes + query = ( + "SELECT a.start + 1 AS s " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" ) # Act - rows = conn.execute(sql).fetchall() + sql = _assert_agrees_with_naive(conn, query) + rows = sorted(conn.execute(sql).fetchall()) # Assert - assert rows == [(100, 99)] + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [(101,), (101,), (501,)] - def test_query_should_apply_one_based_offset_when_only_left_table_is_one_based( - self, conn + def test_query_should_match_naive_plan_when_expression_sits_alongside_plain_columns( # noqa: E501 + self, peaks_genes ): - """Test that the 1-based offset is applied per side, not globally. + """Test that an expression mixed with plain columns matches the naive plan. Given: - ``peaks`` declared as 1-based half-open and ``genes`` declared - as 0-based half-open, with values that overlap iff the offset - is applied to the peaks side - (``peaks[100, 101)`` and ``genes[99, 100)``). + The ``peaks`` / ``genes`` fixture tables and the ``bedtools + intersect -wo`` shape: plain columns from both sides alongside the + overlap width ``LEAST(a."end", b."end") - GREATEST(a.start, + b.start) AS ov``. When: - The DuckDB-dialect SQL is executed. + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. Then: - It should report the overlap; if the offset were skipped no - overlap would be reported. + It should engage the IEJoin and return the naive plan's rows and + schema, with the plain columns and the expression interleaved in + SELECT-list order (#109). """ # Arrange - _make_table(conn, "peaks", [("chr1", 100, 101, "p", 0, "+")]) - _make_table(conn, "genes", [("chr1", 99, 100, "g", 0, "+")]) - sql_offset = transpile( - """ - SELECT a.start AS a_s, b.start AS b_s - FROM peaks a - JOIN genes b ON a.interval INTERSECTS b.interval - """, - tables=[ - Table( - "peaks", - coordinate_system="1based", - interval_type="half_open", - ), - Table( - "genes", - coordinate_system="0based", - interval_type="half_open", - ), - ], - dialect="duckdb", - ) - sql_no_offset = transpile( - """ - SELECT a.start AS a_s, b.start AS b_s - FROM peaks a - JOIN genes b ON a.interval INTERSECTS b.interval - """, - tables=[ - Table( - "peaks", - coordinate_system="0based", - interval_type="half_open", - ), - Table( - "genes", - coordinate_system="0based", - interval_type="half_open", - ), - ], - dialect="duckdb", + conn = peaks_genes + query = ( + "SELECT a.chrom, a.start, " + 'LEAST(a."end", b."end") - GREATEST(a.start, b.start) AS ov, ' + "b.name " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" ) # Act - rows_offset = conn.execute(sql_offset).fetchall() - rows_no_offset = conn.execute(sql_no_offset).fetchall() + sql = _assert_agrees_with_naive(conn, query) + rows = sorted(conn.execute(sql).fetchall()) # Assert - assert rows_offset == [(100, 99)] - assert rows_no_offset == [] + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [ + ("chr1", 100, 50, "g1"), + ("chr1", 500, 100, "g2"), + ("chr2", 100, 50, "g4"), + ] - def test_query_should_return_empty_when_left_table_is_empty(self, conn): - """Test the empty-left input case for the IEJoin path. + def test_query_should_match_naive_plan_for_expression_under_semi_join( + self, peaks_genes + ): + """Test that a left-side expression under SEMI JOIN matches the naive plan. Given: - ``peaks`` empty and ``genes`` populated with one row. + The ``peaks`` / ``genes`` fixture tables and a ``a."end" - a.start + AS len`` projection under a SEMI JOIN INTERSECTS. When: - The DuckDB-dialect SQL is executed. + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. Then: - It should return an empty result without raising. + It should engage the IEJoin (the correlated ``EXISTS`` form) and + return the naive plan's rows and schema — one width per left row + with at least one overlap (#109). """ # Arrange - _make_table(conn, "peaks", []) - _make_table(conn, "genes", [("chr1", 150, 250, "g1", 0, "+")]) - sql = transpile( - """ - SELECT a.start AS a_s, b.start AS b_s - FROM peaks a - JOIN genes b ON a.interval INTERSECTS b.interval - """, - tables=["peaks", "genes"], - dialect="duckdb", + conn = peaks_genes + query = ( + 'SELECT a.chrom, a."end" - a.start AS len ' + "FROM peaks a SEMI JOIN genes b ON a.interval INTERSECTS b.interval" ) # Act - rows = conn.execute(sql).fetchall() + sql = _assert_agrees_with_naive(conn, query) + rows = sorted(conn.execute(sql).fetchall()) # Assert - assert rows == [] + assert "SET VARIABLE __giql_iejoin_" in sql + assert "EXISTS" in sql + assert rows == [("chr1", 100), ("chr1", 100), ("chr2", 100)] - def test_query_should_return_empty_when_right_table_is_empty(self, conn): - """Test the empty-right input case for the IEJoin path. + def test_query_should_match_naive_plan_when_expression_combined_with_group_by_and_order_by( # noqa: E501 + self, peaks_genes + ): + """Test that an expression composes with GROUP BY / ORDER BY on the IEJoin. Given: - ``peaks`` populated with one row and ``genes`` empty. + The ``peaks`` / ``genes`` fixture tables and a query projecting + ``a.start + 1 AS s`` next to ``COUNT(*)``, grouped on the leaf column + and ordered by the expression's alias. When: - The DuckDB-dialect SQL is executed. + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. Then: - It should return an empty result without raising. + It should engage the IEJoin and return the naive plan's rows and + schema, in ``ORDER BY`` order — the modifier rewriter and the + expression rebuild share one inner-alias map (#109). """ # Arrange - _make_table(conn, "peaks", [("chr1", 100, 200, "p1", 0, "+")]) - _make_table(conn, "genes", []) - sql = transpile( - """ - SELECT a.start AS a_s, b.start AS b_s - FROM peaks a - JOIN genes b ON a.interval INTERSECTS b.interval - """, - tables=["peaks", "genes"], - dialect="duckdb", + conn = peaks_genes + query = ( + "SELECT a.chrom, a.start + 1 AS s, COUNT(*) AS n " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom, a.start ORDER BY s, a.chrom" ) # Act + sql = _assert_agrees_with_naive(conn, query) rows = conn.execute(sql).fetchall() # Assert - assert rows == [] + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [("chr1", 101, 1), ("chr2", 101, 1), ("chr1", 501, 1)] - def test_query_should_return_typed_empty_result_when_chromosome_intersection_is_empty( - self, conn - ): - """Test that the empty-schema fallback exposes correctly typed columns. + def test_query_should_plan_expression_projection_through_ie_join(self, conn): + """Test that an expression projection's dynamic SQL still reaches IE_JOIN. Given: - Two tables whose chromosome sets are disjoint so the - chromosome-intersection ``string_agg`` is NULL. + An INTERSECTS INNER join over enough rows for the planner to choose + a range join, projecting the clipped overlap span. When: - The DuckDB-dialect SQL is executed. + The emitted session variable's dynamic SQL is planned. Then: - It should fetch an empty result whose column descriptions - match the source-table schema declared by ``_make_table`` - (chrom as VARCHAR, start/end as INTEGER). + It should plan through ``IE_JOIN`` and not the quadratic + ``BLOCKWISE_NL_JOIN`` — the projection rebuild leaves the + per-chromosome join untouched (#109). """ # Arrange - _make_table(conn, "peaks", [("chrA", 100, 200, "p", 0, "+")]) - _make_table(conn, "genes", [("chrB", 100, 200, "g", 0, "+")]) + _make_planner_scale_tables(conn) sql = transpile( - """ - SELECT a.chrom AS a_chrom, a.start AS a_start, a.end AS a_end, + "SELECT a.chrom, GREATEST(a.start, b.start) AS start, " + 'LEAST(a."end", b."end") AS "end" ' + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + plan = _explain_dynamic_sql(conn, sql) + + # Assert + assert "IE_JOIN" in plan + assert "BLOCKWISE_NL_JOIN" not in plan + + @settings( + max_examples=30, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + peaks=_interval_lists(), + genes=_interval_lists(), + expression=_expression_trees(_SCALAR_LEAVES), + ) + def test_query_should_match_naive_plan_for_random_scalar_expressions( + self, conn, peaks, genes, expression + ): + """Test cross-plan equivalence over randomized scalar expression projections. + + Given: + A Hypothesis-generated pair of small interval lists and a random + scalar expression tree over the ``a`` / ``b`` start and end columns + (arithmetic, ``GREATEST`` / ``LEAST``, ``ABS``, ``CASE``). + When: + The same query is transpiled with ``dialect=None`` and with + ``dialect='duckdb'`` and both are executed. + Then: + The DuckDB plan should engage the IEJoin and return the naive + plan's rows and result schema (#109). + """ + # Arrange + peak_rows = [(c, s, s + length) for (c, s, length) in peaks] + gene_rows = [(c, s, s + length) for (c, s, length) in genes] + _make_interval_tables(conn, peak_rows, gene_rows) + query = ( + f"SELECT a.chrom AS c, {expression} AS v, b.start AS bs " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = _assert_agrees_with_naive(conn, query) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + + @settings( + max_examples=30, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + peaks=_interval_lists(), + genes=_interval_lists(), + kind=st.sampled_from(["SEMI", "ANTI"]), + expression=_expression_trees(_LEFT_ONLY_LEAVES), + threshold=st.integers(min_value=0, max_value=100), + ) + def test_query_should_match_naive_plan_for_random_left_only_expressions( + self, conn, peaks, genes, kind, expression, threshold + ): + """Test cross-plan equivalence for random expressions under SEMI / ANTI. + + Given: + A Hypothesis-generated pair of small interval lists, a SEMI or ANTI + join, a random scalar expression tree over the left side's start + and end columns, a ``WHERE`` threshold on the left start, and an + ``ORDER BY`` on the expression's alias. + When: + The same query is transpiled with ``dialect=None`` and with + ``dialect='duckdb'`` and both are executed. + Then: + The DuckDB plan should engage the correlated ``EXISTS`` / + ``NOT EXISTS`` IEJoin and return the naive plan's rows, schema + and order — the rebuilt expression composes with the outer-wrapper + WHERE filter (#109, #200). + """ + # Arrange + peak_rows = [(c, s, s + length) for (c, s, length) in peaks] + gene_rows = [(c, s, s + length) for (c, s, length) in genes] + _make_interval_tables(conn, peak_rows, gene_rows) + query = ( + f"SELECT a.chrom AS c, {expression} AS v FROM peaks a " + f"{kind} JOIN genes b ON a.interval INTERSECTS b.interval " + f"WHERE a.start >= {threshold} ORDER BY v, a.chrom, a.start" + ) + + # Act + sql = _assert_agrees_with_naive(conn, query) + ordered_dd = conn.execute(sql).fetchall() + ordered_naive = conn.execute( + transpile(query, tables=["peaks", "genes"]) + ).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert "EXISTS" in sql + assert ordered_dd == ordered_naive + + @settings( + max_examples=30, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + peaks=_interval_lists(), + genes=_interval_lists(), + ) + def test_query_should_match_python_clipped_overlap_for_random_inputs( + self, conn, peaks, genes + ): + """Test the clipped-span projection against a Python-native reference. + + Given: + A Hypothesis-generated pair of small interval lists and the + ``bedtools intersect`` clipped-span plus ``-wo`` width projection + (``GREATEST`` / ``LEAST`` over both sides' endpoints). + When: + The DuckDB-dialect SQL is executed and compared against a + Python-native reference computing the same span per overlapping + pair. + Then: + It should return exactly the reference multiset, with every width + strictly positive (#109). + """ + # Arrange + peak_rows = [(c, s, s + length) for (c, s, length) in peaks] + gene_rows = [(c, s, s + length) for (c, s, length) in genes] + _make_interval_tables(conn, peak_rows, gene_rows) + sql = transpile( + "SELECT a.chrom AS c, GREATEST(a.start, b.start) AS s, " + 'LEAST(a."end", b."end") AS e, ' + 'LEAST(a."end", b."end") - GREATEST(a.start, b.start) AS ov ' + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == _python_clipped_overlap(peak_rows, gene_rows) + assert all(row[3] > 0 for row in rows) + + def test_query_should_match_naive_plan_for_non_numeric_expressions( + self, peaks_genes + ): + """Test that string, boolean and cast expressions match the naive plan. + + Given: + The ``peaks`` / ``genes`` fixture tables and a SELECT list mixing + a ``CASE`` yielding string literals, ``||`` concatenation, a + boolean comparison, and a ``CAST`` to ``DOUBLE``. + When: + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. + Then: + It should engage the IEJoin and return the naive plan's rows and + schema — the rebuild is type-agnostic, not integer-only (#109). + """ + # Arrange + conn = peaks_genes + query = ( + "SELECT CASE WHEN a.start < b.start THEN 'a' ELSE 'b' END AS who, " + "a.name || '_' || b.name AS pair, a.start < b.start AS flag, " + "CAST(a.start AS DOUBLE) AS s " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = _assert_agrees_with_naive(conn, query) + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [ + ("a", "p1_g1", True, 100.0), + ("b", "p3_g2", False, 500.0), + ("b", "p4_g4", False, 100.0), + ] + + def test_query_should_match_naive_plan_for_expression_under_anti_join_with_where( + self, peaks_genes + ): + """Test that an expression under ANTI JOIN with a WHERE residual matches naive. + + Given: + The ``peaks`` / ``genes`` fixture tables and a ``a."end" - a.start + AS len`` projection under an ANTI JOIN filtered by ``WHERE a.score > + 20``. + When: + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. + Then: + It should engage the ``NOT EXISTS`` IEJoin and return the naive + plan's rows and schema — the WHERE residual is applied on the outer + wrapper beside the rebuilt expression (#109, #200). + """ + # Arrange + conn = peaks_genes + query = ( + 'SELECT a.chrom, a."end" - a.start AS len ' + "FROM peaks a ANTI JOIN genes b ON a.interval INTERSECTS b.interval " + "WHERE a.score > 20" + ) + + # Act + sql = _assert_agrees_with_naive(conn, query) + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert "NOT EXISTS" in sql + assert rows == [("chr2", 100)] + + def test_query_should_match_naive_plan_for_expression_in_implicit_cross_join( + self, peaks_genes + ): + """Test that the ``-wo`` width over a comma join matches the naive plan. + + Given: + The ``peaks`` / ``genes`` fixture tables and an implicit comma join + whose INTERSECTS sits in the WHERE, projecting the overlap width + and ordering by its alias descending. + When: + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. + Then: + It should engage the IEJoin and return the naive plan's rows, + schema and order (#109). + """ + # Arrange + conn = peaks_genes + query = ( + "SELECT a.name, b.name AS gname, " + 'LEAST(a."end", b."end") - GREATEST(a.start, b.start) AS ov ' + "FROM peaks a, genes b WHERE a.interval INTERSECTS b.interval " + "ORDER BY ov DESC, a.name" + ) + + # Act + sql = _assert_agrees_with_naive(conn, query) + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [("p3", "g2", 100), ("p1", "g1", 50), ("p4", "g4", 50)] + + def test_query_should_match_naive_plan_when_distinct_dedups_expression( + self, peaks_genes + ): + """Test that DISTINCT over an expression projection matches the naive plan. + + Given: + The ``peaks`` / ``genes`` fixture tables and ``SELECT DISTINCT + a.chrom, a."end" - a.start AS len`` ordered by the alias. + When: + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. + Then: + It should engage the IEJoin and return the naive plan's rows in + order — DISTINCT dedups on the rebuilt value, not the leaves (#109). + """ + # Arrange + conn = peaks_genes + query = ( + 'SELECT DISTINCT a.chrom, a."end" - a.start AS len ' + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval " + "ORDER BY len, a.chrom" + ) + + # Act + sql = _assert_agrees_with_naive(conn, query) + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [("chr1", 100), ("chr2", 100)] + + def test_query_should_match_naive_plan_when_group_by_is_expression( + self, peaks_genes + ): + """Test that grouping by an expression with HAVING matches the naive plan. + + Given: + The ``peaks`` / ``genes`` fixture tables and a query projecting the + interval length with ``COUNT(*)``, grouped by the length expression + itself and filtered by ``HAVING``. + When: + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. + Then: + It should engage the IEJoin and return the naive plan's rows — the + GROUP BY expression is rewritten over the same inner aliases as + the projection (#109). + """ + # Arrange + conn = peaks_genes + query = ( + 'SELECT a."end" - a.start AS len, COUNT(*) AS n ' + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval " + 'GROUP BY a."end" - a.start HAVING COUNT(*) > 2' + ) + + # Act + sql = _assert_agrees_with_naive(conn, query) + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [(100, 3)] + + def test_query_should_match_naive_plan_when_limit_offset_follow_expression_order( # noqa: E501 + self, peaks_genes + ): + """Test that LIMIT / OFFSET over an expression-ordered result matches naive. + + Given: + The ``peaks`` / ``genes`` fixture tables and ``a.start + 1 AS s`` + ordered by that alias with ``LIMIT 2 OFFSET 1``. + When: + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. + Then: + It should engage the IEJoin and return the naive plan's rows in + order — the second and third rows by the rebuilt value (#109). + """ + # Arrange + conn = peaks_genes + query = ( + "SELECT a.chrom, a.start + 1 AS s " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval " + "ORDER BY s, a.chrom LIMIT 2 OFFSET 1" + ) + + # Act + sql = _assert_agrees_with_naive(conn, query) + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [("chr2", 101), ("chr1", 501)] + + def test_query_should_match_naive_plan_for_expression_over_using_join( + self, peaks_genes + ): + """Test that an expression over a ``USING (chrom)`` join matches naive. + + Given: + The ``peaks`` / ``genes`` fixture tables joined with ``USING + (chrom)`` and a WHERE-INTERSECTS, projecting a concatenation over + the USING column and the right side plus an arithmetic expression. + When: + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. + Then: + It should engage the IEJoin and return the naive plan's rows and + schema (#109). + """ + # Arrange + conn = peaks_genes + query = ( + "SELECT a.chrom || ':' || b.name AS c, a.start + 1 AS s " + "FROM peaks a JOIN genes b USING (chrom) " + "WHERE a.interval INTERSECTS b.interval" + ) + + # Act + sql = _assert_agrees_with_naive(conn, query) + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [("chr1:g1", 101), ("chr1:g2", 501), ("chr2:g4", 101)] + + def test_query_should_match_naive_plan_for_expression_over_custom_columns( + self, conn + ): + """Test that expression leaves resolve through custom column names. + + Given: + Tables whose genomic columns are ``chromosome`` / ``start_pos`` / + ``end_pos`` registered with matching ``Table`` configs, and the + clipped-span plus width projection over those columns. + When: + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. + Then: + It should engage the IEJoin and return the naive plan's rows and + schema (#109). + """ + # Arrange + for table, rows in ( + ("peaks", [("chr1", 100, 200), ("chr1", 1000, 1100)]), + ("genes", [("chr1", 150, 250), ("chr1", 5000, 6000)]), + ): + conn.execute( + f"CREATE TABLE {table} " + "(chromosome VARCHAR, start_pos INTEGER, end_pos INTEGER)" + ) + conn.executemany(f"INSERT INTO {table} VALUES (?, ?, ?)", rows) + tables = [ + Table(name, chrom_col="chromosome", start_col="start_pos", end_col="end_pos") + for name in ("peaks", "genes") + ] + query = ( + "SELECT a.chromosome AS c, GREATEST(a.start_pos, b.start_pos) AS s, " + "LEAST(a.end_pos, b.end_pos) - GREATEST(a.start_pos, b.start_pos) AS ov " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = _assert_agrees_with_naive(conn, query, tables=tables) + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [("chr1", 150, 50)] + + def test_query_should_match_naive_plan_for_expression_over_one_based_closed( + self, conn + ): + """Test that the rebuilt expression sees raw one-based closed coordinates. + + Given: + One-based closed tables where a peak ends exactly where a gene + starts, and the clipped-span projection over their endpoints. + When: + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. + Then: + It should engage the IEJoin, canonicalise the join predicate so the + touching closed endpoint counts as an overlap, and return the raw + stored values in the expression like the naive plan (#109). + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 200, "p", 1, "+")]) + _make_table(conn, "genes", [("chr1", 200, 300, "g", 2, "-")]) + tables = [ + Table(name, coordinate_system="1based", interval_type="closed") + for name in ("peaks", "genes") + ] + query = ( + 'SELECT GREATEST(a.start, b.start) AS s, LEAST(a."end", b."end") AS e ' + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = _assert_agrees_with_naive(conn, query, tables=tables) + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert "- 1" in sql + assert rows == [(200, 200)] + + def test_query_should_match_naive_plan_when_expression_leaves_use_mixed_case_aliases( # noqa: E501 + self, peaks_genes + ): + """Test that case-folded aliases resolve inside expression leaves. + + Given: + The ``peaks`` / ``genes`` fixture tables aliased ``p`` and ``G`` + and referenced as ``P`` / ``G`` / ``g`` across the SELECT list and + the INTERSECTS. + When: + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. + Then: + It should engage the IEJoin and return the naive plan's rows and + schema — alias matching inside expression leaves is case-folded + like DuckDB's own (#109). + """ + # Arrange + conn = peaks_genes + query = ( + "SELECT P.name, P.start < G.start AS flag " + "FROM peaks p JOIN genes G ON p.interval INTERSECTS g.interval" + ) + + # Act + sql = _assert_agrees_with_naive(conn, query) + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [("p1", True), ("p3", False), ("p4", False)] + + def test_query_should_execute_unaliased_expression_under_synthesized_name( + self, peaks_genes + ): + """Test that an unaliased expression projection executes under its name. + + Given: + The ``peaks`` / ``genes`` fixture tables and an unaliased + ``a.start + 1`` at SELECT position 1. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + It should return one shifted start per overlapping pair under the + output name ``__giql_expr_1`` (#109). + """ + # Arrange + conn = peaks_genes + sql = transpile( + "SELECT a.chrom, a.start + 1 " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + result = conn.execute(sql) + column_names = [d[0] for d in result.description] + rows = sorted(result.fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert column_names == ["chrom", "__giql_expr_1"] + assert rows == [("chr1", 101), ("chr1", 501), ("chr2", 101)] + + def test_query_should_match_naive_plan_when_projection_is_distance_operator( + self, peaks_genes + ): + """Test that a DISTANCE projection now rides the IEJoin path. + + Given: + The ``peaks`` / ``genes`` fixture tables and ``DISTANCE(a.interval, + b.interval) AS d`` in the SELECT list — a GIQL operator pass 3 + expands to a scalar expression before the join is rewritten. + When: + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. + Then: + It should engage the IEJoin (the expanded operator is a + rebuildable expression rather than a decline) and return the naive + plan's rows and schema (#109). + """ + # Arrange + conn = peaks_genes + query = ( + "SELECT a.chrom, DISTANCE(a.interval, b.interval) AS d " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = _assert_agrees_with_naive(conn, query) + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [("chr1", 0), ("chr1", 0), ("chr2", 0)] + + @pytest.mark.parametrize("kind", ["SEMI", "ANTI"]) + def test_query_should_plan_left_only_expression_projection_through_ie_join( + self, conn, kind + ): + """Test that an expression under SEMI / ANTI keeps the range-join plan. + + Given: + An INTERSECTS SEMI / ANTI join over enough rows for the planner to + choose a range join, projecting the left interval's length. + When: + The emitted session variable's dynamic SQL is planned. + Then: + It should plan through ``IE_JOIN`` and not the quadratic + ``BLOCKWISE_NL_JOIN`` — the rebuild leaves the correlated + ``EXISTS`` form untouched (#109, #208). + """ + # Arrange + _make_planner_scale_tables(conn) + sql = transpile( + f'SELECT a."end" - a.start AS len FROM peaks a {kind} JOIN genes b ' + "ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + plan = _explain_dynamic_sql(conn, sql) + + # Assert + assert "IE_JOIN" in plan + assert "BLOCKWISE_NL_JOIN" not in plan + + @pytest.mark.parametrize( + "kind, projection", + [ + ( + "", + "GREATEST(a.start, b.start) AS s, " + 'LEAST(a."end", b."end") - GREATEST(a.start, b.start) AS ov', + ), + ("SEMI", 'a."end" - a.start AS len'), + ("ANTI", 'a."end" - a.start AS len'), + ], + ids=["inner", "semi", "anti"], + ) + def test_query_should_match_naive_plan_for_expression_when_partition_exceeds_ceiling( # noqa: E501 + self, conn, kind, projection + ): + """Test that the unpartitioned fallback carries the rebuilt expression. + + Given: + 3,000 intervals per side spread over 300 shared chromosomes -- + above ``MAX_PER_CHROM_PARTITIONS`` -- and an INNER / SEMI / ANTI + join projecting a scalar expression. + When: + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. + Then: + It should take the unpartitioned branch and return the naive + plan's rows and schema — that branch projects the same inner leaf + list the expression is rebuilt over (#109). + """ + # Arrange + for table, offset in (("peaks", 0), ("genes", 173)): + conn.execute(f"DROP TABLE IF EXISTS {table}") + conn.execute(f""" + CREATE TABLE {table} AS + SELECT 'chr' || (i % 300) AS chrom, + ((i * 37 + {offset}) % 5000)::INTEGER AS "start", + (((i * 37 + {offset}) % 5000) + 300)::INTEGER AS "end" + FROM range(3000) t(i) + """) + query = ( + f"SELECT a.chrom, {projection} FROM peaks a {kind} JOIN genes b " + "ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = _assert_agrees_with_naive(conn, query) + + # Assert + assert f"> {MAX_PER_CHROM_PARTITIONS}" in sql + + def test_query_should_type_expression_columns_in_empty_schema_fallback(self, conn): + """Test that the typed empty result carries rebuilt expression columns. + + Given: + Tables whose chromosomes are disjoint, so no per-chromosome branch + is emitted, and a SELECT list of expressions yielding VARCHAR, + INTEGER, DOUBLE and BIGINT values. + When: + The query is transpiled with ``dialect='duckdb'`` and executed, and + separately with ``dialect=None``. + Then: + It should return no rows with the naive plan's column names and + types — the empty-schema fallback evaluates each expression over + the ``WHERE FALSE`` leaves (#109). + """ + # Arrange + _make_table(conn, "peaks", [("chrA", 100, 200, "p", 1, "+")]) + _make_table(conn, "genes", [("chrB", 150, 250, "g", 2, "-")]) + query = ( + "SELECT a.chrom AS c, GREATEST(a.start, b.start) AS s, " + 'LEAST(a."end", b."end") - GREATEST(a.start, b.start) AS ov, ' + "a.name || b.name AS nm, a.start / 2 AS h, CAST(a.start AS BIGINT) AS big " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = _assert_agrees_with_naive(conn, query) + rows, schema = _rows_and_schema(conn, sql) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [] + assert schema == [ + ("c", "VARCHAR"), + ("s", "INTEGER"), + ("ov", "INTEGER"), + ("nm", "VARCHAR"), + ("h", "DOUBLE"), + ("big", "BIGINT"), + ] + + def test_query_should_isolate_results_when_expressions_share_a_session_variable( # noqa: E501 + self, conn + ): + """Test that two expressions over one leaf set share a variable safely. + + Given: + One overlapping pair, and ``GREATEST(a.start, b.start)`` and + ``LEAST(a.start, b.start)`` transpiled as separate queries whose + inner projections are identical. + When: + Both setups run on one connection and then both final SELECTs. + Then: + It should mint one content-addressed session variable for both + (the name keys on the inner leaf list) and each SELECT should still + return its own expression's value (#109). + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 200, "p", 1, "+")]) + _make_table(conn, "genes", [("chr1", 150, 250, "g", 2, "-")]) + template = ( + "SELECT {fn}(a.start, b.start) AS v " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" + ) + greatest_sql = transpile( + template.format(fn="GREATEST"), tables=["peaks", "genes"], dialect="duckdb" + ) + least_sql = transpile( + template.format(fn="LEAST"), tables=["peaks", "genes"], dialect="duckdb" + ) + greatest_setup, greatest_select = greatest_sql.split(";\n") + least_setup, least_select = least_sql.split(";\n") + + # Act + conn.execute(greatest_setup) + conn.execute(least_setup) + greatest_rows = conn.execute(greatest_select).fetchall() + least_rows = conn.execute(least_select).fetchall() + + # Assert + assert _dynamic_sql_variables(greatest_sql) == _dynamic_sql_variables(least_sql) + assert greatest_rows == [(150,)] + assert least_rows == [(100,)] + + def test_query_should_treat_touching_half_open_intervals_as_non_overlapping( + self, conn + ): + """Test that half-open intervals that share an endpoint do not overlap. + + Given: + Two half-open tables where ``peaks[100, 200)`` ends exactly + where ``genes[200, 300)`` begins. + When: + The DuckDB-dialect SQL is executed. + Then: + It should return an empty result (touching half-open + intervals do not overlap). + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 200, "p", 0, "+")]) + _make_table(conn, "genes", [("chr1", 200, 300, "g", 0, "+")]) + sql = transpile( + """ + SELECT a.start AS a_s, b.start AS b_s + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=[ + Table("peaks", coordinate_system="0based", interval_type="half_open"), + Table("genes", coordinate_system="0based", interval_type="half_open"), + ], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert rows == [] + + def test_query_should_handle_mixed_coordinate_systems(self, conn): + """Test semantic equivalence across mixed coordinate systems. + + Given: + ``peaks`` declared as 1-based closed (``[100, 200]``) and + ``genes`` declared as 0-based half-open (``[99, 200)``) + describing the same genomic span. + When: + The DuckDB-dialect SQL is executed. + Then: + It should report the pair as overlapping (both canonicalize + to the same 0-based half-open span). + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 200, "p", 0, "+")]) + _make_table(conn, "genes", [("chr1", 99, 200, "g", 0, "+")]) + sql = transpile( + """ + SELECT a.start AS a_s, b.start AS b_s + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=[ + Table("peaks", coordinate_system="1based", interval_type="closed"), + Table("genes", coordinate_system="0based", interval_type="half_open"), + ], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert rows == [(100, 99)] + + def test_query_should_apply_one_based_offset_when_only_left_table_is_one_based( + self, conn + ): + """Test that the 1-based offset is applied per side, not globally. + + Given: + ``peaks`` declared as 1-based half-open and ``genes`` declared + as 0-based half-open, with values that overlap iff the offset + is applied to the peaks side + (``peaks[100, 101)`` and ``genes[99, 100)``). + When: + The DuckDB-dialect SQL is executed. + Then: + It should report the overlap; if the offset were skipped no + overlap would be reported. + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 101, "p", 0, "+")]) + _make_table(conn, "genes", [("chr1", 99, 100, "g", 0, "+")]) + sql_offset = transpile( + """ + SELECT a.start AS a_s, b.start AS b_s + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=[ + Table( + "peaks", + coordinate_system="1based", + interval_type="half_open", + ), + Table( + "genes", + coordinate_system="0based", + interval_type="half_open", + ), + ], + dialect="duckdb", + ) + sql_no_offset = transpile( + """ + SELECT a.start AS a_s, b.start AS b_s + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=[ + Table( + "peaks", + coordinate_system="0based", + interval_type="half_open", + ), + Table( + "genes", + coordinate_system="0based", + interval_type="half_open", + ), + ], + dialect="duckdb", + ) + + # Act + rows_offset = conn.execute(sql_offset).fetchall() + rows_no_offset = conn.execute(sql_no_offset).fetchall() + + # Assert + assert rows_offset == [(100, 99)] + assert rows_no_offset == [] + + def test_query_should_return_empty_when_left_table_is_empty(self, conn): + """Test the empty-left input case for the IEJoin path. + + Given: + ``peaks`` empty and ``genes`` populated with one row. + When: + The DuckDB-dialect SQL is executed. + Then: + It should return an empty result without raising. + """ + # Arrange + _make_table(conn, "peaks", []) + _make_table(conn, "genes", [("chr1", 150, 250, "g1", 0, "+")]) + sql = transpile( + """ + SELECT a.start AS a_s, b.start AS b_s + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert rows == [] + + def test_query_should_return_empty_when_right_table_is_empty(self, conn): + """Test the empty-right input case for the IEJoin path. + + Given: + ``peaks`` populated with one row and ``genes`` empty. + When: + The DuckDB-dialect SQL is executed. + Then: + It should return an empty result without raising. + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 200, "p1", 0, "+")]) + _make_table(conn, "genes", []) + sql = transpile( + """ + SELECT a.start AS a_s, b.start AS b_s + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert rows == [] + + def test_query_should_return_typed_empty_result_when_chromosome_intersection_is_empty( + self, conn + ): + """Test that the empty-schema fallback exposes correctly typed columns. + + Given: + Two tables whose chromosome sets are disjoint so the + chromosome-intersection ``string_agg`` is NULL. + When: + The DuckDB-dialect SQL is executed. + Then: + It should fetch an empty result whose column descriptions + match the source-table schema declared by ``_make_table`` + (chrom as VARCHAR, start/end as INTEGER). + """ + # Arrange + _make_table(conn, "peaks", [("chrA", 100, 200, "p", 0, "+")]) + _make_table(conn, "genes", [("chrB", 100, 200, "g", 0, "+")]) + sql = transpile( + """ + SELECT a.chrom AS a_chrom, a.start AS a_start, a.end AS a_end, b.chrom AS b_chrom, b.start AS b_start, b.end AS b_end FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval @@ -6910,18 +8300,7 @@ def test_query_should_match_python_native_overlap_for_random_inputs( # Arrange peak_rows = [(c, s, s + length) for (c, s, length) in peaks] gene_rows = [(c, s, s + length) for (c, s, length) in genes] - conn.execute("DROP TABLE IF EXISTS peaks") - conn.execute("DROP TABLE IF EXISTS genes") - conn.execute( - 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' - ) - if peak_rows: - conn.executemany("INSERT INTO peaks VALUES (?, ?, ?)", peak_rows) - conn.execute( - 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' - ) - if gene_rows: - conn.executemany("INSERT INTO genes VALUES (?, ?, ?)", gene_rows) + _make_interval_tables(conn, peak_rows, gene_rows) sql = transpile( """ SELECT a.chrom AS a_chrom, a.start AS a_start, a.end AS a_end, @@ -6947,24 +8326,8 @@ def test_query_should_match_python_native_overlap_for_random_inputs( suppress_health_check=[HealthCheck.function_scoped_fixture], ) @given( - peaks=st.lists( - st.tuples( - st.sampled_from(["chr1", "chr2"]), - st.integers(min_value=0, max_value=100), - st.integers(min_value=1, max_value=30), - ), - min_size=0, - max_size=6, - ), - genes=st.lists( - st.tuples( - st.sampled_from(["chr1", "chr2"]), - st.integers(min_value=0, max_value=100), - st.integers(min_value=1, max_value=30), - ), - min_size=0, - max_size=6, - ), + peaks=_interval_lists(), + genes=_interval_lists(), ) def test_query_should_match_naive_predicate_plan_for_random_inputs( self, conn, peaks, genes @@ -6983,18 +8346,7 @@ def test_query_should_match_naive_predicate_plan_for_random_inputs( # Arrange peak_rows = [(c, s, s + length) for (c, s, length) in peaks] gene_rows = [(c, s, s + length) for (c, s, length) in genes] - conn.execute("DROP TABLE IF EXISTS peaks") - conn.execute("DROP TABLE IF EXISTS genes") - conn.execute( - 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' - ) - if peak_rows: - conn.executemany("INSERT INTO peaks VALUES (?, ?, ?)", peak_rows) - conn.execute( - 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' - ) - if gene_rows: - conn.executemany("INSERT INTO genes VALUES (?, ?, ?)", gene_rows) + _make_interval_tables(conn, peak_rows, gene_rows) query = """ SELECT a.chrom AS a_chrom, a.start AS a_start, a.end AS a_end, b.chrom AS b_chrom, b.start AS b_start, b.end AS b_end @@ -7595,18 +8947,7 @@ def test_query_should_match_python_native_semi_overlap_for_random_inputs( # Arrange peak_rows = [(c, s, s + length) for (c, s, length) in peaks] gene_rows = [(c, s, s + length) for (c, s, length) in genes] - conn.execute("DROP TABLE IF EXISTS peaks") - conn.execute("DROP TABLE IF EXISTS genes") - conn.execute( - 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' - ) - if peak_rows: - conn.executemany("INSERT INTO peaks VALUES (?, ?, ?)", peak_rows) - conn.execute( - 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' - ) - if gene_rows: - conn.executemany("INSERT INTO genes VALUES (?, ?, ?)", gene_rows) + _make_interval_tables(conn, peak_rows, gene_rows) sql = transpile( "SELECT a.chrom, a.start, a.end " "FROM peaks a " @@ -7628,24 +8969,8 @@ def test_query_should_match_python_native_semi_overlap_for_random_inputs( suppress_health_check=[HealthCheck.function_scoped_fixture], ) @given( - peaks=st.lists( - st.tuples( - st.sampled_from(["chr1", "chr2"]), - st.integers(min_value=0, max_value=100), - st.integers(min_value=1, max_value=30), - ), - min_size=0, - max_size=6, - ), - genes=st.lists( - st.tuples( - st.sampled_from(["chr1", "chr2"]), - st.integers(min_value=0, max_value=100), - st.integers(min_value=1, max_value=30), - ), - min_size=0, - max_size=6, - ), + peaks=_interval_lists(), + genes=_interval_lists(), ) def test_query_should_match_python_native_semi_overlap_for_narrow_random_inputs( self, conn, peaks, genes @@ -7658,24 +8983,13 @@ def test_query_should_match_python_native_semi_overlap_for_narrow_random_inputs( A ``SEMI JOIN ... ON a.interval INTERSECTS b.interval`` is transpiled with ``dialect='duckdb'`` and executed. Then: - The set of returned rows should equal the Python reference of - distinct left rows with at least one overlapping right row. - """ - # Arrange - peak_rows = [(c, s, s + length) for (c, s, length) in peaks] - gene_rows = [(c, s, s + length) for (c, s, length) in genes] - conn.execute("DROP TABLE IF EXISTS peaks") - conn.execute("DROP TABLE IF EXISTS genes") - conn.execute( - 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' - ) - if peak_rows: - conn.executemany("INSERT INTO peaks VALUES (?, ?, ?)", peak_rows) - conn.execute( - 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' - ) - if gene_rows: - conn.executemany("INSERT INTO genes VALUES (?, ?, ?)", gene_rows) + The set of returned rows should equal the Python reference of + distinct left rows with at least one overlapping right row. + """ + # Arrange + peak_rows = [(c, s, s + length) for (c, s, length) in peaks] + gene_rows = [(c, s, s + length) for (c, s, length) in genes] + _make_interval_tables(conn, peak_rows, gene_rows) query = ( "SELECT a.chrom, a.start, a.end " "FROM peaks a " @@ -7734,18 +9048,7 @@ def test_query_should_match_python_native_anti_overlap_for_random_inputs( # Arrange peak_rows = [(c, s, s + length) for (c, s, length) in peaks] gene_rows = [(c, s, s + length) for (c, s, length) in genes] - conn.execute("DROP TABLE IF EXISTS peaks") - conn.execute("DROP TABLE IF EXISTS genes") - conn.execute( - 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' - ) - if peak_rows: - conn.executemany("INSERT INTO peaks VALUES (?, ?, ?)", peak_rows) - conn.execute( - 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' - ) - if gene_rows: - conn.executemany("INSERT INTO genes VALUES (?, ?, ?)", gene_rows) + _make_interval_tables(conn, peak_rows, gene_rows) sql = transpile( "SELECT a.chrom, a.start, a.end " "FROM peaks a " @@ -7767,24 +9070,8 @@ def test_query_should_match_python_native_anti_overlap_for_random_inputs( suppress_health_check=[HealthCheck.function_scoped_fixture], ) @given( - peaks=st.lists( - st.tuples( - st.sampled_from(["chr1", "chr2"]), - st.integers(min_value=0, max_value=100), - st.integers(min_value=1, max_value=30), - ), - min_size=0, - max_size=6, - ), - genes=st.lists( - st.tuples( - st.sampled_from(["chr1", "chr2"]), - st.integers(min_value=0, max_value=100), - st.integers(min_value=1, max_value=30), - ), - min_size=0, - max_size=6, - ), + peaks=_interval_lists(), + genes=_interval_lists(), ) def test_query_should_match_python_native_anti_overlap_for_narrow_random_inputs( self, conn, peaks, genes @@ -7805,18 +9092,7 @@ def test_query_should_match_python_native_anti_overlap_for_narrow_random_inputs( # Arrange peak_rows = [(c, s, s + length) for (c, s, length) in peaks] gene_rows = [(c, s, s + length) for (c, s, length) in genes] - conn.execute("DROP TABLE IF EXISTS peaks") - conn.execute("DROP TABLE IF EXISTS genes") - conn.execute( - 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' - ) - if peak_rows: - conn.executemany("INSERT INTO peaks VALUES (?, ?, ?)", peak_rows) - conn.execute( - 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' - ) - if gene_rows: - conn.executemany("INSERT INTO genes VALUES (?, ?, ?)", gene_rows) + _make_interval_tables(conn, peak_rows, gene_rows) query = ( "SELECT a.chrom, a.start, a.end " "FROM peaks a " @@ -7976,24 +9252,8 @@ def test_query_should_match_naive_plan_for_semi_join_with_where_residual(self, c suppress_health_check=[HealthCheck.function_scoped_fixture], ) @given( - peaks=st.lists( - st.tuples( - st.sampled_from(["chr1", "chr2"]), - st.integers(min_value=0, max_value=100), - st.integers(min_value=1, max_value=30), - ), - min_size=0, - max_size=6, - ), - genes=st.lists( - st.tuples( - st.sampled_from(["chr1", "chr2"]), - st.integers(min_value=0, max_value=100), - st.integers(min_value=1, max_value=30), - ), - min_size=0, - max_size=6, - ), + peaks=_interval_lists(), + genes=_interval_lists(), ) def test_query_should_match_python_native_anti_overlap_with_where_residual_random_inputs( self, conn, peaks, genes @@ -8014,18 +9274,7 @@ def test_query_should_match_python_native_anti_overlap_with_where_residual_rando # Arrange peak_rows = [(c, s, s + length) for (c, s, length) in peaks] gene_rows = [(c, s, s + length) for (c, s, length) in genes] - conn.execute("DROP TABLE IF EXISTS peaks") - conn.execute("DROP TABLE IF EXISTS genes") - conn.execute( - 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' - ) - if peak_rows: - conn.executemany("INSERT INTO peaks VALUES (?, ?, ?)", peak_rows) - conn.execute( - 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' - ) - if gene_rows: - conn.executemany("INSERT INTO genes VALUES (?, ?, ?)", gene_rows) + _make_interval_tables(conn, peak_rows, gene_rows) query = ( "SELECT a.chrom, a.start, a.end " "FROM peaks a " @@ -8721,14 +9970,23 @@ class TestTranspileDuckDBIEJoinUnsupportedProjectionFallback: """Regression tests for #204 and #205. Projections the naive-predicate plan compiles but the IEJoin projection - rebuild cannot express — expressions (``a.start + 1``), window aggregates, - ``FILTER`` clauses, scalar subqueries, aggregates nested in expressions - (``COUNT(*) * 2``), and stars nested in an aggregate argument - (``COUNT(a.*)`` / ``MIN(COLUMNS(*))``) — must decline the IEJoin and fall - back to the naive plan, keeping ``dialect="duckdb"`` consistent with every - other backend rather than hard-erroring (#205) or miscompiling (#204). A - projection with an out-of-scope column reference stays a clean transpile - error (the naive plan rejects it too). + rebuild cannot express — window aggregates, ``FILTER`` clauses, scalar + subqueries, aggregates nested in expressions (``COUNT(*) * 2``), bare + literals, and stars nested in an aggregate argument (``COUNT(a.*)`` / + ``MIN(COLUMNS(*))``) — must decline the IEJoin and fall back to the naive + plan, keeping ``dialect="duckdb"`` consistent with every other backend + rather than hard-erroring (#205) or miscompiling (#204). A projection with + an out-of-scope column reference stays a clean transpile error (the naive + plan rejects it too). A scalar expression over qualified columns + (``a.start + 1``) is no longer in this set: it is rebuilt on the IEJoin + path (#109), and these declines must not over-reach onto it. + + #109's review added three more members, each a shape the first cut of the + expression rebuild admitted and answered wrongly: a literal subscript + (re-rendered shifted by one), a lambda (rebound against the wrapper + relation's alias scope), and an aggregate sqlglot models outside + :class:`~sqlglot.expressions.AggFunc` (rebuilt as an expression, so named + ``__giql_expr_`` against the published ``__giql_agg_`` contract). """ @pytest.mark.parametrize( @@ -8738,13 +9996,43 @@ class TestTranspileDuckDBIEJoinUnsupportedProjectionFallback: "COUNT(b.*)", "COUNT(a.* EXCLUDE (name))", "MIN(COLUMNS(*))", - "a.start + 1", "SUM(a.score) OVER ()", "SUM(a.score) FILTER (WHERE a.score > 0)", "COUNT(*) * 2", + "COUNT(a.*) + 1", "(SELECT 1) AS s", + "a.start + (SELECT 1)", + "COLUMNS(*) + 1", "100", "a.name, COUNT(a.*)", + "GREATEST(a.start, SUM(b.start) OVER ())", + "CASE WHEN EXISTS (SELECT 1) THEN a.start END", + "a.start + COUNT(*)", + "a.start IN (SELECT start FROM genes)", + "GREATEST(a.start, b.*)", + "1 + 1", + "NULL", + "CAST(100 AS INT)", + # A literal subscript re-renders shifted by one, because the query + # is read with GIQLDialect (INDEX_OFFSET 0) and the rebuild is + # emitted with DuckDB's generator (INDEX_OFFSET 1) — #109 B1. + "a.arr[1]", + "a.name[1]", + "a.arr[1] + a.start", + # A lambda parameter would rebind against the wrapper relation, + # whose alias scope differs from the join's — #109 B2. + "list_transform(a.arr, x -> x + a.start)", + "list_transform(b.arr, a -> a.start)", + # Aggregates sqlglot models outside AggFunc, as Anonymous or as a + # dedicated non-AggFunc class — #109 B3. + "product(a.score)", + "histogram(a.score)", + "list(a.score)", + "quantile_cont(a.score, 0.5)", + "quantile_cont(a.score, 0.5) * 2", + "a.start + 1 AS v, 100 AS k", + "a.*, a.start + 1 AS v", + "a.start + 1 AS v, SUM(a.score) OVER () AS w", ], ) def test_transpile_should_decline_unsupported_projection_to_naive_plan( @@ -8845,12 +10133,16 @@ def test_query_should_match_naive_plan_for_min_columns_star(self, conn): @pytest.mark.parametrize( "projection, expected", [ - ("a.start + 1", [(101,)]), + ("a.start + (SELECT 1)", [(101,)]), ("SUM(a.score) OVER ()", [(7,)]), ("SUM(a.score) FILTER (WHERE a.score > 0)", [(7,)]), ("COUNT(*) * 2", [(2,)]), ("(SELECT 1) AS s", [(1,)]), ("COUNT(a.*)", [(1,)]), + ("GREATEST(a.start, SUM(b.start) OVER ())", [(150,)]), + ("CASE WHEN EXISTS (SELECT 1) THEN a.start END", [(100,)]), + ("a.start IN (SELECT start FROM genes)", [(False,)]), + ("1 + 1", [(2,)]), ], ) def test_query_should_match_naive_plan_for_unsupported_projection( @@ -8860,9 +10152,10 @@ def test_query_should_match_naive_plan_for_unsupported_projection( Given: A projection the IEJoin cannot rebuild but the naive plan handles — - an expression, a window aggregate, a FILTER clause, an - arithmetic-over-aggregate, a scalar subquery, or an aggregate over a - qualified star — over an INTERSECTS join with one overlapping pair. + an expression over a scalar subquery, a window aggregate, a FILTER + clause, an arithmetic-over-aggregate, a scalar subquery, or an + aggregate over a qualified star — over an INTERSECTS join with one + overlapping pair. When: The query is transpiled with ``dialect='duckdb'`` and executed, and separately with ``dialect=None``. @@ -8895,6 +10188,10 @@ def test_query_should_match_naive_plan_for_unsupported_projection( "score + 1", "SUM(score) OVER ()", "SUM(score) FILTER (WHERE score > 0)", + "GREATEST(a.start, start)", + "start + (SELECT 1)", + "start + COUNT(*)", + "a.start + 1 AS v, score", ], ) def test_transpile_should_raise_when_unsupported_wrapper_has_unqualified_column( @@ -8905,7 +10202,8 @@ def test_transpile_should_raise_when_unsupported_wrapper_has_unqualified_column( Given: An expression / window aggregate / FILTER clause whose argument references an unqualified column the dialect cannot attribute to a - join side. + join side — at the top level, nested beside a qualified leaf, inside + a declinable wrapper, or as a sibling of a rebuildable expression. When: ``transpile`` is called with ``dialect='duckdb'``. Then: @@ -8946,19 +10244,19 @@ def test_transpile_should_raise_left_side_only_for_right_ref_in_wrapper(self, ki ) @pytest.mark.parametrize("kind", ["SEMI", "ANTI"]) - def test_transpile_should_decline_unsupported_projection_under_left_only_join( + def test_transpile_should_engage_iejoin_for_expression_projection_under_left_only_join( # noqa: E501 self, kind ): - """Test that an unsupported projection under SEMI/ANTI declines to naive. + """Test that a left-side expression under SEMI/ANTI keeps the IEJoin plan. Given: An ``a.start + 1`` expression projection under a SEMI / ANTI join. When: ``transpile`` is called with ``dialect='duckdb'``. Then: - It should decline the IEJoin (no ``SET VARIABLE`` scaffolding) so - the naive-predicate plan handles it — the decline applies under - left-only joins as under INNER. + It should engage the IEJoin (``SET VARIABLE __giql_iejoin_`` + emitted) — the scalar-expression rebuild applies under left-only + joins as under INNER (#109). """ # Arrange query = ( @@ -8970,8 +10268,8 @@ def test_transpile_should_decline_unsupported_projection_under_left_only_join( sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") # Assert - assert "SET VARIABLE" not in sql.upper() - assert "getvariable" not in sql + assert "SET VARIABLE __giql_iejoin_" in sql + assert "getvariable" in sql def test_transpile_should_still_engage_iejoin_for_plain_aggregate(self): """Test that a plain qualified aggregate still engages the IEJoin. @@ -8996,3 +10294,364 @@ def test_transpile_should_still_engage_iejoin_for_plain_aggregate(self): # Assert assert "SET VARIABLE __giql_iejoin_" in sql + + +def _make_typed_interval_tables(conn) -> None: + """Recreate peaks/genes with the array and struct columns the gate needs. + + ``peaks.arr`` is a plain list and ``peaks.name`` a string, so a subscript + has something to shift; ``genes.arr`` is a list of structs, so a lambda + body can reference a field and shadow the ``a`` join alias. + """ + conn.execute("DROP TABLE IF EXISTS peaks") + conn.execute(""" + CREATE TABLE peaks ( + chrom VARCHAR, "start" INTEGER, "end" INTEGER, + name VARCHAR, score INTEGER, arr INTEGER[] + ) + """) + conn.execute("INSERT INTO peaks VALUES ('chr1', 100, 200, 'abcdef', 10, [7, 8, 9])") + conn.execute("DROP TABLE IF EXISTS genes") + conn.execute(""" + CREATE TABLE genes ( + chrom VARCHAR, "start" INTEGER, "end" INTEGER, + name VARCHAR, score INTEGER, arr STRUCT("start" INTEGER)[] + ) + """) + conn.execute( + "INSERT INTO genes VALUES " + "('chr1', 150, 250, 'g1', 1, [{'start': 1}, {'start': 2}])" + ) + + +class TestTranspileDuckDBIEJoinProjectionContract: + """Regression tests for #109's review findings B1-B5. + + The expression rebuild re-emits a user-authored tree with the DuckDB + generator, though the query was read with :class:`~giql.dialect.GIQLDialect`. + Its first cut gated that on a blocklist of six sqlglot node classes while + publishing the contract as a property, so every node class the blocklist + did not name was admitted by default. Three shapes reached the gap and + answered wrongly. These tests pin the property instead: a shape either + engages and agrees with the naive plan, or declines — never engages and + diverges. + """ + + @pytest.mark.parametrize( + "projection", + ["a.arr[1]", "a.name[1]", "a.arr[1] + a.start", "a.arr[1][1]"], + ) + def test_transpile_should_decline_subscript_projection(self, projection): + """Test that a subscript projection is not rebuilt on the IEJoin path. + + Given: + A column-to-column INTERSECTS join projecting a literal subscript + of an array or string column. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should decline the IEJoin. GIQL reads with ``INDEX_OFFSET`` 0 + and DuckDB writes with 1, so re-emitting the subtree shifts every + literal subscript by one; the rebuild must not be a source of that + (#109 B1). + + This asserts the decline rather than cross-plan value agreement, + because the shift is not the rebuild's to fix: ``dialect="duckdb"`` + renders the whole statement with DuckDB's generator, so a bare + ``SELECT arr[1] FROM peaks`` with no join at all already returns the + second element. That is tracked as its own defect; declining here + keeps the rebuild from becoming a second source of it. + """ + # Arrange + query = ( + f"SELECT {projection} AS v FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert "SET VARIABLE" not in sql.upper() + + @pytest.mark.parametrize( + "projection, expected", + [ + ('list_transform(b.arr, a -> a."start")', [100, 100]), + ('list_transform(b.arr, x -> x."start" + a.start)', [101, 102]), + ], + ids=["parameter_shadows_join_alias", "parameter_distinct_from_alias"], + ) + def test_query_should_bind_a_lambda_body_against_the_join_scope( + self, conn, projection, expected + ): + """Test that a lambda body still binds against the join's alias scope. + + Given: + A column-to-column INTERSECTS join projecting a lambda whose + parameter either shares the left join alias's name or does not. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + It should decline the IEJoin and return the value the join scope + gives — for the shadowing case ``[100, 100]``, DuckDB resolving + ``a."start"`` to the table rather than to the parameter's struct + field. Rebuilding would swap the join for the wrapper relation, + which carries no ``a`` alias, so the parameter would capture those + references and silently return ``[1, 2]`` instead (#109 B2). + """ + # Arrange + _make_typed_interval_tables(conn) + query = ( + f"SELECT {projection} AS v FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE" not in sql.upper() + assert rows == [(expected,)] + + def test_transpile_should_never_rebuild_an_aggregate_as_an_expression(self, conn): + """Test that no DuckDB aggregate is rebuilt through the expression path. + + Given: + Every function name DuckDB's own catalog reports as an aggregate. + When: + Each is projected over a qualified column and transpiled with + ``dialect='duckdb'``. + Then: + Each should either decline or engage through the aggregate path, + never the expression path. sqlglot models only some aggregates as + :class:`~sqlglot.expressions.AggFunc` — the rest parse as + ``Anonymous`` or as a dedicated non-``AggFunc`` class — so one + slipping through would be named ``__giql_expr_`` against the + published ``__giql_agg_`` contract, and would bypass the + dedicated "requires qualified aggregate arguments" diagnostic + (#109 B3). Generated from the catalog rather than a fixed list, so + a sqlglot reclassification fails here and not in a user's query. + """ + # Arrange + names = [ + row[0] + for row in conn.execute( + "SELECT DISTINCT function_name FROM duckdb_functions() " + "WHERE function_type = 'aggregate' ORDER BY function_name" + ).fetchall() + ] + assert len(names) > 50, "DuckDB should expose a substantial aggregate catalog" + + # Act + misnamed = [] + for name in names: + query = ( + f"SELECT {name}(a.score) FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval" + ) + try: + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + except ValueError: + continue # a qualifier diagnosis, not a rebuild + if "__giql_expr_" in sql: + misnamed.append(name) + + # Assert + assert misnamed == [] + + @pytest.mark.parametrize( + "prefix", + ["__giql_expr_1", "__giql_agg_1"], + ) + def test_query_should_not_repeat_an_output_name_when_user_alias_collides( + self, peaks_genes, prefix + ): + """Test that a synthesized output name never shadows a user's alias. + + Given: + A SELECT list carrying a user alias spelled exactly like the name + the dialect synthesizes for an unaliased expression at that + position. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + The result columns should all be distinctly named. DuckDB accepts + duplicates silently, so a collision would make ``fetchdf()`` and + any name-keyed access collapse or shadow a column (#109 B4). + """ + # Arrange + conn = peaks_genes + query = ( + f"SELECT a.chrom AS {prefix}, a.start + 1 FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + cursor = conn.execute(sql) + names = [column[0] for column in cursor.description] + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert len(names) == len(set(names)), names + assert prefix in names + + @pytest.mark.parametrize( + "projection, engages", + [ + ("a.start + 1", True), + ('LEAST(a."end", b."end") - GREATEST(a.start, b.start)', True), + ("CASE WHEN a.start < b.start THEN 1 ELSE 2 END", True), + ("CAST(a.start AS BIGINT)", True), + ("TRY_CAST(a.start AS BIGINT)", True), + ("a.name || '_' || b.name", True), + ("COALESCE(a.start, 100)", True), + ("LENGTH(a.name)", True), + ("UPPER(a.name)", True), + ("SUBSTRING(a.name, 1, 3)", True), + ("a.start IS DISTINCT FROM b.start", True), + ("a.start < b.start", True), + ("a.arr[1:2]", True), + ("{'k': a.start}", True), + ("[a.start, b.start]", True), + ("a.arr[1]", False), + ("list_transform(a.arr, x -> x)", False), + ("SUM(a.score) OVER ()", False), + ("COUNT(*) * 2", False), + ("unnest([a.start, b.start])", False), + ("epoch(a.start)", False), + ("COLUMNS(*) + 1", False), + ("a.start + (SELECT 1)", False), + ("100", False), + ], + ) + def test_transpile_should_hold_its_published_projection_contract( + self, projection, engages + ): + """Test the engage/decline boundary the performance guide publishes. + + Given: + One projection per node class the contract names, on both sides of + the boundary. + When: + The query is transpiled with ``dialect='duckdb'``. + Then: + It should engage exactly when the guide says it does. A sqlglot + release that reclassifies a node, or a future relaxation of the + gate, fails here rather than silently moving a query between plans + or renaming its result column (#109 B5). + """ + # Arrange + query = ( + f"SELECT {projection} AS v FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert ("SET VARIABLE __giql_iejoin_" in sql) is engages + + def test_transpile_should_allocate_one_inner_column_for_case_variant_leaves(self): + """Test that case-variant leaf spellings share one inner column. + + Given: + An expression whose two leaves name one physical column in + different cases. + When: + The query is transpiled with ``dialect='duckdb'``. + Then: + Exactly one ``__giql_p`` should be allocated. DuckDB resolves + identifiers case-insensitively, so two entries would carry a + duplicate of the same value per row through every branch of the + per-chromosome union (#109 A1). + """ + # Act + sql = transpile( + "SELECT GREATEST(a.START, a.start) AS g FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert len(set(re.findall(r"__giql_p\d+", sql))) == 1 + + @pytest.mark.parametrize( + "projection, expected", + [ + ("GREATEST(a.start, c.start)", "'c.start' references unknown table"), + ('GREATEST(a.start, LEAST(a."end", oops))', "column 'oops' must be"), + ("mycat.myschema.a.start + 1", "carries a catalog/schema qualifier"), + ], + ids=["unknown_qualifier", "unqualified_leaf", "catalog_qualified"], + ) + def test_transpile_should_name_the_offending_leaf_when_projection_is_unresolvable( + self, projection, expected + ): + """Test that the diagnosis names the leaf, not the tree around it. + + Given: + An expression projection with one unresolvable leaf, nested at + varying depth. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should raise ``ValueError`` naming that leaf and its specific + fault. Echoing the whole projection would prescribe qualifying a + column the user already qualified (#109 A3), and a catalog- + qualified leaf must raise rather than engage, since the naive plan + rejects it at bind time (#109 A12). + """ + # Arrange + query = ( + f"SELECT {projection} AS v FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act & assert + with pytest.raises(ValueError) as excinfo: + transpile(query, tables=["peaks", "genes"], dialect="duckdb") + assert expected in str(excinfo.value) + + @pytest.mark.parametrize("kind", ["SEMI", "ANTI"]) + @pytest.mark.parametrize( + "projection", + [ + "b.start + 1", + "CASE WHEN a.start > 0 THEN b.start ELSE a.start END", + 'LEAST(a."end", GREATEST(a.start, b.start))', + ], + ) + def test_transpile_should_raise_left_side_only_for_a_nested_right_side_leaf( + self, kind, projection + ): + """Test that a right-side leaf raises however deeply it is nested. + + Given: + A SEMI or ANTI join projecting an expression with a right-side + leaf at the top level or nested inside a function or ``CASE``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should raise the dedicated left-side-only ``ValueError`` naming + the leaf. Before #109's review this rule was written in three + places and the two that ran later depended on the first having + run (#109 B7, A10). + """ + # Arrange + query = ( + f"SELECT {projection} AS v FROM peaks a {kind} JOIN genes b " + "ON a.interval INTERSECTS b.interval" + ) + + # Act & assert + with pytest.raises(ValueError) as excinfo: + transpile(query, tables=["peaks", "genes"], dialect="duckdb") + message = str(excinfo.value) + assert "only projects left-side columns" in message + assert "'b.start'" in message From a43e4271d55f5ece0f5264a76b6e7eb9a38aeb3e Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 3 Sep 2026 08:17:00 -0400 Subject: [PATCH 3/5] test: Oracle the bedtools clipped-span and -wo shapes against bedtools These are the two modes the expression rebuild exists to serve, and nothing in the lane exercised them: the intersect oracle covered the join shapes whose projections are plain columns, which is every shape that already reached the fast path. Both new cases assert the dialect engaged as well as that the rows agree, because either would keep passing on the naive predicate and so would report nothing about the rewrite. The wrapper gains a clip mode for the first of them. Every existing caller selects a bedtools flag, and the default output with no flag at all -- each A feature clipped to its overlap -- had no way to be requested; the branch passes no flag and reads the result back through the same bed6 path the other flagless modes use. --- tests/integration/bedtools/conftest.py | 29 +++++++++ tests/integration/bedtools/test_intersect.py | 61 +++++++++++++++++++ .../bedtools/utils/bedtools_wrapper.py | 8 ++- 3 files changed, 97 insertions(+), 1 deletion(-) diff --git a/tests/integration/bedtools/conftest.py b/tests/integration/bedtools/conftest.py index 9581fd9..c5551b7 100644 --- a/tests/integration/bedtools/conftest.py +++ b/tests/integration/bedtools/conftest.py @@ -17,6 +17,7 @@ pytestmark = pytest.mark.integration +from .utils.data_models import GenomicInterval # noqa: E402 from .utils.duckdb_loader import load_intervals # noqa: E402 @@ -31,6 +32,34 @@ def duckdb_connection(): conn.close() +@pytest.fixture +def partial_overlap_intervals(duckdb_connection): + """Load an A/B interval pair with partial overlaps on two chromosomes. + + Also carries one A interval with no overlap and one on a chromosome absent + from B, so a dialect rewrite that drops unmatched contigs is visible. Yields + the two tuple lists for handing to the bedtools wrapper. + """ + intervals_a = [ + GenomicInterval("chr1", 100, 200, "a1", 100, "+"), + GenomicInterval("chr1", 300, 400, "a2", 200, "-"), + GenomicInterval("chr2", 50, 150, "a3", 300, "+"), + GenomicInterval("chr1", 5000, 6000, "a_lonely", 400, "-"), + GenomicInterval("chrX", 10, 20, "a_other_chrom", 500, "+"), + ] + intervals_b = [ + GenomicInterval("chr1", 150, 250, "b1", 100, "+"), + GenomicInterval("chr1", 350, 450, "b2", 200, "+"), + GenomicInterval("chr2", 100, 120, "b3", 300, "-"), + GenomicInterval("chrY", 1, 5, "b_other_chrom", 400, "-"), + ] + rows_a = [i.to_tuple() for i in intervals_a] + rows_b = [i.to_tuple() for i in intervals_b] + load_intervals(duckdb_connection, "intervals_a", rows_a) + load_intervals(duckdb_connection, "intervals_b", rows_b) + yield rows_a, rows_b + + @pytest.fixture(scope="function") def giql_query(duckdb_connection): """Provide a helper that loads data, transpiles GIQL, and executes. diff --git a/tests/integration/bedtools/test_intersect.py b/tests/integration/bedtools/test_intersect.py index c66ea3f..92ee80b 100644 --- a/tests/integration/bedtools/test_intersect.py +++ b/tests/integration/bedtools/test_intersect.py @@ -451,6 +451,67 @@ def test_intersect_right_outer_join_duckdb_dialect(duckdb_connection): assert comparison.match, comparison.failure_message() +@pytest.mark.parametrize( + "bedtools_kwargs, projection", + [ + ( + {"clip": True}, + """ + a.chrom, + GREATEST(a.start, b.start) AS start, + LEAST(a."end", b."end") AS "end", + a.name, a.score, a.strand + """, + ), + ( + {"write_overlap": True}, + """ + a.chrom, a.start, a.end, a.name, a.score, a.strand, + b.chrom AS b_chrom, b.start AS b_start, b.end AS b_end, + b.name AS b_name, b.score AS b_score, b.strand AS b_strand, + LEAST(a."end", b."end") - GREATEST(a.start, b.start) AS overlap + """, + ), + ], + ids=["clipped_span", "write_overlap"], +) +def test_intersect_expression_projection_duckdb_dialect( + duckdb_connection, partial_overlap_intervals, bedtools_kwargs, projection +): + """ + GIVEN interval sets with partial overlaps on two chromosomes, plus one A + interval with no overlap and one on a chromosome absent from B + WHEN a scalar expression projection over both sides' columns is transpiled + with dialect="duckdb", keeping the IEJoin plan under an expression + projection (#109) — the clipped span (GREATEST of the starts, LEAST of + the ends) or that span's width alongside both sides' columns + THEN results match the corresponding bedtools intersect output exactly + + The dialect assertion matters as much as the comparison: these projection + shapes used to decline to the naive-predicate plan, so the rebuild is only + exercised when the fast path actually fires. + """ + rows_a, rows_b = partial_overlap_intervals + + bedtools_result = intersect(rows_a, rows_b, **bedtools_kwargs) + + sql = transpile( + f""" + SELECT + {projection} + FROM intervals_a a + JOIN intervals_b b ON a.interval INTERSECTS b.interval + """, + tables=["intervals_a", "intervals_b"], + dialect="duckdb", + ) + assert "SET VARIABLE __giql_iejoin_" in sql + giql_result = duckdb_connection.execute(sql).fetchall() + + comparison = compare_results(giql_result, bedtools_result) + assert comparison.match, comparison.failure_message() + + def test_intersect_left_outer_join_duckdb_dialect_with_duplicate_rows( duckdb_connection, ): diff --git a/tests/integration/bedtools/utils/bedtools_wrapper.py b/tests/integration/bedtools/utils/bedtools_wrapper.py index 5a75304..5a741ea 100644 --- a/tests/integration/bedtools/utils/bedtools_wrapper.py +++ b/tests/integration/bedtools/utils/bedtools_wrapper.py @@ -37,6 +37,7 @@ def intersect( count: bool = False, write_overlap: bool = False, write_all_overlap: bool = False, + clip: bool = False, fraction_a: float | None = None, fraction_b: float | None = None, reciprocal: bool = False, @@ -58,6 +59,11 @@ def intersect( write_all_overlap : bool Write overlap amount for all A features including non-overlapping (-wao). + clip : bool + Suppress this wrapper's default ``-u`` so bedtools writes each A + feature clipped to its overlap with B. bedtools does that when given + no flags at all; this wrapper otherwise passes ``-u``, so the + parameter is what reaches that output, not a no-op. fraction_a : float or None Minimum overlap as fraction of A (-f). fraction_b : float or None @@ -85,7 +91,7 @@ def intersect( kwargs["wao"] = True elif reciprocal: kwargs["wa"] = True - else: + elif not clip: kwargs["u"] = True if strand_mode == "same": From 23ca2f8edcb8c8a07da21fef5c6a91a270a6f721 Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 3 Sep 2026 08:17:09 -0400 Subject: [PATCH 4/5] docs: Move expression projections onto the supported dialect list The guide listed an expression among the shapes that fall back, which is now the opposite of what the dialect does, and a caller reading it would spell out a clipped overlap span expecting the naive predicate and get the fast path instead. Expression projections join the decorations the dialect absorbs, naming the bedtools modes they serve and the synthesized output name an unaliased one carries. The fallback entry keeps the shapes that still decline and now says so as a closed list rather than an open one, since the distinction between a rebuildable expression and a wrapper the rebuild cannot express is the whole of what a caller needs to predict the plan. Two boundaries are stated where they would otherwise surprise: an expression under a LEFT or RIGHT join still falls back, because the decomposition's unmatched half only NULL-fills columns; and an unqualified column inside an expression still raises, where the guide previously said every such wrapper falls back once qualified. --- docs/transpilation/performance.rst | 123 +++++++++++++++++++++++------ 1 file changed, 101 insertions(+), 22 deletions(-) diff --git a/docs/transpilation/performance.rst b/docs/transpilation/performance.rst index af319c5..65e9911 100644 --- a/docs/transpilation/performance.rst +++ b/docs/transpilation/performance.rst @@ -388,13 +388,14 @@ results. rows = conn.execute(sql).fetchall() The dialect rewrites the whole query, so the supported shape is -``SELECT FROM -{INNER|SEMI|ANTI|LEFT|RIGHT} JOIN {ON|WHERE} `` — the JOIN and the column-to-column -``INTERSECTS`` are both required (literal-range ``INTERSECTS`` against a -single table falls through to the default predicate generator). ``LEFT`` -and ``RIGHT`` accept ``ON`` only: the outer-join decomposition rejects any -top-level ``WHERE``, for the reason given under **Outer joins** below. +``SELECT FROM {INNER|SEMI|ANTI|LEFT|RIGHT} JOIN {ON|WHERE} `` — the JOIN and the +column-to-column ``INTERSECTS`` are both required (literal-range +``INTERSECTS`` against a single table falls through to the default +predicate generator). ``LEFT`` and ``RIGHT`` accept ``ON`` only: the +outer-join decomposition rejects any top-level ``WHERE``, for the reason +given under **Outer joins** below. **Join variants.** ``INNER JOIN`` is the default. ``SEMI JOIN`` returns distinct left rows that have at least one overlapping match; ``ANTI @@ -516,6 +517,44 @@ them): a.col)``, ...) are appended to the outer SELECT. Aggregate arguments must be table-qualified (``COUNT(*)`` and ``COUNT(a.col)`` are both accepted). +- **Scalar expression projections.** A SELECT-list expression keeps the + IEJoin plan when four properties hold: it is row-wise, it binds no + names, it carries no star, and at least one leaf is a qualified column. + ``GREATEST(a.start, b.start)``, + ``LEAST(a."end", b."end") - GREATEST(a.start, b.start)``, + ``a.start + 1``, a ``CASE``, a cast, string concatenation, a comparison, + and the ordinary scalar functions (``COALESCE``, ``LENGTH``, ``UPPER``, + ``SUBSTRING``, ``DATE_TRUNC``, ...) all qualify. Each leaf column is + projected into the per-chromosome subquery and the expression is rebuilt + on the outer SELECT over those inner columns, so the join itself is + untouched. This is what puts the ``bedtools intersect`` default (clipped + overlap span) and ``-wo`` (pairs plus overlap width) shapes on the fast + path. + + The leaves rather than the computed value go inward deliberately. + Computing the expression inside the subquery would narrow the wrapper + relation, but the session-variable name is a content digest of that + subquery's SQL, so the expression's text would fold into the digest and + ``a.start + 1`` / ``a.start + 2`` / ... would each mint a distinct + ``SET VARIABLE`` on a long-lived connection — trading the bounded + variable set above for an unbounded one. Leaf dedup means the bedtools + shapes pay nothing for this, since they already project their leaves as + plain columns. + + An aliased expression keeps its alias; an unaliased one is named + ``__giql_expr_`` by position, as an unaliased aggregate is named + ``__giql_agg_``, because DuckDB's automatic name would be derived + from the rewritten text. A synthesized name is chosen around the names + the caller wrote, so it never duplicates one — DuckDB accepts duplicate + result-column names silently, which would make ``fetchdf()`` and any + name-keyed access collapse or shadow a column. Two same-named *user* + columns (``SELECT a.start, b.start``) are left as the caller wrote them, + because the naive plan reports the duplicate too. + + A projection failing any of the four properties falls back; see + **Projections the IEJoin cannot rebuild** below. Under a ``LEFT`` / + ``RIGHT`` outer join an expression projection still declines, since the + decomposition's unmatched half only NULL-fills columns. - **Extra JOIN / WHERE predicates.** Additional non-INTERSECTS predicates ANDed onto the join ON or WHERE (e.g. ``a.score > b.score`` or ``WHERE a.score > 100``) are inlined into each @@ -570,16 +609,46 @@ The soft-fallback shapes are: star against DuckDB's live schema, so declining keeps the projection identical across backends rather than silently narrowing it. - **Projections the IEJoin cannot rebuild.** The dialect's projection - rebuild handles a qualified column (``a.col``) or a plain aggregate - over qualified columns (``SUM(a.score)`` / ``COUNT(*)``). Any other - SELECT-list shape the naive plan compiles for free — an expression - (``a.start + 1``), a window aggregate (``SUM(a.score) OVER (...)``), a - ``FILTER`` clause, a scalar subquery, an aggregate nested in an - expression (``COUNT(*) * 2``), a bare literal, or a star nested in an - aggregate argument (``COUNT(a.*)`` / ``MIN(COLUMNS(*))``) — falls back, - so ``dialect="duckdb"`` stays consistent with every other backend - instead of hard-erroring or, for the aggregate-nested star, - miscompiling. A projection whose column the rebuild cannot attribute to + rebuild handles a qualified column (``a.col``), a scalar expression + over qualified columns (``GREATEST(a.start, b.start)``, ``a.start + 1`` + — see **Scalar expression projections** above), or a plain aggregate + over qualified columns (``SUM(a.score)`` / ``COUNT(*)``). The remaining + SELECT-list shapes the naive plan compiles for free fall back, so + ``dialect="duckdb"`` stays consistent with every other backend instead + of hard-erroring or, for the aggregate-nested star, miscompiling. Each + falls back for one of the four properties above: + + - *not row-wise* — a window aggregate (``SUM(a.score) OVER (...)``), a + ``FILTER`` clause, an aggregate nested in an expression + (``COUNT(*) * 2``), or a row-multiplying call + (``unnest([a.start, b.start])``); + - *binds names* — a scalar subquery, bare or inside an expression, and a + lambda (``list_transform(b.arr, x -> x + a.start)``). A lambda's body + resolves against the surrounding relation's aliases, and the rebuild + replaces the join with the wrapper relation, so a parameter sharing a + join alias's name would bind differently; + - *carries a star* — a star nested in an aggregate argument or + expression (``COUNT(a.*)`` / ``MIN(COLUMNS(*))`` / ``COLUMNS(*) + 1``), + which a schema-less rebuild cannot enumerate; + - *no qualified column leaf* — a bare literal (``100``, ``1 + 1``, + ``CAST(100 AS INT)``). There is nothing to attribute to a join side, + and one such SELECT item declines the whole query, so + ``SELECT a.start, 'tag' AS src`` takes the naive plan. + + Two further shapes fall back for reasons specific to the round trip + through sqlglot. A **subscript** (``a.arr[1]``) falls back because GIQL + reads with a zero-based index offset and DuckDB writes with a one-based + one, so re-emitting the subtree would shift the literal by one. And a + **function sqlglot does not model** — parsed as an anonymous call, which + includes DuckDB aggregates such as ``product``, ``histogram`` and + ``quantile_cont`` — falls back because nothing tells the rebuild whether + it aggregates, multiplies rows, or binds names. Beyond these named + cases the rebuild verifies the round trip rather than predicting it: it + re-emits the tree with DuckDB's generator, re-reads it, and declines + unless the tree comes back unchanged. An expression projection under + a ``LEFT`` / ``RIGHT`` outer join also falls back, because the + decomposition's unmatched half only NULL-fills side-attributable + columns. A projection whose column the rebuild cannot attribute to a join side (unqualified, an unknown table, or the right side under a SEMI / ANTI join) raises at transpile time instead (see the hard-error list below) — the unknown-table and right-side cases are rejected by the @@ -637,12 +706,17 @@ these rather than silently miscompile): every star it falls back to the naive-predicate plan — see the soft-fallback list above. An expression / window aggregate / ``FILTER`` clause over an *unqualified* column raises the same "requires qualified - projections" error — once qualified, the wrapper itself falls back to the - naive plan rather than erroring. A scalar subquery always falls back, even - over an unqualified inner column, since that column resolves against the - subquery's own scope.) + projections" error — once qualified, a scalar expression is rebuilt on + the IEJoin path, and a window / ``FILTER`` wrapper falls back to the + naive plan rather than erroring. A scalar subquery always falls back, + even over an unqualified inner column, since that column resolves + against the subquery's own scope.) + Every one of these diagnostics names the offending *leaf* rather than + the projection around it, so ``GREATEST(a.start, c.start)`` reports + ``c.start`` instead of prescribing that a column already qualified be + qualified. - **Unknown table qualifier.** ``c.col`` when the only sides are ``a`` - and ``b``. + and ``b``, at any depth inside an expression. - **Unknown alias in GROUP BY / HAVING / ORDER BY.** A modifier-clause column qualified with a table alias outside the join's two sides (``c.col``) raises with a ``cannot resolve … in `` message @@ -653,6 +727,11 @@ these rather than silently miscompile): SEMI / ANTI return left-side columns only. (A right-side ``b.*`` falls back to the naive-predicate plan with every other star — see the soft-fallback list above.) +- **Catalog- or schema-qualified column.** ``mycat.myschema.a.col`` + raises, in a projection as well as in an extra predicate. The + alias-rewriter remaps ``a`` to the inner subquery's own ``a``, and the + ``mycat.myschema`` prefix would survive that rewrite and address a + different relation; the naive plan rejects the form at bind time too. - **Aggregate of unqualified column.** ``SUM(score)`` raises; use ``SUM(a.score)`` or ``SUM(b.score)``. (A qualified aggregate the IEJoin cannot rebuild — ``SUM(a.score) OVER (...)``, ``SUM(a.score) FILTER From ad54b7dd4caabfc8c623cc4b4b2dd374b822e376 Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 3 Sep 2026 13:13:48 -0400 Subject: [PATCH 5/5] test: Estimate the outer-join scale ratios from the best of several runs The LEFT-join scale test divides two wall-clock measurements and asserts the ratio stays under 2.5x. That bound has to sit between the 2.0x the decomposition should achieve and the 3.3x regression it exists to catch (#95), so the window it discriminates is narrow -- narrower than a shared runner's variance on a single sample. CI produced 2.64x for an implementation that measures 1.6x unloaded, failing the assertion without any change to the SQL either plan emits. The two plans also do not degrade alike under load. The dialect plan is a multi-statement per-chromosome UNION ALL with far more parallelism to lose than the naive plan's single hash join, so contention inflates the ratio by construction rather than symmetrically. Widening the bound is therefore not an option: it would have to cross into the band the regression occupies. Take the minimum of several runs per plan instead. Contention inflates a sample upward and never downward, so the minimum estimates the uncontended cost and the ratio stops depending on whether a stall happened to land in one measurement and not the other. The 2.5x bound is unchanged and still catches the regression. Costs about 14s on the two tests that measure ratios; the structural INTERSECT / NOT IN assertions that pin the mechanism are untouched. --- tests/test_duckdb_iejoin.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/tests/test_duckdb_iejoin.py b/tests/test_duckdb_iejoin.py index 022db7e..ecd1415 100644 --- a/tests/test_duckdb_iejoin.py +++ b/tests/test_duckdb_iejoin.py @@ -207,11 +207,32 @@ def _assert_agrees_with_naive(conn, query: str, tables=("peaks", "genes")) -> st return dialect_sql -def _timed_rows(conn, sql: str) -> tuple[float, list[tuple]]: - """Return the wall-clock seconds and rows from executing *sql* on *conn*.""" - started = time.perf_counter() - rows = conn.execute(sql).fetchall() - return time.perf_counter() - started, rows +def _timed_rows(conn, sql: str, *, repeats: int = 3) -> tuple[float, list[tuple]]: + """Return the best-of-*repeats* wall-clock seconds and rows for *sql*. + + Callers divide two of these measurements and assert on the ratio, so the + measurement has to survive a shared CI runner. Contention inflates a + sample upward and never downward, which makes the minimum of several runs + an estimate of the uncontended cost; a single sample is not, and the two + plans do not degrade alike. The dialect plan is a multi-statement + per-chromosome ``UNION ALL`` with far more parallelism to lose than the + naive plan's single hash join, so contention inflates the ratio by + construction rather than symmetrically. + + That matters because the window these ratios discriminate is narrow -- + 2.0x for the plan the decomposition should pick against 3.3x for the one + it must not (#95) -- and narrower than one sample's variance: CI has + produced 2.64x for an implementation that measures 1.6x unloaded. + Repeating is what keeps the bound tight enough to still catch the + regression. + """ + timings: list[float] = [] + rows: list[tuple] = [] + for _ in range(repeats): + started = time.perf_counter() + rows = conn.execute(sql).fetchall() + timings.append(time.perf_counter() - started) + return min(timings), rows def _dynamic_sql_variables(sql: str) -> list[str]: @@ -5623,7 +5644,9 @@ def test_query_should_not_regress_against_naive_when_preserved_side_has_many_con ``IE_JOIN`` for *every* chromosome -- including the 24 that carry all the rows. Measured here: 8.3s partitioning on preserved-distinct, 5.0s on the shared INTERSECT, against 2.5s naive (#95). The bound is a ratio - rather than a wall clock so it calibrates to the machine. + rather than a wall clock so it calibrates to the machine, and each + side is the best of several runs so a transient stall on a shared + runner cannot land in one measurement and not the other. """ # Arrange conn.execute("DROP TABLE IF EXISTS peaks")