From c1dab3260e3777c5793364c1b29a5acd1d5d1526 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:31:29 +0100 Subject: [PATCH 01/22] refactor: move scenario SQL and artifact planning to native code --- .../src/planner_scenario/_helpers/markers.rs | 25 + .../src/planner_scenario/_helpers/mod.rs | 3 + .../src/planner_scenario/constants.rs | 5 + .../src/planner_scenario/errors.rs | 10 + .../src/planner_scenario/main/hash_prefix.rs | 18 + .../src/planner_scenario/main/mod.rs | 3 + .../main/parse_artifact_name.rs | 21 + .../planner_scenario/main/replace_markers.rs | 45 ++ .../src/planner_scenario/mod.rs | 3 + .../src/planner_scenario/tests.rs | 2 + .../planner_scenario/tests/test_markers.rs | 69 ++ .../src/planner_scenario/tests/test_types.rs | 8 + .../_helpers/compiled_project/loop_classes.rs | 5 +- .../_helpers/planner_scenario/artifacts.rs | 263 +++++++ .../bindings/_helpers/planner_scenario/cli.rs | 8 +- .../_helpers/planner_scenario/constants.rs | 2 + .../_helpers/planner_scenario/entries.rs | 159 ++++ .../_helpers/planner_scenario/execution.rs | 331 ++++++++ .../_helpers/planner_scenario/fixtures.rs | 147 ++++ .../bindings/_helpers/planner_scenario/mod.rs | 6 + .../_helpers/planner_scenario/models.rs | 11 + .../_helpers/planner_scenario/registration.rs | 49 +- .../bindings/_helpers/planner_scenario/sql.rs | 144 ++++ .../src/resolution/main/mod.rs | 1 + .../src/resolution/main/quoted_name.rs | 11 + .../src/resolution/main/reference_spans.rs | 10 +- src/sqlbuild/_native.pyi | 72 ++ .../_helpers/scenario/artifact_names.py | 62 -- .../planner/_helpers/scenario/artifacts.py | 145 ---- .../planner/_helpers/scenario/relations.py | 744 ------------------ .../_build_scenario_artifact_physical_name.py | 2 +- .../is_scenario_artifact_physical_name.py | 2 +- .../cli/commands/main/scenario/helpers.py | 25 + .../commands/main/scenario/test_namespace.py | 2 +- .../commands/main/scenario/test_scenario.py | 10 + .../compiler/planner/_helpers/helpers.py | 2 +- .../_helpers/test_scenario_artifacts.py | 6 +- .../_helpers/test_scenario_relations.py | 8 +- .../executor/janitor/main/test_plan.py | 4 +- 39 files changed, 1460 insertions(+), 983 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_scenario/_helpers/markers.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scenario/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scenario/constants.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scenario/errors.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scenario/main/hash_prefix.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scenario/main/parse_artifact_name.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scenario/main/replace_markers.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scenario/tests/test_markers.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs create mode 100644 crates/sqlbuild-sqltext/src/resolution/main/quoted_name.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/scenario/artifact_names.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/scenario/artifacts.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/scenario/relations.py diff --git a/crates/sqlbuild-analysis/src/planner_scenario/_helpers/markers.rs b/crates/sqlbuild-analysis/src/planner_scenario/_helpers/markers.rs new file mode 100644 index 0000000000..4f9ff8fb48 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/_helpers/markers.rs @@ -0,0 +1,25 @@ +//! Exact authored marker recognition at a byte offset. + +use crate::planner_scenario::constants::{DBT_REF_MARKER, MARKERS}; +use sqlbuild_sqltext::resolution::main::quoted_name::quoted_name; + +pub(crate) fn marker_at(sql: &str, kind: usize) -> Option<(usize, String)> { + let tail = sql.strip_prefix(MARKERS[kind])?; + let (first, tail) = quoted_name(tail)?; + if tail.starts_with(')') { + return Some((sql.len() - tail.len() + 1, first.to_owned())); + } + if kind != DBT_REF_MARKER { + return None; + } + let tail = tail + .trim_start_matches(python_whitespace) + .strip_prefix(',')?; + let (second, tail) = quoted_name(tail.trim_start_matches(python_whitespace))?; + tail.starts_with(')') + .then(|| (sql.len() - tail.len() + 1, format!("{first}__{second}"))) +} + +fn python_whitespace(character: char) -> bool { + character.is_whitespace() || matches!(character, '\u{1c}'..='\u{1f}') +} diff --git a/crates/sqlbuild-analysis/src/planner_scenario/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_scenario/_helpers/mod.rs new file mode 100644 index 0000000000..dd0049eb8b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/_helpers/mod.rs @@ -0,0 +1,3 @@ +//! Shared scenario SQL text operations. + +pub(super) mod markers; diff --git a/crates/sqlbuild-analysis/src/planner_scenario/constants.rs b/crates/sqlbuild-analysis/src/planner_scenario/constants.rs new file mode 100644 index 0000000000..6809122df1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/constants.rs @@ -0,0 +1,5 @@ +//! Exact authored scenario marker grammar. + +pub(super) const MARKERS: [&str; 4] = ["__ref(", "__seed(", "__source(", "__dbt_ref("]; +pub(super) const DBT_REF_MARKER: usize = 3; +pub(super) const HASH_LENGTH: usize = 12; diff --git a/crates/sqlbuild-analysis/src/planner_scenario/errors.rs b/crates/sqlbuild-analysis/src/planner_scenario/errors.rs new file mode 100644 index 0000000000..c3203fe1bf --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/errors.rs @@ -0,0 +1,10 @@ +//! Scenario marker callback and lexical failures. + +use sqlbuild_sqltext::sql_scan::models::Unclosed; + +/// Preserve the first callback or lexical failure in authored SQL order. +#[derive(Debug)] +pub enum MarkerError { + Scan(Unclosed), + Resolve(E), +} diff --git a/crates/sqlbuild-analysis/src/planner_scenario/main/hash_prefix.rs b/crates/sqlbuild-analysis/src/planner_scenario/main/hash_prefix.rs new file mode 100644 index 0000000000..bcbc07359c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/main/hash_prefix.rs @@ -0,0 +1,18 @@ +//! Stable scenario artifact identity hashing. + +use sha2::{Digest, Sha256}; + +/// Hash a project, optional namespace and scenario without changing the stored identity format. +pub fn hash_prefix( + project: &str, + scenario: &str, + namespace: Option<&str>, + length: usize, +) -> String { + let input = namespace.map_or_else( + || format!("{project}:{scenario}"), + |namespace| format!("{project}:{namespace}:{scenario}"), + ); + let hash = format!("{:x}", Sha256::digest(input.as_bytes())); + hash[..length.min(hash.len())].to_owned() +} diff --git a/crates/sqlbuild-analysis/src/planner_scenario/main/mod.rs b/crates/sqlbuild-analysis/src/planner_scenario/main/mod.rs index 97c2acc407..cf8afe82ee 100644 --- a/crates/sqlbuild-analysis/src/planner_scenario/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_scenario/main/mod.rs @@ -1,3 +1,6 @@ //! Scenario planning entry points. pub mod graph; +pub mod hash_prefix; +pub mod parse_artifact_name; +pub mod replace_markers; diff --git a/crates/sqlbuild-analysis/src/planner_scenario/main/parse_artifact_name.rs b/crates/sqlbuild-analysis/src/planner_scenario/main/parse_artifact_name.rs new file mode 100644 index 0000000000..0aacd463bc --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/main/parse_artifact_name.rs @@ -0,0 +1,21 @@ +//! Recognize persisted scenario relation names. + +/// Parse a scenario artifact name using the original fixed hash length and accepted kinds. +pub fn parse_artifact_name(name: &str) -> Option<(&str, &str, &str)> { + let (hash, rest) = name.strip_prefix("__sqb_")?.split_once("__")?; + if hash.len() != crate::planner_scenario::constants::HASH_LENGTH + || !hash + .bytes() + .all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c)) + { + return None; + } + let (kind, logical) = rest.split_once("__")?; + if !["model", "source", "ref", "seed", "dbt_ref"].contains(&kind) + || logical.is_empty() + || logical.contains('\n') + { + return None; + } + Some((hash, kind, logical)) +} diff --git a/crates/sqlbuild-analysis/src/planner_scenario/main/replace_markers.rs b/crates/sqlbuild-analysis/src/planner_scenario/main/replace_markers.rs new file mode 100644 index 0000000000..6f105d8713 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/main/replace_markers.rs @@ -0,0 +1,45 @@ +//! Replace exact scenario relation markers without regenerating authored SQL. + +use crate::planner_scenario::{_helpers::markers::marker_at, errors::MarkerError}; +use sqlbuild_sqltext::code_scanner::main::{ + code_scanner::code_scanner, scanner_non_code_end::scanner_non_code_end, +}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +/// Resolve markers in execution order, optionally retaining the legacy unscanned dbt-ref policy. +pub fn replace_markers( + sql: &str, + kind: usize, + syntax: Option, + mut resolve: impl FnMut(&str, &str) -> Result, +) -> Result> { + let scanner = syntax.map(|syntax| code_scanner(sql, Some(syntax))); + let offsets: Vec = sql + .char_indices() + .map(|(offset, _)| offset) + .chain(std::iter::once(sql.len())) + .collect(); + let mut index = 0; + let mut last = 0; + let mut result = String::new(); + while index + 1 < offsets.len() { + if let Some(scanner) = &scanner + && let Some(end) = scanner_non_code_end(scanner, index).map_err(MarkerError::Scan)? + { + index = end; + continue; + } + let start = offsets[index]; + if let Some((end, name)) = marker_at(&sql[start..], kind) { + let found = &sql[start..start + end]; + result.push_str(&sql[last..start]); + result.push_str(&resolve(&name, found).map_err(MarkerError::Resolve)?); + last = start + end; + index += found.chars().count(); + continue; + } + index += 1; + } + result.push_str(&sql[last..]); + Ok(result) +} diff --git a/crates/sqlbuild-analysis/src/planner_scenario/mod.rs b/crates/sqlbuild-analysis/src/planner_scenario/mod.rs index 2c241db51f..6c2bb84a2d 100644 --- a/crates/sqlbuild-analysis/src/planner_scenario/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_scenario/mod.rs @@ -3,5 +3,8 @@ pub mod main; pub mod models; +mod _helpers; +mod constants; +pub mod errors; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-analysis/src/planner_scenario/tests.rs b/crates/sqlbuild-analysis/src/planner_scenario/tests.rs index 59d9eea16f..59074e360f 100644 --- a/crates/sqlbuild-analysis/src/planner_scenario/tests.rs +++ b/crates/sqlbuild-analysis/src/planner_scenario/tests.rs @@ -2,5 +2,7 @@ #[path = "tests/test_graph.rs"] mod test_graph; +#[path = "tests/test_markers.rs"] +mod test_markers; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_scenario/tests/test_markers.rs b/crates/sqlbuild-analysis/src/planner_scenario/tests/test_markers.rs new file mode 100644 index 0000000000..2f4fe2c466 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/tests/test_markers.rs @@ -0,0 +1,69 @@ +//! Scenario marker text and failure-order regression coverage. + +use crate::planner_scenario::main::replace_markers::replace_markers; +use crate::planner_scenario::tests::test_types::MarkerTestCase; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +#[test] +fn given_scenario_markers_when_resolving_then_preserves_authored_text_and_legacy_dbt_policy() { + let test_cases = [ + MarkerTestCase { + description: "Unicode markers preserve comments and aliases", + sql: "SELECT 'é', * FROM __ref(\"订单\") x -- __ref(\"ignored\")", + kind: 0, + scanned: true, + expected_sql: Ok("SELECT 'é', * FROM target_订单 x -- __ref(\"ignored\")"), + }, + MarkerTestCase { + description: "model dbt refs retain legacy unscanned substitution", + sql: "SELECT '__dbt_ref(\"orders\")'", + kind: 3, + scanned: false, + expected_sql: Ok("SELECT 'target_orders'"), + }, + MarkerTestCase { + description: "check dbt refs preserve string literals", + sql: "SELECT '__dbt_ref(\"orders\")'", + kind: 3, + scanned: true, + expected_sql: Ok("SELECT '__dbt_ref(\"orders\")'"), + }, + MarkerTestCase { + description: "dbt refs accept Python whitespace grammar", + sql: "SELECT * FROM __dbt_ref(\"shop\"\u{1c},\u{2003}\"orders\")", + kind: 3, + scanned: true, + expected_sql: Ok("SELECT * FROM target_shop__orders"), + }, + MarkerTestCase { + description: "uppercase dbt calls stay authored", + sql: "SELECT * FROM __DBT_REF(\"orders\")", + kind: 3, + scanned: true, + expected_sql: Ok("SELECT * FROM __DBT_REF(\"orders\")"), + }, + MarkerTestCase { + description: "unterminated strings fail scanning", + sql: "SELECT 'unterminated", + kind: 0, + scanned: true, + expected_sql: Err("failure"), + }, + ]; + for test_case in test_cases { + let syntax = test_case.scanned.then(|| LexicalSyntax { + line_comment_prefixes: vec!["--".to_owned()], + ..LexicalSyntax::default() + }); + let actual = replace_markers(test_case.sql, test_case.kind, syntax, |name, _| { + Ok::<_, ()>(format!("target_{name}")) + }) + .map_err(|_| "failure"); + assert_eq!( + actual.as_deref(), + test_case.expected_sql.map_err(|_| &"failure"), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_scenario/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_scenario/tests/test_types.rs index eb99591ff8..bfdd023dc9 100644 --- a/crates/sqlbuild-analysis/src/planner_scenario/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_scenario/tests/test_types.rs @@ -10,3 +10,11 @@ pub(super) struct GraphTestCase { pub expected_functions: Vec<(&'static str, &'static str)>, pub expected_warnings: Vec<(&'static str, bool)>, } +#[derive(Debug)] +pub(super) struct MarkerTestCase { + pub description: &'static str, + pub sql: &'static str, + pub kind: usize, + pub scanned: bool, + pub expected_sql: Result<&'static str, &'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs index 219cf1d045..ba5e0ba839 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs @@ -365,7 +365,10 @@ fn input_view_classes<'py>(py: Python<'py>) -> PyResult> { } /// `{item.: item for item in items}`. -fn by_attribute<'py>(items: &Bound<'py, PyAny>, attribute: &str) -> PyResult> { +pub(crate) fn by_attribute<'py>( + items: &Bound<'py, PyAny>, + attribute: &str, +) -> PyResult> { let keyed: Bound<'py, PyDict> = PyDict::new(items.py()); for item in items.try_iter()? { let item: Bound<'py, PyAny> = item?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs new file mode 100644 index 0000000000..20a3aac300 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs @@ -0,0 +1,263 @@ +//! Scenario artifact dataclasses and adapter identifier callbacks. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_scenario::values::{MODELS, TYPES, error, string}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_analysis::planner_scenario::main::{hash_prefix, parse_artifact_name}; + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn compute_scenario_hash_prefix( + py: Python<'_>, + kwargs: Option>, +) -> PyResult { + let args = KeywordArguments::read( + py, + "compute_scenario_hash_prefix", + kwargs, + ( + &["project_name", "scenario_name"], + &["run_namespace", "prefix_length"], + ), + )?; + let project: String = args.value("project_name")?.extract()?; + let scenario: String = args.value("scenario_name")?.extract()?; + let namespace: Option = args.value("run_namespace")?.extract()?; + let length: Option = args.value("prefix_length")?.extract()?; + compute_prefix( + py, + (&project, &scenario, namespace.as_deref()), + length.unwrap_or(12), + ) +} + +fn compute_prefix( + py: Python<'_>, + identity: (&str, &str, Option<&str>), + prefix_length: i64, +) -> PyResult { + if prefix_length < 1 { + return Err(error( + py, + "Scenario hash prefix length must be at least 1", + "SCENARIO_PLAN_INVALID_HASH_PREFIX", + None, + )?); + } + Ok(hash_prefix::hash_prefix( + identity.0, + identity.1, + identity.2, + usize::try_from(prefix_length).unwrap_or(usize::MAX), + )) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn build_scenario_hash_index<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_scenario_hash_index", + kwargs, + ( + &["project_name", "scenarios"], + &["run_namespace", "prefix_length"], + ), + )?; + let project_name: String = args.value("project_name")?.extract()?; + let scenarios = args.value("scenarios")?; + let run_namespace: Option = args.value("run_namespace")?.extract()?; + let prefix_length: Option = args.value("prefix_length")?.extract()?; + let result = PyDict::new(py); + let mut seen: std::collections::HashMap = std::collections::HashMap::new(); + for scenario in scenarios.try_iter()? { + let name: String = scenario?.getattr("name")?.extract()?; + let prefix = compute_prefix( + py, + (&project_name, &name, run_namespace.as_deref()), + prefix_length.unwrap_or(12), + )?; + if let Some(existing) = seen.insert(prefix.clone(), name.clone()) { + return Err(error( + py, + &format!( + "Scenario artifact hash collision: scenarios '{existing}' and '{name}' both map to hash prefix '{prefix}'. Rename one scenario file so SQLBuild can generate distinct warehouse artifact prefixes." + ), + "SCENARIO_PLAN_HASH_COLLISION", + Some( + "Rename one scenario file so SQLBuild can generate distinct artifact prefixes.", + ), + )?); + } + result.set_item(name, prefix)?; + } + Ok(result) +} + +#[pyfunction] +#[pyo3(signature = (*, logical_name, fixed_prefix, identifier_limit))] +pub(crate) fn fit_scenario_artifact_logical_name<'py>( + py: Python<'py>, + logical_name: Bound<'py, PyAny>, + fixed_prefix: Bound<'py, PyAny>, + identifier_limit: Bound<'py, PyAny>, +) -> PyResult> { + call_with( + &imported( + py, + "sqlbuild.adapter.relations.main.fit_artifact_logical_name", + "fit_artifact_logical_name", + )?, + &[ + ("logical_name", logical_name), + ("fixed_prefix", fixed_prefix), + ("identifier_limit", identifier_limit), + ("artifact_label", string(py, "Scenario artifact")), + ], + ) +} + +#[pyfunction] +#[pyo3(signature = (*, hash_prefix, kind, logical_name, identifier_limit))] +pub(crate) fn build_scenario_artifact_physical_name( + hash_prefix: &str, + kind: &str, + logical_name: Bound<'_, PyAny>, + identifier_limit: Bound<'_, PyAny>, +) -> PyResult { + let py = logical_name.py(); + let prefix = format!("__sqb_{hash_prefix}__{kind}__"); + let logical: String = fit_scenario_artifact_logical_name( + py, + logical_name, + string(py, &prefix), + identifier_limit, + )? + .extract()?; + Ok(format!("{prefix}{logical}")) +} + +#[pyfunction] +#[pyo3(signature = (*, hash_prefix, kind, logical_name, identifier_limit=63))] +pub(crate) fn build_scenario_artifact_name( + hash_prefix: &str, + kind: Bound<'_, PyAny>, + logical_name: Bound<'_, PyAny>, + identifier_limit: i64, +) -> PyResult { + let py = kind.py(); + let kind: String = imported(py, TYPES, "ScenarioArtifactKind")? + .call1((kind,))? + .getattr("value")? + .extract()?; + let limit = pyo3::IntoPyObjectExt::into_bound_py_any(identifier_limit, py)?; + build_scenario_artifact_physical_name(hash_prefix, &kind, logical_name, limit) +} + +#[pyfunction] +pub(crate) fn parse_scenario_artifact_physical_name<'py>( + py: Python<'py>, + name: &str, +) -> PyResult> { + let Some((hash, kind, logical)) = parse_artifact_name::parse_artifact_name(name) else { + return Ok(py.None().into_bound(py)); + }; + call_with( + &imported(py, MODELS, "ParsedScenarioArtifactName")?, + &[ + ("hash_prefix", string(py, hash)), + ("kind", string(py, kind)), + ("logical_name", string(py, logical)), + ], + ) +} + +#[pyfunction] +pub(crate) fn is_scenario_artifact_physical_name(name: &str) -> bool { + parse_artifact_name::parse_artifact_name(name).is_some() +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn build_scenario_relation_map<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_scenario_relation_map", + kwargs, + ( + &["scenario_name", "hash_prefix", "artifacts"], + &["identifier_limit", "normalize_identifier"], + ), + )?; + let scenario: String = args.value("scenario_name")?.extract()?; + let hash: String = args.value("hash_prefix")?.extract()?; + let limit = args.value("identifier_limit")?; + let limit = if limit.is_none() { + 63 + } else { + limit.extract()? + }; + let normalize = args.value("normalize_identifier")?; + let seen = PyDict::new(py); + let mut artifacts: Vec> = Vec::new(); + for artifact in args.value("artifacts")?.try_iter()? { + let artifact = artifact?; + let physical = build_scenario_artifact_name( + &hash, + artifact.getattr("kind")?, + artifact.getattr("logical_name")?, + limit, + )?; + let normalized = if normalize.is_none() { + string(py, &physical) + } else { + normalize.call1((&physical,))? + }; + if let Some(existing) = seen.get_item(&normalized)? { + let kind = imported(py, TYPES, "ScenarioArtifactKind")?; + let old_kind: String = kind + .call1((existing.getattr("kind")?,))? + .getattr("value")? + .extract()?; + let new_kind: String = kind + .call1((artifact.getattr("kind")?,))? + .getattr("value")? + .extract()?; + let old_name: String = existing.getattr("logical_name")?.extract()?; + let new_name: String = artifact.getattr("logical_name")?.extract()?; + return Err(error( + py, + &format!( + "Scenario relation name collision in scenario '{scenario}'. Both scenario artifacts map to '{physical}': {old_kind} {old_name} and {new_kind} {new_name}. Rename one artifact or shorten one model/source name." + ), + "SCENARIO_PLAN_RELATION_COLLISION", + Some("Rename or shorten one of the colliding scenario artifacts."), + )?); + } + seen.set_item(normalized, &artifact)?; + artifacts.push(call_with( + &imported(py, MODELS, "ScenarioArtifactName")?, + &[ + ("identity", artifact), + ("physical_name", string(py, &physical)), + ], + )?); + } + call_with( + &imported(py, MODELS, "ScenarioRelationMap")?, + &[ + ("scenario_name", args.value("scenario_name")?), + ("hash_prefix", args.value("hash_prefix")?), + ("artifacts", PyTuple::new(py, artifacts)?.into_any()), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index e07ff15c71..7d7bc1c889 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -9,7 +9,7 @@ use crate::bindings::_helpers::planner_scenario::values::{CONSTANTS, MODELS, TYP use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -const ARTIFACTS: &str = "sqlbuild.compiler.planner._helpers.scenario.artifacts"; +const ARTIFACTS: &str = "sqlbuild._native"; #[pyfunction] #[pyo3(signature = (**kwargs))] @@ -98,11 +98,7 @@ pub(crate) fn build_cli_scenario_plan<'py>( } let relations = build_scenario_relation_plan(py, Some(relation_args))?; let result = call_with( - &imported( - py, - "sqlbuild.compiler.planner._helpers.scenario.relations", - "build_scenario_execution_plan", - )?, + &imported(py, "sqlbuild._native", "build_scenario_execution_plan")?, &[ ("scenario", scenario), ("project", project), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs index bdb5f14345..22fd61edbd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs @@ -2,3 +2,5 @@ pub(super) const REF: &str = "ref"; pub(super) const SOURCE: &str = "source"; +pub(super) const SOURCE_MARKER: usize = 2; +pub(super) const DBT_REF: &str = "dbt_ref"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs new file mode 100644 index 0000000000..69ad9e77e2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs @@ -0,0 +1,159 @@ +//! Existing seed and function plan dataclass construction. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::compiled_project::loop_classes::by_attribute; +use crate::bindings::_helpers::planner_scenario::{ + models::SqlContext, + sql::required_target, + values::{MODELS, error}, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +pub(super) fn seed_entries<'py>( + project: &Bound<'py, PyAny>, + graph: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let seeds = by_attribute(&project.getattr("seeds")?, "name")?; + let fixtures = graph.getattr("seed_fixture_names")?; + let mut result: Vec> = Vec::new(); + for name in graph.getattr("seed_names")?.try_iter()? { + let name = name?; + if fixtures.contains(&name)? { + continue; + } + let Some(seed) = seeds.get_item(&name)? else { + let scenario: String = graph.getattr("name")?.extract()?; + let name: String = name.extract()?; + return Err(error( + py, + &format!("Scenario '{scenario}' requires unknown seed '{name}'"), + "SCENARIO_PLAN_UNKNOWN_SEED", + None, + )?); + }; + let destination = required_target( + relations, + "seed_locations", + &name.extract::()?, + "seed", + )?; + result.push(call_with( + &imported(py, MODELS, "SeedPlanEntry")?, + &[ + ("key", seed.getattr("key")?), + ("name", seed.getattr("name")?), + ("destination", destination), + ( + "file_path", + seed.getattr("seed_file")?.getattr("file_path")?, + ), + ( + "columns", + imported( + py, + "sqlbuild.compiler.planner._helpers.output.plan_entry", + "extract_seed_columns", + )? + .call1((&seed,))?, + ), + ( + "csv_settings", + seed.getattr("schema_entry")?.getattr("csv_settings")?, + ), + ], + )?); + } + PyTuple::new(py, result) +} + +pub(super) fn function_entry<'py>( + function: &Bound<'py, PyAny>, + context: &SqlContext<'py>, + locations: &Bound<'py, PyAny>, + warehouse_columns: &Bound<'py, PyAny>, +) -> PyResult> { + let py = function.py(); + let body = call_with( + &imported( + py, + "sqlbuild.compiler.planner._helpers.resolve.resolve", + "resolve_function_sql", + )?, + &[ + ("adapter", context.adapter.clone()), + ("function", function.clone()), + ( + "model_locations", + context.relations.getattr("model_locations")?, + ), + ( + "seed_locations", + context.relations.getattr("seed_locations")?, + ), + ("function_locations", locations.clone()), + ("source_map", context.relations.getattr("source_map")?), + ("source_warehouse_columns", warehouse_columns.clone()), + ( + "star_exclude_keyword", + context.adapter.call_method0("star_exclude_keyword")?, + ), + ], + )?; + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = Vec::new(); + for name in [ + "key", + "name", + "relative_path", + "destination", + "arguments", + "returns", + ] { + fields.push((name, function.getattr(name)?)); + } + fields.push(("body_sql", body)); + fields.push(( + "fingerprint_query_sql", + imported( + py, + "sqlbuild._native", + "build_compiled_function_fingerprint_sql", + )? + .call1((function,))?, + )); + for name in [ + "fingerprint_destination", + "return_columns", + "language", + "source_file_path", + "runtime_version", + "entry_point", + "packages", + ] { + fields.push((name, function.getattr(name)?)); + } + call_with(&imported(py, MODELS, "FunctionPlanEntry")?, &fields) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn build_scenario_seed_entries<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_scenario_seed_entries", + kwargs, + (&["project", "graph_plan", "relation_plan"], &[]), + )?; + seed_entries( + &args.value("project")?, + &args.value("graph_plan")?, + &args.value("relation_plan")?, + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs new file mode 100644 index 0000000000..75113052c3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -0,0 +1,331 @@ +//! Assemble scenario execution plans through existing planner and adapter APIs. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::compiled_project::loop_classes::by_attribute; +use crate::bindings::_helpers::planner_scenario::{ + entries::{function_entry, seed_entries}, + fixtures::{extract_fixtures, fixture_plans}, + models::SqlContext, + sql::{required_target, resolve_model_dbt_refs, resolve_sql, source_label}, + values::{MODELS, error, string}, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn build_scenario_execution_plan<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_scenario_execution_plan", + kwargs, + ( + &[ + "scenario", + "project", + "adapter", + "graph_plan", + "relation_plan", + "source_lexical_syntax", + ], + &[ + "snapshot", + "source_warehouse_columns", + "sql_analysis_enabled", + ], + ), + )?; + let scenario = args.value("scenario")?; + let project = args.value("project")?; + let graph = args.value("graph_plan")?; + let context = SqlContext { + adapter: args.value("adapter")?, + relations: args.value("relation_plan")?, + syntax: args.value("source_lexical_syntax")?, + label: source_label(&scenario)?, + }; + let snapshot = args.value("snapshot")?; + let snapshot = if snapshot.is_truthy()? { + snapshot + } else { + imported(py, MODELS, "WarehouseSnapshot")?.call0()? + }; + let columns = args.value("source_warehouse_columns")?; + let columns = if columns.is_truthy()? { + columns + } else { + PyDict::new(py).into_any() + }; + let analysis = args.value("sql_analysis_enabled")?; + let analysis = if analysis.is_none() { + true + } else { + analysis.is_truthy()? + }; + let models = by_attribute(&project.getattr("models")?, "name")?; + let functions = by_attribute(&project.getattr("functions")?, "key")?; + let locations = imported(py, "sqlbuild._native", "build_function_locations")? + .call1((project.getattr("functions")?,))?; + let upstream = + imported(py, "sqlbuild._native", "build_execution_upstream_deps")?.call1((&project,))?; + let ordered = call_with( + &imported(py, "sqlbuild._native", "topologically_order_keys")?, + &[("upstream", upstream)], + )?; + let mut model_keys: Vec> = Vec::new(); + for key in ordered.try_iter()? { + let key = key?; + if key.getattr("resource_type")?.eq("model")? + && graph + .getattr("model_names")? + .contains(key.getattr("name")?)? + { + model_keys.push(key); + } + } + let overrides = if analysis { + complete_fixtures(&scenario, &project, (&context, &model_keys))? + } else { + py.None().into_bound(py) + }; + let fixtures = fixture_plans(&scenario, &graph, &context, &overrides)?; + let seeds = seed_entries(&project, &graph, &context.relations)?; + let mut function_entries: Vec> = Vec::new(); + for key in ordered.try_iter()? { + let key = key?; + if graph.getattr("function_deps")?.contains(&key)? + && let Some(function) = functions.get_item(key)? + { + function_entries.push(function_entry(&function, &context, &locations, &columns)?); + } + } + let mut entries: Vec> = Vec::new(); + let mut warnings: Vec> = Vec::new(); + for key in model_keys { + let model = models + .get_item(key.getattr("name")?)? + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err("scenario model"))?; + let destination = required_target( + &context.relations, + "model_locations", + &model.getattr("name")?.extract::()?, + "model", + )?; + let query = resolve_model_dbt_refs( + &model.getattr("query_sql")?.extract::()?, + &context.relations, + )?; + let record = imported(py, "sqlbuild._native", "compiled_model_record")?.call1((&model,))?; + let changes = PyDict::new(py); + changes.set_item("destination", destination)?; + changes.set_item("query_sql", query)?; + let model = imported(py, "dataclasses", "replace")?.call((record,), Some(&changes))?; + let model_context = model_context(&project, &context, (&models, &locations, &columns))?; + let result = call_with( + &imported( + py, + "sqlbuild.compiler.planner._helpers.output.plan_entry", + "plan_model", + )?, + &[ + ("model", model), + ("snapshot", snapshot.clone()), + ("adapter", context.adapter.clone()), + ("context", model_context), + ( + "sql_analysis_enabled", + PyBool::new(py, analysis).to_owned().into_any(), + ), + ( + "query_change_tracking", + PyBool::new(py, false).to_owned().into_any(), + ), + ("full_refresh", PyBool::new(py, true).to_owned().into_any()), + ( + "cursor_overrides", + imported(py, MODELS, "CursorOverridePair")?.call0()?, + ), + ], + )?; + entries.push(result.get_item(0)?); + for warning in result.get_item(1)?.try_iter()? { + warnings.push(warning?); + } + } + let (expected, assertions) = expectations(&scenario, &context)?; + let plan = call_with( + &imported(py, MODELS, "ScenarioExecutionPlan")?, + &[ + ("key", scenario.getattr("key")?), + ("name", scenario.getattr("name")?), + ("graph_plan", graph), + ("relation_plan", context.relations), + ("fixture_plans", fixtures.into_any()), + ("seed_entries", seeds.into_any()), + ( + "function_entries", + PyTuple::new(py, function_entries)?.into_any(), + ), + ("model_entries", PyTuple::new(py, entries)?.into_any()), + ("hook_functions", project.getattr("hook_functions")?), + ("expected_expectations", expected), + ("assertion_expectations", assertions), + ], + )?; + Ok(PyTuple::new(py, [plan, PyTuple::new(py, warnings)?.into_any()])?.into_any()) +} + +fn complete_fixtures<'py>( + scenario: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, + inputs: (&SqlContext<'py>, &[Bound<'py, PyAny>]), +) -> PyResult> { + let (context, keys) = inputs; + let py = scenario.py(); + let mut groups: Vec> = Vec::new(); + for (kind, prefix) in [ + ("SOURCE", "__source__"), + ("MODEL", "__ref__"), + ("SEED", "__seed__"), + ] { + let kind = imported( + py, + "sqlbuild.compiler.compile.types", + "CompiledResourceType", + )? + .getattr(kind)?; + groups.push(PyTuple::new( + py, + [kind, extract_fixtures(scenario, prefix)?.into_any()], + )?); + } + let names = keys + .iter() + .map(|key| key.getattr("name")) + .collect::>>()?; + let completion = call_with( + &imported( + py, + "sqlbuild.compiler.planner._helpers.fixtures.completion", + "build_relation_fixture_completion", + )?, + &[ + ("project", project.clone()), + ("adapter", context.adapter.clone()), + ("ordered_model_names", PyTuple::new(py, names)?.into_any()), + ("fixture_groups", PyTuple::new(py, groups)?.into_any()), + ], + )?; + let diagnostics = completion.getattr("diagnostics")?; + if diagnostics.is_truthy()? { + let mut details: Vec = Vec::new(); + for diagnostic in diagnostics.try_iter()? { + details.push(format!( + "- {}", + diagnostic?.getattr("message")?.extract::()? + )); + } + let name: String = scenario.getattr("name")?.extract()?; + return Err(error( + py, + &format!( + "Scenario '{name}' has invalid relation fixtures in {}:\n{}", + context.label, + details.join("\n") + ), + "SCENARIO_PLAN_INVALID_FIXTURE", + None, + )?); + } + completion.getattr("fixture_sql_by_key") +} + +fn model_context<'py>( + project: &Bound<'py, PyAny>, + context: &SqlContext<'py>, + inputs: (&Bound<'py, PyDict>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let (models, locations, columns) = inputs; + let py = project.py(); + call_with( + &imported(py, MODELS, "ModelPlanContext")?, + &[ + ( + "model_locations", + context.relations.getattr("model_locations")?, + ), + ("models_by_name", models.clone().into_any()), + ( + "functions_by_name", + by_attribute(&project.getattr("functions")?, "name")?.into_any(), + ), + ( + "seed_locations", + context.relations.getattr("seed_locations")?, + ), + ("function_locations", locations.clone()), + ("source_map", context.relations.getattr("source_map")?), + ("source_warehouse_columns", columns.clone()), + ( + "star_exclude_keyword", + context.adapter.call_method0("star_exclude_keyword")?, + ), + ], + ) +} + +fn expectations<'py>( + scenario: &Bound<'py, PyAny>, + context: &SqlContext<'py>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py = scenario.py(); + let mut expected: Vec> = Vec::new(); + for cte in scenario.getattr("expected_ctes")?.try_iter()? { + let cte = cte?; + let name: String = cte.getattr("name")?.extract()?; + let name = name.strip_prefix("__expected__").unwrap_or(&name); + let target = required_target(&context.relations, "model_locations", name, "model")?; + let sql = resolve_sql( + context, + &cte.getattr("sql_body")?.extract::()?, + false, + )?; + expected.push(call_with( + &imported(py, MODELS, "ScenarioExpectedExpectationPlan")?, + &[ + ("model_name", string(py, name)), + ("actual_destination", target), + ("expected_sql", string(py, &sql)), + ], + )?); + } + let mut assertions: Vec> = Vec::new(); + for cte in scenario.getattr("assertion_ctes")?.try_iter()? { + let cte = cte?; + let name: String = cte.getattr("name")?.extract()?; + let sql = resolve_sql( + context, + &cte.getattr("sql_body")?.extract::()?, + false, + )?; + assertions.push(call_with( + &imported(py, MODELS, "ScenarioAssertionExpectationPlan")?, + &[ + ( + "name", + string(py, name.strip_prefix("__assert__").unwrap_or(&name)), + ), + ("sql", string(py, &sql)), + ], + )?); + } + Ok(( + PyTuple::new(py, expected)?.into_any(), + PyTuple::new(py, assertions)?.into_any(), + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs new file mode 100644 index 0000000000..e78120379a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs @@ -0,0 +1,147 @@ +//! Materialize fixture SQL plans while preserving adapter and completion boundaries. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_scenario::{ + models::SqlContext, + sql::{required_target, resolve_sql, source_label}, + values::{MODELS, TYPES, error, string}, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +pub(super) fn extract_fixtures<'py>( + scenario: &Bound<'py, PyAny>, + prefix: &str, +) -> PyResult> { + let result = PyDict::new(scenario.py()); + for cte in scenario.getattr("authored_ctes")?.try_iter()? { + let cte = cte?; + let name: String = cte.getattr("name")?.extract()?; + if let Some(name) = name.strip_prefix(prefix) { + result.set_item(name, cte.getattr("sql_body")?)?; + } + } + Ok(result) +} + +pub(super) fn fixture_plans<'py>( + scenario: &Bound<'py, PyAny>, + graph: &Bound<'py, PyAny>, + context: &SqlContext<'py>, + overrides: &Bound<'py, PyAny>, +) -> PyResult> { + let py = scenario.py(); + let mut helpers: Vec = Vec::new(); + for cte in scenario.getattr("authored_ctes")?.try_iter()? { + let cte = cte?; + let name: String = cte.getattr("name")?.extract()?; + if ["__source__", "__ref__", "__seed__", "__dbt_ref__"] + .iter() + .any(|prefix| name.starts_with(prefix)) + { + continue; + } + let sql = resolve_sql( + context, + &cte.getattr("sql_body")?.extract::()?, + true, + )?; + helpers.push(format!("{name} AS ({sql})")); + } + let mut result: Vec> = Vec::new(); + for (kind, resource, names, locations) in [ + ( + "source", + "source", + "source_fixture_names", + "source_fixture_locations", + ), + ("ref", "model", "ref_fixture_names", "ref_fixture_locations"), + ( + "seed", + "seed", + "seed_fixture_names", + "seed_fixture_locations", + ), + ( + "dbt_ref", + "dbt_ref", + "dbt_ref_fixture_names", + "dbt_ref_fixture_locations", + ), + ] { + let fixtures = extract_fixtures(scenario, &format!("__{kind}__"))?; + for name in graph.getattr(names)?.try_iter()? { + let name: String = name?.extract()?; + let target = required_target(&context.relations, locations, &name, kind)?; + let Some(mut sql) = fixtures.get_item(&name)? else { + return Err(error( + py, + &format!("Scenario is missing {kind} fixture SQL '{name}'"), + "SCENARIO_PLAN_MISSING_FIXTURE_SQL", + None, + )?); + }; + if kind != super::constants::DBT_REF && !overrides.is_none() { + sql = overrides.call_method1("get", ((resource, &name), sql))?; + } + let sql = resolve_sql(context, &sql.extract::()?, true)?; + let sql = if helpers.is_empty() { + sql + } else { + format!("WITH {} {sql}", helpers.join(", ")) + }; + result.push(call_with( + &imported(py, MODELS, "ScenarioFixturePlan")?, + &[ + ( + "kind", + imported(py, TYPES, "ScenarioArtifactKind")?.call1((kind,))?, + ), + ("logical_name", string(py, &name)), + ("destination", target), + ("sql", string(py, &sql)), + ], + )?); + } + } + PyTuple::new(py, result) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn build_scenario_fixture_plans<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_scenario_fixture_plans", + kwargs, + ( + &[ + "scenario", + "graph_plan", + "relation_plan", + "adapter", + "source_lexical_syntax", + ], + &["fixture_sql_overrides"], + ), + )?; + let scenario = args.value("scenario")?; + let context = SqlContext { + adapter: args.value("adapter")?, + relations: args.value("relation_plan")?, + syntax: args.value("source_lexical_syntax")?, + label: source_label(&scenario)?, + }; + fixture_plans( + &scenario, + &args.value("graph_plan")?, + &context, + &args.value("fixture_sql_overrides")?, + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs index 05781c3595..fcbd7bb887 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs @@ -1,8 +1,14 @@ //! Python boundaries for native scenario planning. +mod artifacts; mod cli; mod constants; +mod entries; +mod execution; +mod fixtures; mod graph; +mod models; pub(crate) mod registration; mod relations; +mod sql; mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs new file mode 100644 index 0000000000..0c97f32fd3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs @@ -0,0 +1,11 @@ +//! Borrowed Python callback boundaries for scenario SQL assembly. + +use pyo3::prelude::{Bound, PyAny}; + +#[derive(Debug)] +pub(crate) struct SqlContext<'py> { + pub adapter: Bound<'py, PyAny>, + pub relations: Bound<'py, PyAny>, + pub syntax: Bound<'py, PyAny>, + pub label: String, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs index c7c943ec48..a6e7cbe60c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs @@ -1,10 +1,57 @@ //! Register scenario planning bindings. -use crate::bindings::_helpers::planner_scenario::{cli, graph, relations}; +use crate::bindings::_helpers::planner_scenario::{ + artifacts, cli, entries, execution, fixtures, graph, relations, sql, +}; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + execution::build_scenario_execution_plan, + module + )?)?; + module.add_function(wrap_pyfunction!( + fixtures::build_scenario_fixture_plans, + module + )?)?; + module.add_function(wrap_pyfunction!( + entries::build_scenario_seed_entries, + module + )?)?; + module.add_function(wrap_pyfunction!(sql::resolve_scenario_check_sql, module)?)?; + module.add_function(wrap_pyfunction!( + artifacts::compute_scenario_hash_prefix, + module + )?)?; + module.add_function(wrap_pyfunction!( + artifacts::build_scenario_hash_index, + module + )?)?; + module.add_function(wrap_pyfunction!( + artifacts::build_scenario_artifact_name, + module + )?)?; + module.add_function(wrap_pyfunction!( + artifacts::build_scenario_artifact_physical_name, + module + )?)?; + module.add_function(wrap_pyfunction!( + artifacts::fit_scenario_artifact_logical_name, + module + )?)?; + module.add_function(wrap_pyfunction!( + artifacts::parse_scenario_artifact_physical_name, + module + )?)?; + module.add_function(wrap_pyfunction!( + artifacts::is_scenario_artifact_physical_name, + module + )?)?; + module.add_function(wrap_pyfunction!( + artifacts::build_scenario_relation_map, + module + )?)?; module.add_function(wrap_pyfunction!(graph::plan_scenario_graph, module)?)?; module.add_function(wrap_pyfunction!( relations::build_scenario_relation_plan, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs new file mode 100644 index 0000000000..e3b8b69279 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs @@ -0,0 +1,144 @@ +//! Resolve scenario SQL through the shared native scanner and adapter callbacks. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_scenario::{ + models::SqlContext, + values::{error, string}, +}; +use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::{PyDict, PyStringMethods}; +use sqlbuild_analysis::planner_scenario::{ + errors::MarkerError, main::replace_markers::replace_markers, +}; +use sqlbuild_sqltext::sql_scan::models::Unclosed; + +pub(super) fn resolve_sql( + context: &SqlContext<'_>, + sql: &str, + project_sources: bool, +) -> PyResult { + let py = context.adapter.py(); + let syntax: LexicalSyntaxInput = context.syntax.getattr("native_mapping")?.extract()?; + let mut result = sql.to_owned(); + for (kind, field) in [ + (0, "model_locations"), + (1, "seed_locations"), + ( + 2, + if project_sources { + "project_source_map" + } else { + "source_map" + }, + ), + (3, "dbt_ref_fixture_locations"), + ] { + if project_sources && kind != super::constants::SOURCE_MARKER { + continue; + } + let locations = context.relations.getattr(field)?; + result = replace_markers(&result, kind, Some(syntax.clone().into()), |name, marker| { + let target = locations.call_method1("get", (name,))?; + let target = if target.is_none() {target} else if kind == super::constants::SOURCE_MARKER { + call_with(&imported(py, "sqlbuild._native", "render_source_relation")?, &[("entry", target), ("adapter", context.adapter.clone())])? + } else {target.getattr("qualified_name")?}; + if target.is_none() { + let marker = string(py, marker).repr()?.to_str()?.to_owned(); + return Err(error(py, &format!("Scenario SQL in {} references {marker}, which is not a relation in this scenario", context.label), + "SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER", Some("Reference a model, seed, source, or dbt ref the scenario builds or provides as a fixture."))?); + } + target.extract::() + }).map_err(|failure| match failure { + MarkerError::Resolve(error) => error, + MarkerError::Scan(unclosed) => { + let construct = match unclosed {Unclosed::BlockComment => "block comment", Unclosed::Quote => "quoted string", Unclosed::Parenthesis => "parenthesis"}; + error(py, &format!("Scenario SQL in {} could not be scanned for relation markers: SQL reference resolution contains an unclosed {construct}", context.label), + "SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER", Some("Close every quoted string and block comment using the project dialect's quoting and comment rules.")).unwrap_or_else(|error| error) + } + })?; + } + Ok(result) +} + +pub(super) fn resolve_model_dbt_refs(sql: &str, relations: &Bound<'_, PyAny>) -> PyResult { + let locations = relations.getattr("dbt_ref_fixture_locations")?; + replace_markers(sql, 3, None, |name, marker| { + let target = locations.call_method1("get", (name,))?; + let qualified = if target.is_none() { + target + } else { + target.getattr("qualified_name")? + }; + Ok(if qualified.is_none() { + marker.to_owned() + } else { + qualified.extract()? + }) + }) + .map_err(|failure| match failure { + MarkerError::Resolve(error) => error, + MarkerError::Scan(_) => { + pyo3::exceptions::PyRuntimeError::new_err("unscanned marker replacement failed") + } + }) +} + +pub(super) fn source_label(scenario: &Bound<'_, PyAny>) -> PyResult { + let source = scenario.getattr("source_path")?; + if source.is_truthy()? { + Ok(source.str()?.to_str()?.to_owned()) + } else { + scenario.getattr("name")?.extract() + } +} + +pub(super) fn required_target<'py>( + relations: &Bound<'py, PyAny>, + field: &str, + name: &str, + kind: &str, +) -> PyResult> { + let target = relations.getattr(field)?.call_method1("get", (name,))?; + if target.is_none() { + return Err(error( + relations.py(), + &format!("Scenario relation plan is missing {kind} target '{name}'"), + "SCENARIO_PLAN_MISSING_RELATION_TARGET", + Some("This is likely a SQLBuild bug. Please file an issue with the scenario name."), + )?); + } + Ok(target) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn resolve_scenario_check_sql( + py: Python<'_>, + kwargs: Option>, +) -> PyResult { + let args = KeywordArguments::read( + py, + "resolve_scenario_check_sql", + kwargs, + ( + &[ + "sql", + "relation_plan", + "adapter", + "source_label", + "source_lexical_syntax", + ], + &[], + ), + )?; + let context = SqlContext { + adapter: args.value("adapter")?, + relations: args.value("relation_plan")?, + syntax: args.value("source_lexical_syntax")?, + label: args.value("source_label")?.extract()?, + }; + resolve_sql(&context, &args.value("sql")?.extract::()?, false) +} diff --git a/crates/sqlbuild-sqltext/src/resolution/main/mod.rs b/crates/sqlbuild-sqltext/src/resolution/main/mod.rs index 8b223a84d6..4b60837f85 100644 --- a/crates/sqlbuild-sqltext/src/resolution/main/mod.rs +++ b/crates/sqlbuild-sqltext/src/resolution/main/mod.rs @@ -1,6 +1,7 @@ //! Public SQL resolution decisions. pub mod cursor_subquery; +pub mod quoted_name; pub mod ref_alias; pub mod reference_spans; pub mod source_alias; diff --git a/crates/sqlbuild-sqltext/src/resolution/main/quoted_name.rs b/crates/sqlbuild-sqltext/src/resolution/main/quoted_name.rs new file mode 100644 index 0000000000..2bd599f113 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/main/quoted_name.rs @@ -0,0 +1,11 @@ +//! Shared exact double-quoted reference marker argument parsing. + +/// Read a nonempty marker argument and return the unconsumed text. +pub fn quoted_name(text: &str) -> Option<(&str, &str)> { + let text = text.strip_prefix('"')?; + let end = text.find('"')?; + if end == 0 { + return None; + } + Some((&text[..end], &text[end + 1..])) +} diff --git a/crates/sqlbuild-sqltext/src/resolution/main/reference_spans.rs b/crates/sqlbuild-sqltext/src/resolution/main/reference_spans.rs index 7f1d881d41..9970c9f2ac 100644 --- a/crates/sqlbuild-sqltext/src/resolution/main/reference_spans.rs +++ b/crates/sqlbuild-sqltext/src/resolution/main/reference_spans.rs @@ -1,6 +1,7 @@ //! Locate planner reference markers with their historical lexical policy. use crate::resolution::constants::EXTERNAL_REFERENCE; +use crate::resolution::main::quoted_name::quoted_name; use crate::resolution::models::ReferenceSpan; /// Locate exact reference markers, retaining the external-reference whitespace grammar. @@ -49,12 +50,3 @@ pub fn reference_spans(sql: &str, kind: &str) -> Vec { } spans } - -fn quoted_name(text: &str) -> Option<(&str, &str)> { - let text = text.strip_prefix('"')?; - let end = text.find('"')?; - if end == 0 { - return None; - } - Some((&text[..end], &text[end + 1..])) -} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 2d2fc3b2f9..25af267edf 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -233,6 +233,7 @@ from sqlbuild.compiler.planner.models import ( ModelPlanContext, ModelPlanEntry, OldNameView, + ParsedScenarioArtifactName, ParsedSelector, PathSelector, PlannerChangeResults, @@ -247,12 +248,15 @@ from sqlbuild.compiler.planner.models import ( RelationFixtureCompletion, RelationFixtureDiagnostic, RelationFixturePlanningContext, + ScenarioArtifactIdentity, ScenarioExecutionPlan, + ScenarioFixturePlan, ScenarioGraphPlan, ScenarioRelationMap, ScenarioRelationPlan, SchemaAction, SchemaFinding, + SeedPlanEntry, SelectionStalenessGraph, SelectionStalenessWarning, SelectorExpansion, @@ -3957,6 +3961,74 @@ def build_cli_scenario_plan( source_lexical_syntax: SqlLexicalSyntax, ) -> ScenarioExecutionPlan: ... def run_scenario_test_command(*, request: ScenarioTestCommandRequest) -> int: ... +def compute_scenario_hash_prefix( + *, + project_name: str, + scenario_name: str, + run_namespace: str | None = None, + prefix_length: int = 12, +) -> str: ... +def build_scenario_hash_index( + *, + project_name: str, + scenarios: tuple[CompiledSqlScenario, ...], + run_namespace: str | None = None, + prefix_length: int = 12, +) -> dict[str, str]: ... +def build_scenario_artifact_name( + *, hash_prefix: str, kind: str, logical_name: str, identifier_limit: int = 63 +) -> str: ... +def build_scenario_artifact_physical_name( + *, hash_prefix: str, kind: str, logical_name: str, identifier_limit: int +) -> str: ... +def fit_scenario_artifact_logical_name( + *, logical_name: str, fixed_prefix: str, identifier_limit: int +) -> str: ... +def parse_scenario_artifact_physical_name(name: str) -> ParsedScenarioArtifactName | None: ... +def is_scenario_artifact_physical_name(name: str) -> bool: ... +def build_scenario_relation_map( + *, + scenario_name: str, + hash_prefix: str, + artifacts: tuple[ScenarioArtifactIdentity, ...], + identifier_limit: int = 63, + normalize_identifier: Callable[[str], str] | None = None, +) -> ScenarioRelationMap: ... +def resolve_scenario_check_sql( + *, + sql: str, + relation_plan: ScenarioRelationPlan, + adapter: BaseAdapter, + source_label: str, + source_lexical_syntax: SqlLexicalSyntax, +) -> str: ... +def build_scenario_fixture_plans( + *, + scenario: CompiledSqlScenario, + graph_plan: ScenarioGraphPlan, + relation_plan: ScenarioRelationPlan, + adapter: BaseAdapter, + source_lexical_syntax: SqlLexicalSyntax, + fixture_sql_overrides: dict[tuple[CompiledResourceType, str], str] | None = None, +) -> tuple[ScenarioFixturePlan, ...]: ... +def build_scenario_seed_entries( + *, + project: CompiledProjectView, + graph_plan: ScenarioGraphPlan, + relation_plan: ScenarioRelationPlan, +) -> tuple[SeedPlanEntry, ...]: ... +def build_scenario_execution_plan( + *, + scenario: CompiledSqlScenario, + project: CompiledProjectView, + adapter: BaseAdapter, + graph_plan: ScenarioGraphPlan, + relation_plan: ScenarioRelationPlan, + source_lexical_syntax: SqlLexicalSyntax, + snapshot: WarehouseSnapshot | None = None, + source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]] | None = None, + sql_analysis_enabled: bool = True, +) -> tuple[ScenarioExecutionPlan, tuple[PlanWarning, ...]]: ... def run_scenario_capture_command(*, request: ScenarioCaptureCommandRequest) -> int: ... def sql_analysis_off_guidance( *, discovered_inputs: DiscoveredProjectInputs, purpose: str diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/artifact_names.py b/src/sqlbuild/compiler/planner/_helpers/scenario/artifact_names.py deleted file mode 100644 index 1de215dfe6..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/artifact_names.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Scenario artifact physical-name helper implementations.""" - -from __future__ import annotations - -import re - -from sqlbuild.adapter.relations.main.fit_artifact_logical_name import fit_artifact_logical_name -from sqlbuild.compiler.planner.constants import ( - SCENARIO_ARTIFACT_KINDS, - SCENARIO_ARTIFACT_PREFIX, - SCENARIO_HASH_PREFIX_LENGTH, -) -from sqlbuild.compiler.planner.models import ParsedScenarioArtifactName - -_SCENARIO_ARTIFACT_NAME_RE: re.Pattern[str] = re.compile( - rf"^{re.escape(SCENARIO_ARTIFACT_PREFIX)}" - rf"(?P[0-9a-f]{{{SCENARIO_HASH_PREFIX_LENGTH}}})__" - rf"(?P{'|'.join(SCENARIO_ARTIFACT_KINDS)})__" - r"(?P.+)$" -) - - -def build_scenario_artifact_physical_name( - *, - hash_prefix: str, - kind: str, - logical_name: str, - identifier_limit: int, -) -> str: - fixed_prefix: str = f"{SCENARIO_ARTIFACT_PREFIX}{hash_prefix}__{kind}__" - logical_part: str = fit_scenario_artifact_logical_name( - logical_name=logical_name, - fixed_prefix=fixed_prefix, - identifier_limit=identifier_limit, - ) - return f"{fixed_prefix}{logical_part}" - - -def parse_scenario_artifact_physical_name(name: str) -> ParsedScenarioArtifactName | None: - match: re.Match[str] | None = _SCENARIO_ARTIFACT_NAME_RE.fullmatch(name) - if match is None: - return None - return ParsedScenarioArtifactName( - hash_prefix=match.group("hash_prefix"), - kind=match.group("kind"), - logical_name=match.group("logical_name"), - ) - - -def is_scenario_artifact_physical_name(name: str) -> bool: - return parse_scenario_artifact_physical_name(name) is not None - - -def fit_scenario_artifact_logical_name( - *, logical_name: str, fixed_prefix: str, identifier_limit: int -) -> str: - return fit_artifact_logical_name( - logical_name=logical_name, - fixed_prefix=fixed_prefix, - identifier_limit=identifier_limit, - artifact_label="Scenario artifact", - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/artifacts.py b/src/sqlbuild/compiler/planner/_helpers/scenario/artifacts.py deleted file mode 100644 index f55697843f..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/artifacts.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Deterministic physical artifact naming for SQL scenarios.""" - -from __future__ import annotations - -import hashlib -from collections.abc import Callable - -from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.planner.constants import ( - SCENARIO_DEFAULT_IDENTIFIER_LIMIT, - SCENARIO_HASH_PREFIX_LENGTH, - SCENARIO_PLAN_HASH_COLLISION, - SCENARIO_PLAN_INVALID_HASH_PREFIX, - SCENARIO_PLAN_RELATION_COLLISION, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.scenarios._build_scenario_artifact_physical_name import ( - build_scenario_artifact_physical_name, -) -from sqlbuild.compiler.planner.models import ( - ScenarioArtifactIdentity, - ScenarioArtifactName, - ScenarioRelationMap, -) -from sqlbuild.compiler.planner.types import ScenarioArtifactKind - - -def compute_scenario_hash_prefix( - *, - project_name: str, - scenario_name: str, - run_namespace: str | None = None, - prefix_length: int = SCENARIO_HASH_PREFIX_LENGTH, -) -> str: - """Return a stable scenario artifact hash prefix.""" - - if prefix_length < 1: - raise PlannerInputError( - "Scenario hash prefix length must be at least 1", - code=SCENARIO_PLAN_INVALID_HASH_PREFIX, - ) - hash_input: str = f"{project_name}:{scenario_name}" - if run_namespace is not None: - hash_input = f"{project_name}:{run_namespace}:{scenario_name}" - return hashlib.sha256(hash_input.encode("utf-8")).hexdigest()[:prefix_length] - - -def build_scenario_hash_index( - *, - project_name: str, - scenarios: tuple[CompiledSqlScenario, ...], - run_namespace: str | None = None, - prefix_length: int = SCENARIO_HASH_PREFIX_LENGTH, -) -> dict[str, str]: - """Return scenario name to hash prefix, failing clearly on prefix collisions.""" - - by_prefix: dict[str, CompiledSqlScenario] = {} - result: dict[str, str] = {} - scenario: CompiledSqlScenario - for scenario in scenarios: - prefix: str = compute_scenario_hash_prefix( - project_name=project_name, - scenario_name=scenario.name, - run_namespace=run_namespace, - prefix_length=prefix_length, - ) - existing: CompiledSqlScenario | None = by_prefix.get(prefix) - if existing is not None: - raise PlannerInputError( - "Scenario artifact hash collision: scenarios " - f"'{existing.name}' and '{scenario.name}' both map to hash prefix " - f"'{prefix}'. Rename one scenario file so SQLBuild can generate " - "distinct warehouse artifact prefixes.", - code=SCENARIO_PLAN_HASH_COLLISION, - help=( - "Rename one scenario file so SQLBuild can generate distinct artifact prefixes." - ), - ) - by_prefix[prefix] = scenario - result[scenario.name] = prefix - return result - - -def build_scenario_artifact_name( - *, - hash_prefix: str, - kind: ScenarioArtifactKind | str, - logical_name: str, - identifier_limit: int = SCENARIO_DEFAULT_IDENTIFIER_LIMIT, -) -> str: - """Build one deterministic scenario physical relation name.""" - - normalized_kind: ScenarioArtifactKind = ScenarioArtifactKind(kind) - return build_scenario_artifact_physical_name( - hash_prefix=hash_prefix, - kind=normalized_kind.value, - logical_name=logical_name, - identifier_limit=identifier_limit, - ) - - -def build_scenario_relation_map( - *, - scenario_name: str, - hash_prefix: str, - artifacts: tuple[ScenarioArtifactIdentity, ...], - identifier_limit: int = SCENARIO_DEFAULT_IDENTIFIER_LIMIT, - normalize_identifier: Callable[[str], str] | None = None, -) -> ScenarioRelationMap: - """Build and collision-check physical relation names for one scenario.""" - - normalizer: Callable[[str], str] = normalize_identifier or (lambda value: value) - resolved_artifacts: list[ScenarioArtifactName] = [] - seen: dict[str, ScenarioArtifactIdentity] = {} - artifact: ScenarioArtifactIdentity - for artifact in artifacts: - physical_name: str = build_scenario_artifact_name( - hash_prefix=hash_prefix, - kind=artifact.kind, - logical_name=artifact.logical_name, - identifier_limit=identifier_limit, - ) - normalized_physical_name: str = normalizer(physical_name) - existing: ScenarioArtifactIdentity | None = seen.get(normalized_physical_name) - if existing is not None: - raise PlannerInputError( - f"Scenario relation name collision in scenario '{scenario_name}'. " - f"Both scenario artifacts map to '{physical_name}': " - f"{ScenarioArtifactKind(existing.kind).value} {existing.logical_name} and " - f"{ScenarioArtifactKind(artifact.kind).value} {artifact.logical_name}. " - "Rename one artifact " - "or shorten one model/source name.", - code=SCENARIO_PLAN_RELATION_COLLISION, - help="Rename or shorten one of the colliding scenario artifacts.", - ) - seen[normalized_physical_name] = artifact - resolved_artifacts.append( - ScenarioArtifactName(identity=artifact, physical_name=physical_name) - ) - - return ScenarioRelationMap( - scenario_name=scenario_name, - hash_prefix=hash_prefix, - artifacts=tuple(resolved_artifacts), - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py deleted file mode 100644 index da8e69b766..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py +++ /dev/null @@ -1,744 +0,0 @@ -"""Scenario relation override planning helpers.""" - -from __future__ import annotations - -import re -from collections.abc import Callable -from dataclasses import replace - -from sqlbuild._native import ( - build_compiled_function_fingerprint_sql, - build_execution_upstream_deps, - compiled_model_record, - render_source_relation, - topologically_order_keys, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.constants import ( - DBT_REF_TEST_CTE_PREFIX, - REF_TEST_CTE_PREFIX, - SEED_TEST_CTE_PREFIX, - SOURCE_TEST_CTE_PREFIX, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, - CompiledSqlScenario, - CompileSqlScenarioCte, -) -from sqlbuild.compiler.compile.types import ( - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - CompiledSeedView, -) -from sqlbuild.compiler.planner._helpers.fixtures.completion import ( - build_relation_fixture_completion, -) -from sqlbuild.compiler.planner._helpers.output.plan_entry import extract_seed_columns, plan_model -from sqlbuild.compiler.planner._helpers.resolve.refs import ( - build_function_locations, - replace_executable_matches, -) -from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_function_sql -from sqlbuild.compiler.planner.constants import ( - SCENARIO_PLAN_INVALID_FIXTURE, - SCENARIO_PLAN_MISSING_FIXTURE_SQL, - SCENARIO_PLAN_MISSING_RELATION_TARGET, - SCENARIO_PLAN_UNKNOWN_SEED, - SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - CursorOverridePair, - FunctionPlanEntry, - ModelPlanContext, - ModelPlanEntry, - PlanWarning, - RelationFixtureCompletion, - ScenarioAssertionExpectationPlan, - ScenarioExecutionPlan, - ScenarioExpectedExpectationPlan, - ScenarioFixturePlan, - ScenarioGraphPlan, - ScenarioRelationPlan, - SeedPlanEntry, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import ( - FixtureKey, - ScenarioArtifactKind, -) -from sqlbuild.compiler.references.constants import ( - DBT_REF_CALL_PREFIX, - REF_CALL_PREFIX, - SEED_CALL_PREFIX, - SOURCE_CALL_PREFIX, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import SourceEntry - -_NAME_ARGUMENT_PATTERN_TEXT: str = r'"(?P[^"]+)"\)' -_REF_PATTERN: re.Pattern[str] = re.compile(REF_CALL_PREFIX + _NAME_ARGUMENT_PATTERN_TEXT) -_SEED_PATTERN: re.Pattern[str] = re.compile(SEED_CALL_PREFIX + _NAME_ARGUMENT_PATTERN_TEXT) -_SOURCE_PATTERN: re.Pattern[str] = re.compile(SOURCE_CALL_PREFIX + _NAME_ARGUMENT_PATTERN_TEXT) -_DBT_REF_PATTERN: re.Pattern[str] = re.compile( - DBT_REF_CALL_PREFIX + r'"(?P[^"]+)"(?:\s*,\s*"(?P[^"]+)")?\)' -) - - -def resolve_scenario_check_sql( - *, - sql: str, - relation_plan: ScenarioRelationPlan, - adapter: BaseAdapter, - source_label: str, - source_lexical_syntax: SqlLexicalSyntax, -) -> str: - """Resolve refs, seeds, and sources in authored scenario expected/assertion SQL.""" - - def _ref_target(match: re.Match[str]) -> str | None: - return _location_name(relation_plan.model_locations.get(match.group("name"))) - - def _seed_target(match: re.Match[str]) -> str | None: - return _location_name(relation_plan.seed_locations.get(match.group("name"))) - - def _dbt_ref_target(match: re.Match[str]) -> str | None: - return _location_name( - relation_plan.dbt_ref_fixture_locations.get(_dbt_ref_fixture_name(match)) - ) - - return _replace_relation_markers( - sql=sql, - resolvers=( - (_REF_PATTERN, _ref_target), - (_SEED_PATTERN, _seed_target), - ( - _SOURCE_PATTERN, - _source_resolver(source_map=relation_plan.source_map, adapter=adapter), - ), - (_DBT_REF_PATTERN, _dbt_ref_target), - ), - source_label=source_label, - source_lexical_syntax=source_lexical_syntax, - ) - - -def _source_resolver( - *, source_map: dict[str, SourceEntry], adapter: BaseAdapter -) -> Callable[[re.Match[str]], str | None]: - def _source_target(match: re.Match[str]) -> str | None: - source: SourceEntry | None = source_map.get(match.group("name")) - return None if source is None else render_source_relation(entry=source, adapter=adapter) - - return _source_target - - -def _location_name(location: CompiledRelationLocation | None) -> str | None: - return None if location is None else location.qualified_name - - -def _marker_replacer( - *, resolve_target: Callable[[re.Match[str]], str | None], source_label: str -) -> Callable[[re.Match[str]], str]: - def _replace(match: re.Match[str]) -> str: - target: str | None = resolve_target(match) - if target is None: - raise PlannerInputError( - f"Scenario SQL in {source_label} references {match.group(0)!r}, which is " - "not a relation in this scenario", - code=SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER, - help="Reference a model, seed, source, or dbt ref the scenario builds or " - "provides as a fixture.", - ) - return target - - return _replace - - -def _replace_relation_markers( - *, - sql: str, - resolvers: tuple[tuple[re.Pattern[str], Callable[[re.Match[str]], str | None]], ...], - source_label: str, - source_lexical_syntax: SqlLexicalSyntax, -) -> str: - result: str = sql - pattern: re.Pattern[str] - resolve_target: Callable[[re.Match[str]], str | None] - for pattern, resolve_target in resolvers: - try: - result = replace_executable_matches( - sql=result, - pattern=pattern, - replace=_marker_replacer(resolve_target=resolve_target, source_label=source_label), - lexical_syntax=source_lexical_syntax, - ) - except CompileInputError as error: - raise PlannerInputError( - f"Scenario SQL in {source_label} could not be scanned for relation markers: " - f"{error}", - code=SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER, - help="Close every quoted string and block comment using the project dialect's " - "quoting and comment rules.", - ) from None - return result - - -def build_scenario_execution_plan( - *, - scenario: CompiledSqlScenario, - project: CompiledProjectView, - adapter: BaseAdapter, - graph_plan: ScenarioGraphPlan, - relation_plan: ScenarioRelationPlan, - source_lexical_syntax: SqlLexicalSyntax, - snapshot: WarehouseSnapshot | None = None, - source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]] | None = None, - sql_analysis_enabled: bool = True, -) -> tuple[ScenarioExecutionPlan, tuple[PlanWarning, ...]]: - """Build a dry-run execution plan for one SQL scenario.""" - - effective_snapshot: WarehouseSnapshot = snapshot or WarehouseSnapshot() - effective_source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]] = ( - source_warehouse_columns or {} - ) - models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} - functions_by_key: dict[CompiledObjectKey, CompiledFunctionView] = { - function.key: function for function in project.functions - } - function_locations: dict[str, CompiledRelationLocation] = build_function_locations( - project.functions - ) - scenario_model_names: frozenset[str] = frozenset(graph_plan.model_names) - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_execution_upstream_deps(project) - ) - ordered_model_keys: tuple[CompiledObjectKey, ...] = tuple( - key - for key in topologically_order_keys(upstream=upstream_deps) - if key.resource_type == CompiledResourceType.MODEL and key.name in scenario_model_names - ) - scenario_fixture_groups: tuple[tuple[CompiledResourceType, dict[str, str]], ...] = ( - ( - CompiledResourceType.SOURCE, - _extract_fixture_ctes(scenario=scenario, prefix=SOURCE_TEST_CTE_PREFIX), - ), - ( - CompiledResourceType.MODEL, - _extract_fixture_ctes(scenario=scenario, prefix=REF_TEST_CTE_PREFIX), - ), - ( - CompiledResourceType.SEED, - _extract_fixture_ctes(scenario=scenario, prefix=SEED_TEST_CTE_PREFIX), - ), - ) - fixture_sql_overrides: dict[FixtureKey, str] | None = None - if sql_analysis_enabled: - fixture_completion: RelationFixtureCompletion = build_relation_fixture_completion( - project=project, - adapter=adapter, - ordered_model_names=tuple(key.name for key in ordered_model_keys), - fixture_groups=scenario_fixture_groups, - ) - if fixture_completion.diagnostics: - source_path: str = _scenario_source_label(scenario) - details: str = "\n".join( - f"- {diagnostic.message}" for diagnostic in fixture_completion.diagnostics - ) - raise PlannerInputError( - f"Scenario '{scenario.name}' has invalid relation fixtures in " - f"{source_path}:\n{details}", - code=SCENARIO_PLAN_INVALID_FIXTURE, - ) - fixture_sql_overrides = fixture_completion.fixture_sql_by_key - fixture_plans: tuple[ScenarioFixturePlan, ...] = build_scenario_fixture_plans( - scenario=scenario, - graph_plan=graph_plan, - relation_plan=relation_plan, - adapter=adapter, - source_lexical_syntax=source_lexical_syntax, - fixture_sql_overrides=fixture_sql_overrides, - ) - seed_entries: tuple[SeedPlanEntry, ...] = build_scenario_seed_entries( - project=project, - graph_plan=graph_plan, - relation_plan=relation_plan, - ) - function_entries: tuple[FunctionPlanEntry, ...] = tuple( - _build_scenario_function_entry( - function=functions_by_key[key], - adapter=adapter, - relation_plan=relation_plan, - function_locations=function_locations, - source_warehouse_columns=effective_source_warehouse_columns, - ) - for key in topologically_order_keys(upstream=upstream_deps) - if key in graph_plan.function_deps and key in functions_by_key - ) - - model_entries: list[ModelPlanEntry] = [] - warnings: list[PlanWarning] = [] - key: CompiledObjectKey - for key in ordered_model_keys: - model: CompiledModelView = models_by_name[key.name] - scenario_target: CompiledRelationLocation = _required_target( - targets=relation_plan.model_locations, - name=model.name, - kind=ScenarioArtifactKind.MODEL, - ) - scenario_query_sql: str = _resolve_model_dbt_ref_fixtures( - query_sql=model.query_sql, - relation_plan=relation_plan, - ) - entry: ModelPlanEntry - model_warnings: tuple[PlanWarning, ...] - entry, model_warnings = plan_model( - model=replace( - compiled_model_record(model), - destination=scenario_target, - query_sql=scenario_query_sql, - ), - snapshot=effective_snapshot, - adapter=adapter, - context=ModelPlanContext( - model_locations=relation_plan.model_locations, - models_by_name=models_by_name, - functions_by_name={function.name: function for function in project.functions}, - seed_locations=relation_plan.seed_locations, - function_locations=function_locations, - source_map=relation_plan.source_map, - source_warehouse_columns=effective_source_warehouse_columns, - star_exclude_keyword=adapter.star_exclude_keyword(), - ), - sql_analysis_enabled=sql_analysis_enabled, - query_change_tracking=False, - full_refresh=True, - cursor_overrides=CursorOverridePair(), - ) - model_entries.append(entry) - warnings.extend(model_warnings) - - expected_expectations: tuple[ScenarioExpectedExpectationPlan, ...] = tuple( - _build_expected_check_plan( - expected_cte=expected_cte, - relation_plan=relation_plan, - adapter=adapter, - source_label=_scenario_source_label(scenario), - source_lexical_syntax=source_lexical_syntax, - ) - for expected_cte in scenario.expected_ctes - ) - assertion_expectations: tuple[ScenarioAssertionExpectationPlan, ...] = tuple( - ScenarioAssertionExpectationPlan( - name=assertion_cte.name.removeprefix("__assert__"), - sql=resolve_scenario_check_sql( - sql=assertion_cte.sql_body, - relation_plan=relation_plan, - adapter=adapter, - source_label=_scenario_source_label(scenario), - source_lexical_syntax=source_lexical_syntax, - ), - ) - for assertion_cte in scenario.assertion_ctes - ) - - return ( - ScenarioExecutionPlan( - key=scenario.key, - name=scenario.name, - graph_plan=graph_plan, - relation_plan=relation_plan, - fixture_plans=fixture_plans, - seed_entries=seed_entries, - function_entries=function_entries, - model_entries=tuple(model_entries), - hook_functions=project.hook_functions, - expected_expectations=expected_expectations, - assertion_expectations=assertion_expectations, - ), - tuple(warnings), - ) - - -def _build_scenario_function_entry( - *, - function: CompiledFunctionView, - adapter: BaseAdapter, - relation_plan: ScenarioRelationPlan, - function_locations: dict[str, CompiledRelationLocation], - source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]], -) -> FunctionPlanEntry: - return FunctionPlanEntry( - key=function.key, - name=function.name, - relative_path=function.relative_path, - destination=function.destination, - arguments=function.arguments, - returns=function.returns, - body_sql=resolve_function_sql( - adapter=adapter, - function=function, - model_locations=relation_plan.model_locations, - seed_locations=relation_plan.seed_locations, - function_locations=function_locations, - source_map=relation_plan.source_map, - source_warehouse_columns=source_warehouse_columns, - star_exclude_keyword=adapter.star_exclude_keyword(), - ), - fingerprint_query_sql=build_compiled_function_fingerprint_sql(function), - fingerprint_destination=function.fingerprint_destination, - return_columns=function.return_columns, - language=function.language, - source_file_path=function.source_file_path, - runtime_version=function.runtime_version, - entry_point=function.entry_point, - packages=function.packages, - ) - - -def build_scenario_fixture_plans( - *, - scenario: CompiledSqlScenario, - graph_plan: ScenarioGraphPlan, - relation_plan: ScenarioRelationPlan, - adapter: BaseAdapter, - source_lexical_syntax: SqlLexicalSyntax, - fixture_sql_overrides: dict[FixtureKey, str] | None = None, -) -> tuple[ScenarioFixturePlan, ...]: - """Build self-contained fixture SQL plans, including shared helper CTEs.""" - - source_label: str = _scenario_source_label(scenario) - helper_ctes: tuple[CompileSqlScenarioCte, ...] = _extract_helper_ctes(scenario) - resolved_helper_ctes: tuple[CompileSqlScenarioCte, ...] = tuple( - replace( - helper_cte, - sql_body=_resolve_project_source_refs( - sql=helper_cte.sql_body, - source_map=relation_plan.project_source_map, - adapter=adapter, - source_label=source_label, - source_lexical_syntax=source_lexical_syntax, - ), - ) - for helper_cte in helper_ctes - ) - source_ctes: dict[str, str] = _extract_fixture_ctes( - scenario=scenario, - prefix=SOURCE_TEST_CTE_PREFIX, - ) - ref_ctes: dict[str, str] = _extract_fixture_ctes( - scenario=scenario, - prefix=REF_TEST_CTE_PREFIX, - ) - seed_ctes: dict[str, str] = _extract_fixture_ctes( - scenario=scenario, - prefix=SEED_TEST_CTE_PREFIX, - ) - dbt_ref_ctes: dict[str, str] = _extract_fixture_ctes( - scenario=scenario, - prefix=DBT_REF_TEST_CTE_PREFIX, - ) - overrides: dict[FixtureKey, str] = fixture_sql_overrides or {} - source_ctes = _apply_fixture_overrides( - resource_type=CompiledResourceType.SOURCE, - fixture_sql=source_ctes, - overrides=overrides, - ) - ref_ctes = _apply_fixture_overrides( - resource_type=CompiledResourceType.MODEL, - fixture_sql=ref_ctes, - overrides=overrides, - ) - seed_ctes = _apply_fixture_overrides( - resource_type=CompiledResourceType.SEED, - fixture_sql=seed_ctes, - overrides=overrides, - ) - - plans: list[ScenarioFixturePlan] = [] - source_name: str - for source_name in graph_plan.source_fixture_names: - plans.append( - ScenarioFixturePlan( - kind=ScenarioArtifactKind.SOURCE, - logical_name=source_name, - destination=_required_target( - targets=relation_plan.source_fixture_locations, - name=source_name, - kind=ScenarioArtifactKind.SOURCE, - ), - sql=_wrap_sql_with_helpers( - sql=_resolve_project_source_refs( - sql=_required_fixture_sql( - fixture_sql=source_ctes, logical_name=source_name, kind="source" - ), - source_map=relation_plan.project_source_map, - adapter=adapter, - source_label=source_label, - source_lexical_syntax=source_lexical_syntax, - ), - helper_ctes=resolved_helper_ctes, - ), - ) - ) - - ref_name: str - for ref_name in graph_plan.ref_fixture_names: - plans.append( - ScenarioFixturePlan( - kind=ScenarioArtifactKind.REF, - logical_name=ref_name, - destination=_required_target( - targets=relation_plan.ref_fixture_locations, - name=ref_name, - kind=ScenarioArtifactKind.REF, - ), - sql=_wrap_sql_with_helpers( - sql=_resolve_project_source_refs( - sql=_required_fixture_sql( - fixture_sql=ref_ctes, logical_name=ref_name, kind="ref" - ), - source_map=relation_plan.project_source_map, - adapter=adapter, - source_label=source_label, - source_lexical_syntax=source_lexical_syntax, - ), - helper_ctes=resolved_helper_ctes, - ), - ) - ) - - seed_name: str - for seed_name in graph_plan.seed_fixture_names: - plans.append( - ScenarioFixturePlan( - kind=ScenarioArtifactKind.SEED, - logical_name=seed_name, - destination=_required_target( - targets=relation_plan.seed_fixture_locations, - name=seed_name, - kind=ScenarioArtifactKind.SEED, - ), - sql=_wrap_sql_with_helpers( - sql=_resolve_project_source_refs( - sql=_required_fixture_sql( - fixture_sql=seed_ctes, logical_name=seed_name, kind="seed" - ), - source_map=relation_plan.project_source_map, - adapter=adapter, - source_label=source_label, - source_lexical_syntax=source_lexical_syntax, - ), - helper_ctes=resolved_helper_ctes, - ), - ) - ) - - dbt_ref_name: str - for dbt_ref_name in graph_plan.dbt_ref_fixture_names: - plans.append( - ScenarioFixturePlan( - kind=ScenarioArtifactKind.DBT_REF, - logical_name=dbt_ref_name, - destination=_required_target( - targets=relation_plan.dbt_ref_fixture_locations, - name=dbt_ref_name, - kind=ScenarioArtifactKind.DBT_REF, - ), - sql=_wrap_sql_with_helpers( - sql=_resolve_project_source_refs( - sql=_required_fixture_sql( - fixture_sql=dbt_ref_ctes, - logical_name=dbt_ref_name, - kind="dbt_ref", - ), - source_map=relation_plan.project_source_map, - adapter=adapter, - source_label=source_label, - source_lexical_syntax=source_lexical_syntax, - ), - helper_ctes=resolved_helper_ctes, - ), - ) - ) - - return tuple(plans) - - -def _apply_fixture_overrides( - *, - resource_type: CompiledResourceType, - fixture_sql: dict[str, str], - overrides: dict[FixtureKey, str], -) -> dict[str, str]: - return {name: overrides.get((resource_type, name), sql) for name, sql in fixture_sql.items()} - - -def build_scenario_seed_entries( - *, - project: CompiledProjectView, - graph_plan: ScenarioGraphPlan, - relation_plan: ScenarioRelationPlan, -) -> tuple[SeedPlanEntry, ...]: - """Build project seed load entries for required seeds not overridden by fixtures.""" - - seeds_by_name: dict[str, CompiledSeedView] = {seed.name: seed for seed in project.seeds} - seed_fixture_names: frozenset[str] = frozenset(graph_plan.seed_fixture_names) - seed_entries: list[SeedPlanEntry] = [] - seed_name: str - for seed_name in graph_plan.seed_names: - if seed_name in seed_fixture_names: - continue - seed: CompiledSeedView | None = seeds_by_name.get(seed_name) - if seed is None: - raise PlannerInputError( - f"Scenario '{graph_plan.name}' requires unknown seed '{seed_name}'", - code=SCENARIO_PLAN_UNKNOWN_SEED, - ) - seed_entries.append( - SeedPlanEntry( - key=seed.key, - name=seed.name, - destination=_required_target( - targets=relation_plan.seed_locations, - name=seed_name, - kind=ScenarioArtifactKind.SEED, - ), - file_path=seed.seed_file.file_path, - columns=extract_seed_columns(seed), - csv_settings=seed.schema_entry.csv_settings, - ) - ) - return tuple(seed_entries) - - -def _build_expected_check_plan( - *, - expected_cte: CompileSqlScenarioCte, - relation_plan: ScenarioRelationPlan, - adapter: BaseAdapter, - source_label: str, - source_lexical_syntax: SqlLexicalSyntax, -) -> ScenarioExpectedExpectationPlan: - model_name: str = expected_cte.name.removeprefix("__expected__") - actual_destination: CompiledRelationLocation = _required_target( - targets=relation_plan.model_locations, - name=model_name, - kind=ScenarioArtifactKind.MODEL, - ) - return ScenarioExpectedExpectationPlan( - model_name=model_name, - actual_destination=actual_destination, - expected_sql=resolve_scenario_check_sql( - sql=expected_cte.sql_body, - relation_plan=relation_plan, - adapter=adapter, - source_label=source_label, - source_lexical_syntax=source_lexical_syntax, - ), - ) - - -def _extract_helper_ctes(scenario: CompiledSqlScenario) -> tuple[CompileSqlScenarioCte, ...]: - helpers: list[CompileSqlScenarioCte] = [] - cte: CompileSqlScenarioCte - for cte in scenario.authored_ctes: - if cte.name.startswith(SOURCE_TEST_CTE_PREFIX): - continue - if cte.name.startswith(REF_TEST_CTE_PREFIX): - continue - if cte.name.startswith(SEED_TEST_CTE_PREFIX): - continue - if cte.name.startswith(DBT_REF_TEST_CTE_PREFIX): - continue - helpers.append(cte) - return tuple(helpers) - - -def _extract_fixture_ctes(*, scenario: CompiledSqlScenario, prefix: str) -> dict[str, str]: - result: dict[str, str] = {} - cte: CompileSqlScenarioCte - for cte in scenario.authored_ctes: - if cte.name.startswith(prefix): - result[cte.name.removeprefix(prefix)] = cte.sql_body - return result - - -def _wrap_sql_with_helpers(*, sql: str, helper_ctes: tuple[CompileSqlScenarioCte, ...]) -> str: - if not helper_ctes: - return sql - helper_parts: list[str] = [] - helper_cte: CompileSqlScenarioCte - for helper_cte in helper_ctes: - helper_parts.append(f"{helper_cte.name} AS ({helper_cte.sql_body})") - return f"WITH {', '.join(helper_parts)} {sql}" - - -def _resolve_project_source_refs( - *, - sql: str, - source_map: dict[str, SourceEntry], - adapter: BaseAdapter, - source_label: str, - source_lexical_syntax: SqlLexicalSyntax, -) -> str: - return _replace_relation_markers( - sql=sql, - resolvers=((_SOURCE_PATTERN, _source_resolver(source_map=source_map, adapter=adapter)),), - source_label=source_label, - source_lexical_syntax=source_lexical_syntax, - ) - - -def _scenario_source_label(scenario: CompiledSqlScenario) -> str: - return str(scenario.source_path or scenario.name) - - -def _required_fixture_sql(*, fixture_sql: dict[str, str], logical_name: str, kind: str) -> str: - sql: str | None = fixture_sql.get(logical_name) - if sql is None: - raise PlannerInputError( - f"Scenario is missing {kind} fixture SQL '{logical_name}'", - code=SCENARIO_PLAN_MISSING_FIXTURE_SQL, - ) - return sql - - -def _required_target( - *, - targets: dict[str, CompiledRelationLocation], - name: str, - kind: ScenarioArtifactKind, -) -> CompiledRelationLocation: - target: CompiledRelationLocation | None = targets.get(name) - if target is None: - raise PlannerInputError( - f"Scenario relation plan is missing {kind.value} target '{name}'", - code=SCENARIO_PLAN_MISSING_RELATION_TARGET, - help="This is likely a SQLBuild bug. Please file an issue with the scenario name.", - ) - return target - - -def _resolve_model_dbt_ref_fixtures(*, query_sql: str, relation_plan: ScenarioRelationPlan) -> str: - def _replace_dbt_ref(match: re.Match[str]) -> str: - target: CompiledRelationLocation | None = relation_plan.dbt_ref_fixture_locations.get( - _dbt_ref_fixture_name(match) - ) - if target is None or target.qualified_name is None: - return match.group(0) - return target.qualified_name - - return _DBT_REF_PATTERN.sub(_replace_dbt_ref, query_sql) - - -def _dbt_ref_fixture_name(match: re.Match[str]) -> str: - first: str = match.group("first") - second: str | None = match.group("second") - if second is None: - return first - return f"{first}__{second}" diff --git a/src/sqlbuild/compiler/planner/main/scenarios/_build_scenario_artifact_physical_name.py b/src/sqlbuild/compiler/planner/main/scenarios/_build_scenario_artifact_physical_name.py index 8bcc581641..f1e0241e81 100644 --- a/src/sqlbuild/compiler/planner/main/scenarios/_build_scenario_artifact_physical_name.py +++ b/src/sqlbuild/compiler/planner/main/scenarios/_build_scenario_artifact_physical_name.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.planner._helpers.scenario.artifact_names import ( +from sqlbuild._native import ( build_scenario_artifact_physical_name as _build_scenario_artifact_physical_name, ) diff --git a/src/sqlbuild/compiler/planner/main/scenarios/is_scenario_artifact_physical_name.py b/src/sqlbuild/compiler/planner/main/scenarios/is_scenario_artifact_physical_name.py index d5c430bc29..bced73f581 100644 --- a/src/sqlbuild/compiler/planner/main/scenarios/is_scenario_artifact_physical_name.py +++ b/src/sqlbuild/compiler/planner/main/scenarios/is_scenario_artifact_physical_name.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.planner._helpers.scenario.artifact_names import ( +from sqlbuild._native import ( is_scenario_artifact_physical_name as _is_scenario_artifact_physical_name, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py index 523bbcde73..eb59249a10 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py @@ -200,6 +200,31 @@ def build_scenario_python_hooks_project_files() -> dict[str, str]: return repo_files +def build_scenario_dependency_chain_project_files() -> dict[str, str]: + """Build a three-model scenario with partial source fixtures and final checks.""" + + repo_files: dict[str, str] = build_partial_fixture_scenario_project_files() + repo_files["models/orders_summary.sql"] = ( + 'MODEL (description "Summarize fixture orders.", materialized table);\n' + 'SELECT order_id, status FROM __ref("orders")\n' + ) + repo_files["models/orders_final.sql"] = ( + 'MODEL (description "Final scenario output.", materialized table);\n' + 'SELECT order_id, status FROM __ref("orders_summary")\n' + ) + repo_files["tests/scenarios/partial_orders.sql"] = ( + "SCENARIO (description 'Check a dependency chain.');\n" + "WITH helper AS (SELECT 1 AS order_id),\n" + "__source__raw_orders AS (SELECT order_id FROM helper),\n" + "__expected__orders_final AS (\n" + " SELECT 1 AS order_id, CAST(NULL AS VARCHAR) AS status\n" + "),\n" + '__assert__positive_ids AS (SELECT * FROM __ref("orders_final") WHERE order_id < 1)\n' + "SELECT 1\n" + ) + return repo_files + + def build_capture_safety_project_files(*, use_project_row_limit: bool) -> dict[str, str]: """Build scenario e2e files with optional project snapshot limit config.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_namespace.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_namespace.py index d6c89a3c98..2ce20bfabd 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_namespace.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_namespace.py @@ -11,8 +11,8 @@ import pytest +from sqlbuild._native import compute_scenario_hash_prefix from sqlbuild.cli.entry.main.entry import main -from sqlbuild.compiler.planner._helpers.scenario.artifacts import compute_scenario_hash_prefix from sqlbuild.executor.scenario._helpers.execution import run as scenario_execution from tests.e2e.src.sqlbuild.cli.commands.main.scenario._test_types import ( ScenarioNamespaceE2ETestCase, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py index 70e7a62f47..5b87573170 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py @@ -23,6 +23,7 @@ assert_local_duckdb_state, assert_runtime_artifact_contains, build_partial_fixture_scenario_project_files, + build_scenario_dependency_chain_project_files, build_scenario_project_files, build_scenario_python_hooks_project_files, list_scenario_relation_names, @@ -86,6 +87,15 @@ def test_given_unclosed_scenario_comment_when_running_cli_then_error_names_scena "__sqlbuild_partial_fixture", ), ), + ScenarioPartialFixtureE2ETestCase( + description="three-model dependency chain preserves helper fixtures and final checks", + repo_files=build_scenario_dependency_chain_project_files(), + expected_stdout_fragment="PASS=1 FAIL=0 TOTAL=1", + expected_artifact_fragments=( + 'CAST(NULL AS VARCHAR) AS "status"', + "WITH helper AS (SELECT 1 AS order_id)", + ), + ), ), ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 18f2365be1..bf57006e39 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -14,6 +14,7 @@ import sqlbuild._native as _native from sqlbuild._native import ( build_downstream_deps, + build_scenario_relation_map, compiled_model_record, compute_query_hash, detect_changes, @@ -66,7 +67,6 @@ ) from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner._helpers.scenario.artifacts import build_scenario_relation_map from sqlbuild.compiler.planner.models import ( BackfillResult, ChangeDetectionResult, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py index 13545647f5..f158cefe5b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py @@ -2,14 +2,12 @@ import pytest -from sqlbuild.compiler.planner._helpers.scenario.artifact_names import ( - parse_scenario_artifact_physical_name, -) -from sqlbuild.compiler.planner._helpers.scenario.artifacts import ( +from sqlbuild._native import ( build_scenario_artifact_name, build_scenario_hash_index, build_scenario_relation_map, compute_scenario_hash_prefix, + parse_scenario_artifact_physical_name, ) from sqlbuild.compiler.planner.constants import ( SCENARIO_PLAN_HASH_COLLISION, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py index e307c9bdea..b1e7474eb1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py @@ -7,8 +7,11 @@ import pytest from sqlbuild._native import ( + build_scenario_execution_plan, + build_scenario_fixture_plans, build_scenario_relation_plan, project_with, + resolve_scenario_check_sql, ) from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter @@ -22,11 +25,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner._helpers.scenario.relations import ( - build_scenario_execution_plan, - build_scenario_fixture_plans, - resolve_scenario_check_sql, -) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( ScenarioExecutionPlan, diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py index 8deb2b0c05..cfbbbf145c 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py @@ -4,11 +4,11 @@ import pytest -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.planner._helpers.scenario.artifacts import ( +from sqlbuild._native import ( build_scenario_artifact_name, compute_scenario_hash_prefix, ) +from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.executor.janitor.main.execute import execute_janitor_plan from sqlbuild.executor.janitor.main.plan import build_janitor_plan From 110dc932763afdb7d354b097887f1619ef7c88ff Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:38:00 +0100 Subject: [PATCH 02/22] refactor: resolve fixture reads natively --- .../src/planner_fixtures/_helpers/mod.rs | 2 + .../_helpers/resolved_reads.rs | 48 ++++ .../planner_fixtures/_helpers/stub_markers.rs | 19 ++ .../src/planner_fixtures/constants.rs | 8 + .../src/planner_fixtures/main/mod.rs | 2 + .../planner_fixtures/main/required_columns.rs | 97 +++++++ .../src/planner_fixtures/main/star_keys.rs | 125 +++++++++ .../src/planner_fixtures/mod.rs | 1 + .../src/planner_fixtures/models.rs | 57 +++- .../src/planner_fixtures/tests.rs | 2 + .../src/planner_fixtures/tests/test_reads.rs | 112 ++++++++ .../src/planner_fixtures/tests/test_types.rs | 16 ++ .../src/planner_fixtures/types.rs | 4 + .../_helpers/planner_scenario/execution.rs | 5 +- .../planner_strategy/fixture_completion.rs | 48 ++-- .../planner_strategy/fixture_reads.rs | 230 +++++++++++++++ .../planner_strategy/fixture_values.rs | 1 - .../bindings/_helpers/planner_strategy/mod.rs | 2 + .../_helpers/planner_strategy/models.rs | 11 + .../_helpers/planner_strategy/registration.rs | 1 + src/sqlbuild/_native.pyi | 14 + .../planner/_helpers/fixtures/completion.py | 264 ------------------ .../planner/_helpers/output/plan_output.py | 4 +- .../planner/_helpers/sql_tests/assembly.py | 2 +- .../_helpers/sql_tests/fixture_validation.py | 15 +- .../compiler/planner/_helpers/_test_types.py | 1 + .../planner/_helpers/test_fixture_planning.py | 34 ++- .../_helpers/test_fixture_planning_work.py | 29 +- 28 files changed, 815 insertions(+), 339 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/_helpers/stub_markers.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/mod.rs index 163629ef76..9c1c3c67c6 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/mod.rs @@ -1 +1,3 @@ +pub(crate) mod resolved_reads; pub(crate) mod schema_order; +pub(crate) mod stub_markers; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs new file mode 100644 index 0000000000..5e70d90bcc --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs @@ -0,0 +1,48 @@ +//! Bounded reuse of native compiler read resolution. + +use crate::planner_fixtures::models::{FixtureReadCache, FixtureReadKey}; +use crate::required_columns::errors::RequiredColumnsError; +use crate::required_columns::main::resolve_required_columns::resolve_required_columns; +use crate::required_columns::models::{ColumnRead, ReferenceInput}; + +/// Cache successful read resolution by the full original Python cache identity. +pub(crate) fn resolved_reads( + mut cache: FixtureReadCache, + key: &FixtureReadKey, +) -> ( + FixtureReadCache, + Result, RequiredColumnsError>, +) { + cache.clock += 1; + let clock = cache.clock; + if let Some((used, reads)) = cache.entries.get_mut(key) { + cache.recency.remove(used); + cache.recency.insert(clock, key.clone()); + *used = clock; + let reads = reads.clone(); + return (cache, Ok(reads)); + } + let references: Vec = key + .references + .iter() + .map(|(kind, name, _, _)| ReferenceInput { + kind: kind.clone(), + name: name.clone(), + }) + .collect(); + cache.misses += 1; + let reads = match resolve_required_columns(&key.sql, &references, key.dialect.as_deref()) { + Ok(reads) => reads, + Err(error) => return (cache, Err(error)), + }; + if cache.entries.len() >= crate::planner_fixtures::constants::READ_CACHE_CAPACITY + && let Some((_, oldest)) = cache.recency.pop_first() + { + cache.entries.remove(&oldest); + } + cache.clock += 1; + let clock = cache.clock; + cache.recency.insert(clock, key.clone()); + cache.entries.insert(key.clone(), (clock, reads.clone())); + (cache, Ok(reads)) +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/stub_markers.rs b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/stub_markers.rs new file mode 100644 index 0000000000..8fd50fa5bc --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/stub_markers.rs @@ -0,0 +1,19 @@ +//! Legacy fixture star-analysis marker substitution. + +use crate::planner_fixtures::constants::MARKER_PATTERNS; +use regex::Regex; +use std::sync::LazyLock; + +static MARKERS: LazyLock, regex::Error>> = + LazyLock::new(|| MARKER_PATTERNS.into_iter().map(Regex::new).collect()); + +/// Substitute exact marker forms, including matches in SQL comments and strings. +pub(crate) fn stub_markers(sql: &str) -> Result { + Ok(MARKERS + .as_ref() + .map_err(Clone::clone)? + .iter() + .fold(sql.to_owned(), |sql, pattern| { + pattern.replace_all(&sql, "$1").into_owned() + })) +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/constants.rs b/crates/sqlbuild-analysis/src/planner_fixtures/constants.rs index ec8f90170a..5f69d5e0eb 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/constants.rs @@ -1,3 +1,11 @@ //! Empty fixture marker pattern. +pub(crate) const MARKER_PATTERNS: [&str; 4] = [ + r#"__ref\("([^"]+)"\)"#, + r#"__seed\("([^"]+)"\)"#, + r#"__source\("([^"]+)"\)"#, + r#"__dbt_ref\("[^"]+"[\s\x1c-\x1f]*,[\s\x1c-\x1f]*"([^"]+)"\)"#, +]; + pub(crate) const EMPTY_PATTERN: &str = r"(?i)^[\s\x1c-\x1f]*SELECT[\s\x1c-\x1f]+\*[\s\x1c-\x1f]+FROM[\s\x1c-\x1f]+__empty_fixture[\s\x1c-\x1f]*\([\s\x1c-\x1f]*\)[\s\x1c-\x1f]*$"; +pub(crate) const READ_CACHE_CAPACITY: usize = 4096; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs index e07b486003..9d0320f464 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs @@ -6,5 +6,7 @@ pub mod merge_metadata; pub mod propagate_types; pub mod render_completed; pub mod render_empty; +pub mod required_columns; pub mod requires_analysis; +pub mod star_keys; pub mod unknown_columns; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs new file mode 100644 index 0000000000..d8a2c70a58 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs @@ -0,0 +1,97 @@ +//! Required fixture columns derived from compiler read evidence. + +use sqlbuild_core::text::main::python_casefold::python_casefold; +use std::collections::HashMap; + +use crate::planner_fixtures::_helpers::resolved_reads::resolved_reads; +use crate::planner_fixtures::models::{ + FixtureReadCache, FixtureReadModel, FixtureReadRequirements, FixtureReadSchemas, RequiredColumn, +}; +use crate::required_columns::errors::RequiredColumnsError; + +/// Collect direct reads and authoritative star columns in model discovery order. +pub fn required_columns( + mut cache: FixtureReadCache, + models: &[FixtureReadModel], + schemas: FixtureReadSchemas<'_>, +) -> ( + FixtureReadCache, + Result, +) { + let FixtureReadSchemas { + keys: fixture_keys, + authoritative, + relations, + } = schemas; + let mut result = FixtureReadRequirements::default(); + for model in models { + let mut reads = model.high_confidence.clone(); + if model.key.references.iter().any(|(kind, name, _, _)| { + let resource_type = match kind.as_str() { + "ref" => "model", + "source" => "source", + "seed" => "seed", + _ => return false, + }; + fixture_keys.contains(&(resource_type.to_owned(), name.clone())) + }) { + let (updated, resolved) = resolved_reads(cache, &model.key); + cache = updated; + match resolved { + Ok(resolved) => reads.extend(resolved), + Err(error) => return (cache, Err(error)), + } + } + for read in reads { + let key = (read.resource_type, read.resource_name); + if !fixture_keys.contains(&key) + || authoritative + .get(&key) + .is_some_and(|names| !names.contains(&python_casefold(&read.column_name))) + { + continue; + } + result = add_required(result, key, &read.column_name, &model.name); + } + for key in &model.stars { + result.stars.insert(key.clone()); + if let Some(relation) = relations + .iter() + .find(|relation| &relation.key == key && relation.authoritative) + { + for column in &relation.columns { + result = add_required(result, key.clone(), &column.name, &model.name); + } + } + } + } + (cache, Ok(result)) +} + +fn add_required( + mut result: FixtureReadRequirements, + key: (String, String), + name: &str, + model: &str, +) -> FixtureReadRequirements { + let position = *result.indices.entry(key.clone()).or_insert_with(|| { + result.columns.push((key, Vec::new())); + result.column_indices.push(HashMap::new()); + result.columns.len() - 1 + }); + let columns = &mut result.columns[position].1; + if let Some(index) = result.column_indices[position].get(name) { + let column = &mut columns[*index]; + if column.readers.last().is_none_or(|reader| reader != model) { + column.readers.push(model.to_owned()); + } + } else { + result.column_indices[position].insert(name.to_owned(), columns.len()); + columns.push(RequiredColumn { + name: name.to_owned(), + folded_name: python_casefold(name), + readers: vec![model.to_owned()], + }); + } + result +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs new file mode 100644 index 0000000000..5afda5f19d --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs @@ -0,0 +1,125 @@ +//! Fixture dependencies selected by star projections. + +use std::collections::{HashMap, HashSet}; +use std::str::FromStr; + +use polyglot_sql::{ + AnalyzeQueryOptions, DialectType, SchemaColumn, SchemaTable, ValidationSchema, analyze_query, +}; +use sqlbuild_core::text::main::python_casefold::python_casefold; + +use crate::planner_fixtures::_helpers::stub_markers::stub_markers; +use crate::planner_fixtures::models::FixtureStarRelation; + +/// Resolve star inputs using the same semantic analysis as the compiler. +pub fn star_keys( + sql: &str, + relations: &[FixtureStarRelation], + dialect: Option<&str>, + proven_inputs: Option<&[String]>, +) -> Vec<(String, String)> { + if let Some(inputs) = proven_inputs { + let names: HashSet = inputs.iter().map(|name| python_casefold(name)).collect(); + return relations + .iter() + .filter(|relation| names.contains(&python_casefold(&relation.key.1))) + .map(|relation| relation.key.clone()) + .collect(); + } + if !relations + .iter() + .any(|relation| !relation.columns.is_empty()) + { + return Vec::new(); + } + let Ok(dialect) = + DialectType::from_str(dialect.filter(|name| !name.is_empty()).unwrap_or("generic")) + else { + return Vec::new(); + }; + let tables: Vec = relations + .iter() + .filter(|relation| !relation.columns.is_empty()) + .map(schema_table) + .collect(); + let options = AnalyzeQueryOptions { + dialect, + schema: Some(ValidationSchema { + tables, + strict: None, + }), + complexity_guard: None, + }; + let Ok(sql) = stub_markers(sql) else { + return Vec::new(); + }; + let Ok(analysis) = analyze_query(&sql, options) else { + return Vec::new(); + }; + let mut aliases: HashMap = HashMap::new(); + for relation in analysis.relations { + let name = python_casefold(&relation.name); + aliases.insert(name.clone(), name.clone()); + if let Some(alias) = relation.alias { + aliases.insert(python_casefold(&alias), name); + } + } + let mut names: HashSet = HashSet::new(); + for projection in analysis.star_projections { + if let Some(table) = projection.table { + if let Some(name) = aliases.get(&python_casefold(&table)) { + names.insert(name.clone()); + } + } else { + let expanded: HashSet = projection + .expanded_columns + .iter() + .map(|name| python_casefold(name)) + .collect(); + for relation in relations { + if relation + .columns + .iter() + .any(|column| expanded.contains(&column.folded_name)) + { + names.insert(python_casefold(&relation.key.1)); + } + } + } + } + relations + .iter() + .filter(|relation| names.contains(&python_casefold(&relation.key.1))) + .map(|relation| relation.key.clone()) + .collect() +} + +fn schema_table(relation: &FixtureStarRelation) -> SchemaTable { + SchemaTable { + name: relation.key.1.clone(), + schema: None, + columns: relation + .columns + .iter() + .map(|column| SchemaColumn { + name: column.name.clone(), + data_type: column_type(column.sql_type.as_deref()), + nullable: None, + primary_key: false, + unique: false, + references: None, + }) + .collect(), + aliases: Vec::new(), + primary_key: Vec::new(), + unique_keys: Vec::new(), + foreign_keys: Vec::new(), + } +} + +fn column_type(sql_type: Option<&str>) -> String { + sql_type + .filter(|name| !name.is_empty()) + .unwrap_or("UNKNOWN") + .to_owned() +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs b/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs index b010d05a00..42b1da1020 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs @@ -7,3 +7,4 @@ pub mod main; pub mod models; #[cfg(test)] mod tests; +pub mod types; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/models.rs b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs index 2fbeba43f8..e6995f179a 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/models.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs @@ -1,6 +1,61 @@ //! Passive fixture schema and alignment facts. -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +/// Local fixture schemas used by required-read aggregation. +#[derive(Debug)] +pub struct FixtureReadSchemas<'a> { + pub keys: &'a HashSet<(String, String)>, + pub authoritative: &'a HashMap<(String, String), HashSet>, + pub relations: &'a [FixtureStarRelation], +} + +use crate::required_columns::models::ColumnRead; + +use crate::planner_fixtures::types::FixtureReadReference; + +/// The original SQL, complete references and dialect form the read cache key. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct FixtureReadKey { + pub sql: String, + pub references: Vec, + pub dialect: Option, +} + +/// A bounded LRU of successful fixture read analyses. +#[derive(Debug, Default)] +pub struct FixtureReadCache { + pub entries: HashMap)>, + pub recency: BTreeMap, + pub clock: u64, + pub misses: u64, +} + +/// A model's read evidence and star dependencies in planning order. +#[derive(Debug)] +pub struct FixtureReadModel { + pub name: String, + pub key: FixtureReadKey, + pub high_confidence: Vec, + pub stars: Vec<(String, String)>, +} + +/// Required columns retain model and column insertion order. +#[derive(Debug, Default)] +pub struct FixtureReadRequirements { + pub columns: Vec<((String, String), Vec)>, + pub stars: BTreeSet<(String, String)>, + pub(crate) indices: HashMap<(String, String), usize>, + pub(crate) column_indices: Vec>, +} + +/// One fixture dependency and the schema available to star analysis. +#[derive(Debug)] +pub struct FixtureStarRelation { + pub key: (String, String), + pub columns: Vec, + pub authoritative: bool, +} /// One upstream typed column identified by its resource and folded name. #[derive(Debug)] diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs index a1512cca60..9ea0942c44 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs @@ -4,6 +4,8 @@ mod helpers; mod test_alignment; #[path = "tests/test_propagation.rs"] mod test_propagation; +#[path = "tests/test_reads.rs"] +mod test_reads; #[path = "tests/test_schema.rs"] mod test_schema; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs new file mode 100644 index 0000000000..8d19e9e57e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs @@ -0,0 +1,112 @@ +use crate::planner_fixtures::_helpers::resolved_reads::resolved_reads; +use crate::planner_fixtures::_helpers::stub_markers::stub_markers; +use crate::planner_fixtures::models::{FixtureReadCache, FixtureReadKey}; +use crate::planner_fixtures::tests::test_types::{FixtureMarkerTestCase, FixtureReadCacheTestCase}; + +#[test] +fn given_repeated_reads_when_resolving_then_full_reference_identity_controls_reuse() { + let test_cases = [ + FixtureReadCacheTestCase { + description: "identical request", + sql: "SELECT order_id FROM __source(\"orders\")", + package: None, + argument_count: None, + dialect: Some("duckdb"), + expected_misses: 1, + }, + FixtureReadCacheTestCase { + description: "package changes", + sql: "SELECT order_id FROM __source(\"orders\")", + package: Some("shop"), + argument_count: None, + dialect: Some("duckdb"), + expected_misses: 2, + }, + FixtureReadCacheTestCase { + description: "argument count changes", + sql: "SELECT order_id FROM __source(\"orders\")", + package: None, + argument_count: Some(1), + dialect: Some("duckdb"), + expected_misses: 2, + }, + FixtureReadCacheTestCase { + description: "dialect changes", + sql: "SELECT order_id FROM __source(\"orders\")", + package: None, + argument_count: None, + dialect: None, + expected_misses: 2, + }, + FixtureReadCacheTestCase { + description: "SQL changes", + sql: "SELECT status FROM __source(\"orders\")", + package: None, + argument_count: None, + dialect: Some("duckdb"), + expected_misses: 2, + }, + ]; + for test_case in test_cases { + let key = FixtureReadKey { + sql: "SELECT order_id FROM __source(\"orders\")".to_owned(), + references: vec![("source".into(), "orders".into(), None, None)], + dialect: Some("duckdb".into()), + }; + let (cache, initial) = resolved_reads(FixtureReadCache::default(), &key); + let (cache, repeated) = resolved_reads(cache, &key); + assert_eq!(initial.ok(), repeated.ok(), "{}", test_case.description); + assert_eq!(cache.misses, 1, "{}", test_case.description); + let changed = FixtureReadKey { + sql: test_case.sql.to_owned(), + references: vec![( + "source".into(), + "orders".into(), + test_case.package.map(str::to_owned), + test_case.argument_count, + )], + dialect: test_case.dialect.map(str::to_owned), + }; + let (cache, reads) = resolved_reads(cache, &changed); + assert!(reads.is_ok(), "{}", test_case.description); + assert_eq!( + cache.misses, test_case.expected_misses, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_marker_text_when_stubbing_for_star_analysis_then_preserves_legacy_lexical_semantics() { + let test_cases = [ + FixtureMarkerTestCase { + description: "Unicode reference", + sql: "SELECT * FROM __source(\"Straße\")", + expected_sql: "SELECT * FROM Straße", + }, + FixtureMarkerTestCase { + description: "string and comment matches", + sql: "SELECT '__ref(\"orders\")' -- __seed(\"customers\")", + expected_sql: "SELECT 'orders' -- customers", + }, + FixtureMarkerTestCase { + description: "packaged reference and block comment", + sql: "SELECT * FROM __dbt_ref(\"shop\", \"orders\") /* __source(\"customers\") */", + expected_sql: "SELECT * FROM orders /* customers */", + }, + FixtureMarkerTestCase { + description: "unmatched marker spellings", + sql: "SELECT * FROM __REF(\"orders\") JOIN __ref( 'customers' ) ON TRUE", + expected_sql: "SELECT * FROM __REF(\"orders\") JOIN __ref( 'customers' ) ON TRUE", + }, + ]; + for test_case in test_cases { + assert_eq!( + stub_markers(test_case.sql).ok().as_deref(), + Some(test_case.expected_sql), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs index 0b02326180..62c6427f7e 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs @@ -26,3 +26,19 @@ pub(crate) struct AlignmentTestCase { pub(crate) expected_completed: Option>, pub(crate) expected_diagnostics: &'static [&'static str], } +#[derive(Debug)] +pub(super) struct FixtureReadCacheTestCase { + pub description: &'static str, + pub sql: &'static str, + pub package: Option<&'static str>, + pub argument_count: Option, + pub dialect: Option<&'static str>, + pub expected_misses: u64, +} + +#[derive(Debug)] +pub(super) struct FixtureMarkerTestCase { + pub description: &'static str, + pub sql: &'static str, + pub expected_sql: &'static str, +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/types.rs b/crates/sqlbuild-analysis/src/planner_fixtures/types.rs new file mode 100644 index 0000000000..ec569300e0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/types.rs @@ -0,0 +1,4 @@ +//! Fixture analysis identities. + +/// The complete reference identity retained by the fixture read cache. +pub type FixtureReadReference = (String, String, Option, Option); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index 75113052c3..d48d3c079f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -211,14 +211,13 @@ fn complete_fixtures<'py>( let completion = call_with( &imported( py, - "sqlbuild.compiler.planner._helpers.fixtures.completion", + "sqlbuild._native", "build_relation_fixture_completion", )?, &[ ("project", project.clone()), ("adapter", context.adapter.clone()), - ("ordered_model_names", PyTuple::new(py, names)?.into_any()), - ("fixture_groups", PyTuple::new(py, groups)?.into_any()), + ("request", PyTuple::new(py, [PyTuple::new(py, names)?.into_any(), PyTuple::new(py, groups)?.into_any()])?.into_any()), ], )?; let diagnostics = completion.getattr("diagnostics")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs index 488767569c..1f7562c3f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs @@ -1,29 +1,23 @@ //! Native-hosted relation fixture completion over Python boundary models. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::align_columns::align_columns; use sqlbuild_analysis::planner_fixtures::models::{AlignmentInput, RequiredColumn}; +use crate::bindings::_helpers::planner_strategy::fixture_context::build_relation_fixture_context; use crate::bindings::_helpers::planner_strategy::fixture_preparation::{PreparedFixtures, prepare}; +use crate::bindings::_helpers::planner_strategy::fixture_reads::required_fixture_columns; use crate::bindings::_helpers::planner_strategy::fixture_selection::{ fixture_keys_requiring_analysis, unknown_fixture_column_diagnostics, }; use crate::bindings::_helpers::planner_strategy::fixture_sql::{CompletionSql, render_completed}; use crate::bindings::_helpers::planner_strategy::fixture_types::expected_fixture_types; use crate::bindings::_helpers::planner_strategy::fixture_values::{ - COMPLETION, columns, diagnostic, folded, label, record, strings, + columns, diagnostic, folded, label, record, strings, }; -fn helper<'py>( - py: Python<'py>, - name: &str, - kwargs: &Bound<'py, PyDict>, -) -> PyResult> { - py.import(COMPLETION)?.getattr(name)?.call((), Some(kwargs)) -} - #[pyfunction] #[pyo3(signature = (*, project, adapter, request, planning_context = None))] fn build_relation_fixture_completion<'py>( @@ -33,14 +27,9 @@ fn build_relation_fixture_completion<'py>( planning_context: Option>, ) -> PyResult> { let (ordered_model_names, fixture_groups) = request; - let py = project.py(); let context = match planning_context { Some(context) if context.is_truthy()? => context, - _ => { - let kwargs = PyDict::new(py); - kwargs.set_item("project", project)?; - helper(py, "build_relation_fixture_context", &kwargs)? - } + _ => build_relation_fixture_context(project, None)?, }; let relations = context.getattr("relations")?.cast_into::()?; let prepared = prepare( @@ -55,19 +44,20 @@ fn build_relation_fixture_completion<'py>( .getattr("authoritative_columns")? .cast_into::()?, )?; - let kwargs = PyDict::new(py); - kwargs.set_item("ordered_model_names", &ordered_model_names)?; - kwargs.set_item("model_map", context.getattr("models_by_name")?)?; - kwargs.set_item("fixture_keys", fixture_keys)?; - kwargs.set_item("relations", &relations)?; - kwargs.set_item( - "authoritative_columns", - context.getattr("authoritative_columns")?, + let (required, stars) = required_fixture_columns( + ( + ordered_model_names.extract()?, + context.getattr("models_by_name")?.cast_into::()?, + ), + ( + fixture_keys.into_any(), + relations.clone(), + context + .getattr("authoritative_columns")? + .cast_into::()?, + ), + adapter, )?; - kwargs.set_item("adapter", adapter)?; - let required_and_stars = helper(py, "_required_fixture_columns", &kwargs)?; - let required = required_and_stars.get_item(0)?.cast_into::()?; - let stars = required_and_stars.get_item(1)?; let expected = expected_fixture_types( &context.getattr("expected_types")?.cast_into::()?, ordered_model_names.extract()?, @@ -75,7 +65,7 @@ fn build_relation_fixture_completion<'py>( )?; complete_prepared( prepared, - (relations, required, stars), + (relations, required, stars.into_any()), expected.into_any(), adapter, ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs new file mode 100644 index 0000000000..674d1d0a2a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs @@ -0,0 +1,230 @@ +//! Boundary conversion for native fixture read analysis. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_fixtures::main::required_columns::required_columns; +use sqlbuild_analysis::planner_fixtures::main::star_keys::star_keys; +use sqlbuild_analysis::planner_fixtures::models::{ + FixtureReadKey, FixtureReadModel, FixtureReadSchemas, FixtureStarRelation, +}; +use sqlbuild_analysis::planner_fixtures::types::FixtureReadReference; +use sqlbuild_analysis::required_columns::models::ColumnRead; +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; +use std::collections::{HashMap, HashSet}; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::planner_strategy::fixture_values::columns; +use crate::bindings::_helpers::planner_strategy::models::FixtureReadCacheHost; +use crate::bindings::types::CompilerDetach; + +#[pyfunction] +#[pyo3(signature = (*, model_inputs, fixture_inputs, adapter))] +pub(crate) fn required_fixture_columns<'py>( + model_inputs: (Vec, Bound<'py, PyDict>), + fixture_inputs: (Bound<'py, PyAny>, Bound<'py, PyDict>, Bound<'py, PyDict>), + adapter: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyFrozenSet>)> { + let (ordered_model_names, model_map) = model_inputs; + let (fixture_keys, relations, authoritative_columns) = fixture_inputs; + let py = adapter.py(); + let mut keys: HashMap<(String, String), Bound<'py, PyAny>> = HashMap::new(); + for key in fixture_keys.try_iter()? { + let key = key?; + keys.insert(key.extract::<(String, String)>()?, key); + } + let mut schema: Vec = Vec::new(); + let mut authoritative: HashMap<(String, String), HashSet> = HashMap::new(); + for (key, original) in &keys { + if let Some(value) = relations.get_item(original)? { + schema.push(FixtureStarRelation { + key: key.clone(), + columns: columns(&value.getattr("columns")?, false)?, + authoritative: value.getattr("authoritative_names")?.extract()?, + }); + } else { + schema.push(FixtureStarRelation { + key: key.clone(), + columns: Vec::new(), + authoritative: false, + }); + } + if let Some(value) = authoritative_columns.get_item(original)? { + authoritative.insert(key.clone(), value.extract()?); + } + } + let dialect = adapter + .call_method0("expression_inference_profile")? + .getattr("sql_analysis_dialect")? + .extract::>()?; + let mut models: Vec = Vec::new(); + for name in ordered_model_names { + let Some(model) = model_map.get_item(&name)? else { + continue; + }; + let mut references: Vec = Vec::new(); + for reference in model.getattr("references")?.try_iter()? { + let reference = reference?; + references.push(( + reference.getattr("ref_kind")?.extract()?, + reference.getattr("ref_name")?.extract()?, + reference.getattr("ref_package")?.extract()?, + reference.getattr("call_argument_count")?.extract()?, + )); + } + models.push(FixtureReadModel { + name, + key: FixtureReadKey { + sql: model.getattr("query_sql")?.extract()?, + references, + dialect: dialect.clone(), + }, + high_confidence: lineage_reads(&model)?, + stars: model_stars(&model, &schema, adapter)?, + }); + } + let owner = py + .import("sqlbuild._native")? + .getattr("_fixture_read_cache")? + .cast_into::()?; + let mut owner = owner.borrow_mut(); + let cache = std::mem::take(&mut owner.cache); + let fixture_keys: HashSet<(String, String)> = keys.keys().cloned().collect(); + let (cache, result) = catch_compiler_panic(|| { + Ok(required_columns( + cache, + &models, + FixtureReadSchemas { + keys: &fixture_keys, + authoritative: &authoritative, + relations: &schema, + }, + )) + }) + .map_err(value_error)?; + owner.cache = cache; + let result = result.map_err(|error| value_error(format!("{error:?}")))?; + let required = PyDict::new(py); + for (key, columns) in result.columns { + let values = PyDict::new(py); + for column in columns { + values.set_item(column.name, PySet::new(py, column.readers)?)?; + } + required.set_item(&keys[&key], values)?; + } + let stars = PyFrozenSet::new(py, result.stars.iter().map(|key| &keys[key]))?; + Ok((required, stars)) +} + +fn lineage_reads(model: &Bound<'_, PyAny>) -> PyResult> { + let mut reads: Vec = Vec::new(); + let facts = model.getattr("fast_lineage_columns")?; + if facts.is_none() { + return Ok(reads); + } + for fact in facts.try_iter()? { + let fact = fact?; + if !fact.getattr("confidence")?.eq(model + .py() + .import("sqlbuild.compiler.lineage.types")? + .getattr("ColumnLineageConfidence")? + .getattr("HIGH")?)? + { + continue; + } + for source in fact.getattr("upstream_columns")?.try_iter()? { + let source = source?; + reads.push(ColumnRead { + resource_type: source.getattr("resource_type")?.extract()?, + resource_name: source.getattr("resource_name")?.extract()?, + column_name: source.getattr("column_name")?.extract()?, + }); + } + } + Ok(reads) +} + +fn model_stars( + model: &Bound<'_, PyAny>, + schema: &[FixtureStarRelation], + adapter: &Bound<'_, PyAny>, +) -> PyResult> { + if !model.getattr("fast_lineage_has_star")?.is_truthy()? { + return Ok(Vec::new()); + } + let mut deps: Vec<(String, String)> = Vec::new(); + for dependency in model.getattr("deps")?.try_iter()? { + let dependency = dependency?; + let key = ( + dependency.getattr("resource_type")?.extract::()?, + dependency.getattr("name")?.extract::()?, + ); + if schema.iter().any(|relation| relation.key == key) { + deps.push(key); + } + } + let dependencies = PyFrozenSet::new(model.py(), deps)?; + let mut selected: Vec = Vec::new(); + for dependency in dependencies.try_iter()? { + let dependency: (String, String) = dependency?.extract()?; + if let Some(relation) = schema.iter().find(|relation| relation.key == dependency) { + selected.push(FixtureStarRelation { + key: relation.key.clone(), + columns: relation.columns.clone(), + authoritative: relation.authoritative, + }); + } + } + let contract = model.getattr("dynamic_column_contract")?; + let inputs: Option> = + if !contract.is_none() && contract.getattr("output_proven")?.is_truthy()? { + Some(contract.getattr("input_relations")?.extract()?) + } else { + None + }; + let dialect: Option = + if inputs.is_none() && selected.iter().any(|relation| !relation.columns.is_empty()) { + adapter + .call_method0("expression_inference_profile")? + .getattr("sql_analysis_dialect")? + .extract()? + } else { + None + }; + let sql: String = model.getattr("query_sql")?.extract()?; + model + .py() + .compiler_detach(|| { + Ok(star_keys( + &sql, + &selected, + dialect.as_deref(), + inputs.as_deref(), + )) + }) + .map_err(value_error) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add( + "_fixture_read_cache", + pyo3::Py::new(module.py(), FixtureReadCacheHost::default())?, + )?; + module.add_function(wrap_pyfunction!(fixture_read_cache_info, module)?)?; + module.add_function(wrap_pyfunction!(required_fixture_columns, module)?) +} + +/// Report resolver misses and retained entries, optionally clearing prior fixture reuse. +#[pyfunction] +#[pyo3(signature = (*, reset = false))] +fn fixture_read_cache_info(py: pyo3::Python<'_>, reset: bool) -> PyResult<(u64, usize)> { + let owner = py + .import("sqlbuild._native")? + .getattr("_fixture_read_cache")? + .cast_into::()?; + let mut owner = owner.borrow_mut(); + if reset { + owner.cache = Default::default(); + } + Ok((owner.cache.misses, owner.cache.entries.len())) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs index 0f35b03b71..0bc5231dda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs @@ -5,7 +5,6 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; pub(crate) const MODELS: &str = "sqlbuild.compiler.planner.models"; -pub(crate) const COMPLETION: &str = "sqlbuild.compiler.planner._helpers.fixtures.completion"; pub(crate) fn folded(value: &Bound<'_, PyAny>) -> PyResult { value.call_method0("casefold")?.extract() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs index d1023a4c18..94f6239e04 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs @@ -4,12 +4,14 @@ pub(crate) mod fixture_completion; pub(crate) mod fixture_context; pub(crate) mod fixture_metadata; pub(crate) mod fixture_preparation; +pub(crate) mod fixture_reads; pub(crate) mod fixture_selection; pub(crate) mod fixture_sql; pub(crate) mod fixture_types; pub(crate) mod fixture_values; pub(crate) mod lineage; pub(crate) mod microbatches; +pub(crate) mod models; pub(crate) mod registration; pub(crate) mod strategy; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs new file mode 100644 index 0000000000..f3abaa0b26 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs @@ -0,0 +1,11 @@ +//! Extension-owned fixture analysis reuse, with pure state passed into analysis. + +use pyo3::pyclass; +use sqlbuild_analysis::planner_fixtures::models::FixtureReadCache; + +/// Per-extension cache replacing the Python fixture resolver's LRU. +#[pyclass] +#[derive(Debug, Default)] +pub(crate) struct FixtureReadCacheHost { + pub(crate) cache: FixtureReadCache, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs index c04fcde07b..b6c58fba9f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -14,6 +14,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { fixture_metadata::register(module)?; fixture_sql::register(module)?; super::fixture_context::register(module)?; + super::fixture_reads::register(module)?; super::fixture_selection::register(module)?; super::fixture_types::register(module)?; lineage::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 25af267edf..bae328330d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2506,6 +2506,20 @@ def expected_fixture_types( ordered_model_names: tuple[str, ...], model_map: dict[str, CompiledModelView], ) -> Mapping[tuple[CompiledResourceType, str], dict[str, str]]: ... +def fixture_read_cache_info(*, reset: bool = False) -> tuple[int, int]: ... +def required_fixture_columns( + *, + model_inputs: tuple[tuple[str, ...], dict[str, CompiledModelView]], + fixture_inputs: tuple[ + frozenset[tuple[CompiledResourceType, str]], + dict[tuple[CompiledResourceType, str], FixtureRelationMetadata], + dict[tuple[CompiledResourceType, str], frozenset[str]], + ], + adapter: BaseAdapter, +) -> tuple[ + dict[tuple[CompiledResourceType, str], dict[str, set[str]]], + frozenset[tuple[CompiledResourceType, str]], +]: ... def build_relation_fixture_context( *, project: CompiledProjectView, models_by_name: dict[str, CompiledModelView] | None = None ) -> RelationFixturePlanningContext: ... diff --git a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py b/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py deleted file mode 100644 index 68711fc1df..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py +++ /dev/null @@ -1,264 +0,0 @@ -"""Schema-aware completion for test-only relation fixtures.""" - -from __future__ import annotations - -import re -from collections.abc import Sequence -from functools import lru_cache -from typing import Any - -from sqlbuild._native import ( - analyze_resolved_column_reads, -) -from sqlbuild._native import ( - build_relation_fixture_completion as _native_fixture_completion, -) -from sqlbuild._native import ( - build_relation_fixture_context as build_relation_fixture_context, -) -from sqlbuild._native import ( - complete_empty_fixture_sql as complete_empty_fixture_sql, -) -from sqlbuild._native import ( - complete_typed_null_columns as complete_typed_null_columns, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( - CompiledLineageColumnFact, - CompiledLineageSourceFact, - CompileSqlReference, - DynamicColumnContractProof, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.lineage.types import ( - ColumnLineageConfidence, -) -from sqlbuild.compiler.planner.models import ( - FixtureRelationMetadata, - RelationFixtureCompletion, - RelationFixturePlanningContext, -) -from sqlbuild.compiler.planner.types import FixtureGroups, FixtureKey -from sqlbuild.compiler.references.constants import REF_PATTERN, SEED_PATTERN, SOURCE_PATTERN -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql - -_DBT_REF_PATTERN: re.Pattern[str] = re.compile(r'__dbt_ref\("([^"]+)"\s*,\s*"([^"]+)"\)') - - -def build_relation_fixture_completion( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - ordered_model_names: tuple[str, ...], - fixture_groups: FixtureGroups, - planning_context: RelationFixturePlanningContext | None = None, -) -> RelationFixtureCompletion: - """Complete required omitted columns with typed nulls unless explicitly non-nullable.""" - - return _native_fixture_completion( - project=project, - adapter=adapter, - request=(ordered_model_names, fixture_groups), - planning_context=planning_context, - ) - - -def _required_fixture_columns( - *, - ordered_model_names: tuple[str, ...], - model_map: dict[str, CompiledModelView], - fixture_keys: frozenset[FixtureKey], - relations: dict[FixtureKey, FixtureRelationMetadata], - authoritative_columns: dict[FixtureKey, frozenset[str]], - adapter: BaseAdapter, -) -> tuple[dict[FixtureKey, dict[str, set[str]]], frozenset[FixtureKey]]: - required: dict[FixtureKey, dict[str, set[str]]] = {} - star_fixture_keys: set[FixtureKey] = set() - dialect: str | None = adapter.expression_inference_profile().sql_analysis_dialect - for model_name in ordered_model_names: - model: CompiledModelView | None = model_map.get(model_name) - if model is None: - continue - for upstream in _resolved_fixture_reads( - query_sql=model.query_sql, - references=model.references, - fast_lineage=model.fast_lineage_columns or (), - fixture_keys=fixture_keys, - dialect=dialect, - ): - key: FixtureKey = ( - CompiledResourceType(upstream.resource_type), - upstream.resource_name, - ) - if key not in fixture_keys: - continue - if ( - key in authoritative_columns - and upstream.column_name.casefold() not in authoritative_columns[key] - ): - continue - required.setdefault(key, {}).setdefault(upstream.column_name, set()).add(model_name) - if model.fast_lineage_has_star: - model_star_fixture_keys: frozenset[FixtureKey] = _star_fixture_keys( - model=model, - fixture_keys=fixture_keys, - relations=relations, - adapter=adapter, - ) - star_fixture_keys.update(model_star_fixture_keys) - for key in model_star_fixture_keys: - relation: FixtureRelationMetadata | None = relations.get(key) - if relation is None or not relation.authoritative_names: - continue - for column in relation.columns: - required.setdefault(key, {}).setdefault(column.name, set()).add(model_name) - return required, frozenset(star_fixture_keys) - - -def _resolved_fixture_reads( - *, - query_sql: str, - references: tuple[CompileSqlReference, ...], - fast_lineage: Sequence[CompiledLineageColumnFact], - fixture_keys: frozenset[FixtureKey], - dialect: str | None, -) -> tuple[CompiledLineageSourceFact, ...]: - """Return external model reads without treating fallback lineage as authoritative.""" - - high_confidence_reads: list[CompiledLineageSourceFact] = [] - for fact in fast_lineage: - if fact.confidence == ColumnLineageConfidence.HIGH: - high_confidence_reads.extend(fact.upstream_columns) - needs_resolution: bool = any( - _reference_fixture_key(reference) in fixture_keys for reference in references - ) - if not needs_resolution: - return tuple(high_confidence_reads) - resolved: tuple[CompiledLineageSourceFact, ...] = _cached_resolved_column_reads( - query_sql=query_sql, - references=references, - dialect=dialect, - ) - return tuple(dict.fromkeys([*high_confidence_reads, *resolved])) - - -def _reference_fixture_key(reference: CompileSqlReference) -> FixtureKey | None: - resource_type: CompiledResourceType | None = { - SqlReferenceKind.REF: CompiledResourceType.MODEL, - SqlReferenceKind.SOURCE: CompiledResourceType.SOURCE, - SqlReferenceKind.SEED: CompiledResourceType.SEED, - }.get(SqlReferenceKind(reference.ref_kind)) - return (resource_type, reference.ref_name) if resource_type is not None else None - - -@lru_cache(maxsize=4096) -def _cached_resolved_column_reads( - *, - query_sql: str, - references: tuple[CompileSqlReference, ...], - dialect: str | None, -) -> tuple[CompiledLineageSourceFact, ...]: - return analyze_resolved_column_reads( - query_sql=query_sql, - references=references, - dialect=dialect, - ) - - -def _star_fixture_keys( - *, - model: CompiledModelView, - fixture_keys: frozenset[FixtureKey], - relations: dict[FixtureKey, FixtureRelationMetadata], - adapter: BaseAdapter, -) -> frozenset[FixtureKey]: - dependency_keys: frozenset[FixtureKey] = frozenset( - (CompiledResourceType(dependency.resource_type), dependency.name) - for dependency in model.deps - if (CompiledResourceType(dependency.resource_type), dependency.name) in fixture_keys - ) - dynamic_contract: DynamicColumnContractProof | None = model.dynamic_column_contract - if dynamic_contract is not None and dynamic_contract.output_proven: - input_relations: frozenset[str] = frozenset( - name.casefold() for name in dynamic_contract.input_relations - ) - return frozenset(key for key in dependency_keys if key[1].casefold() in input_relations) - schema_tables: list[dict[str, object]] = [] - for key in dependency_keys: - relation: FixtureRelationMetadata | None = relations.get(key) - if relation is None or not relation.columns: - continue - schema_tables.append( - { - "name": key[1], - "columns": [ - {"name": column.name, "type": column.type or "UNKNOWN"} - for column in relation.columns - ], - } - ) - if not schema_tables: - return frozenset() - polyglot_module: Any = import_polyglot_sql() - try: - analysis: object = polyglot_module.analyze_query( - _replace_relation_markers_with_stubs(model.query_sql), - { - "dialect": adapter.expression_inference_profile().sql_analysis_dialect or "generic", - "schema": {"tables": schema_tables}, - }, - ) - except polyglot_module.PolyglotError: - return frozenset() - if not isinstance(analysis, dict): - return frozenset() - star_projections: object = analysis.get("starProjections") - query_relations: object = analysis.get("relations") - if not isinstance(star_projections, list) or not isinstance(query_relations, list): - return frozenset() - relation_names_by_alias: dict[str, str] = {} - for query_relation in query_relations: - if not isinstance(query_relation, dict): - continue - name: object = query_relation.get("name") - alias: object = query_relation.get("alias") - if not isinstance(name, str): - continue - relation_names_by_alias[name.casefold()] = name.casefold() - if isinstance(alias, str): - relation_names_by_alias[alias.casefold()] = name.casefold() - star_relation_names: set[str] = set() - for projection in star_projections: - if not isinstance(projection, dict): - continue - table: object = projection.get("table") - if table is None: - expanded_columns: object = projection.get("expandedColumns") - if not isinstance(expanded_columns, list): - continue - expanded_names: set[str] = { - column.casefold() for column in expanded_columns if isinstance(column, str) - } - for key in dependency_keys: - relation: FixtureRelationMetadata | None = relations.get(key) - if relation is not None and any( - column.name.casefold() in expanded_names for column in relation.columns - ): - star_relation_names.add(key[1].casefold()) - elif isinstance(table, str): - relation_name: str | None = relation_names_by_alias.get(table.casefold()) - if relation_name is not None: - star_relation_names.add(relation_name) - return frozenset(key for key in dependency_keys if key[1].casefold() in star_relation_names) - - -def _replace_relation_markers_with_stubs(query_sql: str) -> str: - result: str = REF_PATTERN.sub(r"\1", query_sql) - result = SEED_PATTERN.sub(r"\1", result) - result = SOURCE_PATTERN.sub(r"\1", result) - return _DBT_REF_PATTERN.sub(r"\2", result) diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py index 61c803dd83..ffe7a63817 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py @@ -3,6 +3,7 @@ from __future__ import annotations from sqlbuild._native import ( + build_relation_fixture_context, build_source_load_entries, project_lineage_views, upstream_loader_dependency_names, @@ -26,9 +27,6 @@ ) from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.graph.models import LineageGraphViews -from sqlbuild.compiler.planner._helpers.fixtures.completion import ( - build_relation_fixture_context, -) from sqlbuild.compiler.planner._helpers.output.audit_entry import plan_audit from sqlbuild.compiler.planner._helpers.output.plan_entry import ( audit_is_selected, diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py index 3776ff41f7..b79e958faf 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py @@ -4,6 +4,7 @@ from dataclasses import replace +from sqlbuild._native import build_relation_fixture_context from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.constants import ( DBT_REF_TEST_CTE_PREFIX, @@ -26,7 +27,6 @@ CompiledResourceType, SqlTestMode, ) -from sqlbuild.compiler.planner._helpers.fixtures.completion import build_relation_fixture_context from sqlbuild.compiler.planner._helpers.sql_tests.fixture_validation import ( build_validated_test_fixtures, fixture_location, diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py index 7de7a40e3c..03c5108d87 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py @@ -4,6 +4,12 @@ import re +from sqlbuild._native import ( + build_relation_fixture_completion, + build_relation_fixture_context, + complete_empty_fixture_sql, + complete_typed_null_columns, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompiledSqlTest, @@ -11,12 +17,6 @@ InferredColumn, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.planner._helpers.fixtures.completion import ( - build_relation_fixture_completion, - build_relation_fixture_context, - complete_empty_fixture_sql, - complete_typed_null_columns, -) from sqlbuild.compiler.planner.exceptions import SqlTestFixtureValidationError from sqlbuild.compiler.planner.models import ( RelationFixtureCompletion, @@ -54,8 +54,7 @@ def build_validated_test_fixtures( completion: RelationFixtureCompletion = build_relation_fixture_completion( project=project, adapter=adapter, - ordered_model_names=ordered_model_names, - fixture_groups=fixture_groups, + request=(ordered_model_names, fixture_groups), planning_context=fixture_context, ) errors: list[str] = [ diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py index c7e1899610..5a0cdf10b5 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -11,6 +11,7 @@ class FixturePlanningIntegrationTestCase: description: str expected_sql_fragment: str + query_sql: str = 'SELECT order_id, status FROM __source("raw_orders")' @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index 14ea786712..3a910007b9 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -5,14 +5,15 @@ import pytest -from sqlbuild._native import build_compiled_project, discover_project_inputs -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner._helpers.fixtures.completion import ( +from sqlbuild._native import ( + build_compiled_project, build_relation_fixture_completion, build_relation_fixture_context, + discover_project_inputs, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner._helpers.output.plan_output import build_selected_test_entries from sqlbuild.compiler.planner.models import ( RelationFixtureCompletion, @@ -33,6 +34,21 @@ description="partial source fixture", expected_sql_fragment='CAST(NULL AS VARCHAR) AS "status"', ), + FixturePlanningIntegrationTestCase( + description="qualified star through alias", + query_sql='SELECT o.* FROM __source("raw_orders") AS o', + expected_sql_fragment='CAST(NULL AS VARCHAR) AS "status"', + ), + FixturePlanningIntegrationTestCase( + description="CTE and quoted column reads", + query_sql='WITH selected AS (SELECT "order_id", "status" FROM __source("raw_orders")) SELECT * FROM selected', + expected_sql_fragment='CAST(NULL AS VARCHAR) AS "status"', + ), + FixturePlanningIntegrationTestCase( + description="marker-looking strings and comments", + query_sql='SELECT order_id, status, \'__ref("unused")\' AS note FROM __source("raw_orders") /* __source("missing") */', + expected_sql_fragment='CAST(NULL AS VARCHAR) AS "status"', + ), ), ids=lambda case: case.description, ) @@ -51,7 +67,7 @@ def test_given_partial_fixture_when_compiling_and_planning_then_completes_requir 'database = ":memory:"\n' ), "models/orders.sql": ( - 'MODEL (description "Test model orders.");\n\nSELECT order_id, status FROM __source("raw_orders")\n' + f'MODEL (description "Test model orders.");\n\n{test_case.query_sql}\n' ), "sources/raw_orders.yml": ( "sources:\n" @@ -206,9 +222,9 @@ def test_given_source_contract_when_completing_fixture_then_respects_authoritati completed: RelationFixtureCompletion = build_relation_fixture_completion( project=project, adapter=adapter, - ordered_model_names=("orders",), - fixture_groups=( - (CompiledResourceType.SOURCE, {"raw_orders": "SELECT 1 AS ORDER_ID, 2 AS extra"}), + request=( + ("orders",), + ((CompiledResourceType.SOURCE, {"raw_orders": "SELECT 1 AS ORDER_ID, 2 AS extra"}),), ), planning_context=context, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index 8616d21334..38ad5e9037 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -5,6 +5,7 @@ import pytest +from sqlbuild import _native as fixture_completion from sqlbuild._native import ( compiled_model_record, project_with, @@ -19,7 +20,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner._helpers.fixtures import completion as fixture_completion from sqlbuild.compiler.planner._helpers.output import plan_output from sqlbuild.compiler.planner._helpers.output.plan_output import build_selected_test_entries from sqlbuild.compiler.planner._helpers.sql_tests import assembly as sql_test_assembly @@ -102,13 +102,7 @@ def test_given_many_tests_when_building_entries_then_project_fixture_metadata_is ) metadata_builder: Mock = Mock(wraps=fixture_completion.build_relation_fixture_context) monkeypatch.setattr(plan_output, "build_relation_fixture_context", metadata_builder) - fixture_completion._cached_resolved_column_reads.cache_clear() - fallback_analyzer: Mock = Mock(wraps=fixture_completion.analyze_resolved_column_reads) - monkeypatch.setattr( - fixture_completion, - "analyze_resolved_column_reads", - fallback_analyzer, - ) + fixture_completion.fixture_read_cache_info(reset=True) topology_builder: Mock = Mock(wraps=sql_test_assembly.resolve_sql_test_model_chains) monkeypatch.setattr(sql_test_assembly, "resolve_sql_test_model_chains", topology_builder) @@ -124,7 +118,7 @@ def test_given_many_tests_when_building_entries_then_project_fixture_metadata_is assert len(entries) == test_case.test_count assert warnings == [] assert metadata_builder.call_count == test_case.expected_metadata_builds - assert fallback_analyzer.call_count == test_case.expected_analysis_calls + assert fixture_completion.fixture_read_cache_info()[0] == test_case.expected_analysis_calls assert topology_builder.call_count == test_case.expected_topology_builds @@ -239,14 +233,7 @@ def test_given_complete_authoritative_fixture_when_completing_then_skips_column_ (CompiledResourceType.MODEL, model.name): {}, }, ) - fallback_analyzer: Mock = Mock( - side_effect=AssertionError("complete fixtures must not trigger model analysis") - ) - monkeypatch.setattr( - fixture_completion, - "analyze_resolved_column_reads", - fallback_analyzer, - ) + fixture_completion.fixture_read_cache_info(reset=True) fixture_inference: Mock = Mock(wraps=fixture_inferences.infer_fixture_column_facts) monkeypatch.setattr( fixture_inferences, @@ -257,14 +244,16 @@ def test_given_complete_authoritative_fixture_when_completing_then_skips_column_ completed: RelationFixtureCompletion = fixture_completion.build_relation_fixture_completion( project=project_with(project=project, models=(model,)), adapter=PlannerTestAdapter(), - ordered_model_names=(model.name,), - fixture_groups=((CompiledResourceType.SOURCE, {"raw_orders": test_case.fixture_sql}),), + request=( + (model.name,), + ((CompiledResourceType.SOURCE, {"raw_orders": test_case.fixture_sql}),), + ), planning_context=context, ) assert completed.diagnostics == () assert completed.fixture_sql_by_key[source_key] == test_case.expected_fixture_sql - assert fallback_analyzer.call_count == test_case.expected_analysis_calls + assert fixture_completion.fixture_read_cache_info()[0] == test_case.expected_analysis_calls assert fixture_inference.call_count == test_case.expected_inference_calls From d077725c702cd529722025f026480e6eb406e309 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:55:53 +0100 Subject: [PATCH 03/22] refactor: move old-name migration planning to native orchestration --- .../_helpers/planner_migrations/constants.rs | 2 + .../_helpers/planner_migrations/mod.rs | 4 + .../_helpers/planner_migrations/models.rs | 11 + .../planner_migrations/old_name_entries.rs | 372 +++++++++++++ .../planner_migrations/old_name_objects.rs | 123 +++++ .../planner_migrations/old_name_planning.rs | 199 +++++++ .../planner_migrations/old_name_references.rs | 143 +++++ .../_helpers/planner_migrations/old_names.rs | 10 +- .../planner_migrations/registration.rs | 1 + .../_helpers/planner_scenario/execution.rs | 18 +- src/sqlbuild/_native.pyi | 10 + .../planner/_helpers/migrations/old_names.py | 507 ------------------ .../_helpers/planning/warehouse_state.py | 3 +- 13 files changed, 879 insertions(+), 524 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs index 1715fd4546..ab2b48f728 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs @@ -2,3 +2,5 @@ pub(super) const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; pub(super) const MIGRATION_TYPES: &str = "sqlbuild.compiler.migrations.types"; pub(super) const AUTOMATIC: &str = "automatic"; pub(super) const DONE: &str = "done"; +pub(super) const PENDING_ARCHIVE: &str = "pending_archive"; +pub(super) const PENDING_VIEW: &str = "pending_view"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index 18301e2647..328b408540 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -20,6 +20,10 @@ mod model_overlay; mod model_planning; mod model_warnings; mod models; +mod old_name_entries; +mod old_name_objects; +mod old_name_planning; +mod old_name_references; mod old_names; mod planning_boundary; mod planning_execution; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs index a0bf4ad647..1c11e9b2fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs @@ -1,6 +1,17 @@ use pyo3::prelude::{Bound, PyAny}; use pyo3::types::PyDict; +#[derive(Debug)] +pub(crate) struct OldNamePlanner<'py> { + pub runtime: Bound<'py, PyAny>, + pub models: Vec>, + pub histories: Vec>, + pub views: Bound<'py, PyAny>, + pub columns: Bound<'py, PyAny>, + pub retention: Bound<'py, PyAny>, + pub now: Bound<'py, PyAny>, +} + #[derive(Debug)] pub(crate) struct ColumnCandidate<'py> { pub model: Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs new file mode 100644 index 0000000000..b594f187b8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs @@ -0,0 +1,372 @@ +//! Assemble new and resumed old-name view steps from recorded facts. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_migrations::column_objects::{model, native, text}; +use crate::bindings::_helpers::planner_migrations::constants::{PENDING_ARCHIVE, PENDING_VIEW}; +use crate::bindings::_helpers::planner_migrations::models::OldNamePlanner; +use crate::bindings::_helpers::planner_migrations::old_name_objects::{ + expires, location, relation, retention, stores_history, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyBool, PyList, PyTuple}; + +impl<'py> OldNamePlanner<'py> { + pub(super) fn new( + args: &KeywordArguments<'py>, + models: Vec>, + histories: Vec>, + views: Bound<'py, PyAny>, + ) -> PyResult { + let runtime = args.value("runtime")?; + let py = runtime.py(); + let project_config = runtime.getattr("project_config")?; + let config = if project_config.is_none() { + imported(py, "sqlbuild.spec.contracts.models", "MigrationsConfig")?.call0()? + } else { + project_config.getattr("migrations")? + }; + Ok(Self { + models, + histories, + views, + columns: args.value("column_entries")?, + retention: config.getattr("old_name_views")?, + now: runtime.getattr("invocation_time")?, + runtime, + }) + } + + fn newest( + &self, + destination: &Bound<'py, PyAny>, + origin: Option<&Bound<'py, PyAny>>, + ) -> PyResult>> { + let mut newest: Option<(Bound<'py, PyTuple>, Bound<'py, PyAny>)> = None; + for history in &self.histories { + let movement = history.getattr("move")?; + if !movement + .getattr("destination")? + .call_method1("matches", (destination,))? + .is_truthy()? + { + continue; + } + if let Some(origin) = origin + && !movement + .getattr("origin")? + .call_method1("matches", (origin,))? + .is_truthy()? + { + continue; + } + let key = PyTuple::new( + self.runtime.py(), + [ + movement.getattr("created_at")?, + movement.getattr("event_id")?, + ], + )?; + if let Some((previous, _)) = &newest + && !key.gt(previous)? + { + continue; + } + newest = Some((key, history.clone())); + } + Ok(newest.map(|(_, history)| history)) + } + + pub(super) fn for_migration( + &self, + entry: &Bound<'py, PyAny>, + item: &Bound<'py, PyAny>, + ) -> PyResult>> { + if entry.getattr("blocks_build")?.is_truthy()? { + return Ok(None); + } + let decision = entry.getattr("decision")?; + let completed = !entry.getattr("completed_at")?.is_none(); + if decision.getattr("moves_data")?.is_truthy()? || (decision.eq("renamed")? && !completed) { + return self.new_move(entry, item).map(Some); + } + if decision.eq("done")? || (decision.eq("renamed")? && completed) { + let origin = relation(&entry.getattr("origin")?)?; + if let Some(history) = + self.newest(&relation(&entry.getattr("destination")?)?, Some(&origin))? + { + return self.resume(&history, item); + } + } + Ok(None) + } + + pub(super) fn pending_for( + &self, + item: &Bound<'py, PyAny>, + ) -> PyResult>> { + let Some(history) = self.newest(&relation(&item.getattr("destination")?)?, None)? else { + return Ok(None); + }; + if !history + .getattr("move")? + .getattr("destination_model")? + .eq(item.getattr("name")?)? + { + return Ok(None); + } + let Some(planned) = self.resume(&history, item)? else { + return Ok(None); + }; + if planned.getattr("runs_steps")?.is_truthy()? + || (planned.getattr("action")?.eq("none")? + && !planned.getattr("migration_event_id")?.is_none()) + { + Ok(Some(planned)) + } else { + Ok(None) + } + } + + fn skip( + &self, + origin: &Bound<'py, PyAny>, + tracked: Bound<'py, PyAny>, + retention: &Bound<'py, PyAny>, + ) -> PyResult> { + native( + self.runtime.py(), + "old_name_view_skip_reason", + &[ + ("origin", origin.clone()), + ("origin_tracked", tracked), + ("retention", retention.clone()), + ( + "models", + PyTuple::new(self.runtime.py(), &self.models)?.into_any(), + ), + ("views", self.views.clone()), + ], + ) + } + + fn aliases(&self, item: &Bound<'py, PyAny>) -> PyResult> { + let py = self.runtime.py(); + if !stores_history(item)? { + return imported(py, "sqlbuild._native", "declared_column_aliases")?.call1((item,)); + } + let mut aliases: Vec> = Vec::new(); + for entry in self.columns.try_iter()? { + let entry = entry?; + let decision = entry.getattr("decision")?; + if entry.getattr("model_name")?.eq(item.getattr("name")?)? + && (decision.eq("rename")? || decision.eq("record")?) + { + aliases.push(PyTuple::new( + py, + [ + entry.getattr("origin_column")?, + entry.getattr("destination_column")?, + ], + )?); + } + } + let aliases = PyList::new(py, aliases)?; + aliases.call_method0("sort")?; + imported(py, "builtins", "tuple")?.call1((aliases,)) + } + + fn new_move( + &self, + entry: &Bound<'py, PyAny>, + item: &Bound<'py, PyAny>, + ) -> PyResult> { + let py = self.runtime.py(); + let retention = retention(item, &self.retention)?; + let reason = self.skip( + &entry.getattr("origin")?, + entry.getattr("origin_tracked")?, + &retention, + )?; + let mut fields = vec![ + ("model_name", entry.getattr("model_name")?), + ("origin", entry.getattr("origin")?), + ("destination", entry.getattr("destination")?), + ("target_name", entry.getattr("target_name")?), + ]; + if !reason.is_none() || retention.is_none() { + fields.extend([ + ("action", action(&self.runtime, "NONE")?), + ( + "reason", + if reason.is_truthy()? { + reason + } else { + text(py, "old_name_view false") + }, + ), + ]); + } else { + fields.extend([ + ("action", action(&self.runtime, "ARCHIVE_AND_VIEW")?), + ("expires_at", expires(&retention, &self.now)?), + ("retention", retention), + ("column_aliases", self.aliases(item)?), + ( + "stores_history", + PyBool::new(py, stores_history(item)?).to_owned().into_any(), + ), + ( + "records_requirement", + PyBool::new(py, true).to_owned().into_any(), + ), + ( + "grants_supported", + self.runtime + .getattr("adapter")? + .getattr("relation_grants_supported")?, + ), + ]); + } + model(py, "OldNameViewPlanEntry", &fields) + } + + fn resume( + &self, + history: &Bound<'py, PyAny>, + item: &Bound<'py, PyAny>, + ) -> PyResult>> { + let py = self.runtime.py(); + let status: String = + call_with(&history.getattr("status")?, &[("now", self.now.clone())])?.extract()?; + let action_name = match status.as_str() { + "pending_archive" => "ARCHIVE_AND_VIEW", + "pending_view" => "VIEW_ONLY", + "live" | "expired" => "LIVE", + _ => return Ok(None), + }; + if status == PENDING_ARCHIVE || status == PENDING_VIEW { + let reason = self.skip( + &location(&self.runtime, &history.getattr("old")?)?, + py.None().into_bound(py), + &retention(item, &self.retention)?, + )?; + if !reason.is_none() { + return model( + py, + "OldNameViewPlanEntry", + &[ + ( + "model_name", + history.getattr("move")?.getattr("destination_model")?, + ), + ("origin", location(&self.runtime, &history.getattr("old")?)?), + ( + "destination", + location(&self.runtime, &history.getattr("new")?)?, + ), + ("action", action(&self.runtime, "NONE")?), + ( + "target_name", + history.getattr("move")?.getattr("target_name")?, + ), + ( + "migration_event_id", + history.getattr("move")?.getattr("event_id")?, + ), + ("reason", reason), + ], + ) + .map(Some); + } + } + let retention = history.getattr("required")?.getattr("view_retention")?; + let duration = imported(py, "sqlbuild.cursor_algebra.models", "Duration")?.call_method1( + "parse", + (if retention.is_truthy()? { + retention.clone() + } else { + text(py, "") + },), + )?; + let created = history.getattr("created")?; + let expires_at = if !created.is_none() { + created.getattr("expires_at")? + } else if duration.is_none() { + py.None().into_bound(py) + } else { + duration.call_method1("add_to", (&self.now,))? + }; + let grants = if created.is_none() { + py.None().into_bound(py) + } else { + created.getattr("grants_copied")? + }; + model( + py, + "OldNameViewPlanEntry", + &[ + ("model_name", item.getattr("name")?), + ("origin", location(&self.runtime, &history.getattr("old")?)?), + ( + "destination", + location(&self.runtime, &history.getattr("new")?)?, + ), + ("action", action(&self.runtime, action_name)?), + ( + "target_name", + history.getattr("move")?.getattr("target_name")?, + ), + ("retention", retention), + ("expires_at", expires_at), + ( + "column_aliases", + if created.is_none() { + self.aliases(item)? + } else { + created.getattr("column_aliases")? + }, + ), + ( + "stores_history", + PyBool::new(py, stores_history(item)?).to_owned().into_any(), + ), + ( + "migration_event_id", + history.getattr("move")?.getattr("event_id")?, + ), + ( + "grants_copied", + if grants.is_none() { + grants + } else { + imported(py, "builtins", "len")?.call1((grants,))? + }, + ), + ( + "grants_supported", + self.runtime + .getattr("adapter")? + .getattr("relation_grants_supported")?, + ), + ( + "archived", + PyBool::new(py, !history.getattr("archived")?.is_none()) + .to_owned() + .into_any(), + ), + ], + ) + .map(Some) + } +} + +fn action<'py>(runtime: &Bound<'py, PyAny>, name: &str) -> PyResult> { + imported( + runtime.py(), + "sqlbuild.compiler.migrations.types", + "OldNameViewAction", + )? + .getattr(name) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs new file mode 100644 index 0000000000..8cef6f25ce --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs @@ -0,0 +1,123 @@ +//! Old-name planner object conversions and existing native helper calls. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_migrations::column_objects::{config, model}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { + imported( + location.py(), + "sqlbuild.compiler.migrations.main.relation_for_location", + "migration_relation_for_location", + )? + .call1((location,)) +} + +pub(super) fn location<'py>( + runtime: &Bound<'py, PyAny>, + relation: &Bound<'py, PyAny>, +) -> PyResult> { + let mut fields = vec![ + ("database", relation.getattr("database")?), + ("schema", relation.getattr("schema")?), + ("name", relation.getattr("name")?), + ]; + let qualified = call_with( + &runtime + .getattr("adapter")? + .getattr("render_qualified_name")?, + &fields, + )?; + fields.push(("qualified_name", qualified)); + call_with( + &imported( + runtime.py(), + "sqlbuild.compiler.compile.models", + "CompiledRelationLocation", + )?, + &fields, + ) +} + +pub(super) fn stores_history(item: &Bound<'_, PyAny>) -> PyResult { + let materialization = config(item, "materialized")?; + Ok(materialization.eq("incremental")? || materialization.eq("snapshot")?) +} + +pub(super) fn retention<'py>( + item: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + call_with( + &imported( + item.py(), + "sqlbuild.compiler.migrations.main._resolve_old_name_view_retention", + "resolve_old_name_view_retention", + )?, + &[ + ("config_values", item.getattr("config")?.getattr("values")?), + ("project_retention", project.clone()), + ], + ) +} + +pub(super) fn expires<'py>( + retention: &Bound<'py, PyAny>, + now: &Bound<'py, PyAny>, +) -> PyResult> { + let duration = imported(now.py(), "sqlbuild.cursor_algebra.models", "Duration")? + .call_method1("parse", (retention,))?; + if duration.is_none() { + Ok(now.py().None().into_bound(now.py())) + } else { + duration.call_method1("add_to", (now,)) + } +} + +pub(super) fn recorded_view<'py>( + runtime: &Bound<'py, PyAny>, + history: &Bound<'py, PyAny>, + models: &[Bound<'py, PyAny>], +) -> PyResult> { + let py = runtime.py(); + let movement = history.getattr("move")?; + let created = history.getattr("created")?; + let mut reused = py.None().into_bound(py); + for item in models { + if relation(&item.getattr("destination")?)? + .call_method1("matches", (history.getattr("old")?,))? + .is_truthy()? + { + reused = item.getattr("name")?; + break; + } + } + model( + py, + "OldNameView", + &[ + ("destination_model", movement.getattr("destination_model")?), + ("old", location(runtime, &history.getattr("old")?)?), + ("new", location(runtime, &history.getattr("new")?)?), + ( + "expires_at", + if created.is_none() { + py.None().into_bound(py) + } else { + created.getattr("expires_at")? + }, + ), + ( + "column_aliases", + if created.is_none() { + pyo3::types::PyTuple::empty(py).into_any() + } else { + created.getattr("column_aliases")? + }, + ), + ("migration_event_id", movement.getattr("event_id")?), + ("target_name", movement.getattr("target_name")?), + ("name_reused_by", reused), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs new file mode 100644 index 0000000000..90ee923e10 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -0,0 +1,199 @@ +//! Old-name planning orchestration over existing live state inspection. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_migrations::column_objects::{model, native}; +use crate::bindings::_helpers::planner_migrations::old_name_objects::recorded_view; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +fn schemas<'py>( + args: &KeywordArguments<'py>, + selected: &[Bound<'py, PyAny>], +) -> PyResult> { + let runtime = args.value("runtime")?; + let py = runtime.py(); + let snapshot = args.value("snapshot")?; + let needed = PySet::empty(py)?; + for item in selected { + let schema = item.getattr("destination")?.getattr("schema")?; + if !schema.is_none() { + needed.add(schema)?; + } + } + for entry in args.value("migration_entries")?.try_iter()? { + let schema = entry?.getattr("origin")?.getattr("schema")?; + if !schema.is_none() { + needed.add(schema)?; + } + } + let mut include_project = runtime + .getattr("project")? + .getattr("unmatched_literal_sql_relations")? + .is_truthy()?; + if !include_project { + for item in selected { + let existing = snapshot + .getattr("existing_relations")? + .call_method1("get", (item.getattr("name")?,))?; + if !existing.is_none() + && imported( + py, + "sqlbuild.adapter.type_system.main.normalize_relation_type", + "normalize_relation_type", + )? + .call1((existing.getattr("relation_type")?,))? + .eq("view")? + { + include_project = true; + break; + } + } + } + if include_project { + let project = native(py, "migration_project_schemas", &[("runtime", runtime)])?; + for schema in project.try_iter()? { + needed.add(schema?)?; + } + } + native( + py, + "old_name_schemas_to_read", + &[("snapshot", snapshot), ("needed", needed.into_any())], + ) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +fn plan_old_name_views<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "plan_old_name_views", + options, + ( + &[ + "runtime", + "scope", + "snapshot", + "migration_entries", + "column_entries", + ], + &[], + ), + )?; + let runtime = args.value("runtime")?; + let scope = args.value("scope")?; + let project = runtime.getattr("project")?; + let models: Vec> = project + .getattr("models")? + .try_iter()? + .collect::>()?; + let by_name = PyDict::new(py); + for item in &models { + by_name.set_item(item.getattr("name")?, item)?; + } + let mut selected: Vec> = Vec::new(); + for key in scope.getattr("execution_order")?.try_iter()? { + let key = key?; + if scope.getattr("selected_keys")?.contains(&key)? + && key.getattr("resource_type")?.eq("model")? + && let Some(item) = by_name.get_item(key.getattr("name")?)? + { + selected.push(item); + } + } + let state = call_with( + &imported( + py, + "sqlbuild.compiler.planner.classes.migration_state_inspection", + "MigrationStateInspection", + )?, + &[ + ("adapter", runtime.getattr("adapter")?), + ("connection", runtime.getattr("connection")?), + ( + "database", + native( + py, + "migration_planning_database", + &[("runtime", runtime.clone())], + )?, + ), + ], + )?; + let needed = schemas(&args, &selected)?; + if needed.is_truthy()? { + call_with( + &state.getattr("inspect_old_name_views")?, + &[("schemas", needed)], + )?; + } + let target = project.getattr("effective_target_name")?; + let mut histories: Vec> = Vec::new(); + let mut views: Vec> = Vec::new(); + for history in state.getattr("old_name_histories")?.try_iter()? { + let history = history?; + let history_target = history.getattr("move")?.getattr("target_name")?; + if !history_target.is_none() && !history_target.eq(&target)? { + continue; + } + if !history.getattr("created")?.is_none() && history.getattr("dropped")?.is_none() { + views.push(recorded_view(&runtime, &history, &models)?); + } + histories.push(history); + } + let views = PyTuple::new(py, views)?.into_any(); + let planner = super::models::OldNamePlanner::new(&args, models, histories, views.clone())?; + let mut entries: Vec> = Vec::new(); + let covered = PySet::empty(py)?; + for entry in args.value("migration_entries")?.try_iter()? { + let entry = entry?; + let name = entry.getattr("model_name")?; + if let Some(item) = by_name.get_item(&name)? + && let Some(planned) = planner.for_migration(&entry, &item)? + { + entries.push(planned); + covered.add(name)?; + } + } + for item in &selected { + if !covered.contains(item.getattr("name")?)? + && let Some(planned) = planner.pending_for(item)? + { + entries.push(planned); + } + } + let entries = PyTuple::new(py, entries)?.into_any(); + let claims = native( + py, + "old_name_claim_errors", + &[ + ("selected", PyTuple::new(py, selected)?.into_any()), + ("snapshot", args.value("snapshot")?), + ("views", views.clone()), + ("now", runtime.getattr("invocation_time")?), + ], + )?; + let mut warnings: Vec> = claims.try_iter()?.collect::>()?; + warnings.extend(super::old_name_references::errors( + &runtime, &views, &entries, + )?); + model( + py, + "OldNameViewPlanning", + &[ + ("entries", entries), + ("views", views), + ("warnings", PyTuple::new(py, warnings)?.into_any()), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(plan_old_name_views, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs new file mode 100644 index 0000000000..c87e7b0639 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs @@ -0,0 +1,143 @@ +//! Diagnose literal references to compatibility-only old names. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_migrations::column_objects::{model, native, text}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; + +fn old_relation<'py>( + old: &Bound<'py, PyAny>, + name: Bound<'py, PyAny>, +) -> PyResult> { + let py = old.py(); + let reference = call_with( + &imported(py, "sqlbuild.python_nodes.models", "SqlResourceRef")?, + &[ + ( + "kind", + imported(py, "sqlbuild.python_nodes.types", "SqlResourceRefKind")? + .getattr("MODEL")?, + ), + ("name", name.clone()), + ], + )?; + let relation = call_with( + &imported(py, "sqlbuild.compiler.references.models", "RelationName")?, + &[ + ("name", old.getattr("name")?), + ("schema", old.getattr("schema")?), + ("database", old.getattr("database")?), + ], + )?; + call_with( + &imported(py, "sqlbuild.compiler.references.models", "ProjectRelation")?, + &[ + ("ref", reference), + ("relation", relation), + ("compatibility_for", name), + ], + ) +} + +pub(super) fn errors<'py>( + runtime: &Bound<'py, PyAny>, + views: &Bound<'py, PyAny>, + entries: &Bound<'py, PyAny>, +) -> PyResult>> { + let py = runtime.py(); + let literals = runtime + .getattr("project")? + .getattr("unmatched_literal_sql_relations")?; + if !literals.is_truthy()? { + return Ok(Vec::new()); + } + let relations = PyDict::new(py); + for view in views.try_iter()? { + let view = view?; + relations.set_item( + old_relation(&view.getattr("old")?, view.getattr("destination_model")?)?, + py.None(), + )?; + } + for entry in entries.try_iter()? { + let entry = entry?; + if entry.getattr("runs_steps")?.is_truthy()? { + relations.set_item( + old_relation(&entry.getattr("origin")?, entry.getattr("model_name")?)?, + py.None(), + )?; + } + } + let index = call_with( + &imported( + py, + "sqlbuild.compiler.references.models", + "ProjectRelationIndex", + )?, + &[("relations", PyTuple::new(py, relations.keys())?.into_any())], + )?; + let mut errors: Vec> = Vec::new(); + for literal in literals.try_iter()? { + let literal = literal?; + let relation = literal.getattr("relation")?; + let matched = native( + py, + "match_project_relation", + &[("index", index.clone()), ("relation", relation.clone())], + )?; + if matched.is_none() || matched.getattr("compatibility_for")?.is_none() { + continue; + } + let mut parts: Vec = Vec::new(); + for field in ["database", "schema", "name"] { + let part = relation.getattr(field)?; + if part.is_truthy()? { + parts.push(part.extract()?); + } + } + let message: String = native( + py, + "old_name_reference_message", + &[ + ("owner_label", literal.getattr("owner_label")?), + ("written", text(py, &parts.join("."))), + ("method", literal.getattr("method")?), + ("model_name", matched.getattr("compatibility_for")?), + ], + )? + .extract()?; + let remedy: String = native( + py, + "hard_coded_relation_remedy", + &[ + ("owner_kind", literal.getattr("owner_kind")?), + ("ref", matched.getattr("ref")?), + ], + )? + .extract()?; + let path = literal + .getattr("relative_path")? + .str()? + .to_str()? + .to_owned(); + let line = literal.getattr("line")?.str()?.to_str()?.to_owned(); + errors.push(model( + py, + "PlanWarning", + &[ + ("model_name", py.None().into_bound(py)), + ( + "severity", + imported(py, "sqlbuild.compiler.planner.types", "WarningSeverity")? + .getattr("ERROR")?, + ), + ( + "message", + text(py, &format!("{path}:{line}: {message}; {remedy}")), + ), + ("code", text(py, "P008")), + ], + )?); + } + Ok(errors) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs index fd047a8ba9..43f7f0a255 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::planner_migrations::old_name_objects::relation; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PySet, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; @@ -93,15 +94,6 @@ fn claim_message(model: &str, view: &Bound<'_, PyAny>, now: &Bound<'_, PyAny>) - )) } -fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { - imported( - location.py(), - "sqlbuild.compiler.migrations.main.relation_for_location", - "migration_relation_for_location", - )? - .call1((location,)) -} - #[pyfunction] #[pyo3(signature = (**options))] fn old_name_claim_errors<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs index b2850d000d..6516b29d70 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs @@ -7,6 +7,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::compatibility::register(module)?; super::matching::register(module)?; super::old_names::register(module)?; + super::old_name_planning::register(module)?; super::columns::register(module)?; super::column_planning::register(module)?; super::model_planning::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index d48d3c079f..88eb774c88 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -209,15 +209,21 @@ fn complete_fixtures<'py>( .map(|key| key.getattr("name")) .collect::>>()?; let completion = call_with( - &imported( - py, - "sqlbuild._native", - "build_relation_fixture_completion", - )?, + &imported(py, "sqlbuild._native", "build_relation_fixture_completion")?, &[ ("project", project.clone()), ("adapter", context.adapter.clone()), - ("request", PyTuple::new(py, [PyTuple::new(py, names)?.into_any(), PyTuple::new(py, groups)?.into_any()])?.into_any()), + ( + "request", + PyTuple::new( + py, + [ + PyTuple::new(py, names)?.into_any(), + PyTuple::new(py, groups)?.into_any(), + ], + )? + .into_any(), + ), ], )?; let diagnostics = completion.getattr("diagnostics")?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index bae328330d..d84978540b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -209,6 +209,7 @@ from sqlbuild.compiler.planner.models import ( AuditPlanEntry, BackfillResult, ChangeDetectionResult, + ColumnMigrationPlanEntry, ColumnMigrationPlanning, ColumnRenameHint, CursorBounds, @@ -233,6 +234,7 @@ from sqlbuild.compiler.planner.models import ( ModelPlanContext, ModelPlanEntry, OldNameView, + OldNameViewPlanning, ParsedScenarioArtifactName, ParsedSelector, PathSelector, @@ -4753,6 +4755,14 @@ def migration_destination_fingerprints( ) -> dict[str, tuple[str, ...]]: ... def stored_migration_fingerprint(fingerprint: Fingerprint) -> str | None: ... def declared_column_aliases(model: CompiledModelView) -> tuple[tuple[str, str], ...]: ... +def plan_old_name_views( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + migration_entries: tuple[ModelMigrationPlanEntry, ...], + column_entries: tuple[ColumnMigrationPlanEntry, ...], +) -> OldNameViewPlanning: ... def manual_migration_requests(*, scope: PlannerScope) -> tuple[ModelMigrationRequest, ...]: ... def discover_model_migrations( *, diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py b/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py deleted file mode 100644 index 81fda8457b..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py +++ /dev/null @@ -1,507 +0,0 @@ -"""Plan what a build does at migrated models' old names.""" - -from __future__ import annotations - -from collections.abc import Iterable -from datetime import datetime - -from sqlbuild._native import ( - declared_column_aliases, - get_config_str, - hard_coded_relation_remedy, - match_project_relation, - old_name_claim_errors, - old_name_reference_message, - old_name_schemas_to_read, - old_name_view_skip_reason, -) -from sqlbuild._native import ( - migration_planning_database as planning_database, -) -from sqlbuild._native import ( - migration_project_schemas as project_schemas, -) -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.adapter.contract.types import RelationType -from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.migrations.main._resolve_old_name_view_retention import ( - resolve_old_name_view_retention, -) -from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location -from sqlbuild.compiler.migrations.models import MigrationRelation, OldNameViewHistory -from sqlbuild.compiler.migrations.types import ( - ColumnMigrationDecision, - MigrationDecision, - OldNameViewAction, - OldNameViewStatus, -) -from sqlbuild.compiler.planner.classes.migration_state_inspection import ( - MigrationStateInspection, -) -from sqlbuild.compiler.planner.models import ( - ColumnMigrationPlanEntry, - ModelMigrationPlanEntry, - OldNameView, - OldNameViewPlanEntry, - OldNameViewPlanning, - PlannerRuntime, - PlannerScope, - PlanWarning, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import MaterializationType, WarningSeverity -from sqlbuild.compiler.references.models import ( - LiteralSqlRelation, - ProjectRelation, - ProjectRelationIndex, - RelationName, -) -from sqlbuild.cursor_algebra.models import Duration -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.spec.contracts.models import MigrationsConfig - -_HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( - {MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT} -) -_ALIAS_DECISIONS: frozenset[ColumnMigrationDecision] = frozenset( - {ColumnMigrationDecision.RENAME, ColumnMigrationDecision.RECORD} -) -_RESUMED_STATUSES: frozenset[OldNameViewStatus] = frozenset( - {OldNameViewStatus.PENDING_ARCHIVE, OldNameViewStatus.PENDING_VIEW} -) -_PENDING_ACTIONS: dict[OldNameViewStatus, OldNameViewAction] = { - OldNameViewStatus.PENDING_ARCHIVE: OldNameViewAction.ARCHIVE_AND_VIEW, - OldNameViewStatus.PENDING_VIEW: OldNameViewAction.VIEW_ONLY, - OldNameViewStatus.LIVE: OldNameViewAction.LIVE, - OldNameViewStatus.EXPIRED: OldNameViewAction.LIVE, -} - - -def plan_old_name_views( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - migration_entries: tuple[ModelMigrationPlanEntry, ...], - column_entries: tuple[ColumnMigrationPlanEntry, ...], -) -> OldNameViewPlanning: - """Decide old-name steps for selected models and collect recorded compatibility views.""" - - models_by_name: dict[str, CompiledModelView] = { - model.name: model for model in runtime.project.models - } - selected: tuple[CompiledModelView, ...] = tuple( - models_by_name[key.name] - for key in scope.execution_order - if key in scope.selected_keys - and key.resource_type == CompiledResourceType.MODEL - and key.name in models_by_name - ) - state: MigrationStateInspection = MigrationStateInspection( - adapter=runtime.adapter, - connection=runtime.connection, - database=planning_database(runtime=runtime), - ) - schemas: set[str] = old_name_schemas_to_read( - snapshot=snapshot, - needed=_needed_schemas( - runtime=runtime, - selected=selected, - snapshot=snapshot, - migration_entries=migration_entries, - ), - ) - if schemas: - state.inspect_old_name_views(schemas=schemas) - target_name: str | None = runtime.project.effective_target_name - histories: tuple[OldNameViewHistory, ...] = tuple( - history - for history in state.old_name_histories - if history.move.target_name in (None, target_name) - ) - views: tuple[OldNameView, ...] = tuple( - _recorded_view(runtime=runtime, history=history, models=runtime.project.models) - for history in histories - if history.created is not None and history.dropped is None - ) - planner: _OldNamePlanner = _OldNamePlanner( - runtime=runtime, - models_by_name=models_by_name, - histories=histories, - views=views, - column_entries=column_entries, - ) - entries: list[OldNameViewPlanEntry] = [] - covered: set[str] = set() - entry: ModelMigrationPlanEntry - for entry in migration_entries: - planned: OldNameViewPlanEntry | None = planner.for_migration(entry) - if planned is not None: - entries.append(planned) - covered.add(entry.model_name) - model: CompiledModelView - for model in selected: - if model.name in covered: - continue - pending: OldNameViewPlanEntry | None = planner.pending_for(model) - if pending is not None: - entries.append(pending) - return OldNameViewPlanning( - entries=tuple(entries), - views=views, - warnings=( - *old_name_claim_errors( - selected=selected, snapshot=snapshot, views=views, now=runtime.invocation_time - ), - *_old_name_reference_errors(runtime=runtime, views=views, entries=tuple(entries)), - ), - ) - - -class _OldNamePlanner: - """Per-plan inputs shared by every old-name decision.""" - - def __init__( - self, - *, - runtime: PlannerRuntime, - models_by_name: dict[str, CompiledModelView], - histories: tuple[OldNameViewHistory, ...], - views: tuple[OldNameView, ...], - column_entries: tuple[ColumnMigrationPlanEntry, ...], - ) -> None: - self._runtime: PlannerRuntime = runtime - self._models_by_name: dict[str, CompiledModelView] = models_by_name - self._histories: tuple[OldNameViewHistory, ...] = histories - self._views: tuple[OldNameView, ...] = views - self._column_entries: tuple[ColumnMigrationPlanEntry, ...] = column_entries - config: MigrationsConfig = ( - runtime.project_config.migrations - if runtime.project_config is not None - else MigrationsConfig() - ) - self._project_retention: str | None = config.old_name_views - self._now: datetime = runtime.invocation_time - - def for_migration(self, entry: ModelMigrationPlanEntry) -> OldNameViewPlanEntry | None: - model: CompiledModelView | None = self._models_by_name.get(entry.model_name) - if model is None or entry.blocks_build: - return None - if _starts_move(entry): - return self._new_move(entry=entry, model=model) - if entry.decision == MigrationDecision.DONE or ( - entry.decision == MigrationDecision.RENAMED and entry.completed_at is not None - ): - history: OldNameViewHistory | None = self._newest_history( - destination=migration_relation_for_location(entry.destination), - origin=migration_relation_for_location(entry.origin), - ) - return None if history is None else self._from_history(history=history, model=model) - return None - - def pending_for(self, model: CompiledModelView) -> OldNameViewPlanEntry | None: - history: OldNameViewHistory | None = self._newest_history( - destination=migration_relation_for_location(model.destination), origin=None - ) - if history is None or history.move.destination_model != model.name: - return None - planned: OldNameViewPlanEntry | None = self._from_history(history=history, model=model) - if planned is None or not (planned.runs_steps or _skips_pending(planned)): - return None - return planned - - def _new_move( - self, *, entry: ModelMigrationPlanEntry, model: CompiledModelView - ) -> OldNameViewPlanEntry: - retention: str | None = resolve_old_name_view_retention( - config_values=model.config.values, project_retention=self._project_retention - ) - reason: str | None = self._skip_reason( - origin=entry.origin, origin_tracked=entry.origin_tracked, retention=retention - ) - if reason is not None or retention is None: - return _skipped_entry(entry=entry, reason=reason or "old_name_view false") - duration: Duration | None = Duration.parse(retention) - return OldNameViewPlanEntry( - model_name=entry.model_name, - origin=entry.origin, - destination=entry.destination, - action=OldNameViewAction.ARCHIVE_AND_VIEW, - target_name=entry.target_name, - retention=retention, - expires_at=None if duration is None else duration.add_to(self._now), - column_aliases=self._planned_aliases(model), - stores_history=_stores_history(model), - records_requirement=True, - grants_supported=self._runtime.adapter.relation_grants_supported, - ) - - def _skip_reason( - self, - *, - origin: CompiledRelationLocation, - origin_tracked: bool | None, - retention: str | None, - ) -> str | None: - """Return why the old name gets no steps; ``origin_tracked`` None means unknown.""" - - return old_name_view_skip_reason( - origin=origin, - origin_tracked=origin_tracked, - retention=retention, - models=self._models_by_name.values(), - views=self._views, - ) - - def _from_history( - self, *, history: OldNameViewHistory, model: CompiledModelView - ) -> OldNameViewPlanEntry | None: - """Resume a recorded move; its record proves SQLBuild owned the origin when it moved.""" - - status: OldNameViewStatus = history.status(now=self._now) - action: OldNameViewAction | None = _PENDING_ACTIONS.get(status) - if action is None: - return None - if status in _RESUMED_STATUSES: - reason: str | None = self._skip_reason( - origin=_location(runtime=self._runtime, relation=history.old), - origin_tracked=None, - retention=resolve_old_name_view_retention( - config_values=model.config.values, project_retention=self._project_retention - ), - ) - if reason is not None: - return _skipped_resume(runtime=self._runtime, history=history, reason=reason) - duration: Duration | None = Duration.parse(history.required.view_retention or "") - expires_at: datetime | None = ( - history.created.expires_at - if history.created is not None - else (None if duration is None else duration.add_to(self._now)) - ) - return OldNameViewPlanEntry( - model_name=model.name, - origin=_location(runtime=self._runtime, relation=history.old), - destination=_location(runtime=self._runtime, relation=history.new), - action=action, - target_name=history.move.target_name, - retention=history.required.view_retention, - expires_at=expires_at, - column_aliases=( - history.created.column_aliases - if history.created is not None - else self._planned_aliases(model) - ), - stores_history=_stores_history(model), - migration_event_id=history.move.event_id, - grants_copied=( - None - if history.created is None or history.created.grants_copied is None - else len(history.created.grants_copied) - ), - grants_supported=self._runtime.adapter.relation_grants_supported, - archived=history.archived is not None, - ) - - def _newest_history( - self, *, destination: MigrationRelation, origin: MigrationRelation | None - ) -> OldNameViewHistory | None: - matching: tuple[OldNameViewHistory, ...] = tuple( - history - for history in self._histories - if history.move.destination.matches(destination) - and (origin is None or history.move.origin.matches(origin)) - ) - if not matching: - return None - return max(matching, key=lambda item: (item.move.created_at, item.move.event_id)) - - def _planned_aliases(self, model: CompiledModelView) -> tuple[tuple[str, str], ...]: - if _stores_history(model): - return tuple( - sorted( - (entry.origin_column, entry.destination_column) - for entry in self._column_entries - if entry.model_name == model.name and entry.decision in _ALIAS_DECISIONS - ) - ) - return declared_column_aliases(model) - - -def _starts_move(entry: ModelMigrationPlanEntry) -> bool: - if entry.blocks_build: - return False - return entry.decision.moves_data or ( - entry.decision == MigrationDecision.RENAMED and entry.completed_at is None - ) - - -def _skips_pending(entry: OldNameViewPlanEntry) -> bool: - return entry.action == OldNameViewAction.NONE and entry.migration_event_id is not None - - -def _skipped_resume( - *, runtime: PlannerRuntime, history: OldNameViewHistory, reason: str -) -> OldNameViewPlanEntry: - return OldNameViewPlanEntry( - model_name=history.move.destination_model, - origin=_location(runtime=runtime, relation=history.old), - destination=_location(runtime=runtime, relation=history.new), - action=OldNameViewAction.NONE, - target_name=history.move.target_name, - migration_event_id=history.move.event_id, - reason=reason, - ) - - -def _stores_history(model: CompiledModelView) -> bool: - return ( - get_config_str(values=model.config.values, key="materialized") in _HISTORY_MATERIALIZATIONS - ) - - -def _skipped_entry(*, entry: ModelMigrationPlanEntry, reason: str) -> OldNameViewPlanEntry: - return OldNameViewPlanEntry( - model_name=entry.model_name, - origin=entry.origin, - destination=entry.destination, - action=OldNameViewAction.NONE, - target_name=entry.target_name, - reason=reason, - ) - - -def _needed_schemas( - *, - runtime: PlannerRuntime, - selected: tuple[CompiledModelView, ...], - snapshot: WarehouseSnapshot, - migration_entries: tuple[ModelMigrationPlanEntry, ...], -) -> set[str]: - """Return schemas whose old-name facts this plan needs.""" - - schemas: set[str] = { - model.destination.schema for model in selected if model.destination.schema is not None - } - schemas.update( - entry.origin.schema for entry in migration_entries if entry.origin.schema is not None - ) - if runtime.project.unmatched_literal_sql_relations or any( - _may_claim_view(model=model, snapshot=snapshot) for model in selected - ): - schemas.update(project_schemas(runtime=runtime)) - return schemas - - -def _may_claim_view(*, model: CompiledModelView, snapshot: WarehouseSnapshot) -> bool: - relation: RelationInfo | None = snapshot.existing_relations.get(model.name) - return relation is not None and _is_view(relation) - - -def _is_view(relation: RelationInfo) -> bool: - return normalize_relation_type(relation.relation_type) == RelationType.VIEW - - -def _old_name_reference_errors( - *, - runtime: PlannerRuntime, - views: tuple[OldNameView, ...], - entries: tuple[OldNameViewPlanEntry, ...], -) -> tuple[PlanWarning, ...]: - """Refuse literal Python SQL that reads a name kept only as a compatibility view.""" - - literals: tuple[LiteralSqlRelation, ...] = runtime.project.unmatched_literal_sql_relations - if not literals: - return () - index: ProjectRelationIndex = ProjectRelationIndex( - relations=tuple( - dict.fromkeys( - ( - *( - _old_name_relation(old=view.old, model_name=view.destination_model) - for view in views - ), - *( - _old_name_relation(old=entry.origin, model_name=entry.model_name) - for entry in entries - if entry.runs_steps - ), - ) - ) - ) - ) - errors: list[PlanWarning] = [] - literal: LiteralSqlRelation - for literal in literals: - match: ProjectRelation | None = match_project_relation( - index=index, relation=literal.relation - ) - if match is None or match.compatibility_for is None: - continue - written: str = ".".join( - part - for part in (literal.relation.database, literal.relation.schema, literal.relation.name) - if part - ) - errors.append( - PlanWarning( - model_name=None, - severity=WarningSeverity.ERROR, - message=( - f"{literal.relative_path}:{literal.line}: " - + old_name_reference_message( - owner_label=literal.owner_label, - written=written, - method=literal.method, - model_name=match.compatibility_for, - ) - + "; " - + hard_coded_relation_remedy(owner_kind=literal.owner_kind, ref=match.ref) - ), - code="P008", - ) - ) - return tuple(errors) - - -def _old_name_relation(*, old: CompiledRelationLocation, model_name: str) -> ProjectRelation: - return ProjectRelation( - ref=SqlResourceRef(kind=SqlResourceRefKind.MODEL, name=model_name), - relation=RelationName(name=old.name, schema=old.schema, database=old.database), - compatibility_for=model_name, - ) - - -def _recorded_view( - *, runtime: PlannerRuntime, history: OldNameViewHistory, models: Iterable[CompiledModelView] -) -> OldNameView: - return OldNameView( - destination_model=history.move.destination_model, - old=_location(runtime=runtime, relation=history.old), - new=_location(runtime=runtime, relation=history.new), - expires_at=None if history.created is None else history.created.expires_at, - column_aliases=() if history.created is None else history.created.column_aliases, - migration_event_id=history.move.event_id, - target_name=history.move.target_name, - name_reused_by=next( - ( - model.name - for model in models - if migration_relation_for_location(model.destination).matches(history.old) - ), - None, - ), - ) - - -def _location(*, runtime: PlannerRuntime, relation: MigrationRelation) -> CompiledRelationLocation: - return CompiledRelationLocation( - database=relation.database, - schema=relation.schema, - name=relation.name, - qualified_name=runtime.adapter.render_qualified_name( - database=relation.database, schema=relation.schema, name=relation.name - ), - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py index 51064cf363..8e1dc9dff7 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py @@ -4,8 +4,7 @@ import time -from sqlbuild._native import plan_column_migrations, plan_model_migrations -from sqlbuild.compiler.planner._helpers.migrations.old_names import plan_old_name_views +from sqlbuild._native import plan_column_migrations, plan_model_migrations, plan_old_name_views from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( effectively_full_refreshed_model_names, From b3a2174814a875bf50ee12ca0d9a831ad6ffe392 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:57:13 +0100 Subject: [PATCH 04/22] chore: reconcile scenario and fixture port integration --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index e8fb54d60b..840a6447a3 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 33506 +remaining_lines = 31780 [[stays]] path = "executor" From ffc82fae346f08e5615ab086760c039c14dbcfdd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:11:29 +0100 Subject: [PATCH 05/22] refactor: assemble audit plan entries natively --- .../planner_strategy/audit_entries.rs | 240 ++++++++++++++++++ .../_helpers/planner_strategy/constants.rs | 9 + .../bindings/_helpers/planner_strategy/mod.rs | 2 + .../_helpers/planner_strategy/registration.rs | 10 + src/sqlbuild/_native.pyi | 23 ++ .../_helpers/command_planning/static.py | 2 +- .../planner/_helpers/output/audit_entry.py | 129 ---------- .../planner/_helpers/output/plan_entry.py | 18 +- .../planner/_helpers/output/plan_output.py | 46 +--- .../planner/_helpers/test_audit_entry.py | 2 +- .../planner/_helpers/test_audit_entry.py | 2 +- 11 files changed, 289 insertions(+), 194 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs new file mode 100644 index 0000000000..d4d74b7dde --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs @@ -0,0 +1,240 @@ +//! Assemble audit plan dataclasses using native rendering and scheduling. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{ + PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple, PyTupleMethods, +}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +use crate::bindings::_helpers::planner_strategy::constants::{ + AUDIT_TYPES_MODULE, COMPILE_MODELS_MODULE, COMPILE_TYPES_MODULE, MODEL_ATTACHMENT, + NATIVE_MODULE, PLAN_MODELS_MODULE, SOURCE_ATTACHMENT, +}; + +/// Render and validate one audit query with the supplied adapter callbacks. +fn render_query<'py>( + request: &Request<'_, 'py>, + sql: Bound<'py, PyAny>, + context: &str, +) -> PyResult> { + let py = request.py(); + let resolved = call_with( + &imported(py, NATIVE_MODULE, "render_audit_sql")?, + &[ + ("unresolved_sql", sql), + ("model_locations", request.get("model_locations")?), + ("seed_locations", request.get("seed_locations")?), + ("source_map", request.get("source_map")?), + ("adapter", request.get("adapter")?), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "assert_no_unresolved_sql_markers")?, + &[ + ("sql", resolved.clone()), + ("context", PyString::new(py, context).into_any()), + ], + )?; + Ok(resolved) +} + +/// Resolve an audit enum, applying the default only to absent values. +fn audit_enum<'py>( + audit: &Bound<'py, PyAny>, + attribute: &str, + enum_name: &str, + default: &str, +) -> PyResult> { + let py = audit.py(); + let value = audit.getattr(attribute)?; + imported(py, AUDIT_TYPES_MODULE, enum_name)?.call1((if value.is_none() { + PyString::new(py, default).into_any() + } else { + value + },)) +} + +/// Build an audit entry while retaining the public Python plan dataclass. +#[pyfunction] +#[pyo3(signature = (**request))] +pub(crate) fn plan_audit<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("plan_audit", request)?; + let py = request.py(); + let audit = request.get("audit")?; + let name: String = audit.getattr("name")?.extract()?; + let resolved = render_query( + &request, + audit.getattr("sql_body")?, + &format!("audit '{name}' planned SQL"), + )?; + let evidence = audit.getattr("evidence_sql")?; + let evidence_resolved = if evidence.is_none() { + py.None().into_bound(py) + } else { + render_query( + &request, + evidence, + &format!("audit '{name}' planned evidence SQL"), + )? + }; + let attachment = call_with( + &imported(py, NATIVE_MODULE, "resolve_attachment_kind")?, + &[ + ("audit", audit.clone()), + ("downstream_deps", request.get("lineage_downstream_deps")?), + ], + )?; + let attachment = attachment.cast::()?; + let kind = attachment.get_item(0)?; + let target = attachment.get_item(1)?; + let kind_text: String = kind.extract()?; + let materialization = if kind_text == MODEL_ATTACHMENT && !target.is_none() { + request + .get("model_materializations")? + .call_method1("get", (&target,))? + } else { + py.None().into_bound(py) + }; + let requested = audit_enum(&audit, "run_scope", "AuditRunScope", "final")?; + let severity = audit_enum(&audit, "severity", "AuditSeverity", "warn")?; + let effective = call_with( + &imported(py, NATIVE_MODULE, "resolve_effective_run_scope")?, + &[ + ("requested_run_scope", requested.clone()), + ("attached_model_materialization", materialization), + ], + )?; + let mut target_kind = audit.getattr("attached_target_kind")?; + if target_kind.is_none() && (kind_text == MODEL_ATTACHMENT || kind_text == SOURCE_ATTACHMENT) { + target_kind = + imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind_text,))?; + } + let values = PyDict::new(py); + for field in [ + "key", + "name", + "definition_name", + "evaluation_mode", + "thresholds", + "minimum_samples", + "evidence_limit", + "scope_deps", + "attached_column_name", + "always_run", + "description", + ] { + values.set_item(field, audit.getattr(field)?)?; + } + let measurement = audit.getattr("measurement_contract")?; + for field in ["value_column", "sample_count_column", "sample_unit"] { + values.set_item( + field, + if measurement.is_none() { + py.None().into_bound(py) + } else { + measurement.getattr(field)? + }, + )?; + } + for (field, value) in [ + ("resolved_sql", resolved), + ("unresolved_sql", audit.getattr("sql_body")?), + ("attachment_kind", kind), + ("severity", severity), + ("requested_run_scope", requested), + ("effective_run_scope", effective), + ("evidence_resolved_sql", evidence_resolved), + ("evidence_unresolved_sql", audit.getattr("evidence_sql")?), + ("attached_target_kind", target_kind), + ("attached_target_name", target), + ( + "reads_outside_target_lineage", + call_with( + &imported(py, NATIVE_MODULE, "reads_outside_target_lineage")?, + &[ + ("audit", audit), + ("upstream_deps", request.get("lineage_upstream_deps")?), + ], + )?, + ), + ] { + values.set_item(field, value)?; + } + imported(py, PLAN_MODELS_MODULE, "AuditPlanEntry")?.call((), Some(&values)) +} + +/// Select attached audits by their logical target and singular audits by dependency overlap. +#[pyfunction] +#[pyo3(signature = (*, audit, selected_keys))] +pub(crate) fn audit_is_selected( + audit: &Bound<'_, PyAny>, + selected_keys: &Bound<'_, PyAny>, +) -> PyResult { + let py = audit.py(); + let kind = audit.getattr("attached_target_kind")?; + let name = audit.getattr("attached_target_name")?; + if !kind.is_none() && !name.is_none() { + let resource_type = imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")? + .call1((kind,))? + .getattr("resource_type")?; + let key = call_with( + &imported(py, COMPILE_MODELS_MODULE, "CompiledObjectKey")?, + &[("resource_type", resource_type), ("name", name)], + )?; + return selected_keys.contains(key); + } + for dependency in audit.getattr("scope_deps")?.try_iter()? { + if selected_keys.contains(dependency?)? { + return Ok(true); + } + } + Ok(false) +} + +/// Assemble selected audits in compiled-project order without querying lineage for an empty selection. +#[pyfunction] +#[pyo3(signature = (**request))] +pub(crate) fn build_selected_audit_entries<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("build_selected_audit_entries", request)?; + let py = request.py(); + let project = request.get("project")?; + let selected_keys = request.get("scope")?.getattr("selected_keys")?; + let mut selected: Vec> = Vec::new(); + for audit in project.getattr("audits")?.try_iter()? { + let audit = audit?; + if audit_is_selected(&audit, &selected_keys)? { + selected.push(audit); + } + } + let entries = PyList::empty(py); + if selected.is_empty() { + return Ok(entries); + } + let lineage = imported(py, NATIVE_MODULE, "project_lineage_views")?.call1((project,))?; + let relations = request.get("relations")?; + let arguments = PyDict::new(py); + for field in ["model_locations", "seed_locations"] { + arguments.set_item(field, relations.getattr(field)?)?; + } + arguments.set_item("source_map", relations.getattr("source_read_map")?)?; + arguments.set_item("adapter", request.get("adapter")?)?; + arguments.set_item( + "model_materializations", + request.get("model_materializations")?, + )?; + arguments.set_item("lineage_upstream_deps", lineage.getattr("upstream_deps")?)?; + arguments.set_item( + "lineage_downstream_deps", + lineage.getattr("downstream_deps")?, + )?; + for audit in selected { + arguments.set_item("audit", audit)?; + entries.append(plan_audit(Some(&arguments))?)?; + } + Ok(entries) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs new file mode 100644 index 0000000000..350d9284d5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs @@ -0,0 +1,9 @@ +//! Audit plan boundary modules and attachment values. + +pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; +pub(crate) const AUDIT_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; +pub(crate) const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +pub(crate) const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +pub(crate) const PLAN_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +pub(crate) const MODEL_ATTACHMENT: &str = "model"; +pub(crate) const SOURCE_ATTACHMENT: &str = "source"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs index 94f6239e04..1b18069a05 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs @@ -1,4 +1,6 @@ +pub(crate) mod audit_entries; pub(crate) mod audits; +pub(crate) mod constants; pub(crate) mod cursor_types; pub(crate) mod fixture_completion; pub(crate) mod fixture_context; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs index b6c58fba9f..3f85873151 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -1,6 +1,7 @@ //! Register the planner strategy. use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::{prelude::PyModuleMethods, wrap_pyfunction}; use crate::bindings::_helpers::planner_strategy::{ audits, cursor_types, fixture_completion, fixture_metadata, fixture_sql, lineage, microbatches, @@ -8,6 +9,15 @@ use crate::bindings::_helpers::planner_strategy::{ }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(super::audit_entries::plan_audit, module)?)?; + module.add_function(wrap_pyfunction!( + super::audit_entries::audit_is_selected, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::audit_entries::build_selected_audit_entries, + module + )?)?; audits::register(module)?; cursor_types::register(module)?; fixture_completion::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index d84978540b..c507458e73 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -240,6 +240,7 @@ from sqlbuild.compiler.planner.models import ( PathSelector, PlannerChangeResults, PlannerOverrides, + PlannerRelationsContext, PlannerResolvedActions, PlannerRuntime, PlannerScope, @@ -4889,6 +4890,28 @@ def build_model_warnings( on_schema_change: OnSchemaChange | None, type_enforcement: bool, ) -> tuple[PlanWarning, ...]: ... +def plan_audit( + *, + audit: CompiledAuditView, + model_locations: dict[str, CompiledRelationLocation], + seed_locations: dict[str, CompiledRelationLocation], + source_map: dict[str, SourceEntry], + adapter: BaseAdapter, + lineage_upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + lineage_downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + model_materializations: dict[str, str], +) -> AuditPlanEntry: ... +def audit_is_selected( + *, audit: CompiledAuditView, selected_keys: frozenset[CompiledObjectKey] +) -> bool: ... +def build_selected_audit_entries( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + scope: PlannerScope, + relations: PlannerRelationsContext, + model_materializations: dict[str, str], +) -> list[AuditPlanEntry]: ... def resolve_attachment_kind( *, audit: CompiledAuditView, diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py index 642a82b8af..fe32ee724d 100644 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py +++ b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py @@ -5,6 +5,7 @@ from sqlbuild._native import ( build_compiled_function_fingerprint_sql, build_seed_identity, + build_selected_audit_entries, get_materialization_type, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -16,7 +17,6 @@ extract_seed_columns, ) from sqlbuild.compiler.planner._helpers.output.plan_output import ( - build_selected_audit_entries, build_selected_test_entries, plan_function, sql_test_is_selected, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py deleted file mode 100644 index 93ec534f32..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Audit plan entry construction.""" - -from __future__ import annotations - -from sqlbuild._native import ( - assert_no_unresolved_sql_markers, - reads_outside_target_lineage, - render_audit_sql, - resolve_attachment_kind, - resolve_effective_run_scope, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.auditing.types import ( - AuditAttachmentKind, - AuditRunScope, - AuditSeverity, -) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledAuditView -from sqlbuild.compiler.planner.models import AuditPlanEntry -from sqlbuild.spec.contracts.models import SourceEntry - - -def plan_audit( - *, - audit: CompiledAuditView, - model_locations: dict[str, CompiledRelationLocation], - seed_locations: dict[str, CompiledRelationLocation], - source_map: dict[str, SourceEntry], - adapter: BaseAdapter, - lineage_upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - lineage_downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - model_materializations: dict[str, str], -) -> AuditPlanEntry: - """Build an audit plan entry with refs and sources resolved.""" - - resolved_sql: str = render_audit_sql( - unresolved_sql=audit.sql_body, - model_locations=model_locations, - seed_locations=seed_locations, - source_map=source_map, - adapter=adapter, - ) - assert_no_unresolved_sql_markers( - sql=resolved_sql, - context=f"audit '{audit.name}' planned SQL", - ) - evidence_resolved_sql: str | None = None - if audit.evidence_sql is not None: - evidence_resolved_sql = render_audit_sql( - unresolved_sql=audit.evidence_sql, - model_locations=model_locations, - seed_locations=seed_locations, - source_map=source_map, - adapter=adapter, - ) - assert_no_unresolved_sql_markers( - sql=evidence_resolved_sql, - context=f"audit '{audit.name}' planned evidence SQL", - ) - - attachment_kind: AuditAttachmentKind - attached_target_name: str | None - attachment_kind, attached_target_name = resolve_attachment_kind( - audit=audit, - downstream_deps=lineage_downstream_deps, - ) - - attached_materialization: str | None = None - if attachment_kind == AuditAttachmentKind.MODEL and attached_target_name is not None: - attached_materialization = model_materializations.get(attached_target_name) - - requested_run_scope: AuditRunScope = AuditRunScope( - audit.run_scope if audit.run_scope is not None else AuditRunScope.FINAL - ) - severity: AuditSeverity = AuditSeverity( - audit.severity if audit.severity is not None else AuditSeverity.WARN - ) - - effective_run_scope: AuditRunScope = resolve_effective_run_scope( - requested_run_scope=requested_run_scope, - attached_model_materialization=attached_materialization, - ) - attached_target_kind: AttachedAuditTargetKind | None = audit.attached_target_kind - if attached_target_kind is None and attachment_kind == AuditAttachmentKind.MODEL: - attached_target_kind = AttachedAuditTargetKind.MODEL - elif attached_target_kind is None and attachment_kind == AuditAttachmentKind.SOURCE: - attached_target_kind = AttachedAuditTargetKind.SOURCE - - return AuditPlanEntry( - key=audit.key, - name=audit.name, - definition_name=audit.definition_name, - resolved_sql=resolved_sql, - unresolved_sql=audit.sql_body, - attachment_kind=attachment_kind, - severity=severity, - requested_run_scope=requested_run_scope, - effective_run_scope=effective_run_scope, - evaluation_mode=audit.evaluation_mode, - value_column=( - None if audit.measurement_contract is None else audit.measurement_contract.value_column - ), - sample_count_column=( - None - if audit.measurement_contract is None - else audit.measurement_contract.sample_count_column - ), - sample_unit=( - None if audit.measurement_contract is None else audit.measurement_contract.sample_unit - ), - thresholds=audit.thresholds, - minimum_samples=audit.minimum_samples, - evidence_resolved_sql=evidence_resolved_sql, - evidence_unresolved_sql=audit.evidence_sql, - evidence_limit=audit.evidence_limit, - scope_deps=audit.scope_deps, - attached_target_kind=attached_target_kind, - attached_target_name=attached_target_name, - attached_column_name=audit.attached_column_name, - always_run=audit.always_run, - description=audit.description, - reads_outside_target_lineage=reads_outside_target_lineage( - audit=audit, upstream_deps=lineage_upstream_deps - ), - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index 1d7271da0a..4e676c5eb5 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -8,6 +8,7 @@ from typing import Any from sqlbuild._native import ( + audit_is_selected, build_model_warnings, build_source_load_map, cap_microbatch_bounds, @@ -49,7 +50,6 @@ CursorInputRoles, ) from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, CompiledAuditView, CompiledFunctionView, CompiledModelView, @@ -1848,22 +1848,6 @@ def scope_overlaps( return False -def audit_is_selected( - *, - audit: CompiledAuditView, - selected_keys: frozenset[CompiledObjectKey], -) -> bool: - """Return whether an audit belongs to the selected logical target scope.""" - - if audit.attached_target_kind is not None and audit.attached_target_name is not None: - target_key: CompiledObjectKey = CompiledObjectKey( - resource_type=AttachedAuditTargetKind(audit.attached_target_kind).resource_type, - name=audit.attached_target_name, - ) - return target_key in selected_keys - return scope_overlaps(scope_deps=audit.scope_deps, selected_keys=selected_keys) - - def _selected_audit_relation_keys( *, project: CompiledProjectView, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py index ffe7a63817..6316d35768 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py @@ -4,8 +4,8 @@ from sqlbuild._native import ( build_relation_fixture_context, + build_selected_audit_entries, build_source_load_entries, - project_lineage_views, upstream_loader_dependency_names, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -14,7 +14,6 @@ CompiledSqlTest, ) from sqlbuild.compiler.compile.types import ( - CompiledAuditView, CompiledFunctionView, CompiledProjectView, ) @@ -26,10 +25,7 @@ PythonHookEntry, ) from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.graph.models import LineageGraphViews -from sqlbuild.compiler.planner._helpers.output.audit_entry import plan_audit from sqlbuild.compiler.planner._helpers.output.plan_entry import ( - audit_is_selected, build_model_materializations, extract_seed_columns, scope_overlaps, @@ -418,46 +414,6 @@ def plan_function( ) -def build_selected_audit_entries( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - relations: PlannerRelationsContext, - model_materializations: dict[str, str], -) -> list[AuditPlanEntry]: - entries: list[AuditPlanEntry] = [] - selected_audits: tuple[CompiledAuditView, ...] = tuple( - audit - for audit in project.audits - if audit_is_selected(audit=audit, selected_keys=scope.selected_keys) - ) - if not selected_audits: - return entries - lineage: LineageGraphViews = project_lineage_views(project) - lineage_upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - lineage.upstream_deps - ) - lineage_downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - lineage.downstream_deps - ) - audit: CompiledAuditView - for audit in selected_audits: - entries.append( - plan_audit( - audit=audit, - model_locations=relations.model_locations, - seed_locations=relations.seed_locations, - source_map=relations.source_read_map, - adapter=adapter, - lineage_upstream_deps=lineage_upstream_deps, - lineage_downstream_deps=lineage_downstream_deps, - model_materializations=model_materializations, - ) - ) - return entries - - def plan_selected_sql_tests( *, project: CompiledProjectView, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index e02a9b154d..f1b2b4c26d 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -5,6 +5,7 @@ import pytest +from sqlbuild._native import plan_audit from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledAudit, @@ -16,7 +17,6 @@ DiscoveredAuditBlock, DiscoveredAuditFile, ) -from sqlbuild.compiler.planner._helpers.output.audit_entry import plan_audit from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.spec.contracts.models import SourceEntry from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index a88912efab..a371b8a5d6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -2,6 +2,7 @@ import pytest +from sqlbuild._native import plan_audit from sqlbuild.compiler.auditing.types import AuditAttachmentKind, AuditRunScope from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -9,7 +10,6 @@ CompileSqlReference, ) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledAuditView -from sqlbuild.compiler.planner._helpers.output.audit_entry import plan_audit from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import SourceEntry From 927094cb51907f62e209e181e1ccdec4aa5fe536 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:29:45 +0100 Subject: [PATCH 06/22] chore: update Python boundary after audit assembly port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 840a6447a3..56a9ed331e 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 31780 +remaining_lines = 31591 [[stays]] path = "executor" From 2b22f7955c5f41b219a12a59a05291d63b7453ae Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:31:04 +0100 Subject: [PATCH 07/22] refactor: render migration plan presentation natively --- crates/sqlbuild-core/src/cli_text/models.rs | 69 ++++ .../_helpers/migration_sections.rs | 218 +++++++++++++ .../src/plan_display/_helpers/mod.rs | 1 + .../src/plan_display/constants.rs | 8 + .../main/format_model_migrations.rs | 95 ++++++ .../src/plan_display/main/mod.rs | 2 + .../src/plan_display/main/old_name_rows.rs | 19 ++ .../_helpers/plan_output/migrations.rs | 141 +++++++++ .../src/bindings/_helpers/plan_output/mod.rs | 1 + .../_helpers/plan_output/plan_text.rs | 2 +- .../_helpers/plan_output/registration.rs | 3 +- src/sqlbuild/_native.pyi | 5 + .../planner/_helpers/migrations/display.py | 294 ------------------ .../main/pre_build/pre_build_work_display.py | 3 +- .../presentation/main/tree_connector.py | 12 - .../_helpers/migrations/test_old_name_rows.py | 2 +- 16 files changed, 564 insertions(+), 311 deletions(-) create mode 100644 crates/sqlbuild-core/src/plan_display/_helpers/migration_sections.rs create mode 100644 crates/sqlbuild-core/src/plan_display/main/format_model_migrations.rs create mode 100644 crates/sqlbuild-core/src/plan_display/main/old_name_rows.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/migrations/display.py delete mode 100644 src/sqlbuild/presentation/main/tree_connector.py diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 4ff7b7d80e..bb689cf0d2 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -55,6 +55,11 @@ impl CliStyle { self.apply(BLUE, text) } + #[must_use] + pub fn accent_strong(self, text: &str) -> String { + self.title(text) + } + #[must_use] pub fn plan_section(self, text: &str) -> String { self.apply(BOLD, text) @@ -547,3 +552,67 @@ pub struct DbtInteropPlanText { pub path_translations: Vec<(String, String)>, pub warnings: Vec, } +/// Migration decisions already resolved by the planner's model boundary. +#[derive(Clone, Debug, Default)] +pub struct MigrationDecisionText { + pub value: String, + pub label: String, + pub blocks_build: bool, + pub origin_missing: bool, + pub decision_blocks_build: bool, + pub checks_compatibility: bool, +} + +/// One model identity handover's presentation facts. +#[derive(Clone, Debug, Default)] +pub struct MigrationText { + pub model_name: String, + pub origin: String, + pub destination: String, + pub decision: MigrationDecisionText, + pub compatibility: String, + pub discovery: String, + pub completed_at: Option, + pub target_label: String, + pub transfer: Option, + pub transfer_fallback: Option, + pub storage_transition: Option, + pub promotion: Option, + pub findings: Vec, + pub materialization: Option, +} + +/// One old-name compatibility view's presentation facts. +#[derive(Clone, Debug, Default)] +pub struct OldNameText { + pub model_name: String, + pub origin: String, + pub destination: String, + pub action: String, + pub expires_date: Option, + pub column_aliases: Vec<(String, String)>, + pub grants_supported: bool, + pub grants_copied: Option, + pub reason: Option, +} + +/// One column rename's presentation facts. +#[derive(Clone, Debug, Default)] +pub struct ColumnMigrationText { + pub model_name: String, + pub origin_column: String, + pub destination_column: String, + pub decision: MigrationDecisionText, + pub discovery: String, + pub completed_at: Option, +} + +/// Ordered migration sections appended to pre-build work. +#[derive(Clone, Debug)] +pub struct MigrationTextReport { + pub migrations: Vec, + pub old_names: Vec, + pub columns: Vec, + pub display: DisplayLimit, + pub manual_rename_hint: String, +} diff --git a/crates/sqlbuild-core/src/plan_display/_helpers/migration_sections.rs b/crates/sqlbuild-core/src/plan_display/_helpers/migration_sections.rs new file mode 100644 index 0000000000..3d63718b4c --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/_helpers/migration_sections.rs @@ -0,0 +1,218 @@ +//! Migration rows using the shared plan colors and layout. + +use crate::cli_text::main::split_count_header::split_count_header; + +use crate::cli_text::models::{CliStyle, ColumnMigrationText, MigrationText, OldNameText}; +use crate::plan_display::constants::{ + MIGRATION_AUTOMATIC, MIGRATION_DONE, MIGRATION_INCOMPATIBLE, MIGRATION_MANUAL, + MIGRATION_RENAMED, OLD_NAME_LIVE, OLD_NAME_NONE, OLD_NAME_VIEW_ONLY, +}; +use crate::plan_display::main::old_name_rows::old_name_rows; + +pub(crate) fn property_row(label: &str, value: &str, style: CliStyle) -> String { + format!(" {} {value}", style.muted(label)) +} + +pub(crate) fn old_name_facts(entry: &OldNameText) -> Vec<(&str, String)> { + if entry.action == OLD_NAME_NONE { + return vec![( + "left for janitor", + entry + .reason + .as_deref() + .filter(|value| !value.is_empty()) + .unwrap_or("no compatibility view") + .to_owned(), + )]; + } + let until = entry + .expires_date + .as_ref() + .map_or_else(String::new, |date| format!("until {date}")); + let view = if entry.action == OLD_NAME_LIVE { + format!("live {until}").trim().to_owned() + } else { + until + }; + let mut facts = vec![("view", view)]; + if !entry.column_aliases.is_empty() { + facts.push(( + "columns", + entry + .column_aliases + .iter() + .map(|(old, new)| format!("{old} <- {new}")) + .collect::>() + .join(", "), + )); + } + if entry.grants_supported { + let text = entry.grants_copied.as_ref().map_or_else( + || "copied from the old table".to_owned(), + |count| format!("{count} copied from the old table"), + ); + facts.push(("grants", text)); + } + facts.push(( + "old table", + if entry.action == OLD_NAME_VIEW_ONLY { + "already archived" + } else { + "archived" + } + .to_owned(), + )); + facts +} + +fn decision_text(entry: &MigrationText, style: CliStyle) -> String { + let decision = &entry.decision; + if decision.blocks_build && decision.decision_blocks_build { + return style.error_strong(&decision.label); + } + if decision.origin_missing + || matches!( + decision.value.as_str(), + "superseded_replace" | "forced_replace" + ) + { + return style.warning_strong(&decision.label); + } + if decision.value == MIGRATION_DONE { + return style.muted(&decision.label); + } + style.accent_strong(&decision.label) +} + +pub(crate) fn migration_lines( + entry: &MigrationText, + old_name: Option<&OldNameText>, + style: CliStyle, + hint: &str, +) -> Vec { + let renamed = entry.decision.value == MIGRATION_RENAMED; + let decision = if renamed { + if entry.completed_at.is_some() { + style.muted("done") + } else { + style.accent_strong("migrate") + } + } else { + decision_text(entry, style) + }; + let mut rows = vec![format!( + " {} {decision} {} {}", + style.object_name(&entry.model_name), + style.muted(&format!("{} ->", entry.origin)), + entry.destination + )]; + if renamed && entry.completed_at.is_none() { + rows.push(property_row( + "transfer", + if entry.materialization.as_deref() == Some("view") { + "recreate (view)" + } else { + "rebuild (table)" + }, + style, + )); + } + if entry.decision.checks_compatibility { + let value = match entry.compatibility.as_str() { + "compatible" => style.success(&entry.compatibility), + "incompatible" => style.error(&entry.compatibility), + _ => entry.compatibility.clone(), + }; + rows.push(property_row("compatibility", &value, style)); + } + if let Some(transfer) = &entry.transfer { + let fallback = entry + .transfer_fallback + .as_ref() + .map_or_else(String::new, |value| format!(" ({value} if refused)")); + let mut details = vec![format!("{transfer}{fallback}")]; + if let Some(value) = &entry.storage_transition { + details.push(value.clone()); + } + if let Some(value) = &entry.promotion { + details.push(format!("promote by {value}")); + } + rows.push(property_row( + "transfer", + &details + .into_iter() + .filter(|value| !value.is_empty()) + .collect::>() + .join(", "), + style, + )); + } + if entry.discovery != MIGRATION_MANUAL { + rows.push(property_row("discovery", &entry.discovery, style)); + } + if entry.discovery == MIGRATION_AUTOMATIC && entry.completed_at.is_none() { + rows.push(property_row("hint", hint, style)); + } + if let Some(completed) = &entry.completed_at { + rows.push(property_row( + "completed", + &format!("{completed} on target '{}'", entry.target_label), + style, + )); + } + if let Some(old_name) = old_name { + rows.extend(old_name_rows(old_name, style)); + } + for finding in &entry.findings { + let value = format!("! {finding}"); + let styled = if entry.decision.blocks_build || entry.compatibility == MIGRATION_INCOMPATIBLE + { + style.error(&value) + } else { + style.warning(&value) + }; + rows.push(format!(" {styled}")); + } + rows +} + +pub(crate) fn column_migration_text(entry: &ColumnMigrationText, style: CliStyle) -> String { + let decision = &entry.decision; + let styled = if decision.blocks_build { + style.error_strong(&decision.label) + } else if decision.origin_missing { + style.warning_strong(&decision.label) + } else if decision.value == MIGRATION_DONE { + style.muted(&decision.label) + } else { + style.accent_strong(&decision.label) + }; + let mut details: Vec = Vec::new(); + if entry.discovery != MIGRATION_MANUAL { + details.push(entry.discovery.clone()); + } + if decision.value == MIGRATION_DONE + && let Some(completed) = &entry.completed_at + { + details.push(format!("completed {completed}")); + } + let suffix = if details.is_empty() { + String::new() + } else { + format!(" {}", style.muted(&format!("({})", details.join(", ")))) + }; + format!( + "{} -> {} {styled}{suffix}", + entry.origin_column, entry.destination_column + ) +} +pub(crate) fn section_header(style: CliStyle, text: &str) -> String { + if let Some((title, count)) = split_count_header(text) { + return format!( + "{} {}", + style.plan_section(title), + style.muted(&format!("({count}")) + ); + } + style.plan_section(text) +} diff --git a/crates/sqlbuild-core/src/plan_display/_helpers/mod.rs b/crates/sqlbuild-core/src/plan_display/_helpers/mod.rs index 3715888d75..3d544339f8 100644 --- a/crates/sqlbuild-core/src/plan_display/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/plan_display/_helpers/mod.rs @@ -1 +1,2 @@ pub(crate) mod labels; +pub(crate) mod migration_sections; diff --git a/crates/sqlbuild-core/src/plan_display/constants.rs b/crates/sqlbuild-core/src/plan_display/constants.rs index e771ddcad6..ad1ed55a95 100644 --- a/crates/sqlbuild-core/src/plan_display/constants.rs +++ b/crates/sqlbuild-core/src/plan_display/constants.rs @@ -24,3 +24,11 @@ pub(crate) const GRAIN_BATCH_SIZE: [(&str, &str); 6] = [ ("month", "1mo"), ("year", "1y"), ]; +pub(crate) const OLD_NAME_NONE: &str = "none"; +pub(crate) const OLD_NAME_LIVE: &str = "live"; +pub(crate) const OLD_NAME_VIEW_ONLY: &str = "view_only"; +pub(crate) const MIGRATION_DONE: &str = "done"; +pub(crate) const MIGRATION_RENAMED: &str = "renamed"; +pub(crate) const MIGRATION_MANUAL: &str = "manual"; +pub(crate) const MIGRATION_AUTOMATIC: &str = "automatic"; +pub(crate) const MIGRATION_INCOMPATIBLE: &str = "incompatible"; diff --git a/crates/sqlbuild-core/src/plan_display/main/format_model_migrations.rs b/crates/sqlbuild-core/src/plan_display/main/format_model_migrations.rs new file mode 100644 index 0000000000..d68dade41e --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/format_model_migrations.rs @@ -0,0 +1,95 @@ +//! Migration sections in planner order. + +use crate::cli_text::main::format_overflow_line::format_overflow_line; +use crate::cli_text::models::{CliStyle, MigrationTextReport}; +use crate::plan_display::_helpers::migration_sections::{ + column_migration_text, migration_lines, section_header, +}; +use crate::plan_display::main::old_name_rows::old_name_rows; +use std::collections::{BTreeMap, BTreeSet}; + +/// Append model migrations, resumed old names and grouped column migrations. +pub fn format_model_migrations( + mut lines: Vec, + report: &MigrationTextReport, +) -> Vec { + let style = CliStyle { use_color: true }; + let old_names: BTreeMap<_, _> = report + .old_names + .iter() + .map(|entry| (&entry.model_name, entry)) + .collect(); + if !report.migrations.is_empty() { + lines.push(section_header( + style, + &format!("Migrations ({})", report.migrations.len()), + )); + let visible = report + .display + .max_entries + .map_or(report.migrations.len(), |limit| { + limit.min(report.migrations.len()) + }); + for entry in &report.migrations[..visible] { + lines.extend(migration_lines( + entry, + old_names.get(&entry.model_name).copied(), + style, + &report.manual_rename_hint, + )); + } + lines.extend(format_overflow_line( + (report.migrations.len() - visible) as i64, + " ", + &report.display.overflow_flag, + )); + } + let migrated: BTreeSet<_> = report + .migrations + .iter() + .map(|entry| &entry.model_name) + .collect(); + let resumed: Vec<_> = report + .old_names + .iter() + .filter(|entry| !migrated.contains(&entry.model_name)) + .collect(); + if !resumed.is_empty() { + lines.push(section_header( + style, + &format!("Old names ({})", resumed.len()), + )); + for entry in resumed { + lines.push(format!( + " {} {} {}", + style.object_name(&entry.model_name), + style.muted(&format!("{} ->", entry.origin)), + entry.destination + )); + lines.extend(old_name_rows(entry, style)); + } + } + if !report.columns.is_empty() { + lines.push(section_header( + style, + &format!("Column migrations ({})", report.columns.len()), + )); + let mut seen: BTreeSet<&str> = BTreeSet::new(); + for first in &report.columns { + if !seen.insert(&first.model_name) { + continue; + } + lines.push(format!( + " {} {}", + style.object_name(&first.model_name), + style.muted("migrate columns") + )); + for entry in &report.columns { + if entry.model_name == first.model_name { + lines.push(format!(" {}", column_migration_text(entry, style))); + } + } + } + } + lines +} diff --git a/crates/sqlbuild-core/src/plan_display/main/mod.rs b/crates/sqlbuild-core/src/plan_display/main/mod.rs index b2035bea98..a1921ecd30 100644 --- a/crates/sqlbuild-core/src/plan_display/main/mod.rs +++ b/crates/sqlbuild-core/src/plan_display/main/mod.rs @@ -1,7 +1,9 @@ pub mod cursor_bound_display; pub mod effective_microbatch_batch_size; +pub mod format_model_migrations; pub mod format_no_input_rows; pub mod format_waiting_on_empty_inputs; pub mod model_execution_annotation; pub mod model_materialization_label; pub mod model_resource_type; +pub mod old_name_rows; diff --git a/crates/sqlbuild-core/src/plan_display/main/old_name_rows.rs b/crates/sqlbuild-core/src/plan_display/main/old_name_rows.rs new file mode 100644 index 0000000000..aad5d4128f --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/old_name_rows.rs @@ -0,0 +1,19 @@ +//! Render an old relation name and its nested facts. + +use crate::cli_text::main::format_tree_connector::format_tree_connector as tree_connector; +use crate::cli_text::models::{CliStyle, OldNameText}; +use crate::plan_display::_helpers::migration_sections::{old_name_facts, property_row}; + +/// Render the old name as a leaf with one fact per nested row. +pub fn old_name_rows(entry: &OldNameText, style: CliStyle) -> Vec { + let facts = old_name_facts(entry); + let mut rows = vec![property_row("old name", &entry.origin, style)]; + for (index, (label, value)) in facts.iter().enumerate() { + rows.push(format!( + " {} {} {value}", + tree_connector(style, index + 1 == facts.len()), + style.muted(label) + )); + } + rows +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs new file mode 100644 index 0000000000..4e2f85dae8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs @@ -0,0 +1,141 @@ +//! Convert public migration models at the Python boundary for native presentation. + +use crate::bindings::_helpers::plan_output::plan_text::display_limit; +use crate::bindings::_helpers::plan_output::values::{ + iso, items, optional_text, relation_name, text, texts, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::{ + CliStyle, ColumnMigrationText, MigrationDecisionText, MigrationText, MigrationTextReport, + OldNameText, +}; +use sqlbuild_core::plan_display::main::{format_model_migrations, old_name_rows}; +use std::collections::BTreeMap; + +fn label(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + text(&value.getattr("label")?).map(Some) +} + +fn decision(entry: &Bound<'_, PyAny>) -> PyResult { + let value = entry.getattr("decision")?; + Ok(MigrationDecisionText { + value: text(&value)?, + label: text(&value.getattr("label")?)?, + blocks_build: entry.getattr("blocks_build")?.extract()?, + origin_missing: entry.getattr("origin_missing")?.extract()?, + ..MigrationDecisionText::default() + }) +} + +fn migration(entry: &Bound<'_, PyAny>, materialization: Option) -> PyResult { + let mut decision = decision(entry)?; + let value = entry.getattr("decision")?; + decision.decision_blocks_build = value.getattr("blocks_build")?.extract()?; + decision.checks_compatibility = value.getattr("checks_compatibility")?.extract()?; + Ok(MigrationText { + model_name: text(&entry.getattr("model_name")?)?, + origin: text(&relation_name(&entry.getattr("origin")?)?)?, + destination: text(&relation_name(&entry.getattr("destination")?)?)?, + decision, + compatibility: text(&entry.getattr("compatibility")?.getattr("value")?)?, + discovery: text(&entry.getattr("discovery")?.getattr("value")?)?, + completed_at: optional_text(&iso(&entry.getattr("completed_at")?)?)?, + target_label: text(&entry.getattr("target_label")?)?, + transfer: label(&entry.getattr("transfer")?)?, + transfer_fallback: label(&entry.getattr("transfer_fallback")?)?, + storage_transition: optional_text(&entry.getattr("storage_transition")?)?, + promotion: label(&entry.getattr("promotion")?)?, + findings: texts(&entry.getattr("compatibility_findings")?)?, + materialization, + }) +} + +fn old_name(entry: &Bound<'_, PyAny>) -> PyResult { + let expires = entry.getattr("expires_at")?; + let expires_date = if expires.is_none() { + None + } else { + Some(expires.call_method1("strftime", ("%Y-%m-%d",))?.extract()?) + }; + Ok(OldNameText { + model_name: text(&entry.getattr("model_name")?)?, + origin: text(&relation_name(&entry.getattr("origin")?)?)?, + destination: text(&relation_name(&entry.getattr("destination")?)?)?, + action: text(&entry.getattr("action")?)?, + expires_date, + column_aliases: entry.getattr("column_aliases")?.extract()?, + grants_supported: entry.getattr("grants_supported")?.extract()?, + grants_copied: optional_text(&entry.getattr("grants_copied")?)?, + reason: optional_text(&entry.getattr("reason")?)?, + }) +} + +fn column(entry: &Bound<'_, PyAny>) -> PyResult { + Ok(ColumnMigrationText { + model_name: text(&entry.getattr("model_name")?)?, + origin_column: text(&entry.getattr("origin_column")?)?, + destination_column: text(&entry.getattr("destination_column")?)?, + decision: decision(entry)?, + discovery: text(&entry.getattr("discovery")?.getattr("value")?)?, + completed_at: optional_text(&iso(&entry.getattr("completed_at")?)?)?, + }) +} + +#[pyfunction(name = "format_model_migrations", signature = (*, lines, plan, display_options))] +fn format_migrations( + lines: Vec, + plan: &Bound<'_, PyAny>, + display_options: &Bound<'_, PyAny>, +) -> PyResult> { + let mut materializations: BTreeMap = BTreeMap::new(); + for entry in items(&plan.getattr("model_entries")?)? { + materializations.insert( + text(&entry.getattr("name")?)?, + text(&entry.getattr("materialization_type")?)?, + ); + } + let mut migrations: Vec = Vec::new(); + for entry in items(&plan.getattr("migration_entries")?)? { + let name = text(&entry.getattr("model_name")?)?; + migrations.push(migration(&entry, materializations.get(&name).cloned())?); + } + let report = MigrationTextReport { + migrations, + old_names: items(&plan.getattr("old_name_view_entries")?)? + .iter() + .map(old_name) + .collect::>()?, + columns: items(&plan.getattr("column_migration_entries")?)? + .iter() + .map(column) + .collect::>()?, + display: display_limit(display_options)?, + manual_rename_hint: plan + .py() + .import("sqlbuild.compiler.planner.constants")? + .getattr("MANUAL_RENAME_HINT")? + .extract()?, + }; + Ok(format_model_migrations::format_model_migrations( + lines, &report, + )) +} + +#[pyfunction(name = "old_name_rows", signature = (*, entry, style))] +fn render_old_name(entry: &Bound<'_, PyAny>, style: &Bound<'_, PyAny>) -> PyResult> { + Ok(old_name_rows::old_name_rows( + &old_name(entry)?, + CliStyle { + use_color: style.getattr("use_color")?.extract()?, + }, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_migrations, module)?)?; + module.add_function(wrap_pyfunction!(render_old_name, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs index fd834ed5b9..a4063abaf5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod cursor_details; pub(crate) mod display; +pub(crate) mod migrations; pub(crate) mod plan_json; pub(crate) mod plan_text; pub(crate) mod python_identity; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs index 11392a2f4e..e4377d901e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs @@ -391,7 +391,7 @@ fn pruned_models(metadata: &Bound<'_, PyAny>) -> PyResult> { Ok(kept) } -fn display_limit(options: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn display_limit(options: &Bound<'_, PyAny>) -> PyResult { Ok(DisplayLimit { max_entries: options.getattr("max_entries_per_section")?.extract()?, overflow_flag: text(&options.getattr("overflow_flag")?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs index cd0fab2819..96c0593e33 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs @@ -2,10 +2,11 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::plan_output::{display, plan_json, plan_text, safety}; +use crate::bindings::_helpers::plan_output::{display, migrations, plan_json, plan_text, safety}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { display::register(module)?; + migrations::register(module)?; plan_json::register(module)?; plan_text::register(module)?; safety::register(module) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index c507458e73..c21c5a0e4c 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -234,6 +234,7 @@ from sqlbuild.compiler.planner.models import ( ModelPlanContext, ModelPlanEntry, OldNameView, + OldNameViewPlanEntry, OldNameViewPlanning, ParsedScenarioArtifactName, ParsedSelector, @@ -5140,3 +5141,7 @@ def build_source_load_entries( source_map: dict[str, SourceEntry], is_reload: bool, ) -> tuple[SourceLoadPlanEntry, ...]: ... +def format_model_migrations( + *, lines: list[str], plan: PlanOutput, display_options: DisplayOptions +) -> list[str]: ... +def old_name_rows(*, entry: OldNameViewPlanEntry, style: CliStyle) -> list[str]: ... diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/display.py b/src/sqlbuild/compiler/planner/_helpers/migrations/display.py deleted file mode 100644 index da617d68bc..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/display.py +++ /dev/null @@ -1,294 +0,0 @@ -"""Plan text for model migrations and identity handovers.""" - -from __future__ import annotations - -from collections.abc import Callable, Sequence - -from sqlbuild._native import append_overflow_line, count_header_style, visible_entries -from sqlbuild.compiler.migrations.types import ( - ColumnMigrationDecision, - MigrationCompatibility, - MigrationDecision, - MigrationDiscovery, - OldNameViewAction, -) -from sqlbuild.compiler.planner.constants import MANUAL_RENAME_HINT -from sqlbuild.compiler.planner.models import ( - ColumnMigrationPlanEntry, - ModelMigrationPlanEntry, - OldNameViewPlanEntry, - PlanOutput, -) -from sqlbuild.compiler.planner.types import MaterializationType -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.tree_connector import tree_connector -from sqlbuild.presentation.models import DisplayOptions - - -def format_model_migrations( - *, lines: list[str], plan: PlanOutput, display_options: DisplayOptions -) -> list[str]: - """Format declared and discovered model migrations, then identity handovers.""" - - result: list[str] = list(lines) - style: CliStyle = CliStyle(use_color=True) - header_style: Callable[[str], str] = count_header_style( - style=style, title_style=style.plan_section - ) - entries: tuple[ModelMigrationPlanEntry, ...] = plan.migration_entries - old_names: dict[str, OldNameViewPlanEntry] = { - entry.model_name: entry for entry in plan.old_name_view_entries - } - materializations: dict[str, str] = { - entry.name: str(entry.materialization_type) for entry in plan.model_entries - } - if entries: - result.append(header_style(f"Migrations ({len(entries)})")) - visible: Sequence[ModelMigrationPlanEntry] = visible_entries( - entries=entries, options=display_options - ) - entry: ModelMigrationPlanEntry - for entry in visible: - result.extend( - _migration_lines( - entry=entry, - style=style, - old_name=old_names.get(entry.model_name), - materialization=materializations.get(entry.model_name), - ) - ) - result = append_overflow_line( - lines=result, - total_count=len(entries), - visible_count=len(visible), - indent=" ", - options=display_options, - ) - result = _format_resumed_old_names( - lines=result, - plan=plan, - style=style, - header_style=header_style, - ) - return _format_column_migrations( - lines=result, plan=plan, style=style, header_style=header_style - ) - - -def _format_resumed_old_names( - *, - lines: list[str], - plan: PlanOutput, - style: CliStyle, - header_style: Callable[[str], str], -) -> list[str]: - """Format old-name steps that resume a move no longer listed as a migration.""" - - migrated: frozenset[str] = frozenset(entry.model_name for entry in plan.migration_entries) - resumed: tuple[OldNameViewPlanEntry, ...] = tuple( - entry for entry in plan.old_name_view_entries if entry.model_name not in migrated - ) - if not resumed: - return lines - result: list[str] = list(lines) - result.append(header_style(f"Old names ({len(resumed)})")) - entry: OldNameViewPlanEntry - for entry in resumed: - origin: str = entry.origin.qualified_name or entry.origin.name - destination: str = entry.destination.qualified_name or entry.destination.name - result.append( - f" {style.object_name(entry.model_name)} {style.muted(f'{origin} ->')} {destination}" - ) - result.extend(old_name_rows(entry=entry, style=style)) - return result - - -def old_name_rows(*, entry: OldNameViewPlanEntry, style: CliStyle) -> list[str]: - """Render the old name of a migration as a leaf with one fact per nested row.""" - - origin: str = entry.origin.qualified_name or entry.origin.name - facts: list[tuple[str, str]] = old_name_facts(entry) - rows: list[str] = [_property_row(label="old name", value=origin, style=style)] - index: int - label: str - value: str - for index, (label, value) in enumerate(facts): - connector: str = tree_connector(style=style, last=index == len(facts) - 1) - rows.append(f" {connector} {style.muted(label)} {value}") - return rows - - -def old_name_facts(entry: OldNameViewPlanEntry) -> list[tuple[str, str]]: - """Return what happens at a migrated model's old name, one labelled fact per row.""" - - if entry.action == OldNameViewAction.NONE: - return [("left for janitor", entry.reason or "no compatibility view")] - until: str = "" if entry.expires_at is None else f"until {entry.expires_at:%Y-%m-%d}" - view: str = f"live {until}".strip() if entry.action == OldNameViewAction.LIVE else until - facts: list[tuple[str, str]] = [("view", view)] - if entry.column_aliases: - facts.append(("columns", ", ".join(f"{old} <- {new}" for old, new in entry.column_aliases))) - if entry.grants_supported: - facts.append(("grants", _grants_text(entry))) - archived: str = ( - "already archived" if entry.action == OldNameViewAction.VIEW_ONLY else "archived" - ) - facts.append(("old table", archived)) - return facts - - -def _grants_text(entry: OldNameViewPlanEntry) -> str: - if entry.grants_copied is None: - return "copied from the old table" - return f"{entry.grants_copied} copied from the old table" - - -def _format_column_migrations( - *, - lines: list[str], - plan: PlanOutput, - style: CliStyle, - header_style: Callable[[str], str], -) -> list[str]: - """Format in-place column renames grouped under the model whose table they change.""" - - entries: tuple[ColumnMigrationPlanEntry, ...] = plan.column_migration_entries - if not entries: - return lines - result: list[str] = list(lines) - result.append(header_style(f"Column migrations ({len(entries)})")) - model_names: tuple[str, ...] = tuple(dict.fromkeys(entry.model_name for entry in entries)) - model_name: str - for model_name in model_names: - result.append(f" {style.object_name(model_name)} {style.muted('migrate columns')}") - entry: ColumnMigrationPlanEntry - for entry in entries: - if entry.model_name == model_name: - result.append(f" {_column_migration_text(entry=entry, style=style)}") - return result - - -def _column_migration_text(*, entry: ColumnMigrationPlanEntry, style: CliStyle) -> str: - rename: str = f"{entry.origin_column} -> {entry.destination_column}" - decision: str - if entry.blocks_build: - decision = style.error_strong(entry.decision.label) - elif entry.origin_missing: - decision = style.warning_strong(entry.decision.label) - elif entry.decision == ColumnMigrationDecision.DONE: - decision = style.muted(entry.decision.label) - else: - decision = style.accent_strong(entry.decision.label) - details: list[str] = [] - if entry.discovery != MigrationDiscovery.MANUAL: - details.append(entry.discovery.value) - if entry.decision == ColumnMigrationDecision.DONE and entry.completed_at is not None: - details.append(f"completed {entry.completed_at.isoformat()}") - suffix: str = f" {style.muted(f'({", ".join(details)})')}" if details else "" - return f"{rename} {decision}{suffix}" - - -def _migration_lines( - *, - entry: ModelMigrationPlanEntry, - style: CliStyle, - old_name: OldNameViewPlanEntry | None, - materialization: str | None, -) -> list[str]: - origin: str = entry.origin.qualified_name or entry.origin.name - destination: str = entry.destination.qualified_name or entry.destination.name - relation_move: str = f"{style.muted(f'{origin} ->')} {destination}" - name: str = style.object_name(entry.model_name) - decision: str = _decision_text(entry=entry, style=style) - if entry.decision == MigrationDecision.RENAMED: - decision = ( - style.muted(MigrationDecision.DONE.label) - if entry.completed_at is not None - else style.accent_strong(MigrationDecision.MIGRATE.label) - ) - rows: list[str] = [f" {name} {decision} {relation_move}"] - if entry.decision == MigrationDecision.RENAMED and entry.completed_at is None: - rows.append( - _property_row( - label="transfer", value=identity_transfer_label(materialization), style=style - ) - ) - if entry.decision.checks_compatibility: - rows.append( - _property_row( - label="compatibility", - value=_compatibility_text(compatibility=entry.compatibility, style=style), - style=style, - ) - ) - if entry.transfer is not None: - details: tuple[str | None, ...] = ( - entry.transfer.label - + ( - "" - if entry.transfer_fallback is None - else f" ({entry.transfer_fallback.label} if refused)" - ), - entry.storage_transition, - None if entry.promotion is None else f"promote by {entry.promotion.label}", - ) - rows.append( - _property_row( - label="transfer", - value=", ".join(detail for detail in details if detail), - style=style, - ) - ) - if entry.discovery != MigrationDiscovery.MANUAL: - rows.append(_property_row(label="discovery", value=entry.discovery.value, style=style)) - if entry.discovery == MigrationDiscovery.AUTOMATIC and entry.completed_at is None: - rows.append(_property_row(label="hint", value=MANUAL_RENAME_HINT, style=style)) - if entry.completed_at is not None: - rows.append( - _property_row( - label="completed", - value=f"{entry.completed_at.isoformat()} on target '{entry.target_label}'", - style=style, - ) - ) - if old_name is not None: - rows.extend(old_name_rows(entry=old_name, style=style)) - blocking: bool = entry.blocks_build or ( - entry.compatibility == MigrationCompatibility.INCOMPATIBLE - ) - finding_style: Callable[[str], str] = style.error if blocking else style.warning - rows.extend(f" {finding_style(f'! {finding}')}" for finding in entry.compatibility_findings) - return rows - - -def identity_transfer_label(materialization: str | None) -> str: - """Describe how a renamed table or view reaches its new name.""" - - if materialization == MaterializationType.VIEW: - return "recreate (view)" - return "rebuild (table)" - - -def _property_row(*, label: str, value: str, style: CliStyle) -> str: - return f" {style.muted(label)} {value}" - - -def _decision_text(*, entry: ModelMigrationPlanEntry, style: CliStyle) -> str: - decision: MigrationDecision = entry.decision - if entry.blocks_build and decision.blocks_build: - return style.error_strong(decision.label) - if entry.origin_missing: - return style.warning_strong(decision.label) - if decision in (MigrationDecision.SUPERSEDED_REPLACE, MigrationDecision.FORCED_REPLACE): - return style.warning_strong(decision.label) - if decision == MigrationDecision.DONE: - return style.muted(decision.label) - return style.accent_strong(decision.label) - - -def _compatibility_text(*, compatibility: MigrationCompatibility, style: CliStyle) -> str: - if compatibility == MigrationCompatibility.COMPATIBLE: - return style.success(compatibility.value) - if compatibility == MigrationCompatibility.INCOMPATIBLE: - return style.error(compatibility.value) - return compatibility.value diff --git a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py index 5caddd6e1f..7915fda271 100644 --- a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py +++ b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py @@ -2,8 +2,7 @@ from __future__ import annotations -from sqlbuild._native import append_overflow_line, visible_entries -from sqlbuild.compiler.planner._helpers.migrations.display import format_model_migrations +from sqlbuild._native import append_overflow_line, format_model_migrations, visible_entries from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.presentation.models import DisplayOptions diff --git a/src/sqlbuild/presentation/main/tree_connector.py b/src/sqlbuild/presentation/main/tree_connector.py deleted file mode 100644 index 549f493cda..0000000000 --- a/src/sqlbuild/presentation/main/tree_connector.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public tree connector rendering entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.structure import tree_connector as _tree_connector -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def tree_connector(*, style: CliStyle, last: bool) -> str: - """Render a dim tree connector for a group entry.""" - - return _tree_connector(style=style, last=last) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_old_name_rows.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_old_name_rows.py index 929f6a6529..443abdb3a2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_old_name_rows.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_old_name_rows.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.planner._helpers.migrations.display import old_name_rows +from sqlbuild._native import old_name_rows from sqlbuild.presentation.classes.cli_style import CliStyle from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( OldNameRowsTestCase, From 76ee2e3f81f5dc599a97052cf55e6226195c779b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:32:08 +0100 Subject: [PATCH 08/22] chore: update Python boundary after migration presentation port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 56a9ed331e..9e3381174b 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 31591 +remaining_lines = 31284 [[stays]] path = "executor" From 0c5cd096489b77fc0ebe527736b6caaab84d60cf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:19:49 +0100 Subject: [PATCH 09/22] refactor: host SQL test plan assembly in native code --- .../_helpers/planner_scenario/fixtures.rs | 13 +- .../_helpers/sql_test_glue/authored_ctes.rs | 18 +- .../_helpers/sql_test_glue/constants.rs | 13 + .../bindings/_helpers/sql_test_glue/mod.rs | 10 + .../_helpers/sql_test_glue/plan_assembly.rs | 133 ++++++ .../_helpers/sql_test_glue/plan_driver.rs | 264 ++++++++++++ .../_helpers/sql_test_glue/plan_entries.rs | 181 ++++++++ .../_helpers/sql_test_glue/plan_errors.rs | 101 +++++ .../_helpers/sql_test_glue/plan_fixtures.rs | 131 ++++++ .../_helpers/sql_test_glue/plan_rendering.rs | 204 +++++++++ .../_helpers/sql_test_glue/plan_rows.rs | 84 ++++ .../_helpers/sql_test_glue/plan_values.rs | 24 ++ .../_helpers/sql_test_glue/registration.rs | 23 +- .../bindings/_helpers/sql_test_glue/types.rs | 6 + src/sqlbuild/_native.pyi | 36 ++ .../planner/_helpers/output/plan_output.py | 2 +- .../planner/_helpers/sql_tests/assembly.py | 356 ---------------- .../_helpers/sql_tests/native_planning.py | 396 ------------------ .../_plan_compiled_sql_test_artifacts.py | 6 +- .../main/execution/sql_test_model_chain.py | 6 +- .../_helpers/sql_test_assembly/helpers.py | 14 +- .../sql_test_assembly/test_plan_test.py | 6 +- .../_helpers/test_fixture_planning_work.py | 2 +- 23 files changed, 1247 insertions(+), 782 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/types.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs index e78120379a..786df6d4ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs @@ -15,15 +15,10 @@ pub(super) fn extract_fixtures<'py>( scenario: &Bound<'py, PyAny>, prefix: &str, ) -> PyResult> { - let result = PyDict::new(scenario.py()); - for cte in scenario.getattr("authored_ctes")?.try_iter()? { - let cte = cte?; - let name: String = cte.getattr("name")?.extract()?; - if let Some(name) = name.strip_prefix(prefix) { - result.set_item(name, cte.getattr("sql_body")?)?; - } - } - Ok(result) + crate::bindings::_helpers::sql_test_glue::authored_ctes::prefixed_ctes( + &scenario.getattr("authored_ctes")?, + prefix, + ) } pub(super) fn fixture_plans<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs index 1546c8daee..d0adb4dbcd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::PyDict; +use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::main::text_position::text_position; use sqlbuild_analysis::compiled_project::models::TextPosition; @@ -18,6 +18,22 @@ const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_INPUT_ERROR: &str = "CompileInputError"; const MODEL_MODE: &str = "model"; +/// Extract fixture bodies from authored test or scenario CTEs in authored order. +pub(crate) fn prefixed_ctes<'py>( + ctes: &Bound<'py, PyAny>, + prefix: &str, +) -> PyResult> { + let result = PyDict::new(ctes.py()); + for cte in ctes.try_iter()? { + let cte = cte?; + let name: String = cte.getattr("name")?.extract()?; + if let Some(name) = name.strip_prefix(prefix) { + result.set_item(name, cte.getattr("sql_body")?)?; + } + } + Ok(result) +} + /// One authored SQL body to read: its text, file contents, relative path and body span. struct AuthoredBody { sql: String, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs new file mode 100644 index 0000000000..941f615f19 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs @@ -0,0 +1,13 @@ +//! SQL-test plan resource and severity decisions. + +pub(super) const UDF: &str = "udf"; +pub(super) const TABLE_FN: &str = "table_fn"; +pub(super) const ERROR_SEVERITY: &str = "error"; +pub(super) const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; +pub(super) const PLAN_MODELS: &str = "sqlbuild.compiler.planner.models"; +pub(super) const NATIVE_PLANNING: &str = "sqlbuild._native"; +pub(super) const FIXTURE_VALIDATION: &str = + "sqlbuild.compiler.planner._helpers.sql_tests.fixture_validation"; +pub(super) const CURSOR_WINDOW: &str = "sqlbuild.compiler.planner._helpers.sql_tests.cursor_window"; +pub(super) const SENTINEL: &str = "__SQLBUILD_CALL_SUFFIX__"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index 2964e1bb08..410f1a68e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -1,5 +1,15 @@ pub(crate) mod assembly; pub(crate) mod authored_ctes; mod compiled_tests; +mod constants; +mod plan_assembly; +mod plan_driver; +mod plan_entries; +mod plan_errors; +mod plan_fixtures; +mod plan_rendering; +mod plan_rows; +mod plan_values; pub(crate) mod planning; pub(crate) mod registration; +mod types; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs new file mode 100644 index 0000000000..357f762f9b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs @@ -0,0 +1,133 @@ +//! Assemble per-test outcomes around native planning and adapter fixture validation. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::sql_test_glue::constants::{NATIVE_PLANNING, PLAN_MODELS}; + +#[pyfunction(name = "plan_sql_tests")] +#[pyo3(signature = (**kwargs))] +pub(crate) fn assemble_sql_test_plans<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "plan_sql_tests", + kwargs, + ( + &["tests", "project", "adapter"], + &[ + "sql_analysis_enabled", + "validate_fixtures", + "fixture_planning_context", + ], + ), + )?; + let tests: Vec> = args.value("tests")?.extract()?; + let project = args.value("project")?; + let adapter = args.value("adapter")?; + let mut context = py.None().into_bound(py); + let (planned, diagnostics) = + if args.flag("sql_analysis_enabled")? && args.flag("validate_fixtures")? { + context = args.value("fixture_planning_context")?; + if !context.is_truthy()? { + context = call_with( + &imported( + py, + "sqlbuild._native", + "build_relation_fixture_context", + )?, + &[("project", project.clone())], + )?; + } + super::plan_fixtures::validate(&tests, (&project, &adapter), &context)? + } else { + (tests.clone(), Default::default()) + }; + let plannable: Vec<_> = planned + .into_iter() + .enumerate() + .filter_map(|(index, test)| (!diagnostics.contains_key(&index)).then_some(test)) + .collect(); + let plans = call_with( + &imported(py, NATIVE_PLANNING, "plan_sql_tests_natively")?, + &[ + ("project", project.clone()), + ("tests", PyTuple::new(py, plannable)?.into_any()), + ("adapter", adapter), + ( + "sql_analysis_enabled", + args.value_or_false("sql_analysis_enabled")?, + ), + ( + "render_sql", + pyo3::types::PyBool::new(py, false).to_owned().into_any(), + ), + ], + )?; + let plans: Vec> = plans.extract()?; + if plans.len() != tests.len() - diagnostics.len() { + return Err(pyo3::exceptions::PyValueError::new_err( + "zip() argument lengths differ", + )); + } + let mut plans = plans.into_iter(); + let models = PyDict::new(py); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + models.set_item(model.getattr("name")?, model)?; + } + let result_class = imported(py, PLAN_MODELS, "SqlTestPlanResult")?; + let mut results: Vec> = Vec::new(); + for (index, test) in tests.iter().enumerate() { + if let Some(diagnostic) = diagnostics.get(&index) { + results.push(call_with( + &result_class, + &[ + ("entry", py.None().into_bound(py)), + ("fixture_diagnostics", diagnostic.clone()), + ], + )?); + continue; + } + let Some(plan) = plans.next() else { + return Err(pyo3::exceptions::PyValueError::new_err( + "missing SQL-test plan", + )); + }; + let mut errors: Vec = plan.getattr("error_messages")?.extract()?; + errors.extend(super::plan_entries::expected_diagnostics( + test, &plan, &context, + )?); + if !errors.is_empty() { + results.push(call_with( + &result_class, + &[ + ("entry", py.None().into_bound(py)), + ("fixture_diagnostics", PyTuple::new(py, errors)?.into_any()), + ], + )?); + continue; + } + results.push(call_with( + &result_class, + &[ + ( + "entry", + super::plan_entries::entry( + test, + &plan, + (&project, &models), + &args.value_or_false("sql_analysis_enabled")?, + )?, + ), + ("warnings", plan.getattr("warnings")?), + ], + )?); + } + PyTuple::new(py, results) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs new file mode 100644 index 0000000000..deb6b75c96 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs @@ -0,0 +1,264 @@ +//! Host SQL-test request preparation, native planning, and Python result conversion. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::sql_test_glue::constants::PLAN_MODELS; +use crate::bindings::_helpers::sql_test_glue::plan_values::tuple; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; +use pyo3::types::{PyBool, PyDict, PyTuple}; + +pub(super) fn chains<'py>( + project: &Bound<'py, PyAny>, + tests: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + if !tests.is_truthy()? { + return Ok(PyTuple::empty(py).into_any()); + } + let request = call_with( + &imported( + py, + "sqlbuild.compiler.sql_test_glue.models", + "NativeSqlTestChainRequest", + )?, + &[ + ("models", project.getattr("models")?), + ("tests", tests.clone()), + ( + "lexical_syntax", + project + .getattr("sql_lexical_syntax")? + .getattr("native_mapping")?, + ), + ], + )?; + let native = + imported(py, "sqlbuild._native", "resolve_compiled_sql_test_chains")?.call1((request,)); + let native = match native { + Ok(value) => value, + Err(error) if error.is_instance_of::(py) => { + return Err(super::plan_errors::planning_error(py, error, tests)?); + } + Err(error) => return Err(error), + }; + let mut result: Vec> = Vec::new(); + for chain in native.try_iter()? { + result.push(tuple(&chain?)?); + } + Ok(PyTuple::new(py, result)?.into_any()) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn resolve_sql_test_model_chains<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "resolve_sql_test_model_chains", + kwargs, + (&["project", "tests"], &[]), + )?; + chains(&args.value("project")?, &args.value("tests")?) +} + +fn planned<'py>(args: &KeywordArguments<'py>, artifact: bool) -> PyResult> { + let project = args.value("project")?; + let py = project.py(); + let tests = args.value("tests")?; + if !tests.is_truthy()? { + return Ok(PyTuple::empty(py)); + } + let start: u64 = imported(py, "time", "perf_counter_ns")? + .call0()? + .extract()?; + let adapter = args.value("adapter")?; + let model_sql = super::plan_rendering::models(&project, &adapter)?; + let functions = super::plan_rendering::functions(&project, &adapter)?; + let overrides = super::plan_rendering::overrides(&project, &tests, &adapter)?; + let request = call_with( + &imported( + py, + "sqlbuild.compiler.sql_test_glue.models", + "NativeSqlTestPlanningRequest", + )?, + &[ + ("models", project.getattr("models")?), + ("rendered_model_sql", model_sql.into_any()), + ("functions", functions.into_any()), + ("tests", tests.clone()), + ("rendered_test_overrides", overrides.into_any()), + ("sql_analysis_enabled", args.value("sql_analysis_enabled")?), + ( + "sql_analysis_dialect", + adapter.call_method0("sql_analysis_dialect")?, + ), + ( + "set_difference_operator", + adapter.call_method0("render_set_difference_operator")?, + ), + ( + "requires_derived_table_aliases", + adapter.call_method0("requires_derived_table_aliases")?, + ), + ( + "lexical_syntax", + project + .getattr("sql_lexical_syntax")? + .getattr("native_mapping")?, + ), + ( + "render_sql", + PyBool::new(py, artifact || args.flag("render_sql")?) + .to_owned() + .into_any(), + ), + ( + "include_plan", + PyBool::new( + py, + !artifact && (!args.passed("include_plan")? || args.flag("include_plan")?), + ) + .to_owned() + .into_any(), + ), + ], + )?; + let native = imported(py, "sqlbuild._native", "plan_compiled_sql_tests")?.call1((request,)); + let native = match native { + Ok(value) => value, + Err(error) if error.is_instance_of::(py) => { + return Err(super::plan_errors::planning_error(py, error, &tests)?); + } + Err(error) => return Err(error), + }; + let planning_ns: u64 = native.get_item(1)?.extract()?; + let rendering_ns: u64 = native.get_item(2)?.extract()?; + let end: u64 = imported(py, "time", "perf_counter_ns")? + .call0()? + .extract()?; + let collector = imported( + py, + "sqlbuild.compiler.profiling.classes.context", + "CompileTimingContext", + )? + .getattr("active")? + .call_method0("get")?; + if !collector.is_none() { + use pyo3::IntoPyObject; + for (phase, elapsed) in [ + ( + "test_planning_ms", + planning_ns + + end + .saturating_sub(start) + .saturating_sub(planning_ns) + .saturating_sub(rendering_ns), + ), + ("comparison_render_ms", rendering_ns), + ] { + call_with( + &collector.getattr("add")?, + &[ + ("phase", super::plan_values::string(py, phase)), + ("elapsed_ns", elapsed.into_pyobject(py)?.into_any()), + ], + )?; + } + } + let mut plans: Vec> = Vec::new(); + for row in native.get_item(0)?.try_iter()? { + let plan = super::plan_rows::plan(&row?)?; + if artifact { + let sql = plan.getattr("sql")?; + if sql.is_none() { + return Err(PyErr::from_value( + imported( + py, + "sqlbuild.compiler.planner.exceptions", + "NativeSqlTestPlanningError", + )? + .call1(("native SQL-test planning omitted rendered SQL",))?, + )); + } + plans.push(call_with( + &imported(py, PLAN_MODELS, "NativeSqlTestArtifact")?, + &[ + ("sql", sql), + ("model_names", plan.getattr("model_names")?), + ("error_messages", plan.getattr("error_messages")?), + ], + )?); + } else { + plans.push(plan); + } + } + PyTuple::new(py, plans) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn plan_sql_tests_natively<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "plan_sql_tests_natively", + kwargs, + ( + &[ + "project", + "tests", + "adapter", + "sql_analysis_enabled", + "render_sql", + ], + &["include_plan"], + ), + )?; + planned(&args, false) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn plan_compiled_sql_test_artifacts<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "plan_compiled_sql_test_artifacts", + kwargs, + ( + &["project", "tests", "adapter", "sql_analysis_enabled"], + &[], + ), + )?; + planned(&args, true) +} + +#[pyfunction] +#[pyo3(signature = (*, warnings))] +pub(crate) fn sql_test_plan_error_messages<'py>( + warnings: Bound<'py, PyAny>, +) -> PyResult> { + let mut messages: Vec = Vec::new(); + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + for warning in warnings.try_iter()? { + let warning = warning?; + let message: String = warning.getattr("message")?.extract()?; + if warning + .getattr("severity")? + .getattr("value")? + .extract::()? + == super::constants::ERROR_SEVERITY + && seen.insert(message.clone()) + { + messages.push(message); + } + } + PyTuple::new(warnings.py(), messages) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs new file mode 100644 index 0000000000..a2fe69b0e3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs @@ -0,0 +1,181 @@ +//! Assemble existing SQL-test plan dataclasses without changing executor ownership. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::sql_test_glue::constants::{COMPILE_TYPES, PLAN_MODELS}; +use crate::bindings::_helpers::sql_test_glue::constants::{TABLE_FN, UDF}; +use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; + +use crate::bindings::_helpers::sql_test_glue::authored_ctes::prefixed_ctes; + +fn prefixed_names<'py>(test: &Bound<'py, PyAny>, prefix: &str) -> PyResult> { + let py = test.py(); + let mut names: Vec = if model_payload(test)? { + prefixed_ctes(&test.getattr("payload")?.getattr("authored_ctes")?, prefix)? + .keys() + .extract()? + } else { + Vec::new() + }; + names.sort(); + Ok(PyTuple::new(py, names)?.into_any()) +} + +fn function_deps<'py>( + test: &Bound<'py, PyAny>, + plan: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, + inputs: (&Bound<'py, PyAny>, &Bound<'py, PyDict>), +) -> PyResult> { + let py = test.py(); + let deps = PyDict::new(py); + if model_payload(test)? { + let names: Vec = plan.getattr("model_names")?.extract()?; + let mocked: std::collections::HashSet = + inputs.0.extract::>()?.into_iter().collect(); + let models = inputs.1; + for name in names { + let Some(model) = models.get_item(name)? else { + continue; + }; + for dep in model.getattr("deps")?.try_iter()? { + let dep = dep?; + let resource: String = dep.getattr("resource_type")?.getattr("value")?.extract()?; + if resource != UDF && resource != TABLE_FN { + continue; + } + if resource == TABLE_FN + && mocked.contains(&dep.getattr("name")?.extract::()?) + { + continue; + } + deps.set_item(dep, py.None())?; + } + } + } else { + let payload = test.getattr("payload")?; + let mode: String = payload.getattr("mode")?.getattr("value")?.extract()?; + if mode == UDF || mode == TABLE_FN { + let names: std::collections::HashSet = payload + .getattr("tested_resource_names")? + .extract::>()? + .into_iter() + .collect(); + for function in project.getattr("functions")?.try_iter()? { + let function = function?; + let key = function.getattr("key")?; + if key + .getattr("resource_type")? + .getattr("value")? + .extract::()? + == mode + && names.contains(&function.getattr("name")?.extract::()?) + { + deps.set_item(key, py.None())?; + } + } + } + } + Ok(PyTuple::new(py, deps.keys().iter())?.into_any()) +} + +pub(super) fn entry<'py>( + test: &Bound<'py, PyAny>, + plan: &Bound<'py, PyAny>, + inputs: (&Bound<'py, PyAny>, &Bound<'py, PyDict>), + analysis: &Bound<'py, PyAny>, +) -> PyResult> { + let py = test.py(); + let fields = PyDict::new(py); + for name in [ + "key", + "name", + "source_path", + "block_index", + "parent_name", + "case_name", + "case_index", + "case_fingerprint", + "parameter_schema", + "parameter_values", + "scope_deps", + ] { + fields.set_item(name, test.getattr(name)?)?; + } + for (field, prefix) in [ + ("mock_ref_names", "__ref__"), + ("mock_source_names", "__source__"), + ("mock_seed_names", "__seed__"), + ("mock_dbt_ref_names", "__dbt_ref__"), + ("mock_table_function_names", "__table_fn__"), + ] { + fields.set_item(field, prefixed_names(test, prefix)?)?; + } + let mocks = fields + .get_item("mock_table_function_names")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("missing mock names"))?; + fields.set_item( + "function_deps", + function_deps(test, plan, inputs.0, (&mocks, inputs.1))?, + )?; + fields.set_item("chain", plan.getattr("chain")?)?; + fields.set_item("assertions", plan.getattr("assertions")?)?; + fields.set_item("sql_analysis_enabled", analysis)?; + imported(py, PLAN_MODELS, "SqlTestPlanEntry")?.call((), Some(&fields)) +} + +pub(super) fn expected_diagnostics( + test: &Bound<'_, PyAny>, + plan: &Bound<'_, PyAny>, + context: &Bound<'_, PyAny>, +) -> PyResult> { + let mut result: Vec = Vec::new(); + if context.is_none() { + return Ok(result); + } + let py = test.py(); + let resource_types = imported(py, COMPILE_TYPES, "CompiledResourceType")?; + for step in plan.getattr("chain")?.try_iter()? { + let step = step?; + let name = step.getattr("model_name")?; + let available = context + .getattr("authoritative_columns")? + .call_method1("get", ((resource_types.getattr("MODEL")?, &name),))?; + let expected = step.getattr("expected_columns")?; + if expected.is_none() || available.is_none() { + continue; + } + let message = call_with( + &imported( + py, + "sqlbuild.executor.testing.main.missing_expected_columns", + "describe_missing_expected_columns", + )?, + &[ + ("model_name", name.clone()), + ("expected_columns", expected), + ("available_columns", available), + ], + )?; + if message.is_none() { + continue; + } + let location = call_with( + &imported(py, super::constants::FIXTURE_VALIDATION, "fixture_location")?, + &[ + ("test", test.clone()), + ("resource_type", resource_types.getattr("SQL_TEST")?), + ("name", name), + ], + )?; + result.push(format!( + "SQL test '{}': {}: {}", + test.getattr("name")?.extract::()?, + location.extract::()?, + message.extract::()? + )); + } + Ok(result) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs new file mode 100644 index 0000000000..a90e0ead0e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs @@ -0,0 +1,101 @@ +//! Project native planning failures into existing located diagnostic exceptions. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::sql_test_glue::plan_values::string; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; + +pub(super) fn planning_error( + py: Python<'_>, + error: PyErr, + tests: &Bound<'_, PyAny>, +) -> PyResult { + let message = error.value(py).str()?.to_string(); + if let Some(payload) = message.strip_prefix("sql_test_reference:") { + let payload = imported(py, "orjson", "loads")?.call1((payload,))?; + let mut found = None; + for test in tests.try_iter()? { + let test = test?; + if test.getattr("name")?.eq(payload.get_item("testName")?)? + && test + .getattr("test_file")? + .getattr("relative_path")? + .str()? + .into_any() + .eq(payload.get_item("fileLabel")?)? + { + found = Some(test); + break; + } + } + let cte = payload.get_item("cteName")?; + let call = payload.get_item("call")?; + let message = format!( + "SQL test CTE '{}' calls {}, which the test query cannot resolve", + cte.extract::()?, + call.extract::()? + ); + let help = format!( + "Mock it in the test, for example {} AS (SELECT ...), or call __ref(\"\") on a model the test runs.", + payload.get_item("mockCte")?.extract::()? + ); + let kwargs = pyo3::types::PyDict::new(py); + use pyo3::types::PyDictMethods; + kwargs.set_item("help", help)?; + let class = if let Some(test) = found { + kwargs.set_item( + "location", + call_with( + &imported( + py, + "sqlbuild.compiler.compile.classes.sql_test_cte_locator", + "SqlTestCteLocator", + )? + .getattr("locate")?, + &[ + ("test_file", test.getattr("test_file")?), + ("test_block", test.getattr("test_block")?), + ("cte_name", cte), + ("call", call), + ], + )?, + )?; + "SqlTestReferenceError" + } else { + kwargs.set_item( + "code", + imported( + py, + "sqlbuild.compiler.compile.constants", + "SQL_TEST_HELPER_REFERENCE_CODE", + )?, + )?; + "CompileInputError" + }; + return Ok(PyErr::from_value( + imported(py, "sqlbuild.compiler.compile.exceptions", class)? + .call((message,), Some(&kwargs))?, + )); + } + let (module, class, message) = if let Some(message) = message.strip_prefix("compile_input:") { + ( + "sqlbuild.compiler.compile.exceptions", + "CompileInputError", + message.to_owned(), + ) + } else if let Some(message) = message.strip_prefix("planner_input:") { + ( + "sqlbuild.compiler.planner.exceptions", + "PlannerInputError", + message.to_owned(), + ) + } else { + ( + "sqlbuild.compiler.planner.exceptions", + "NativeSqlTestPlanningError", + format!("native SQL-test planning failed: {message}"), + ) + }; + Ok(PyErr::from_value( + imported(py, module, class)?.call1((string(py, &message),))?, + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs new file mode 100644 index 0000000000..ca492e61ad --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs @@ -0,0 +1,131 @@ +//! Complete SQL-test fixtures through the existing fixture completion boundary. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::sql_test_glue::authored_ctes::prefixed_ctes; +use crate::bindings::_helpers::sql_test_glue::constants::{FIXTURE_VALIDATION, NATIVE_PLANNING}; +use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; + +pub(super) fn validated_test<'py>( + test: &Bound<'py, PyAny>, + chain: &Bound<'py, PyAny>, + inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + context: &Bound<'py, PyAny>, +) -> PyResult> { + let py = test.py(); + let payload = test.getattr("payload")?; + let ctes = payload.getattr("authored_ctes")?; + let expected = payload.getattr("expected_ctes")?; + let completed = call_with( + &imported(py, FIXTURE_VALIDATION, "build_validated_test_fixtures")?, + &[ + ("test", test.clone()), + ("project", inputs.0.clone()), + ("adapter", inputs.1.clone()), + ("ordered_model_names", chain.clone()), + ("mock_refs", prefixed_ctes(&ctes, "__ref__")?.into_any()), + ( + "mock_sources", + prefixed_ctes(&ctes, "__source__")?.into_any(), + ), + ("mock_seeds", prefixed_ctes(&ctes, "__seed__")?.into_any()), + ( + "expected_outputs", + prefixed_ctes(&expected, "__expected__")?.into_any(), + ), + ("planning_context", context.clone()), + ], + )?; + let replace = imported(py, "dataclasses", "replace")?; + let mut authored: Vec> = Vec::new(); + for cte in ctes.try_iter()? { + let cte = cte?; + let name: String = cte.getattr("name")?.extract()?; + let mut replacement = cte.clone(); + for (index, prefix) in ["__ref__", "__source__", "__seed__"].iter().enumerate() { + if let Some(name) = name.strip_prefix(prefix) { + let sql = completed + .get_item(index)? + .call_method1("get", (name, cte.getattr("sql_body")?))?; + let kwargs = PyDict::new(py); + kwargs.set_item("sql_body", sql)?; + replacement = replace.call((&cte,), Some(&kwargs))?; + break; + } + } + authored.push(replacement); + } + let mut expected_ctes: Vec> = Vec::new(); + for cte in expected.try_iter()? { + let cte = cte?; + let name: String = cte.getattr("name")?.extract()?; + let name = name.strip_prefix("__expected__").unwrap_or(&name); + let kwargs = PyDict::new(py); + kwargs.set_item( + "sql_body", + completed + .get_item(3)? + .call_method1("get", (name, cte.getattr("sql_body")?))?, + )?; + expected_ctes.push(replace.call((&cte,), Some(&kwargs))?); + } + let kwargs = PyDict::new(py); + kwargs.set_item("authored_ctes", PyTuple::new(py, authored)?)?; + kwargs.set_item("expected_ctes", PyTuple::new(py, expected_ctes)?)?; + let payload = replace.call((payload,), Some(&kwargs))?; + let kwargs = PyDict::new(py); + kwargs.set_item("payload", payload)?; + replace.call((test,), Some(&kwargs)) +} + +pub(super) fn validate<'py>( + tests: &[Bound<'py, PyAny>], + inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + context: &Bound<'py, PyAny>, +) -> PyResult> { + let py = context.py(); + let mut model_tests: Vec> = Vec::new(); + let mut indexes: Vec = Vec::new(); + for (index, test) in tests.iter().enumerate() { + if model_payload(test)? { + indexes.push(index); + model_tests.push(test.clone()); + } + } + if indexes.is_empty() { + return Ok((tests.to_vec(), Default::default())); + } + let chains = call_with( + &imported(py, NATIVE_PLANNING, "resolve_sql_test_model_chains")?, + &[ + ("project", inputs.0.clone()), + ("tests", PyTuple::new(py, model_tests)?.into_any()), + ], + )?; + let chains: Vec> = chains.extract()?; + if chains.len() != indexes.len() { + return Err(pyo3::exceptions::PyValueError::new_err( + "zip() argument lengths differ", + )); + } + let exception = imported( + py, + "sqlbuild.compiler.planner.exceptions", + "SqlTestFixtureValidationError", + )?; + let mut validated = tests.to_vec(); + let mut diagnostics: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for (index, chain) in indexes.into_iter().zip(chains) { + match validated_test(&tests[index], &chain, inputs, context) { + Ok(test) => validated[index] = test, + Err(error) if error.value(py).is_instance(&exception)? => { + diagnostics.insert(index, error.value(py).getattr("diagnostics")?); + } + Err(error) => return Err(error), + } + } + Ok((validated, diagnostics)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs new file mode 100644 index 0000000000..77159f0e5c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs @@ -0,0 +1,204 @@ +//! Adapter rendering and declared cursor-window boundaries for native test planning. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::sql_test_glue::plan_values::{model_payload, string}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +use crate::bindings::_helpers::sql_test_glue::constants::{CURSOR_WINDOW, SENTINEL}; + +pub(super) fn mentions(sql: &str) -> bool { + ["__cursor_start", "__cursor_end"] + .iter() + .any(|name| sql.contains(name)) +} + +fn render<'py>( + sql: Bound<'py, PyAny>, + model: Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + test: Bound<'py, PyAny>, +) -> PyResult> { + call_with( + &imported(adapter.py(), CURSOR_WINDOW, "render_test_cursor_intrinsics")?, + &[ + ("sql", sql), + ("model", model), + ("adapter", adapter.clone()), + ("test", test), + ], + ) +} + +pub(super) fn models<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let result = PyDict::new(py); + for (index, model) in project.getattr("models")?.try_iter()?.enumerate() { + let model = model?; + let sql = model.getattr("query_sql")?; + if mentions(&sql.extract::()?) { + result.set_item( + index, + render(sql, model, adapter, py.None().into_bound(py))?, + )?; + } + } + Ok(result) +} + +pub(super) fn overrides<'py>( + project: &Bound<'py, PyAny>, + tests: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let models = PyDict::new(py); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + models.set_item(model.getattr("name")?, model)?; + } + let tests: Vec> = tests.extract()?; + let mut indexes: Vec = Vec::new(); + let mut windowed: Vec> = Vec::new(); + for (index, test) in tests.iter().enumerate() { + if model_payload(test)? + && call_with( + &imported(py, CURSOR_WINDOW, "declares_cursor_window")?, + &[("test", test.clone())], + )? + .is_truthy()? + { + indexes.push(index); + windowed.push(test.clone()); + } + } + let chains = if indexes.is_empty() { + Vec::new() + } else { + super::plan_driver::chains(project, &PyTuple::new(py, windowed)?.into_any())? + .extract::>>()? + }; + if indexes.len() != chains.len() { + return Err(pyo3::exceptions::PyValueError::new_err( + "zip() argument lengths differ", + )); + } + let chains: std::collections::BTreeMap<_, _> = indexes.into_iter().zip(chains).collect(); + let result = PyDict::new(py); + for (index, test) in tests.iter().enumerate() { + if !model_payload(test)? { + continue; + } + let overrides = test + .getattr("payload")? + .getattr("model_query_overrides")? + .cast_into::()?; + let chain = chains.get(&index); + if chain.is_none() + && !overrides + .values() + .try_iter()? + .map(|value| Ok(mentions(&value?.extract::()?))) + .collect::>>()? + .contains(&true) + { + continue; + } + let chain_names: Vec = chain + .map(|value| value.extract()) + .transpose()? + .unwrap_or_default(); + let rendered = PyDict::new(py); + for (name, sql) in overrides.iter() { + let name: String = name.extract()?; + let sql = if let Some(model) = models.get_item(&name)? { + render( + sql, + model, + adapter, + if chain_names.contains(&name) { + test.clone() + } else { + py.None().into_bound(py) + }, + )? + } else { + sql + }; + rendered.set_item(name, sql)?; + } + if chain.is_some() { + let mut chain_models: Vec> = Vec::new(); + for name in chain_names { + if let Some(model) = models.get_item(name)? { + chain_models.push(model); + } + } + let cursor_models = call_with( + &imported(py, CURSOR_WINDOW, "declared_window_cursor_models")?, + &[ + ("test", test.clone()), + ("chain_models", PyTuple::new(py, chain_models)?.into_any()), + ], + )?; + for model in cursor_models.try_iter()? { + let model = model?; + let name = model.getattr("name")?; + if !rendered.contains(&name)? { + rendered.set_item( + name, + render(model.getattr("query_sql")?, model, adapter, test.clone())?, + )?; + } + } + } + result.set_item(index, rendered)?; + } + Ok(result) +} + +pub(super) fn functions<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let mut result: Vec> = Vec::new(); + for function in project.getattr("functions")?.try_iter()? { + let function = function?; + let target = function.getattr("destination")?.getattr("qualified_name")?; + if target.is_none() { + continue; + } + let mut fields = vec![function.getattr("name")?]; + for method in ["render_udf_call", "render_table_function_call"] { + let rendered: String = call_with( + &adapter.getattr(method)?, + &[ + ("target", target.clone()), + ("call_suffix_sql", string(py, SENTINEL)), + ], + )? + .extract()?; + let Some((prefix, suffix)) = rendered + .split_once(SENTINEL) + .filter(|(_, suffix)| !suffix.contains(SENTINEL)) + else { + return Err(PyErr::from_value( + imported( + py, + "sqlbuild.compiler.planner.exceptions", + "NativeSqlTestPlanningError", + )? + .call1(("adapter SQL function call template is not deterministic",))?, + )); + }; + fields.push(string(py, prefix)); + fields.push(string(py, suffix)); + } + result.push(PyTuple::new(py, fields)?); + } + PyTuple::new(py, result) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs new file mode 100644 index 0000000000..2d599dabcb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs @@ -0,0 +1,84 @@ +//! Materialize native plan rows as the existing executor-facing dataclasses. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::sql_test_glue::constants::PLAN_MODELS; +use crate::bindings::_helpers::sql_test_glue::plan_values::tuple; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +fn step<'py>(row: &Bound<'py, PyAny>, chain: bool) -> PyResult> { + let py = row.py(); + let kwargs = PyDict::new(py); + let fields: &[&str] = if chain { + &[ + "model_name", + "resolved_sql", + "expected_cte_sql", + "lifted_ctes", + "comparison_body_sql", + "expected_columns", + "expected_lifted_ctes", + ] + } else { + &["name", "resolved_sql", "lifted_ctes", "comparison_body_sql"] + }; + for (index, name) in fields.iter().enumerate() { + let value = row.get_item(index)?; + kwargs.set_item( + *name, + if ["lifted_ctes", "expected_columns", "expected_lifted_ctes"].contains(name) + && !value.is_none() + { + tuple(&value)? + } else { + value + }, + )?; + } + imported( + py, + PLAN_MODELS, + if chain { + "ChainStep" + } else { + "SqlTestAssertionStep" + }, + )? + .call((), Some(&kwargs)) +} + +pub(super) fn plan<'py>(row: &Bound<'py, PyAny>) -> PyResult> { + let py = row.py(); + let mut chain: Vec> = Vec::new(); + for value in row.get_item(1)?.try_iter()? { + chain.push(step(&value?, true)?); + } + let mut assertions: Vec> = Vec::new(); + for value in row.get_item(2)?.try_iter()? { + assertions.push(step(&value?, false)?); + } + let mut warnings: Vec> = Vec::new(); + let severity = imported(py, "sqlbuild.compiler.planner.types", "WarningSeverity")?; + for value in row.get_item(4)?.try_iter()? { + let value = value?; + warnings.push(call_with( + &imported(py, PLAN_MODELS, "PlanWarning")?, + &[ + ("model_name", value.get_item(0)?), + ("severity", severity.call1((value.get_item(1)?,))?), + ("message", value.get_item(2)?), + ], + )?); + } + call_with( + &imported(py, PLAN_MODELS, "NativeSqlTestPlan")?, + &[ + ("chain", PyTuple::new(py, chain)?.into_any()), + ("assertions", PyTuple::new(py, assertions)?.into_any()), + ("model_names", tuple(&row.get_item(3)?)?), + ("warnings", PyTuple::new(py, warnings)?.into_any()), + ("error_messages", tuple(&row.get_item(5)?)?), + ("sql", row.get_item(0)?), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs new file mode 100644 index 0000000000..1a3617db47 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs @@ -0,0 +1,24 @@ +//! Python value boundaries shared by SQL-test plan assembly. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +use crate::bindings::_helpers::sql_test_glue::constants::COMPILE_MODELS; + +pub(super) fn string<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +pub(super) fn model_payload(test: &Bound<'_, PyAny>) -> PyResult { + test.getattr("payload")?.is_instance(&imported( + test.py(), + COMPILE_MODELS, + "CompiledModelSqlTestPayload", + )?) +} + +pub(super) fn tuple<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + imported(value.py(), "builtins", "tuple")?.call1((value,)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs index 2c18eb70c4..c6df114194 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs @@ -1,10 +1,31 @@ //! Register the native SQL test planning and assembly functions. -use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; use crate::bindings::_helpers::sql_test_glue::{authored_ctes, compiled_tests, planning}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + super::plan_assembly::assemble_sql_test_plans, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::plan_driver::plan_sql_tests_natively, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::plan_driver::resolve_sql_test_model_chains, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::plan_driver::plan_compiled_sql_test_artifacts, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::plan_driver::sql_test_plan_error_messages, + module + )?)?; authored_ctes::register(module)?; planning::register(module)?; compiled_tests::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/types.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/types.rs new file mode 100644 index 0000000000..707f3f2975 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/types.rs @@ -0,0 +1,6 @@ +//! SQL-test fixture validation boundary values. + +use pyo3::prelude::{Bound, PyAny}; +use std::collections::BTreeMap; + +pub(super) type ValidatedTests<'py> = (Vec>, BTreeMap>); diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index c21c5a0e4c..bc41836f58 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -236,6 +236,8 @@ from sqlbuild.compiler.planner.models import ( OldNameView, OldNameViewPlanEntry, OldNameViewPlanning, + NativeSqlTestArtifact, + NativeSqlTestPlan, ParsedScenarioArtifactName, ParsedSelector, PathSelector, @@ -266,6 +268,7 @@ from sqlbuild.compiler.planner.models import ( SelectorExpansion, SourceLoadPlanEntry, SqlTestPlanEntry, + SqlTestPlanResult, SqlTestSelection, UnitTestSelectorSplit, WarehouseSnapshot, @@ -455,6 +458,39 @@ from sqlbuild.sql_values.models import SqlValue, SqlValueLimits from sqlbuild.sql_values.types import CollectionRendering, StateSqlValueType BUILD_IDENTITY: str + +def plan_sql_tests( + *, + tests: tuple[CompiledSqlTest, ...], + project: CompiledProjectView, + adapter: BaseAdapter, + sql_analysis_enabled: bool = False, + validate_fixtures: bool = False, + fixture_planning_context: RelationFixturePlanningContext | None = None, +) -> tuple[SqlTestPlanResult, ...]: ... +def plan_sql_tests_natively( + *, + project: CompiledProjectView, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + sql_analysis_enabled: bool, + render_sql: bool, + include_plan: bool = True, +) -> tuple[NativeSqlTestPlan, ...]: ... +def plan_compiled_sql_test_artifacts( + *, + project: CompiledProjectView, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + sql_analysis_enabled: bool, +) -> tuple[NativeSqlTestArtifact, ...]: ... +def resolve_sql_test_model_chains( + *, + project: CompiledProjectView, + tests: tuple[CompiledSqlTest, ...], +) -> tuple[tuple[str, ...], ...]: ... +def sql_test_plan_error_messages(*, warnings: tuple[PlanWarning, ...]) -> tuple[str, ...]: ... + STAGE_CAPTURE_OMITTED_ATTRIBUTES: dict[str, frozenset[str]] def migration_planning_database(*, runtime: PlannerRuntime) -> str | None: ... diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py index 6316d35768..6c7cb28659 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py @@ -6,6 +6,7 @@ build_relation_fixture_context, build_selected_audit_entries, build_source_load_entries, + plan_sql_tests, upstream_loader_dependency_names, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -31,7 +32,6 @@ scope_overlaps, ) from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_function_sql -from sqlbuild.compiler.planner._helpers.sql_tests.assembly import plan_sql_tests from sqlbuild.compiler.planner.exceptions import ( PlannerInputError, ) diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py deleted file mode 100644 index b79e958faf..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py +++ /dev/null @@ -1,356 +0,0 @@ -"""SQL-native unit-test plan entries built from native chain planning.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild._native import build_relation_fixture_context -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.constants import ( - DBT_REF_TEST_CTE_PREFIX, - EXPECTED_TEST_CTE_PREFIX, - REF_TEST_CTE_PREFIX, - SEED_TEST_CTE_PREFIX, - SOURCE_TEST_CTE_PREFIX, - TABLE_FN_TEST_CTE_PREFIX, -) -from sqlbuild.compiler.compile.models import ( - CompiledDirectLogicSqlTestPayload, - CompiledModelSqlTestPayload, - CompiledObjectKey, - CompiledSqlTest, - CompileSqlTestCte, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - SqlTestMode, -) -from sqlbuild.compiler.planner._helpers.sql_tests.fixture_validation import ( - build_validated_test_fixtures, - fixture_location, -) -from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( - plan_sql_tests_natively, - resolve_sql_test_model_chains, -) -from sqlbuild.compiler.planner.exceptions import SqlTestFixtureValidationError -from sqlbuild.compiler.planner.models import ( - NativeSqlTestPlan, - RelationFixturePlanningContext, - SqlTestPlanEntry, - SqlTestPlanResult, -) -from sqlbuild.executor.testing.main.missing_expected_columns import ( - describe_missing_expected_columns, -) - -_FUNCTION_RESOURCE_TYPES: frozenset[CompiledResourceType] = frozenset( - {CompiledResourceType.UDF, CompiledResourceType.TABLE_FN} -) - - -def plan_sql_tests( - *, - tests: tuple[CompiledSqlTest, ...], - project: CompiledProjectView, - adapter: BaseAdapter, - sql_analysis_enabled: bool = False, - validate_fixtures: bool = False, - fixture_planning_context: RelationFixturePlanningContext | None = None, -) -> tuple[SqlTestPlanResult, ...]: - """Plan SQL tests in one native batch, reporting fixture diagnostics per test.""" - - planned_tests: tuple[CompiledSqlTest, ...] = tests - diagnostics_by_index: dict[int, tuple[str, ...]] = {} - validation_context: RelationFixturePlanningContext | None = None - if sql_analysis_enabled and validate_fixtures: - validation_context = fixture_planning_context or build_relation_fixture_context( - project=project - ) - planned_tests, diagnostics_by_index = _validate_fixtures( - tests=tests, - project=project, - adapter=adapter, - fixture_planning_context=validation_context, - ) - plannable_indexes: tuple[int, ...] = tuple( - index for index in range(len(tests)) if index not in diagnostics_by_index - ) - plans: tuple[NativeSqlTestPlan, ...] = plan_sql_tests_natively( - project=project, - tests=tuple(planned_tests[index] for index in plannable_indexes), - adapter=adapter, - sql_analysis_enabled=sql_analysis_enabled, - render_sql=False, - ) - plans_by_index: dict[int, NativeSqlTestPlan] = dict(zip(plannable_indexes, plans, strict=True)) - models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} - results: list[SqlTestPlanResult] = [] - for index, test in enumerate(tests): - plan: NativeSqlTestPlan | None = plans_by_index.get(index) - if plan is None: - results.append( - SqlTestPlanResult(entry=None, fixture_diagnostics=diagnostics_by_index[index]) - ) - continue - plan_diagnostics: tuple[str, ...] = ( - *plan.error_messages, - *_expected_column_diagnostics( - test=test, plan=plan, fixture_planning_context=validation_context - ), - ) - if plan_diagnostics: - results.append(SqlTestPlanResult(entry=None, fixture_diagnostics=plan_diagnostics)) - continue - results.append( - SqlTestPlanResult( - entry=_plan_entry( - test=test, - plan=plan, - project=project, - models_by_name=models_by_name, - sql_analysis_enabled=sql_analysis_enabled, - ), - warnings=plan.warnings, - ) - ) - return tuple(results) - - -def _expected_column_diagnostics( - *, - test: CompiledSqlTest, - plan: NativeSqlTestPlan, - fixture_planning_context: RelationFixturePlanningContext | None, -) -> tuple[str, ...]: - """Report expected columns that authoritative model output metadata proves absent.""" - - if fixture_planning_context is None: - return () - diagnostics: list[str] = [] - for step in plan.chain: - available_columns: frozenset[str] | None = ( - fixture_planning_context.authoritative_columns.get( - (CompiledResourceType.MODEL, step.model_name) - ) - ) - if step.expected_columns is None or available_columns is None: - continue - message: str | None = describe_missing_expected_columns( - model_name=step.model_name, - expected_columns=step.expected_columns, - available_columns=available_columns, - ) - if message is None: - continue - location: str = fixture_location( - test=test, resource_type=CompiledResourceType.SQL_TEST, name=step.model_name - ) - diagnostics.append(f"SQL test '{test.name}': {location}: {message}") - return tuple(diagnostics) - - -def _validate_fixtures( - *, - tests: tuple[CompiledSqlTest, ...], - project: CompiledProjectView, - adapter: BaseAdapter, - fixture_planning_context: RelationFixturePlanningContext, -) -> tuple[tuple[CompiledSqlTest, ...], dict[int, tuple[str, ...]]]: - """Return tests with validated fixture completions plus diagnostics by test index.""" - - model_indexes: tuple[int, ...] = tuple( - index - for index, test in enumerate(tests) - if isinstance(test.payload, CompiledModelSqlTestPayload) - ) - if not model_indexes: - return tests, {} - chains: tuple[tuple[str, ...], ...] = resolve_sql_test_model_chains( - project=project, - tests=tuple(tests[index] for index in model_indexes), - ) - chains_by_index: dict[int, tuple[str, ...]] = dict(zip(model_indexes, chains, strict=True)) - context: RelationFixturePlanningContext = fixture_planning_context - validated_tests: list[CompiledSqlTest] = [] - diagnostics_by_index: dict[int, tuple[str, ...]] = {} - for index, test in enumerate(tests): - chain: tuple[str, ...] | None = chains_by_index.get(index) - if chain is None: - validated_tests.append(test) - continue - try: - validated_tests.append( - _with_validated_fixtures( - test=test, - project=project, - adapter=adapter, - ordered_model_names=chain, - fixture_planning_context=context, - ) - ) - except SqlTestFixtureValidationError as error: - validated_tests.append(test) - diagnostics_by_index[index] = error.diagnostics - return tuple(validated_tests), diagnostics_by_index - - -def _with_validated_fixtures( - *, - test: CompiledSqlTest, - project: CompiledProjectView, - adapter: BaseAdapter, - ordered_model_names: tuple[str, ...], - fixture_planning_context: RelationFixturePlanningContext, -) -> CompiledSqlTest: - payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload = test.payload - if not isinstance(payload, CompiledModelSqlTestPayload): - return test - mock_refs, mock_sources, mock_seeds, expected_outputs = build_validated_test_fixtures( - test=test, - project=project, - adapter=adapter, - ordered_model_names=ordered_model_names, - mock_refs=_prefixed_ctes(ctes=payload.authored_ctes, prefix=REF_TEST_CTE_PREFIX), - mock_sources=_prefixed_ctes(ctes=payload.authored_ctes, prefix=SOURCE_TEST_CTE_PREFIX), - mock_seeds=_prefixed_ctes(ctes=payload.authored_ctes, prefix=SEED_TEST_CTE_PREFIX), - expected_outputs=_expected_outputs(payload=payload), - planning_context=fixture_planning_context, - ) - completed_bodies: tuple[tuple[str, dict[str, str]], ...] = ( - (REF_TEST_CTE_PREFIX, mock_refs), - (SOURCE_TEST_CTE_PREFIX, mock_sources), - (SEED_TEST_CTE_PREFIX, mock_seeds), - ) - authored_ctes: tuple[CompileSqlTestCte, ...] = tuple( - _completed_cte(cte=cte, completed_bodies=completed_bodies) for cte in payload.authored_ctes - ) - expected_ctes: tuple[CompileSqlTestCte, ...] = tuple( - replace( - cte, - sql_body=expected_outputs.get( - cte.name.removeprefix(EXPECTED_TEST_CTE_PREFIX), cte.sql_body - ), - ) - for cte in payload.expected_ctes - ) - return replace( - test, - payload=replace(payload, authored_ctes=authored_ctes, expected_ctes=expected_ctes), - ) - - -def _completed_cte( - *, - cte: CompileSqlTestCte, - completed_bodies: tuple[tuple[str, dict[str, str]], ...], -) -> CompileSqlTestCte: - for prefix, bodies in completed_bodies: - if cte.name.startswith(prefix): - return replace(cte, sql_body=bodies.get(cte.name.removeprefix(prefix), cte.sql_body)) - return cte - - -def _expected_outputs(*, payload: CompiledModelSqlTestPayload) -> dict[str, str]: - return { - cte.name.removeprefix(EXPECTED_TEST_CTE_PREFIX): cte.sql_body - for cte in payload.expected_ctes - } - - -def _prefixed_ctes(*, ctes: tuple[CompileSqlTestCte, ...], prefix: str) -> dict[str, str]: - return { - cte.name.removeprefix(prefix): cte.sql_body for cte in ctes if cte.name.startswith(prefix) - } - - -def _prefixed_names(*, test: CompiledSqlTest, prefix: str) -> tuple[str, ...]: - if not isinstance(test.payload, CompiledModelSqlTestPayload): - return () - return tuple(sorted(_prefixed_ctes(ctes=test.payload.authored_ctes, prefix=prefix))) - - -def _plan_entry( - *, - test: CompiledSqlTest, - plan: NativeSqlTestPlan, - project: CompiledProjectView, - models_by_name: dict[str, CompiledModelView], - sql_analysis_enabled: bool, -) -> SqlTestPlanEntry: - mock_table_function_names: tuple[str, ...] = _prefixed_names( - test=test, prefix=TABLE_FN_TEST_CTE_PREFIX - ) - return SqlTestPlanEntry( - key=test.key, - name=test.name, - source_path=test.source_path, - block_index=test.block_index, - parent_name=test.parent_name, - case_name=test.case_name, - case_index=test.case_index, - case_fingerprint=test.case_fingerprint, - parameter_schema=test.parameter_schema, - parameter_values=test.parameter_values, - mock_ref_names=_prefixed_names(test=test, prefix=REF_TEST_CTE_PREFIX), - mock_source_names=_prefixed_names(test=test, prefix=SOURCE_TEST_CTE_PREFIX), - mock_seed_names=_prefixed_names(test=test, prefix=SEED_TEST_CTE_PREFIX), - mock_dbt_ref_names=_prefixed_names(test=test, prefix=DBT_REF_TEST_CTE_PREFIX), - mock_table_function_names=mock_table_function_names, - chain=plan.chain, - assertions=plan.assertions, - scope_deps=test.scope_deps, - function_deps=( - _direct_function_deps(test=test, project=project) - if isinstance(test.payload, CompiledDirectLogicSqlTestPayload) - else _chain_function_deps( - model_names=plan.model_names, - models_by_name=models_by_name, - mocked_table_functions=frozenset(mock_table_function_names), - ) - ), - sql_analysis_enabled=sql_analysis_enabled, - ) - - -def _chain_function_deps( - *, - model_names: tuple[str, ...], - models_by_name: dict[str, CompiledModelView], - mocked_table_functions: frozenset[str], -) -> tuple[CompiledObjectKey, ...]: - deps: dict[CompiledObjectKey, None] = {} - for model_name in model_names: - model: CompiledModelView | None = models_by_name.get(model_name) - if model is None: - continue - for dep in model.deps: - if dep.resource_type not in _FUNCTION_RESOURCE_TYPES: - continue - if dep.resource_type == CompiledResourceType.TABLE_FN and ( - dep.name in mocked_table_functions - ): - continue - deps[dep] = None - return tuple(deps) - - -def _direct_function_deps( - *, test: CompiledSqlTest, project: CompiledProjectView -) -> tuple[CompiledObjectKey, ...]: - if not isinstance(test.payload, CompiledDirectLogicSqlTestPayload): - return () - resource_type: CompiledResourceType | None = { - SqlTestMode.UDF: CompiledResourceType.UDF, - SqlTestMode.TABLE_FN: CompiledResourceType.TABLE_FN, - }.get(test.payload.mode) - if resource_type is None: - return () - tested_names: frozenset[str] = frozenset(test.payload.tested_resource_names) - return tuple( - function.key - for function in project.functions - if function.key.resource_type == resource_type and function.name in tested_names - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py deleted file mode 100644 index 9a8f915454..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py +++ /dev/null @@ -1,396 +0,0 @@ -"""Native SQL-test chain planning and comparison rendering boundary.""" - -from __future__ import annotations - -import time - -import orjson - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.classes.sql_test_cte_locator import SqlTestCteLocator -from sqlbuild.compiler.compile.constants import SQL_TEST_HELPER_REFERENCE_CODE -from sqlbuild.compiler.compile.exceptions import CompileInputError, SqlTestReferenceError -from sqlbuild.compiler.compile.models import ( - CompiledDirectLogicSqlTestPayload, - CompiledModelSqlTestPayload, - CompiledSqlTest, -) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.planner._helpers.sql_tests.cursor_window import ( - declared_window_cursor_models, - declares_cursor_window, - render_test_cursor_intrinsics, -) -from sqlbuild.compiler.planner.exceptions import NativeSqlTestPlanningError, PlannerInputError -from sqlbuild.compiler.planner.models import ( - ChainStep, - NativeSqlTestArtifact, - NativeSqlTestPlan, - PlanWarning, - SqlTestAssertionStep, -) -from sqlbuild.compiler.planner.types import WarningSeverity -from sqlbuild.compiler.profiling.classes.context import CompileTimingContext -from sqlbuild.compiler.profiling.models import CompileTimingCollector -from sqlbuild.compiler.sql_test_glue.constants import CURSOR_INTRINSIC_NAMES -from sqlbuild.compiler.sql_test_glue.models import ( - NativeSqlTestChainRequest, - NativeSqlTestPlanningRequest, -) -from sqlbuild.compiler.sql_test_glue.types import ( - NativeAssertionStepRow, - NativeChainStepRow, - NativeSqlTestPlanRow, -) - -_UNRESOLVED_REFERENCE_PREFIX: str = "sql_test_reference:" - -_CALL_SUFFIX_SENTINEL: str = "__SQLBUILD_CALL_SUFFIX__" -_NATIVE_WORKERS: int = 4 - - -def plan_compiled_sql_test_artifacts( - *, - project: CompiledProjectView, - tests: tuple[CompiledSqlTest, ...], - adapter: BaseAdapter, - sql_analysis_enabled: bool, -) -> tuple[NativeSqlTestArtifact, ...]: - """Plan and render SQL tests from the compiled objects, with natively projected errors.""" - - artifacts: list[NativeSqlTestArtifact] = [] - for plan in _plan_compiled_sql_tests( - project=project, - tests=tests, - adapter=adapter, - sql_analysis_enabled=sql_analysis_enabled, - render_sql=True, - include_plan=False, - ): - if plan.sql is None: - raise NativeSqlTestPlanningError("native SQL-test planning omitted rendered SQL") - artifacts.append( - NativeSqlTestArtifact( - sql=plan.sql, model_names=plan.model_names, error_messages=plan.error_messages - ) - ) - return tuple(artifacts) - - -def sql_test_plan_error_messages(*, warnings: tuple[PlanWarning, ...]) -> tuple[str, ...]: - """Return each distinct ERROR-severity planning message for one SQL test, in order.""" - - return tuple( - dict.fromkeys( - warning.message for warning in warnings if warning.severity is WarningSeverity.ERROR - ) - ) - - -def plan_sql_tests_natively( - *, - project: CompiledProjectView, - tests: tuple[CompiledSqlTest, ...], - adapter: BaseAdapter, - sql_analysis_enabled: bool, - render_sql: bool, - include_plan: bool = True, -) -> tuple[NativeSqlTestPlan, ...]: - """Plan SQL-test chains and assertions in one native batch, optionally rendering SQL.""" - - return _plan_compiled_sql_tests( - project=project, - tests=tests, - adapter=adapter, - sql_analysis_enabled=sql_analysis_enabled, - render_sql=render_sql, - include_plan=include_plan, - ) - - -def _plan_compiled_sql_tests( - *, - project: CompiledProjectView, - tests: tuple[CompiledSqlTest, ...], - adapter: BaseAdapter, - sql_analysis_enabled: bool, - render_sql: bool, - include_plan: bool, -) -> tuple[NativeSqlTestPlan, ...]: - """Plan from the compiled objects natively; Python supplies only the adapter's renderings.""" - - if not tests: - return () - start_ns: int = time.perf_counter_ns() - rendered_model_sql: dict[int, str] = { - index: render_test_cursor_intrinsics( - sql=model.query_sql, model=model, adapter=adapter, test=None - ) - for index, model in enumerate(project.models) - if _mentions_cursor_intrinsic(model.query_sql) - } - functions: tuple[tuple[str, str, str, str, str], ...] = _function_templates( - project=project, adapter=adapter - ) - rendered_test_overrides: dict[int, dict[str, str]] = _rendered_test_overrides( - project=project, tests=tests, adapter=adapter - ) - request: NativeSqlTestPlanningRequest = NativeSqlTestPlanningRequest( - models=project.models, - rendered_model_sql=rendered_model_sql, - functions=functions, - tests=tests, - rendered_test_overrides=rendered_test_overrides, - sql_analysis_enabled=sql_analysis_enabled, - sql_analysis_dialect=adapter.sql_analysis_dialect(), - set_difference_operator=adapter.render_set_difference_operator(), - requires_derived_table_aliases=adapter.requires_derived_table_aliases(), - lexical_syntax=project.sql_lexical_syntax.native_mapping, - render_sql=render_sql, - include_plan=include_plan, - ) - try: - rows, planning_ns, rendering_ns = _native.plan_compiled_sql_tests(request) - except ValueError as error: - raise _planning_error(error=error, tests=tests) from None - _record_planning_timing(start_ns=start_ns, planning_ns=planning_ns, rendering_ns=rendering_ns) - return tuple(_plan_from_row(row=row) for row in rows) - - -def _plan_from_row(*, row: NativeSqlTestPlanRow) -> NativeSqlTestPlan: - sql, chain, assertions, model_names, warnings, error_messages = row - return NativeSqlTestPlan( - chain=tuple(_chain_step_from_row(row=step) for step in chain), - assertions=tuple(_assertion_step_from_row(row=step) for step in assertions), - model_names=tuple(model_names), - warnings=tuple( - PlanWarning(model_name=model_name, severity=WarningSeverity(severity), message=message) - for model_name, severity, message in warnings - ), - error_messages=tuple(error_messages), - sql=sql, - ) - - -def _assertion_step_from_row(*, row: NativeAssertionStepRow) -> SqlTestAssertionStep: - name, resolved_sql, lifted_ctes, comparison_body_sql = row - return SqlTestAssertionStep( - name=name, - resolved_sql=resolved_sql, - lifted_ctes=tuple(lifted_ctes), - comparison_body_sql=comparison_body_sql, - ) - - -def _chain_step_from_row(*, row: NativeChainStepRow) -> ChainStep: - ( - model_name, - resolved_sql, - expected_cte_sql, - lifted_ctes, - comparison_body_sql, - expected_columns, - expected_lifted_ctes, - ) = row - return ChainStep( - model_name=model_name, - resolved_sql=resolved_sql, - expected_cte_sql=expected_cte_sql, - lifted_ctes=tuple(lifted_ctes), - comparison_body_sql=comparison_body_sql, - expected_columns=None if expected_columns is None else tuple(expected_columns), - expected_lifted_ctes=tuple(expected_lifted_ctes), - ) - - -def _mentions_cursor_intrinsic(sql: str) -> bool: - """Whether SQL names a cursor intrinsic; SQL that does not is never rendered differently.""" - - return any(name in sql for name in CURSOR_INTRINSIC_NAMES) - - -def _rendered_test_overrides( - *, project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...], adapter: BaseAdapter -) -> dict[int, dict[str, str]]: - """Cursor-rendered model overrides for the tests whose overrides cursor windows change.""" - - models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} - windowed_indexes: tuple[int, ...] = tuple( - index - for index, test in enumerate(tests) - if isinstance(test.payload, CompiledModelSqlTestPayload) - and declares_cursor_window(test=test) - ) - chains: tuple[tuple[str, ...], ...] = ( - resolve_sql_test_model_chains( - project=project, tests=tuple(tests[index] for index in windowed_indexes) - ) - if windowed_indexes - else () - ) - chains_by_index: dict[int, tuple[str, ...]] = dict(zip(windowed_indexes, chains, strict=True)) - rendered: dict[int, dict[str, str]] = {} - for index, test in enumerate(tests): - payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload = test.payload - if not isinstance(payload, CompiledModelSqlTestPayload): - continue - chain: tuple[str, ...] | None = chains_by_index.get(index) - if chain is None and not any( - _mentions_cursor_intrinsic(sql) for sql in payload.model_query_overrides.values() - ): - continue - overrides: dict[str, str] | None = _windowed_model_query_overrides( - test=test, models_by_name=models_by_name, adapter=adapter, chain=chain - ) - if overrides is not None: - rendered[index] = overrides - return rendered - - -def _record_planning_timing(*, start_ns: int, planning_ns: int, rendering_ns: int) -> None: - elapsed_ns: int = time.perf_counter_ns() - start_ns - boundary_overhead_ns: int = max(0, elapsed_ns - planning_ns - rendering_ns) - collector: CompileTimingCollector | None = CompileTimingContext.active.get() - if collector is not None: - collector.add( - phase="test_planning_ms", - elapsed_ns=planning_ns + boundary_overhead_ns, - ) - collector.add(phase="comparison_render_ms", elapsed_ns=rendering_ns) - - -def resolve_sql_test_model_chains( - *, project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...] -) -> tuple[tuple[str, ...], ...]: - """Return each test's ordered unmocked model chain; direct-logic tests have none.""" - - if not tests: - return () - try: - native_chains: list[list[str]] = _native.resolve_compiled_sql_test_chains( - NativeSqlTestChainRequest( - models=project.models, - tests=tests, - lexical_syntax=project.sql_lexical_syntax.native_mapping, - ) - ) - except ValueError as error: - raise _planning_error(error=error, tests=tests) from None - return tuple(tuple(chain) for chain in native_chains) - - -def _planning_error(*, error: ValueError, tests: tuple[CompiledSqlTest, ...]) -> Exception: - message: str = str(error) - if message.startswith(_UNRESOLVED_REFERENCE_PREFIX): - return _unresolved_reference_error( - payload=orjson.loads(message.removeprefix(_UNRESOLVED_REFERENCE_PREFIX)), tests=tests - ) - if message.startswith("compile_input:"): - return CompileInputError(message.removeprefix("compile_input:")) - if message.startswith("planner_input:"): - return PlannerInputError(message.removeprefix("planner_input:")) - return NativeSqlTestPlanningError(f"native SQL-test planning failed: {message}") - - -def _unresolved_reference_error( - *, payload: dict[str, str], tests: tuple[CompiledSqlTest, ...] -) -> Exception: - """Locate a reference call the native planner could not resolve in its test file.""" - - test: CompiledSqlTest | None = next( - ( - test - for test in tests - if test.name == payload["testName"] - and str(test.test_file.relative_path) == payload["fileLabel"] - ), - None, - ) - cte_name: str = payload["cteName"] - call: str = payload["call"] - message: str = f"SQL test CTE '{cte_name}' calls {call}, which the test query cannot resolve" - help_text: str = ( - f"Mock it in the test, for example {payload['mockCte']} AS (SELECT ...), or call " - '__ref("") on a model the test runs.' - ) - if test is None: - return CompileInputError(message, code=SQL_TEST_HELPER_REFERENCE_CODE, help=help_text) - return SqlTestReferenceError( - message, - location=SqlTestCteLocator.locate( - test_file=test.test_file, test_block=test.test_block, cte_name=cte_name, call=call - ), - help=help_text, - ) - - -def _function_templates( - *, project: CompiledProjectView, adapter: BaseAdapter -) -> tuple[tuple[str, str, str, str, str], ...]: - """Each function's name and the adapter's UDF and table-function call templates.""" - - functions: list[tuple[str, str, str, str, str]] = [] - for function in project.functions: - target: str | None = function.destination.qualified_name - if target is None: - continue - udf_prefix, udf_suffix = _render_call_template( - rendered=adapter.render_udf_call( - target=target, - call_suffix_sql=_CALL_SUFFIX_SENTINEL, - ) - ) - table_function_prefix, table_function_suffix = _render_call_template( - rendered=adapter.render_table_function_call( - target=target, - call_suffix_sql=_CALL_SUFFIX_SENTINEL, - ) - ) - functions.append( - (function.name, udf_prefix, udf_suffix, table_function_prefix, table_function_suffix) - ) - return tuple(functions) - - -def _windowed_model_query_overrides( - *, - test: CompiledSqlTest, - models_by_name: dict[str, CompiledModelView], - adapter: BaseAdapter, - chain: tuple[str, ...] | None, -) -> dict[str, str] | None: - payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload = test.payload - if not isinstance(payload, CompiledModelSqlTestPayload): - return None - chain_names: frozenset[str] = frozenset(chain or ()) - overrides: dict[str, str] = {} - for model_name, sql in payload.model_query_overrides.items(): - model: CompiledModelView | None = models_by_name.get(model_name) - overrides[model_name] = ( - sql - if model is None - else render_test_cursor_intrinsics( - sql=sql, - model=model, - adapter=adapter, - test=test if model_name in chain_names else None, - ) - ) - if chain is not None: - chain_models: tuple[CompiledModelView, ...] = tuple( - models_by_name[name] for name in chain if name in models_by_name - ) - for model in declared_window_cursor_models(test=test, chain_models=chain_models): - if model.name not in overrides: - overrides[model.name] = render_test_cursor_intrinsics( - sql=model.query_sql, model=model, adapter=adapter, test=test - ) - return overrides - - -def _render_call_template(*, rendered: str) -> tuple[str, str]: - prefix, separator, suffix = rendered.partition(_CALL_SUFFIX_SENTINEL) - if not separator or _CALL_SUFFIX_SENTINEL in suffix: - raise NativeSqlTestPlanningError("adapter SQL function call template is not deterministic") - return prefix, suffix diff --git a/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py b/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py index 65b71f266b..62e4997dad 100644 --- a/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py +++ b/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py @@ -1,11 +1,11 @@ """Planner entrypoint for SQL-test artifacts planned natively from the compiled objects.""" +from sqlbuild._native import ( + plan_compiled_sql_test_artifacts as _plan_compiled_sql_test_artifacts, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( - plan_compiled_sql_test_artifacts as _plan_compiled_sql_test_artifacts, -) from sqlbuild.compiler.planner.models import NativeSqlTestArtifact diff --git a/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py b/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py index 94695b3960..4412ed1b5c 100644 --- a/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py +++ b/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py @@ -1,10 +1,10 @@ """Public planner entrypoint for batched SQL-test model closure discovery.""" -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlTest -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( +from sqlbuild._native import ( resolve_sql_test_model_chains, ) +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlTest +from sqlbuild.compiler.compile.types import CompiledProjectView def sql_test_model_chain_names_by_key( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 8106ea1ef6..3484d1dbfc 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -8,7 +8,13 @@ from types import MappingProxyType from typing import Any -from sqlbuild._native import expand_sql_macros +from sqlbuild._native import ( + expand_sql_macros, + plan_compiled_sql_test_artifacts, + plan_sql_tests, + plan_sql_tests_natively, + sql_test_plan_error_messages, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter @@ -44,13 +50,7 @@ DiscoveredSqlTestBlock, DiscoveredSqlTestFile, ) -from sqlbuild.compiler.planner._helpers.sql_tests.assembly import plan_sql_tests from sqlbuild.compiler.planner._helpers.sql_tests.comments import uncommented_pattern_matches -from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( - plan_compiled_sql_test_artifacts, - plan_sql_tests_natively, - sql_test_plan_error_messages, -) from sqlbuild.compiler.planner.models import ( NativeSqlTestPlan, PlanWarning, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index 77a9ebd7ed..5761e52fc5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -6,7 +6,9 @@ import pytest from sqlbuild._native import ( + plan_compiled_sql_test_artifacts, project_with, + sql_test_plan_error_messages, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -24,10 +26,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile -from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( - plan_compiled_sql_test_artifacts, - sql_test_plan_error_messages, -) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.main.commands._relations import resolve_static_relation_context from sqlbuild.compiler.planner.main.commands._scope import resolve_static_command_scope diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index 38ad5e9037..76098d4a9a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -5,6 +5,7 @@ import pytest +import sqlbuild._native as sql_test_assembly from sqlbuild import _native as fixture_completion from sqlbuild._native import ( compiled_model_record, @@ -22,7 +23,6 @@ ) from sqlbuild.compiler.planner._helpers.output import plan_output from sqlbuild.compiler.planner._helpers.output.plan_output import build_selected_test_entries -from sqlbuild.compiler.planner._helpers.sql_tests import assembly as sql_test_assembly from sqlbuild.compiler.planner.classes import fixture_column_inferences as fixture_inferences from sqlbuild.compiler.planner.models import ( FixtureColumnMetadata, From 3ce4d865c9e517a28f1415c3dfe49e67e72cb042 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:32:19 +0100 Subject: [PATCH 10/22] refactor: mark expected-column helper as internal --- .../src/bindings/_helpers/sql_test_glue/plan_entries.rs | 2 +- src/sqlbuild/executor/testing/_helpers/expected_column_probe.py | 2 +- ...missing_expected_columns.py => _missing_expected_columns.py} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/sqlbuild/executor/testing/main/{missing_expected_columns.py => _missing_expected_columns.py} (100%) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs index a2fe69b0e3..398090d144 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs @@ -150,7 +150,7 @@ pub(super) fn expected_diagnostics( let message = call_with( &imported( py, - "sqlbuild.executor.testing.main.missing_expected_columns", + "sqlbuild.executor.testing.main._missing_expected_columns", "describe_missing_expected_columns", )?, &[ diff --git a/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py b/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py index 62b071b5a9..9c193493fe 100644 --- a/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py +++ b/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py @@ -13,7 +13,7 @@ from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.executor.testing._helpers.native_requests import comparison_request -from sqlbuild.executor.testing.main.missing_expected_columns import ( +from sqlbuild.executor.testing.main._missing_expected_columns import ( describe_missing_expected_columns, ) from sqlbuild.executor.testing.types import NativeSqlTestRenderingModule diff --git a/src/sqlbuild/executor/testing/main/missing_expected_columns.py b/src/sqlbuild/executor/testing/main/_missing_expected_columns.py similarity index 100% rename from src/sqlbuild/executor/testing/main/missing_expected_columns.py rename to src/sqlbuild/executor/testing/main/_missing_expected_columns.py From 96e62d1c4c6101972aa82dd47f0499310c55cf52 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:48:47 +0100 Subject: [PATCH 11/22] refactor: render SQL-test cursor windows natively Move SQL-test cursor window validation and intrinsic rendering into the native SQL-test glue and delete the Python cursor window and comment-scanning helpers. Test helpers now read logical references through the native reference extractor. --- .../_helpers/attachments/cursor_intrinsics.rs | 2 +- .../_helpers/sql_test_glue/constants.rs | 12 +- .../_helpers/sql_test_glue/cursor_window.rs | 278 ++++++++++++++++++ .../bindings/_helpers/sql_test_glue/mod.rs | 1 + .../_helpers/sql_test_glue/plan_assembly.rs | 6 +- .../_helpers/sql_test_glue/plan_rendering.rs | 59 +--- src/sqlbuild/_native.pyi | 4 +- .../planner/_helpers/sql_tests/comments.py | 48 --- .../_helpers/sql_tests/cursor_window.py | 166 ----------- .../_helpers/sql_test_assembly/helpers.py | 15 +- .../_helpers/sql_test_assembly/helpers.py | 71 +++-- .../sql_test_assembly/test_cursor_window.py | 15 +- 12 files changed, 373 insertions(+), 304 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/sql_tests/comments.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs index e942a97baf..12a91b34c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs @@ -128,7 +128,7 @@ fn cursor_intrinsics_analysis_sql( /// Whether `sql` uses cursor intrinsics, and `sql` with both bounds rendered when given. #[pyfunction] #[pyo3(signature = (*, sql, start_sql=None, end_sql=None))] -fn resolve_cursor_intrinsics( +pub(crate) fn resolve_cursor_intrinsics( py: Python<'_>, sql: &str, start_sql: Option<&str>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs index 941f615f19..b91d4d24b3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs @@ -9,5 +9,15 @@ pub(super) const PLAN_MODELS: &str = "sqlbuild.compiler.planner.models"; pub(super) const NATIVE_PLANNING: &str = "sqlbuild._native"; pub(super) const FIXTURE_VALIDATION: &str = "sqlbuild.compiler.planner._helpers.sql_tests.fixture_validation"; -pub(super) const CURSOR_WINDOW: &str = "sqlbuild.compiler.planner._helpers.sql_tests.cursor_window"; +pub(super) const COMPILE_EXCEPTIONS: &str = "sqlbuild.compiler.compile.exceptions"; +pub(super) const CURSOR_ALGEBRA_EXCEPTIONS: &str = "sqlbuild.cursor_algebra.exceptions"; +pub(super) const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; +pub(super) const INTEGER_CURSOR_TYPE: &str = "integer"; +pub(super) const TIMESTAMP_CURSOR_TYPE: &str = "timestamp"; +pub(super) const DEFAULT_TIMESTAMP_START: &str = "1900-01-01 00:00:00"; +pub(super) const DEFAULT_TIMESTAMP_END: &str = "2999-12-31 00:00:00"; +pub(super) const DEFAULT_INTEGER_START: &str = "-1000000000000000"; +pub(super) const DEFAULT_INTEGER_END: &str = "1000000000000000"; +pub(super) const CURSOR_START_KEY: &str = "cursor_start"; +pub(super) const CURSOR_END_KEY: &str = "cursor_end"; pub(super) const SENTINEL: &str = "__SQLBUILD_CALL_SUFFIX__"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs new file mode 100644 index 0000000000..fb345d4fd0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs @@ -0,0 +1,278 @@ +//! Cursor windows rendered for models that SQL tests evaluate from their real SQL. + +use std::cmp::Ordering; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_cursor::algebra::main::compare::compare; +use sqlbuild_cursor::algebra::main::floor_to_grain::floor_to_grain; +use sqlbuild_cursor::algebra::models::{Grain, Scalar}; + +use crate::bindings::_helpers::attachments::cursor_intrinsics::resolve_cursor_intrinsics; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cursor::conversion::CursorSession; +use crate::bindings::_helpers::cursor::scalars::{parse_value, render_value}; +use crate::bindings::_helpers::model_config::config_values::config_str; +use crate::bindings::_helpers::sql_test_glue::constants::{ + COMPILE_EXCEPTIONS, CURSOR_ALGEBRA_EXCEPTIONS, CURSOR_END_KEY, CURSOR_START_KEY, + DEFAULT_INTEGER_END, DEFAULT_INTEGER_START, DEFAULT_TIMESTAMP_END, DEFAULT_TIMESTAMP_START, + INTEGER_CURSOR_TYPE, PLANNER_TYPES, TIMESTAMP_CURSOR_TYPE, +}; + +/// One model's cursor name, type and timestamp grain. +struct ModelCursor<'py> { + name: String, + cursor_type: Bound<'py, PyAny>, + type_name: String, + grain: Option<(String, Grain)>, +} + +/// One declared bound with the defaults and key used when it is absent or invalid. +struct BoundRequest<'a, 'py> { + raw: Option>, + default: &'a str, + key: &'a str, +} + +/// Whether SQL calls `__cursor_start()` or `__cursor_end()`. +pub(super) fn uses_cursor_intrinsics(py: Python<'_>, sql: &str) -> PyResult { + Ok(resolve_cursor_intrinsics(py, sql, None, None)?.1) +} + +/// Whether a SQL test header declares its own cursor window. +pub(super) fn declares_cursor_window(test: &Bound<'_, PyAny>) -> PyResult { + let block = test.getattr("test_block")?; + Ok(!block.getattr("cursor_start")?.is_none() || !block.getattr("cursor_end")?.is_none()) +} + +/// SQL with intrinsics rendered from the test's declared window, or the wide default window. +pub(super) fn render_test_cursor_intrinsics<'py>( + sql: Bound<'py, PyAny>, + model: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + test: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let py = sql.py(); + let text: String = sql.extract()?; + if !uses_cursor_intrinsics(py, &text)? { + return Ok(sql); + } + let cursor = model_cursor(model)?; + let (start, end) = test_window(&cursor, test)?; + let literal = |value: &Bound<'py, PyAny>| -> PyResult { + call_with( + &adapter.getattr("render_cursor_bound_literal")?, + &[ + ("value", PyString::new(py, &render_value(value)?).into_any()), + ("cursor_type", cursor.cursor_type.clone()), + ], + )? + .extract() + }; + let (start_sql, end_sql) = (literal(&start)?, literal(&end)?); + let (rendered, _) = resolve_cursor_intrinsics(py, &text, Some(&start_sql), Some(&end_sql))?; + Ok(PyString::new(py, &rendered).into_any()) +} + +/// Chain models using intrinsics, after validating the test's declared window against each. +pub(super) fn declared_window_cursor_models<'py>( + test: &Bound<'py, PyAny>, + chain_models: Vec>, +) -> PyResult>> { + let py = test.py(); + let mut cursor_models: Vec> = Vec::new(); + for model in chain_models { + if uses_cursor_intrinsics(py, &model.getattr("query_sql")?.extract::()?)? { + cursor_models.push(model); + } + } + if cursor_models.is_empty() { + return Err(compile_input_error( + py, + format!( + "{} declares cursor_start or cursor_end, but no model it evaluates uses __cursor_start() or __cursor_end()", + test_label(test)? + ), + )); + } + for model in &cursor_models { + test_window(&model_cursor(model)?, Some(test))?; + } + Ok(cursor_models) +} + +/// The start and exclusive end of one model's window, validating declared bounds. +fn test_window<'py>( + cursor: &ModelCursor<'py>, + test: Option<&Bound<'py, PyAny>>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let block = test.map(|test| test.getattr("test_block")).transpose()?; + let declared = |name: &str| -> PyResult>> { + let Some(block) = &block else { + return Ok(None); + }; + let value = block.getattr(name)?; + Ok((!value.is_none()).then_some(value)) + }; + let integer = cursor.type_name == INTEGER_CURSOR_TYPE; + let start = bound( + BoundRequest { + raw: declared(CURSOR_START_KEY)?, + default: if integer { + DEFAULT_INTEGER_START + } else { + DEFAULT_TIMESTAMP_START + }, + key: CURSOR_START_KEY, + }, + cursor, + test, + )?; + let end = bound( + BoundRequest { + raw: declared(CURSOR_END_KEY)?, + default: if integer { + DEFAULT_INTEGER_END + } else { + DEFAULT_TIMESTAMP_END + }, + key: CURSOR_END_KEY, + }, + cursor, + test, + )?; + if let Some(test) = test + && ordering(&start, &end)? != Ordering::Less + { + return Err(compile_input_error( + test.py(), + format!( + "{}: cursor_start '{}' must be before the exclusive cursor_end '{}' for model '{}'", + test_label(test)?, + render_value(&start)?, + render_value(&end)?, + cursor.name + ), + )); + } + Ok((start, end)) +} + +/// One parsed bound, or the parsed default when the test declares none. +fn bound<'py>( + request: BoundRequest<'_, 'py>, + cursor: &ModelCursor<'py>, + test: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let py = cursor.cursor_type.py(); + let (Some(raw), Some(test)) = (request.raw, test) else { + return parse_value( + &PyString::new(py, request.default).into_any(), + &cursor.cursor_type, + ); + }; + let value = match parse_value(&raw, &cursor.cursor_type) { + Ok(value) => value, + Err(error) + if error.value(py).is_instance(&imported( + py, + CURSOR_ALGEBRA_EXCEPTIONS, + "CursorAlgebraError", + )?)? => + { + let expected = if cursor.type_name == INTEGER_CURSOR_TYPE { + "an integer" + } else { + "an ISO timestamp or date" + }; + return Err(compile_input_error( + py, + format!( + "{}: {} '{}' must be {expected} because model '{}' uses cursor_type {}", + test_label(test)?, + request.key, + raw.str()?, + cursor.name, + cursor.type_name + ), + )); + } + Err(error) => return Err(error), + }; + if let Some((grain_name, grain)) = &cursor.grain { + let session = CursorSession::new(py)?; + let scalar: Scalar = session.scalar(&value)?; + let floored: Scalar = floor_to_grain(&scalar, Some(*grain)); + if session.checked(compare(&floored, &scalar, &session))? != Ordering::Equal { + return Err(compile_input_error( + py, + format!( + "{}: {} '{}' is not aligned to cursor_grain {grain_name} of model '{}'", + test_label(test)?, + request.key, + raw.str()?, + cursor.name + ), + )); + } + } + Ok(value) +} + +/// How two parsed cursor values order. +fn ordering(left: &Bound<'_, PyAny>, right: &Bound<'_, PyAny>) -> PyResult { + let session = CursorSession::new(left.py())?; + let (left, right) = (session.scalar(left)?, session.scalar(right)?); + session.checked(compare(&left, &right, &session)) +} + +/// The cursor type and timestamp grain a model's config declares. +fn model_cursor<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py = model.py(); + let values = model.getattr("config")?.getattr("values")?; + let declared = config_str(&values, "cursor_type")?.filter(|value| { + value + .extract::() + .is_ok_and(|value| !value.is_empty()) + }); + let cursor_type = imported(py, PLANNER_TYPES, "CursorType")?.call1(( + declared.unwrap_or_else(|| PyString::new(py, TIMESTAMP_CURSOR_TYPE).into_any()), + ))?; + let type_name: String = cursor_type.getattr("value")?.extract()?; + let grain = match config_str(&values, "cursor_grain")? { + Some(grain) if type_name == TIMESTAMP_CURSOR_TYPE => { + let grain = imported(py, PLANNER_TYPES, "CursorGrain")?.call1((grain,))?; + let session = CursorSession::new(py)?; + session + .grain(&grain)? + .map(|known| PyResult::Ok((grain.getattr("value")?.extract()?, known))) + .transpose()? + } + _ => None, + }; + Ok(ModelCursor { + name: model.getattr("name")?.extract()?, + cursor_type, + type_name, + grain, + }) +} + +/// `SQL test '' in ''`. +fn test_label(test: &Bound<'_, PyAny>) -> PyResult { + Ok(format!( + "SQL test '{}' in '{}'", + test.getattr("name")?.str()?, + test.getattr("test_file")?.getattr("relative_path")?.str()? + )) +} + +/// A `CompileInputError` carrying `message`. +fn compile_input_error(py: Python<'_>, message: String) -> PyErr { + match imported(py, COMPILE_EXCEPTIONS, "CompileInputError") + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index 410f1a68e7..7fe7199aaf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod assembly; pub(crate) mod authored_ctes; mod compiled_tests; mod constants; +mod cursor_window; mod plan_assembly; mod plan_driver; mod plan_entries; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs index 357f762f9b..6ecfaaac8d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs @@ -36,11 +36,7 @@ pub(crate) fn assemble_sql_test_plans<'py>( context = args.value("fixture_planning_context")?; if !context.is_truthy()? { context = call_with( - &imported( - py, - "sqlbuild._native", - "build_relation_fixture_context", - )?, + &imported(py, "sqlbuild._native", "build_relation_fixture_context")?, &[("project", project.clone())], )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs index 77159f0e5c..f39de53e1e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs @@ -5,7 +5,10 @@ use crate::bindings::_helpers::sql_test_glue::plan_values::{model_payload, strin use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use crate::bindings::_helpers::sql_test_glue::constants::{CURSOR_WINDOW, SENTINEL}; +use crate::bindings::_helpers::sql_test_glue::constants::SENTINEL; +use crate::bindings::_helpers::sql_test_glue::cursor_window::{ + declared_window_cursor_models, declares_cursor_window, render_test_cursor_intrinsics, +}; pub(super) fn mentions(sql: &str) -> bool { ["__cursor_start", "__cursor_end"] @@ -13,23 +16,6 @@ pub(super) fn mentions(sql: &str) -> bool { .any(|name| sql.contains(name)) } -fn render<'py>( - sql: Bound<'py, PyAny>, - model: Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, - test: Bound<'py, PyAny>, -) -> PyResult> { - call_with( - &imported(adapter.py(), CURSOR_WINDOW, "render_test_cursor_intrinsics")?, - &[ - ("sql", sql), - ("model", model), - ("adapter", adapter.clone()), - ("test", test), - ], - ) -} - pub(super) fn models<'py>( project: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, @@ -42,7 +28,7 @@ pub(super) fn models<'py>( if mentions(&sql.extract::()?) { result.set_item( index, - render(sql, model, adapter, py.None().into_bound(py))?, + render_test_cursor_intrinsics(sql, &model, adapter, None)?, )?; } } @@ -64,13 +50,7 @@ pub(super) fn overrides<'py>( let mut indexes: Vec = Vec::new(); let mut windowed: Vec> = Vec::new(); for (index, test) in tests.iter().enumerate() { - if model_payload(test)? - && call_with( - &imported(py, CURSOR_WINDOW, "declares_cursor_window")?, - &[("test", test.clone())], - )? - .is_truthy()? - { + if model_payload(test)? && declares_cursor_window(test)? { indexes.push(index); windowed.push(test.clone()); } @@ -115,15 +95,11 @@ pub(super) fn overrides<'py>( for (name, sql) in overrides.iter() { let name: String = name.extract()?; let sql = if let Some(model) = models.get_item(&name)? { - render( + render_test_cursor_intrinsics( sql, - model, + &model, adapter, - if chain_names.contains(&name) { - test.clone() - } else { - py.None().into_bound(py) - }, + chain_names.contains(&name).then_some(test), )? } else { sql @@ -137,20 +113,17 @@ pub(super) fn overrides<'py>( chain_models.push(model); } } - let cursor_models = call_with( - &imported(py, CURSOR_WINDOW, "declared_window_cursor_models")?, - &[ - ("test", test.clone()), - ("chain_models", PyTuple::new(py, chain_models)?.into_any()), - ], - )?; - for model in cursor_models.try_iter()? { - let model = model?; + for model in declared_window_cursor_models(test, chain_models)? { let name = model.getattr("name")?; if !rendered.contains(&name)? { rendered.set_item( name, - render(model.getattr("query_sql")?, model, adapter, test.clone())?, + render_test_cursor_intrinsics( + model.getattr("query_sql")?, + &model, + adapter, + Some(test), + )?, )?; } } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index bc41836f58..daa3467dd1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -233,11 +233,11 @@ from sqlbuild.compiler.planner.models import ( ModelMigrationRequest, ModelPlanContext, ModelPlanEntry, + NativeSqlTestArtifact, + NativeSqlTestPlan, OldNameView, OldNameViewPlanEntry, OldNameViewPlanning, - NativeSqlTestArtifact, - NativeSqlTestPlan, ParsedScenarioArtifactName, ParsedSelector, PathSelector, diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/comments.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/comments.py deleted file mode 100644 index 14418ad9da..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/comments.py +++ /dev/null @@ -1,48 +0,0 @@ -"""SQL comment handling for textual unit-test fallback resolution.""" - -from __future__ import annotations - -import re - -_SQL_NON_CODE_PATTERN: re.Pattern[str] = re.compile( - r"'(?:''|[^'])*'|\"(?:\"\"|[^\"])*\"|`(?:``|[^`])*`|" - r"\$\$.*?\$\$|--[^\n]*|/\*.*?\*/", - re.DOTALL, -) - - -def uncommented_pattern_matches(*, pattern: re.Pattern[str], sql: str) -> tuple[re.Match[str], ...]: - """Find pattern matches outside SQL comments and quoted values.""" - - return uncommented_matches_by_pattern(patterns=(pattern,), sql=sql)[0] - - -def uncommented_matches_by_pattern( - *, patterns: tuple[re.Pattern[str], ...], sql: str -) -> tuple[tuple[re.Match[str], ...], ...]: - """Find matches for multiple patterns with one protected-region scan.""" - - if all(pattern.search(sql) is None for pattern in patterns): - return tuple(() for _pattern in patterns) - protected_ranges: tuple[tuple[int, int], ...] = tuple( - (match.start(), match.end()) for match in _SQL_NON_CODE_PATTERN.finditer(sql) - ) - - def _matches(pattern: re.Pattern[str]) -> tuple[re.Match[str], ...]: - protected_index: int = 0 - matches: list[re.Match[str]] = [] - match: re.Match[str] - for match in pattern.finditer(sql): - while ( - protected_index < len(protected_ranges) - and protected_ranges[protected_index][1] <= match.start() - ): - protected_index += 1 - if ( - protected_index == len(protected_ranges) - or match.start() < protected_ranges[protected_index][0] - ): - matches.append(match) - return tuple(matches) - - return tuple(_matches(pattern) for pattern in patterns) diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py deleted file mode 100644 index 31bf70ada0..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Cursor windows rendered for models that SQL tests evaluate from their real SQL.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild._native import cursor_compare as compare -from sqlbuild._native import cursor_floor_to_grain as floor_to_grain -from sqlbuild._native import cursor_parse as parse -from sqlbuild._native import cursor_render as render -from sqlbuild._native import get_config_str, resolve_cursor_intrinsics -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompiledSqlTest -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner._helpers.resolve.cursor_intrinsics import ( - render_cursor_intrinsic_bounds, -) -from sqlbuild.compiler.planner.models import CursorBounds -from sqlbuild.compiler.planner.types import CursorGrain, CursorType -from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.types import CursorScalar - -_DEFAULT_TIMESTAMP_START: str = "1900-01-01 00:00:00" -_DEFAULT_TIMESTAMP_END: str = "2999-12-31 00:00:00" -_DEFAULT_INTEGER_START: str = "-1000000000000000" -_DEFAULT_INTEGER_END: str = "1000000000000000" -_CURSOR_START_KEY: str = "cursor_start" -_CURSOR_END_KEY: str = "cursor_end" - - -@dataclass(frozen=True) -class _ModelCursor: - name: str - cursor_type: CursorType - cursor_grain: CursorGrain | None - - -def uses_cursor_intrinsics(*, sql: str) -> bool: - """Return whether SQL calls `__cursor_start()` or `__cursor_end()`.""" - - _, found = resolve_cursor_intrinsics(sql=sql) - return found - - -def declares_cursor_window(*, test: CompiledSqlTest) -> bool: - """Return whether a SQL test header declares its own cursor window.""" - - return test.test_block.cursor_start is not None or test.test_block.cursor_end is not None - - -def render_test_cursor_intrinsics( - *, - sql: str, - model: CompiledModelView, - adapter: BaseAdapter, - test: CompiledSqlTest | None, -) -> str: - """Render cursor intrinsics with the test's declared window, or the wide default window.""" - - if not uses_cursor_intrinsics(sql=sql): - return sql - cursor: _ModelCursor = _model_cursor(model=model) - start, end = _test_window(cursor=cursor, test=test) - return render_cursor_intrinsic_bounds( - sql=sql, - bounds=CursorBounds(start=start, end=end), - cursor_type=cursor.cursor_type, - adapter=adapter, - ) - - -def declared_window_cursor_models( - *, test: CompiledSqlTest, chain_models: tuple[CompiledModelView, ...] -) -> tuple[CompiledModelView, ...]: - """Validate a declared window against every chain model using intrinsics and return them.""" - - cursor_models: tuple[CompiledModelView, ...] = tuple( - model for model in chain_models if uses_cursor_intrinsics(sql=model.query_sql) - ) - if not cursor_models: - raise CompileInputError( - f"{_test_label(test=test)} declares cursor_start or cursor_end, but no model it " - "evaluates uses __cursor_start() or __cursor_end()" - ) - for model in cursor_models: - _test_window(cursor=_model_cursor(model=model), test=test) - return cursor_models - - -def _test_window( - *, cursor: _ModelCursor, test: CompiledSqlTest | None -) -> tuple[CursorScalar, CursorScalar]: - declared_start: str | None = test.test_block.cursor_start if test is not None else None - declared_end: str | None = test.test_block.cursor_end if test is not None else None - integer: bool = cursor.cursor_type == CursorType.INTEGER - start: CursorScalar = _bound( - raw=declared_start, - default=_DEFAULT_INTEGER_START if integer else _DEFAULT_TIMESTAMP_START, - key=_CURSOR_START_KEY, - cursor=cursor, - test=test, - ) - end: CursorScalar = _bound( - raw=declared_end, - default=_DEFAULT_INTEGER_END if integer else _DEFAULT_TIMESTAMP_END, - key=_CURSOR_END_KEY, - cursor=cursor, - test=test, - ) - if test is not None and compare(left=start, right=end) >= 0: - raise CompileInputError( - f"{_test_label(test=test)}: cursor_start '{render(value=start)}' must be before " - f"the exclusive cursor_end '{render(value=end)}' for model '{cursor.name}'" - ) - return start, end - - -def _bound( - *, - raw: str | None, - default: str, - key: str, - cursor: _ModelCursor, - test: CompiledSqlTest | None, -) -> CursorScalar: - if raw is None or test is None: - return parse(raw=default, cursor_type=cursor.cursor_type) - try: - value: CursorScalar = parse(raw=raw, cursor_type=cursor.cursor_type) - except CursorAlgebraError: - expected: str = ( - "an integer" if cursor.cursor_type == CursorType.INTEGER else "an ISO timestamp or date" - ) - raise CompileInputError( - f"{_test_label(test=test)}: {key} '{raw}' must be {expected} because model " - f"'{cursor.name}' uses cursor_type {cursor.cursor_type.value}" - ) from None - if cursor.cursor_grain is not None and compare( - left=floor_to_grain(value=value, grain=cursor.cursor_grain), right=value - ): - raise CompileInputError( - f"{_test_label(test=test)}: {key} '{raw}' is not aligned to cursor_grain " - f"{cursor.cursor_grain.value} of model '{cursor.name}'" - ) - return value - - -def _model_cursor(*, model: CompiledModelView) -> _ModelCursor: - cursor_type: CursorType = CursorType( - get_config_str(values=model.config.values, key="cursor_type") or CursorType.TIMESTAMP - ) - grain: str | None = get_config_str(values=model.config.values, key="cursor_grain") - return _ModelCursor( - name=model.name, - cursor_type=cursor_type, - cursor_grain=( - CursorGrain(grain) - if grain is not None and cursor_type == CursorType.TIMESTAMP - else None - ), - ) - - -def _test_label(*, test: CompiledSqlTest) -> str: - return f"SQL test '{test.name}' in '{test.test_file.relative_path}'" diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index ef7dab0c6b..04f67da526 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -2,7 +2,6 @@ from __future__ import annotations -import re from pathlib import Path from sqlbuild.compiler.compile.constants import ( @@ -29,10 +28,13 @@ DiscoveredSqlTestBlock, DiscoveredSqlTestFile, ) -from sqlbuild.compiler.planner._helpers.sql_tests.comments import uncommented_pattern_matches +from sqlbuild.compiler.references.types import SqlReferenceKind from tests.integration.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( ExecuteChainTestCase, ) +from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly.helpers import ( + reference_names_by_kind, +) _STUB_TEST_FILE: DiscoveredSqlTestFile = DiscoveredSqlTestFile( file_path=Path("tests/unit/test_chain.sql"), @@ -109,12 +111,9 @@ def build_test_and_project( model_name: str query_sql: str for model_name, query_sql in test_case.model_queries.items(): - dependency_names: tuple[str, ...] = tuple( - match.group(1) - for match in uncommented_pattern_matches( - pattern=re.compile(r'__ref\("([^"]+)"\)'), sql=query_sql - ) - ) + dependency_names: tuple[str, ...] = reference_names_by_kind(query_sql=query_sql)[ + SqlReferenceKind.REF + ] dependencies: tuple[CompiledObjectKey, ...] = tuple( CompiledObjectKey( resource_type=CompiledResourceType.MODEL, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 3484d1dbfc..c17745d1c6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -2,7 +2,6 @@ from __future__ import annotations -import re from collections.abc import Callable from pathlib import Path from types import MappingProxyType @@ -10,6 +9,7 @@ from sqlbuild._native import ( expand_sql_macros, + extract_sql_references, plan_compiled_sql_test_artifacts, plan_sql_tests, plan_sql_tests_natively, @@ -50,14 +50,15 @@ DiscoveredSqlTestBlock, DiscoveredSqlTestFile, ) -from sqlbuild.compiler.planner._helpers.sql_tests.comments import uncommented_pattern_matches from sqlbuild.compiler.planner.models import ( NativeSqlTestPlan, PlanWarning, SqlTestPlanEntry, SqlTestPlanResult, ) +from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( PlanTestChainTestCase, @@ -380,25 +381,24 @@ def build_test_and_project( def _model_dependencies(*, query_sql: str) -> tuple[str, ...]: - return tuple( - dict.fromkeys( - match.group(1) - for match in uncommented_pattern_matches( - pattern=re.compile(r'__ref\("([^"]+)"\)'), sql=query_sql - ) - ) - ) + return _reference_names(query_sql=query_sql, kind=SqlReferenceKind.REF) def _table_function_dependencies(*, query_sql: str) -> tuple[str, ...]: - return tuple( - dict.fromkeys( - match.group(1) - for match in uncommented_pattern_matches( - pattern=re.compile(r'__table_fn\("([^"]+)"\)'), sql=query_sql - ) - ) - ) + return _reference_names(query_sql=query_sql, kind=SqlReferenceKind.TABLE_FUNCTION) + + +def _reference_names(*, query_sql: str, kind: SqlReferenceKind) -> tuple[str, ...]: + return reference_names_by_kind(query_sql=query_sql)[kind] + + +def reference_names_by_kind(*, query_sql: str) -> dict[SqlReferenceKind, tuple[str, ...]]: + """Return distinct logical reference names in authored order, grouped by reference kind.""" + + names: dict[SqlReferenceKind, dict[str, None]] = {kind: {} for kind in SqlReferenceKind} + for reference in extract_sql_references(sql=query_sql, syntax=SqlLexicalSyntax()): + names[SqlReferenceKind(reference.ref_kind)][reference.ref_name] = None + return {kind: tuple(kind_names) for kind, kind_names in names.items()} def _build_model_query_overrides( @@ -541,7 +541,9 @@ def build_windowed_sql_test(*, cursor_start: str | None, cursor_end: str | None) return CompiledSqlTest( key=CompiledObjectKey(resource_type=CompiledResourceType.SQL_TEST, name="orders_case"), - scope_deps=(), + scope_deps=( + CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="daily_orders"), + ), name="orders_case", test_file=DiscoveredSqlTestFile( file_path=Path("tests/unit/orders_case.sql"), @@ -556,5 +558,34 @@ def build_windowed_sql_test(*, cursor_start: str | None, cursor_end: str | None) cursor_start=cursor_start, cursor_end=cursor_end, ), - sql_body="", + sql_body=( + f"WITH {EXPECTED_TEST_CTE_PREFIX}daily_orders AS " + "(SELECT TIMESTAMP '2026-02-01' AS order_date) SELECT 1" + ), + payload=CompiledModelSqlTestPayload( + expected_model_names=("daily_orders",), + expected_ctes=( + CompileSqlTestCte( + name=f"{EXPECTED_TEST_CTE_PREFIX}daily_orders", + sql_body="SELECT TIMESTAMP '2026-02-01' AS order_date", + ), + ), + ), + ) + + +def plan_cursor_window_model_sql( + *, model: CompiledModelView, test: CompiledSqlTest, adapter: BaseAdapter +) -> str: + """Plan one windowed SQL test and return the rendered SQL of its cursor model step.""" + + project: CompiledProjectView = CompiledProject( + run_id="test_run", + effective_target_name=None, + effective_connection={}, + effective_vars={}, + models=(model,), ) + entry, _ = plan_single_test(test=test, project=project, adapter=adapter) + (step,) = entry.chain + return step.resolved_sql diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_cursor_window.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_cursor_window.py index 6532187bc8..933806dbd0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_cursor_window.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_cursor_window.py @@ -4,17 +4,14 @@ import pytest -from sqlbuild.compiler.planner._helpers.sql_tests.cursor_window import ( - render_test_cursor_intrinsics, -) from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( CursorWindowRenderingTestCase, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly.helpers import ( CURSOR_WINDOW_ADAPTERS, - CURSOR_WINDOW_MODEL_SQL, build_cursor_model, build_windowed_sql_test, + plan_cursor_window_model_sql, ) @@ -63,13 +60,12 @@ def test_given_model_using_cursor_intrinsics_when_rendering_default_window_then_bounds_are_typed( test_case: CursorWindowRenderingTestCase, ) -> None: - rendered: str = render_test_cursor_intrinsics( - sql=CURSOR_WINDOW_MODEL_SQL, + rendered: str = plan_cursor_window_model_sql( model=build_cursor_model( cursor_type=test_case.cursor_type, cursor_grain=test_case.cursor_grain ), + test=build_windowed_sql_test(cursor_start=None, cursor_end=None), adapter=CURSOR_WINDOW_ADAPTERS[test_case.adapter_name], - test=None, ) assert rendered == test_case.expected_sql @@ -96,15 +92,14 @@ def test_given_model_using_cursor_intrinsics_when_rendering_default_window_then_ def test_given_declared_test_window_when_rendering_then_declared_bounds_replace_defaults( test_case: CursorWindowRenderingTestCase, ) -> None: - rendered: str = render_test_cursor_intrinsics( - sql=CURSOR_WINDOW_MODEL_SQL, + rendered: str = plan_cursor_window_model_sql( model=build_cursor_model( cursor_type=test_case.cursor_type, cursor_grain=test_case.cursor_grain ), - adapter=CURSOR_WINDOW_ADAPTERS[test_case.adapter_name], test=build_windowed_sql_test( cursor_start=test_case.cursor_start, cursor_end=test_case.cursor_end ), + adapter=CURSOR_WINDOW_ADAPTERS[test_case.adapter_name], ) assert rendered == test_case.expected_sql From bf192a3e0c3999c6e6dddf658cdacd28bf7608ed Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:08:06 +0100 Subject: [PATCH 12/22] refactor: reuse native compile input errors for SQL-test cursor windows --- .../_helpers/attachments/cursor_intrinsics.rs | 2 +- .../_helpers/sql_test_glue/constants.rs | 1 - .../_helpers/sql_test_glue/cursor_window.rs | 30 +++++++------------ 3 files changed, 12 insertions(+), 21 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs index 12a91b34c6..332a25bccd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs @@ -31,7 +31,7 @@ pub(crate) fn reserved_cursor_markers() -> Vec { } /// A `CompileInputError` carrying `message`. -fn compile_input_error(py: Python<'_>, message: String) -> PyResult { +pub(crate) fn compile_input_error(py: Python<'_>, message: String) -> PyResult { Ok(PyErr::from_value( imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?.call1((message,))?, )) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs index b91d4d24b3..8c1e8c3318 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs @@ -9,7 +9,6 @@ pub(super) const PLAN_MODELS: &str = "sqlbuild.compiler.planner.models"; pub(super) const NATIVE_PLANNING: &str = "sqlbuild._native"; pub(super) const FIXTURE_VALIDATION: &str = "sqlbuild.compiler.planner._helpers.sql_tests.fixture_validation"; -pub(super) const COMPILE_EXCEPTIONS: &str = "sqlbuild.compiler.compile.exceptions"; pub(super) const CURSOR_ALGEBRA_EXCEPTIONS: &str = "sqlbuild.cursor_algebra.exceptions"; pub(super) const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; pub(super) const INTEGER_CURSOR_TYPE: &str = "integer"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs index fb345d4fd0..6c1ae72432 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs @@ -2,21 +2,23 @@ use std::cmp::Ordering; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyString; use sqlbuild_cursor::algebra::main::compare::compare; use sqlbuild_cursor::algebra::main::floor_to_grain::floor_to_grain; use sqlbuild_cursor::algebra::models::{Grain, Scalar}; -use crate::bindings::_helpers::attachments::cursor_intrinsics::resolve_cursor_intrinsics; +use crate::bindings::_helpers::attachments::cursor_intrinsics::{ + compile_input_error, resolve_cursor_intrinsics, +}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::cursor::conversion::CursorSession; use crate::bindings::_helpers::cursor::scalars::{parse_value, render_value}; use crate::bindings::_helpers::model_config::config_values::config_str; use crate::bindings::_helpers::sql_test_glue::constants::{ - COMPILE_EXCEPTIONS, CURSOR_ALGEBRA_EXCEPTIONS, CURSOR_END_KEY, CURSOR_START_KEY, - DEFAULT_INTEGER_END, DEFAULT_INTEGER_START, DEFAULT_TIMESTAMP_END, DEFAULT_TIMESTAMP_START, - INTEGER_CURSOR_TYPE, PLANNER_TYPES, TIMESTAMP_CURSOR_TYPE, + CURSOR_ALGEBRA_EXCEPTIONS, CURSOR_END_KEY, CURSOR_START_KEY, DEFAULT_INTEGER_END, + DEFAULT_INTEGER_START, DEFAULT_TIMESTAMP_END, DEFAULT_TIMESTAMP_START, INTEGER_CURSOR_TYPE, + PLANNER_TYPES, TIMESTAMP_CURSOR_TYPE, }; /// One model's cursor name, type and timestamp grain. @@ -93,7 +95,7 @@ pub(super) fn declared_window_cursor_models<'py>( "{} declares cursor_start or cursor_end, but no model it evaluates uses __cursor_start() or __cursor_end()", test_label(test)? ), - )); + )?); } for model in &cursor_models { test_window(&model_cursor(model)?, Some(test))?; @@ -153,7 +155,7 @@ fn test_window<'py>( render_value(&end)?, cursor.name ), - )); + )?); } Ok((start, end)) } @@ -195,7 +197,7 @@ fn bound<'py>( cursor.name, cursor.type_name ), - )); + )?); } Err(error) => return Err(error), }; @@ -213,7 +215,7 @@ fn bound<'py>( raw.str()?, cursor.name ), - )); + )?); } } Ok(value) @@ -266,13 +268,3 @@ fn test_label(test: &Bound<'_, PyAny>) -> PyResult { test.getattr("test_file")?.getattr("relative_path")?.str()? )) } - -/// A `CompileInputError` carrying `message`. -fn compile_input_error(py: Python<'_>, message: String) -> PyErr { - match imported(py, COMPILE_EXCEPTIONS, "CompileInputError") - .and_then(|class| class.call1((message,))) - { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } -} From bb5073c7b643eb68d18258bd5c1ed45efe1aa709 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:09:25 +0100 Subject: [PATCH 13/22] chore: update Python boundary after SQL-test planning port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 9e3381174b..72f5982736 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 31284 +remaining_lines = 30318 [[stays]] path = "executor" From 092ab26a2c964c503c243c83eadd43999f59d5a0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:02:32 +0100 Subject: [PATCH 14/22] refactor: project and decode migration state events natively --- Cargo.lock | 1 + .../planner_migrations/column_entries.rs | 21 +- .../planner_migrations/discovery_events.rs | 10 +- .../planner_migrations/model_entries.rs | 13 +- .../planner_migrations/old_name_objects.rs | 13 +- .../bindings/_helpers/state_rows/constants.rs | 12 + .../bindings/_helpers/state_rows/events.rs | 65 +-- .../_helpers/state_rows/migration_objects.rs | 89 ++++ .../state_rows/migration_projections.rs | 191 ++++++++ .../_helpers/state_rows/migration_rows.rs | 437 ++++++++++++++++++ .../src/bindings/_helpers/state_rows/mod.rs | 5 + .../bindings/_helpers/state_rows/models.rs | 20 + .../_helpers/state_rows/registration.rs | 6 +- crates/sqlbuild-state/Cargo.toml | 1 + .../sqlbuild-state/src/events/_helpers/mod.rs | 2 + .../src/events/_helpers/python_text.rs | 21 + .../src/events/_helpers/relations.rs | 34 ++ crates/sqlbuild-state/src/events/constants.rs | 63 +++ .../src/events/main/column_aliases_json.rs | 31 ++ .../src/events/main/decoded_column_aliases.rs | 22 + .../src/events/main/decoded_grants.rs | 14 + .../src/events/main/event_cells.rs | 22 + .../src/events/main/event_read_sql.rs | 10 + .../src/events/main/grants_json.rs | 14 + crates/sqlbuild-state/src/events/main/mod.rs | 10 + .../src/events/main/newest_column_event.rs | 28 ++ .../src/events/main/newest_move_event.rs | 19 + .../src/events/main/old_name_histories.rs | 41 ++ .../src/events/main/readable_columns.rs | 14 + crates/sqlbuild-state/src/events/models.rs | 51 ++ crates/sqlbuild-state/src/events/tests.rs | 4 + .../src/events/tests/helpers.rs | 68 ++- .../src/events/tests/test_codecs.rs | 159 +++++++ .../src/events/tests/test_projections.rs | 138 ++++++ .../src/events/tests/test_types.rs | 56 ++- src/sqlbuild/_native.pyi | 64 ++- .../migrations/_helpers/column_sql.py | 129 ------ .../migrations/_helpers/old_name_sql.py | 204 -------- .../compiler/migrations/_helpers/sql.py | 149 ------ src/sqlbuild/compiler/migrations/constants.py | 75 --- .../migrations/main/_newest_column_event.py | 30 -- .../compiler/migrations/main/_newest_event.py | 25 - .../main/_project_old_name_views.py | 40 -- .../migrations/main/_read_column_events.py | 20 +- .../compiler/migrations/main/_read_events.py | 10 +- .../main/_resolve_old_name_view_retention.py | 20 - .../main/column_create_table_sql.py | 5 +- .../migrations/main/create_table_sql.py | 5 +- .../main/old_name_view_create_table_sql.py | 5 +- .../main/read_old_name_view_events.py | 22 +- .../main/read_old_name_view_histories.py | 2 +- .../migrations/main/write_column_event.py | 18 +- .../compiler/migrations/main/write_event.py | 13 +- .../main/write_old_name_view_event.py | 18 +- .../classes/migration_state_inspection.py | 3 +- .../compiler/migrations/main/_test_types.py | 20 +- .../main/test_column_migration_events.py | 4 +- .../migrations/main/test_migration_events.py | 2 +- .../main/test_old_name_view_events.py | 10 +- .../migrations/main/test_stored_event_rows.py | 135 ++++++ 60 files changed, 1881 insertions(+), 852 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_projections.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/state_rows/models.rs create mode 100644 crates/sqlbuild-state/src/events/_helpers/python_text.rs create mode 100644 crates/sqlbuild-state/src/events/_helpers/relations.rs create mode 100644 crates/sqlbuild-state/src/events/main/column_aliases_json.rs create mode 100644 crates/sqlbuild-state/src/events/main/decoded_column_aliases.rs create mode 100644 crates/sqlbuild-state/src/events/main/decoded_grants.rs create mode 100644 crates/sqlbuild-state/src/events/main/event_cells.rs create mode 100644 crates/sqlbuild-state/src/events/main/event_read_sql.rs create mode 100644 crates/sqlbuild-state/src/events/main/grants_json.rs create mode 100644 crates/sqlbuild-state/src/events/main/newest_column_event.rs create mode 100644 crates/sqlbuild-state/src/events/main/newest_move_event.rs create mode 100644 crates/sqlbuild-state/src/events/main/old_name_histories.rs create mode 100644 crates/sqlbuild-state/src/events/main/readable_columns.rs create mode 100644 crates/sqlbuild-state/src/events/tests/test_codecs.rs create mode 100644 crates/sqlbuild-state/src/events/tests/test_projections.rs delete mode 100644 src/sqlbuild/compiler/migrations/_helpers/column_sql.py delete mode 100644 src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py delete mode 100644 src/sqlbuild/compiler/migrations/_helpers/sql.py delete mode 100644 src/sqlbuild/compiler/migrations/main/_newest_column_event.py delete mode 100644 src/sqlbuild/compiler/migrations/main/_newest_event.py delete mode 100644 src/sqlbuild/compiler/migrations/main/_project_old_name_views.py delete mode 100644 src/sqlbuild/compiler/migrations/main/_resolve_old_name_view_retention.py create mode 100644 tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py diff --git a/Cargo.lock b/Cargo.lock index 8736887afc..14631c2f4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1354,6 +1354,7 @@ dependencies = [ name = "sqlbuild-state" version = "0.156.1" dependencies = [ + "serde_json", "sha2", "sqlbuild-core", "sqlbuild-cursor", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs index 64584d334c..138507dbd4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs @@ -7,6 +7,7 @@ use crate::bindings::_helpers::planner_migrations::column_objects::{ }; use crate::bindings::_helpers::planner_migrations::constants::{AUTOMATIC, DONE, MIGRATION_TYPES}; use crate::bindings::_helpers::planner_migrations::models::{CheckedColumns, ColumnCandidate}; +use crate::bindings::_helpers::state_rows::migration_projections::newest_column; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; use sqlbuild_analysis::planner_migrations::main::column_decision::column_decision; @@ -60,24 +61,18 @@ pub(super) fn decide<'py>( "migration_relation_for_location", )? .call1((candidate.model.getattr("destination")?,))?; - let newest_event = imported( - py, - "sqlbuild.compiler.migrations.main._newest_column_event", - "newest_column_migration_event_mentioning", - )?; let mut entries: Vec> = Vec::new(); for (origin_name, destination, discovery) in pairs { let origin_key = lower(&origin_name)?; let origin = candidate.warehouse.get_item(&origin_key)?; let has_destination = candidate.warehouse.contains(lower(&destination)?)?; - let newest = call_with( - &newest_event, - &[ - ("events", events.clone()), - ("relation", relation.clone()), - ("origin_column", origin_name.clone()), - ("destination_column", destination.clone()), - ], + let newest = newest_column( + events, + &relation, + ( + &origin_name.extract::()?, + &destination.extract::()?, + ), )?; let recorded = !newest.is_none() && call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs index 649ca5f5ed..5810366377 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::planner_migrations::column_objects::config; +use crate::bindings::_helpers::state_rows::migration_projections::newest_move; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; @@ -25,14 +26,7 @@ pub(super) fn newest<'py>( "migration_relation_for_location", )? .call1((location,))?; - call_with( - &imported( - location.py(), - "sqlbuild.compiler.migrations.main._newest_event", - "newest_migration_event_mentioning", - )?, - &[("events", events.clone()), ("relation", relation)], - ) + newest_move(events, &relation) } pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs index d426b7231b..4142649951 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs @@ -10,6 +10,7 @@ use crate::bindings::_helpers::planner_migrations::constants::MIGRATION_TYPES; use crate::bindings::_helpers::planner_migrations::model_handover::{ applies, empty_version, handover, }; +use crate::bindings::_helpers::state_rows::migration_projections::newest_move; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyBool, PyTuple}; use sqlbuild_analysis::planner_migrations::main::model_decision::model_decision; @@ -84,17 +85,7 @@ pub(super) fn decide<'py>(args: &KeywordArguments<'py>) -> PyResult(location: &Bound<'py, PyAny>) -> PyResult> { @@ -48,17 +49,7 @@ pub(super) fn retention<'py>( item: &Bound<'py, PyAny>, project: &Bound<'py, PyAny>, ) -> PyResult> { - call_with( - &imported( - item.py(), - "sqlbuild.compiler.migrations.main._resolve_old_name_view_retention", - "resolve_old_name_view_retention", - )?, - &[ - ("config_values", item.getattr("config")?.getattr("values")?), - ("project_retention", project.clone()), - ], - ) + old_name_retention(&item.getattr("config")?.getattr("values")?, project) } pub(super) fn expires<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs new file mode 100644 index 0000000000..5e23a58eda --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs @@ -0,0 +1,12 @@ +//! Python modules, names and messages of migration state rows. + +pub(crate) const MIGRATION_MODELS: &str = "sqlbuild.compiler.migrations.models"; +pub(crate) const MIGRATION_TYPES: &str = "sqlbuild.compiler.migrations.types"; +pub(crate) const MIGRATION_EXCEPTIONS: &str = "sqlbuild.compiler.migrations.exceptions"; +pub(crate) const STATE_TABLE_DDL: &str = + "sqlbuild.adapter.state_sql.main.render_state_table_create_sql"; +pub(crate) const SQL_VALUE_TYPES: &str = "sqlbuild.sql_values.types"; +pub(crate) const OLD_NAME_VIEW_CONFIG_KEY: &str = "old_name_view"; +pub(crate) const MODEL_KIND: &str = "model"; +pub(crate) const COLUMN_KIND: &str = "column"; +pub(crate) const OLD_NAME_KIND: &str = "old_name"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs index 5b0c5d4001..dded2008a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs @@ -1,4 +1,4 @@ -//! Migration event IDs and the SQL that appends and looks up one event row. +//! Migration event IDs and the typed literals of one appended event row. use pyo3::exceptions::PyValueError; use pyo3::prelude::{ @@ -9,7 +9,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::json::errors::JsonEmitError; use sqlbuild_state::events::errors::StateSqlError; use sqlbuild_state::events::main::column_migration_event_id::column_migration_event_id; -use sqlbuild_state::events::main::event_exists_sql::event_exists_sql; +use sqlbuild_state::events::main::event_cells::event_cells; use sqlbuild_state::events::main::event_insert_sql::event_insert_sql; use sqlbuild_state::events::main::migration_event_id::migration_event_id; use sqlbuild_state::events::main::old_name_view_event_id::old_name_view_event_id; @@ -20,14 +20,12 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::state_rows::timestamps::{date_time_typed, presumed_utc}; const SQL_VALUES_EXCEPTIONS: &str = "sqlbuild.sql_values.exceptions"; -const STRING_TYPE: &str = "string"; -const TEXT_TIMESTAMP_TYPE: &str = "text_timestamp"; fn emit_error(error: JsonEmitError) -> PyErr { PyValueError::new_err(format!("{error:?}")) } -fn relation(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn relation(value: &Bound<'_, PyAny>) -> PyResult { Ok(EventRelation { database: value.getattr("database")?.extract()?, schema: value.getattr("schema")?.extract()?, @@ -77,13 +75,6 @@ fn old_name_id(event_type: &Bound<'_, PyAny>, migration_event_id: &str) -> PyRes old_name_view_event_id(&event_type, migration_event_id).map_err(emit_error) } -/// The lookup that makes an append-only event write idempotent. -#[pyfunction(name = "render_event_exists_sql")] -#[pyo3(signature = (*, table, event_id))] -fn exists_sql(table: &str, event_id: &str) -> String { - event_exists_sql(table, event_id) -} - fn state_value(value: &Bound<'_, PyAny>, column_type: StateColumnType) -> PyResult { if value.is_none() { return Ok(StateValue::Null); @@ -97,49 +88,23 @@ fn state_value(value: &Bound<'_, PyAny>, column_type: StateColumnType) -> PyResu Ok(StateValue::Other) } -fn column_type(declared: &Bound<'_, PyAny>) -> PyResult { - let name: String = declared.str()?.extract()?; - match name.as_str() { - STRING_TYPE => Ok(StateColumnType::String), - TEXT_TIMESTAMP_TYPE => Ok(StateColumnType::TextTimestamp), - _ => Err(PyValueError::new_err(format!( - "migration event columns are string or text timestamps, not {name}" - ))), - } -} - -/// One append-only event row as `INSERT ... VALUES` with typed literals. -#[pyfunction(name = "render_event_insert_sql")] -#[pyo3(signature = (*, table, columns, column_types, values))] -fn insert_sql( +/// One append-only event row as `INSERT ... VALUES` with typed literals, failing like Python. +pub(crate) fn typed_insert_sql( + py: Python<'_>, table: &str, - columns: Vec, - column_types: &Bound<'_, PyAny>, - values: Vec>, + (columns, values): (&[&str], &[Bound<'_, PyAny>]), ) -> PyResult { - if columns.len() != values.len() { - return Err(PyValueError::new_err( - "event columns and values must have the same length", - )); - } - let mut cells: Vec = Vec::with_capacity(columns.len()); - for (column, value) in columns.into_iter().zip(&values) { - let column_type: StateColumnType = column_type(&column_types.get_item(&column)?)?; - let converted: PyResult = state_value(value, column_type); - let value: StateValue = match converted { - Ok(value) => value, + let mut cells: Vec = event_cells(columns, vec![StateValue::Null; values.len()]); + for (index, value) in values.iter().enumerate() { + match state_value(value, cells[index].column_type) { + Ok(converted) => cells[index].value = converted, Err(error) => { - event_insert_sql(table, &cells).map_err(rendering_error(value.py()))?; + event_insert_sql(table, &cells[..index]).map_err(rendering_error(py))?; return Err(error); } - }; - cells.push(StateCell { - column, - column_type, - value, - }); + } } - event_insert_sql(table, &cells).map_err(rendering_error(column_types.py())) + event_insert_sql(table, &cells).map_err(rendering_error(py)) } /// `SqlValueRenderingError(message)` for an ill-typed value. @@ -158,7 +123,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(migration_id, module)?)?; module.add_function(wrap_pyfunction!(column_id, module)?)?; module.add_function(wrap_pyfunction!(old_name_id, module)?)?; - module.add_function(wrap_pyfunction!(exists_sql, module)?)?; - module.add_function(wrap_pyfunction!(insert_sql, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs new file mode 100644 index 0000000000..a0fff30ca8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs @@ -0,0 +1,89 @@ +//! Migration state table layouts and the Python objects stored in them. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_state::events::constants::{ + COLUMN_MIGRATION_COLUMNS, COLUMN_MIGRATION_TABLE_NAME, MIGRATION_COLUMNS, MIGRATION_TABLE_NAME, + OLD_NAME_VIEW_COLUMNS, OLD_NAME_VIEW_TABLE_NAME, +}; +use sqlbuild_state::events::models::EventOrder; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::state_rows::constants::{ + COLUMN_KIND, MIGRATION_EXCEPTIONS, MODEL_KIND, OLD_NAME_KIND, +}; +use crate::bindings::_helpers::state_rows::models::{EventTable, EventTableLayout}; +use crate::bindings::_helpers::state_rows::timestamps::presumed_utc; + +/// The layout of the state table `kind` names: `model`, `column` or `old_name`. +pub(crate) fn layout(kind: &str) -> PyResult { + match kind { + MODEL_KIND => Ok(EventTableLayout { + table: EventTable::Model, + name: MIGRATION_TABLE_NAME, + columns: &MIGRATION_COLUMNS, + relation_attribute: "destination", + state_message: "model migration state requires a target schema", + event_message: "model migration events require a destination schema", + }), + COLUMN_KIND => Ok(EventTableLayout { + table: EventTable::Column, + name: COLUMN_MIGRATION_TABLE_NAME, + columns: &COLUMN_MIGRATION_COLUMNS, + relation_attribute: "relation", + state_message: "column migration state requires a target schema", + event_message: "column migration events require a relation schema", + }), + OLD_NAME_KIND => Ok(EventTableLayout { + table: EventTable::OldName, + name: OLD_NAME_VIEW_TABLE_NAME, + columns: &OLD_NAME_VIEW_COLUMNS, + relation_attribute: "new", + state_message: "old-name view state requires a target schema", + event_message: "old-name view facts require a destination schema", + }), + _ => Err(PyValueError::new_err(format!( + "unknown migration state table kind: {kind}" + ))), + } +} + +/// `MigrationStateError(message)`. +pub(crate) fn state_error(py: Python<'_>, message: &str) -> PyErr { + match imported(py, MIGRATION_EXCEPTIONS, "MigrationStateError") + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// The adapter-qualified name of `layout`'s table in one schema. +pub(crate) fn qualified_table<'py>( + layout: &EventTableLayout, + render_qualified_name: &Bound<'py, PyAny>, + (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult { + let py: Python<'py> = render_qualified_name.py(); + let table: Bound<'py, PyAny> = call_with( + render_qualified_name, + &[ + ("database", database), + ("schema", schema), + ("name", PyString::new(py, layout.name).into_any()), + ], + )?; + if table.is_none() { + return Err(state_error(py, layout.state_message)); + } + table.extract() +} + +/// The history order of a stored event: its creation time in UTC, then its ID. +pub(crate) fn event_order(event: &Bound<'_, PyAny>) -> PyResult { + Ok(EventOrder { + created_at: presumed_utc(&event.getattr("created_at")?)?, + event_id: event.getattr("event_id")?.extract()?, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_projections.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_projections.rs new file mode 100644 index 0000000000..cb87db8e4c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_projections.rs @@ -0,0 +1,191 @@ +//! Current-state projections over recorded migration events. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_state::events::main::newest_column_event::newest_column_event; +use sqlbuild_state::events::main::newest_move_event::newest_move_event; +use sqlbuild_state::events::main::old_name_histories::old_name_histories; +use sqlbuild_state::events::models::{ + ColumnEventKey, EventOrder, EventRelation, MoveEventKey, OldNameFactKey, OldNameStep, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::state_rows::constants::{ + MIGRATION_MODELS, OLD_NAME_VIEW_CONFIG_KEY, +}; +use crate::bindings::_helpers::state_rows::events::relation as event_relation; +use crate::bindings::_helpers::state_rows::migration_objects::event_order; + +fn collected<'py>(events: &Bound<'py, PyAny>) -> PyResult>> { + events.try_iter()?.collect() +} + +fn chosen<'py>( + py: Python<'py>, + events: &[Bound<'py, PyAny>], + index: Option, +) -> Bound<'py, PyAny> { + index + .and_then(|index| events.get(index).cloned()) + .unwrap_or_else(|| py.None().into_bound(py)) +} + +/// The latest recorded move naming `relation` as origin or destination, or `None`. +pub(crate) fn newest_move<'py>( + events: &Bound<'py, PyAny>, + relation: &Bound<'py, PyAny>, +) -> PyResult> { + let events: Vec> = collected(events)?; + let keys: Vec = events + .iter() + .map(|event| { + Ok(MoveEventKey { + order: event_order(event)?, + origin: event_relation(&event.getattr("origin")?)?, + destination: event_relation(&event.getattr("destination")?)?, + }) + }) + .collect::>()?; + let index: Option = newest_move_event(&keys, &event_relation(relation)?); + Ok(chosen(relation.py(), &events, index)) +} + +/// The latest recorded rename on `relation` naming either column, or `None`. +pub(crate) fn newest_column<'py>( + events: &Bound<'py, PyAny>, + relation: &Bound<'py, PyAny>, + (origin_column, destination_column): (&str, &str), +) -> PyResult> { + let events: Vec> = collected(events)?; + let keys: Vec = events + .iter() + .map(|event| { + Ok(ColumnEventKey { + order: event_order(event)?, + relation: event_relation(&event.getattr("relation")?)?, + origin_column: event.getattr("origin_column")?.extract()?, + destination_column: event.getattr("destination_column")?.extract()?, + }) + }) + .collect::>()?; + let target: EventRelation = event_relation(relation)?; + let index: Option = + newest_column_event(&keys, &target, (origin_column, destination_column)); + Ok(chosen(relation.py(), &events, index)) +} + +fn step(fact: &Bound<'_, PyAny>) -> PyResult { + let value: String = fact.getattr("event_type")?.getattr("value")?.extract()?; + match value.as_str() { + "required" => Ok(OldNameStep::Required), + "origin_archived" => Ok(OldNameStep::OriginArchived), + "view_created" => Ok(OldNameStep::ViewCreated), + "view_dropped" => Ok(OldNameStep::ViewDropped), + _ => Err(PyValueError::new_err(format!( + "unknown old-name view event type: {value}" + ))), + } +} + +/// One history per recorded move whose old name needs a compatibility view, in move order. +pub(crate) fn old_name_views<'py>( + moves: &Bound<'py, PyAny>, + facts: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = moves.py(); + let moves: Vec> = collected(moves)?; + let facts: Vec> = collected(facts)?; + let move_orders: Vec = moves.iter().map(event_order).collect::>()?; + let fact_keys: Vec = facts + .iter() + .map(|fact| { + Ok(OldNameFactKey { + order: event_order(fact)?, + migration_event_id: fact.getattr("migration_event_id")?.extract()?, + step: step(fact)?, + }) + }) + .collect::>()?; + let history = imported(py, MIGRATION_MODELS, "OldNameViewHistory")?; + let fact_at = |index: Option| chosen(py, &facts, index); + let histories: Vec> = old_name_histories(&move_orders, &fact_keys) + .into_iter() + .map(|indexes| { + call_with( + &history, + &[ + ("move", moves[indexes.moved].clone()), + ("required", facts[indexes.required].clone()), + ("archived", fact_at(indexes.archived)), + ("created", fact_at(indexes.created)), + ("dropped", fact_at(indexes.dropped)), + ], + ) + }) + .collect::>()?; + PyTuple::new(py, histories) +} + +/// The model's old-name view retention, the project's when unset, or `None` when disabled. +pub(crate) fn old_name_retention<'py>( + config_values: &Bound<'py, PyAny>, + project_retention: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = config_values.py(); + if !config_values.contains(OLD_NAME_VIEW_CONFIG_KEY)? { + return Ok(project_retention.clone()); + } + let value: Bound<'py, PyAny> = config_values.get_item(OLD_NAME_VIEW_CONFIG_KEY)?; + if value.is(PyBool::new(py, false)) { + return Ok(py.None().into_bound(py)); + } + value.str()?.call_method0("strip") +} + +#[pyfunction(name = "newest_migration_event_mentioning")] +#[pyo3(signature = (*, events, relation))] +fn newest_move_binding<'py>( + events: &Bound<'py, PyAny>, + relation: &Bound<'py, PyAny>, +) -> PyResult> { + newest_move(events, relation) +} + +#[pyfunction(name = "newest_column_migration_event_mentioning")] +#[pyo3(signature = (*, events, relation, origin_column, destination_column))] +fn newest_column_binding<'py>( + events: &Bound<'py, PyAny>, + relation: &Bound<'py, PyAny>, + origin_column: &str, + destination_column: &str, +) -> PyResult> { + newest_column(events, relation, (origin_column, destination_column)) +} + +#[pyfunction(name = "project_old_name_views")] +#[pyo3(signature = (*, moves, facts))] +fn old_name_views_binding<'py>( + moves: &Bound<'py, PyAny>, + facts: &Bound<'py, PyAny>, +) -> PyResult> { + old_name_views(moves, facts) +} + +#[pyfunction(name = "resolve_old_name_view_retention")] +#[pyo3(signature = (*, config_values, project_retention))] +fn retention_binding<'py>( + config_values: &Bound<'py, PyAny>, + project_retention: &Bound<'py, PyAny>, +) -> PyResult> { + old_name_retention(config_values, project_retention) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(newest_move_binding, module)?)?; + module.add_function(wrap_pyfunction!(newest_column_binding, module)?)?; + module.add_function(wrap_pyfunction!(old_name_views_binding, module)?)?; + module.add_function(wrap_pyfunction!(retention_binding, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs new file mode 100644 index 0000000000..279c3c9e54 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs @@ -0,0 +1,437 @@ +//! Create, read and append SQL of migration state tables, and stored rows decoded to events. + +use pyo3::exceptions::{PyKeyError, PyValueError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_state::events::constants::TIMESTAMP_COLUMNS; +use sqlbuild_state::events::main::column_aliases_json::column_aliases_json; +use sqlbuild_state::events::main::decoded_column_aliases::decoded_column_aliases; +use sqlbuild_state::events::main::decoded_grants::decoded_grants; +use sqlbuild_state::events::main::event_exists_sql::event_exists_sql; +use sqlbuild_state::events::main::event_read_sql::event_read_sql; +use sqlbuild_state::events::main::grants_json::grants_json; +use sqlbuild_state::events::main::readable_columns::readable_columns; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; +use crate::bindings::_helpers::state_rows::constants::{ + MIGRATION_MODELS, MIGRATION_TYPES, SQL_VALUE_TYPES, STATE_TABLE_DDL, +}; +use crate::bindings::_helpers::state_rows::events::typed_insert_sql; +use crate::bindings::_helpers::state_rows::migration_objects::{ + layout, qualified_table, state_error, +}; +use crate::bindings::_helpers::state_rows::models::{EventTable, EventTableLayout}; +use crate::bindings::_helpers::state_rows::timestamps::date_time_typed; + +/// DDL that creates one migration state table when it is missing. +#[pyfunction(name = "render_migration_state_create_sql")] +#[pyo3(signature = (**request))] +fn create_sql<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("render_migration_state_create_sql", request)?; + let py: Python<'py> = request.py(); + let layout: EventTableLayout = layout(&request.get("kind")?.extract::()?)?; + let table: String = qualified_table( + &layout, + &request.get("render_qualified_name")?, + (request.get("database")?, request.get("schema")?), + )?; + let value_types: Bound<'py, PyAny> = imported(py, SQL_VALUE_TYPES, "StateSqlValueType")?; + let column_types: Bound<'py, PyDict> = PyDict::new(py); + for column in layout.columns { + let member: &str = if TIMESTAMP_COLUMNS.contains(column) { + "TEXT_TIMESTAMP" + } else { + "STRING" + }; + column_types.set_item(*column, value_types.getattr(member)?)?; + } + call_with( + &imported(py, STATE_TABLE_DDL, "render_state_table_create_sql")?, + &[ + ("qualified_name", PyString::new(py, &table).into_any()), + ("columns", PyTuple::new(py, layout.columns)?.into_any()), + ("column_types", column_types.into_any()), + ("required_columns", PyFrozenSet::empty(py)?.into_any()), + ( + "render_framework_type", + request.get("render_framework_type")?, + ), + ], + ) +} + +/// The SQL reading every stored event of one schema, and the columns it selects. +#[pyfunction(name = "render_migration_event_read_sql")] +#[pyo3(signature = (**request))] +fn read_sql<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<(String, Bound<'py, PyTuple>)> { + let request = Request::new("render_migration_event_read_sql", request)?; + let layout: EventTableLayout = layout(&request.get("kind")?.extract::()?)?; + let stored: Option> = match request.optional("stored_columns")? { + Some(stored) if !stored.is_none() => Some(stored.extract()?), + _ => None, + }; + let columns: Vec<&str> = match &stored { + Some(stored) => readable_columns(layout.columns, stored), + None => layout.columns.to_vec(), + }; + let table: String = qualified_table( + &layout, + &request.get("render_qualified_name")?, + (request.get("database")?, request.get("schema")?), + )?; + Ok(( + event_read_sql(&table, &columns), + PyTuple::new(request.py(), columns)?, + )) +} + +fn optional<'py>(value: Bound<'py, PyAny>, attribute: &str) -> PyResult> { + if value.is_none() { + Ok(value) + } else { + value.getattr(attribute) + } +} + +fn text<'py>(py: Python<'py>, value: Option) -> Bound<'py, PyAny> { + match value { + Some(value) => PyString::new(py, &value).into_any(), + None => py.None().into_bound(py), + } +} + +fn row_values<'py>( + table: EventTable, + event: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = event.py(); + let field = |name: &str| event.getattr(name); + let relation = |name: &str| -> PyResult<[Bound<'py, PyAny>; 3]> { + let relation: Bound<'py, PyAny> = event.getattr(name)?; + Ok([ + relation.getattr("database")?, + relation.getattr("schema")?, + relation.getattr("name")?, + ]) + }; + let mut values: Vec> = vec![field("event_id")?, field("target_name")?]; + match table { + EventTable::Model => { + values.push(field("origin_model")?); + values.extend(relation("origin")?); + values.push(field("destination_model")?); + values.extend(relation("destination")?); + values.push(field("origin_version_hash")?); + } + EventTable::Column => { + values.push(field("model_name")?); + values.extend(relation("relation")?); + values.push(field("origin_column")?); + values.push(field("destination_column")?); + } + EventTable::OldName => { + values.push(field("event_type")?.getattr("value")?); + values.push(field("migration_event_id")?); + values.push(field("destination_model")?); + values.extend(relation("old")?); + values.extend(relation("new")?); + values.push(field("view_retention")?); + values.push(field("archive_name")?); + let aliases: Vec<(String, String)> = field("column_aliases")?.extract()?; + let encoded = column_aliases_json(&aliases).map_err(json_error)?; + values.push(text(py, encoded)); + let grants: Option> = field("grants_copied")?.extract()?; + values.push(text( + py, + grants_json(grants.as_deref()).map_err(json_error)?, + )); + values.push(field("view_sql")?); + values.push(field("expires_at")?); + values.push(optional(field("drop_reason")?, "value")?); + values.push(field("run_id")?); + values.push(field("created_at")?); + return Ok(values); + } + } + values.push(field("discovery")?.getattr("value")?); + values.push(field("decision")?.getattr("value")?); + values.push(field("run_id")?); + values.push(field("created_at")?); + Ok(values) +} + +fn json_error(error: sqlbuild_core::json::errors::JsonEmitError) -> PyErr { + PyValueError::new_err(format!("{error:?}")) +} + +/// The idempotency lookup and the insert that append one event to its schema's state table. +#[pyfunction(name = "render_migration_event_write_sql")] +#[pyo3(signature = (*, kind, event, render_qualified_name))] +fn write_sql<'py>( + kind: &str, + event: &Bound<'py, PyAny>, + render_qualified_name: &Bound<'py, PyAny>, +) -> PyResult<(String, String)> { + let py: Python<'py> = event.py(); + let layout: EventTableLayout = layout(kind)?; + let relation: Bound<'py, PyAny> = event.getattr(layout.relation_attribute)?; + let schema: Bound<'py, PyAny> = relation.getattr("schema")?; + if schema.is_none() { + return Err(state_error(py, layout.event_message)); + } + let table: String = qualified_table( + &layout, + render_qualified_name, + (relation.getattr("database")?, schema), + )?; + let event_id: String = event.getattr("event_id")?.extract()?; + let values: Vec> = row_values(layout.table, event)?; + let insert: String = typed_insert_sql(py, &table, (layout.columns, &values))?; + Ok((event_exists_sql(&table, &event_id), insert)) +} + +/// One stored row keyed by the columns its query selected. +struct StoredRow<'a, 'py> { + columns: &'a [String], + values: Vec>, +} + +impl<'py> StoredRow<'_, 'py> { + fn get(&self, column: &str) -> Option<&Bound<'py, PyAny>> { + self.columns + .iter() + .position(|name| name == column) + .and_then(|index| self.values.get(index)) + } + + fn required(&self, column: &str) -> PyResult> { + self.get(column) + .cloned() + .ok_or_else(|| PyKeyError::new_err(column.to_owned())) + } + + fn text(&self, column: &str) -> PyResult> { + Ok(self.required(column)?.str()?.into_any()) + } + + fn optional_text(&self, column: &str) -> PyResult>> { + match self.get(column) { + Some(value) if !value.is_none() => Ok(Some(value.str()?.into_any())), + _ => Ok(None), + } + } + + fn nullable(&self, py: Python<'py>, column: &str) -> PyResult> { + Ok(self + .optional_text(column)? + .unwrap_or_else(|| py.None().into_bound(py))) + } + + fn relation(&self, py: Python<'py>, prefix: &str) -> PyResult> { + call_with( + &imported(py, MIGRATION_MODELS, "MigrationRelation")?, + &[ + ( + "database", + self.nullable(py, &format!("{prefix}_database"))?, + ), + ("schema", self.nullable(py, &format!("{prefix}_schema"))?), + ("name", self.text(&format!("{prefix}_name"))?), + ], + ) + } +} + +/// `decode_event_timestamp`: a stored timestamp as an aware datetime, naive values read as UTC. +fn timestamp<'py>(raw: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = raw.py(); + let parsed: Bound<'py, PyAny> = if date_time_typed(raw)? { + raw.clone() + } else { + imported(py, DATETIME_MODULE, "datetime")?.call_method1("fromisoformat", (raw.str()?,))? + }; + if !parsed.getattr("tzinfo")?.is_none() { + return Ok(parsed); + } + let utc: Bound<'py, PyDict> = PyDict::new(py); + utc.set_item("tzinfo", imported(py, DATETIME_MODULE, "UTC")?)?; + parsed.call_method("replace", (), Some(&utc)) +} + +fn enumerated<'py>( + py: Python<'py>, + class: &str, + value: Bound<'py, PyAny>, +) -> PyResult> { + imported(py, MIGRATION_TYPES, class)?.call1((value,)) +} + +fn known_member<'py>( + py: Python<'py>, + class: &str, + value: Bound<'py, PyAny>, +) -> PyResult>> { + match enumerated(py, class, value) { + Ok(member) => Ok(Some(member)), + Err(error) if error.is_instance_of::(py) => Ok(None), + Err(error) => Err(error), + } +} + +fn model_event<'py>(py: Python<'py>, row: &StoredRow<'_, 'py>) -> PyResult> { + let created_at: Bound<'py, PyAny> = timestamp(&row.required("created_at")?)?; + let version_hash: Bound<'py, PyAny> = row.required("origin_version_hash")?; + let version_hash: Bound<'py, PyAny> = if version_hash.is_truthy()? { + version_hash.str()?.into_any() + } else { + PyString::new(py, "").into_any() + }; + call_with( + &imported(py, MIGRATION_MODELS, "MigrationEvent")?, + &[ + ("event_id", row.text("event_id")?), + ("target_name", row.nullable(py, "target_name")?), + ("origin_model", row.nullable(py, "origin_model")?), + ("origin", row.relation(py, "origin")?), + ("destination_model", row.text("destination_model")?), + ("destination", row.relation(py, "destination")?), + ("origin_version_hash", version_hash), + ( + "discovery", + enumerated(py, "MigrationDiscovery", row.text("discovery")?)?, + ), + ( + "decision", + enumerated(py, "MigrationDecision", row.text("decision")?)?, + ), + ("run_id", row.text("run_id")?), + ("created_at", created_at), + ], + ) +} + +fn column_event<'py>(py: Python<'py>, row: &StoredRow<'_, 'py>) -> PyResult> { + call_with( + &imported(py, MIGRATION_MODELS, "ColumnMigrationEvent")?, + &[ + ("event_id", row.text("event_id")?), + ("target_name", row.nullable(py, "target_name")?), + ("model_name", row.text("model_name")?), + ("relation", row.relation(py, "relation")?), + ("origin_column", row.text("origin_column")?), + ("destination_column", row.text("destination_column")?), + ( + "discovery", + enumerated(py, "MigrationDiscovery", row.text("discovery")?)?, + ), + ( + "decision", + enumerated(py, "ColumnMigrationDecision", row.text("decision")?)?, + ), + ("run_id", row.text("run_id")?), + ("created_at", timestamp(&row.required("created_at")?)?), + ], + ) +} + +fn stored_text(row: &StoredRow<'_, '_>, column: &str) -> PyResult> { + row.optional_text(column)? + .map(|value| value.extract()) + .transpose() +} + +fn old_name_event<'py>( + py: Python<'py>, + row: &StoredRow<'_, 'py>, +) -> PyResult>> { + let raw_type: Bound<'py, PyAny> = match row.get("event_type") { + Some(value) => value.str()?.into_any(), + None => PyString::new(py, "None").into_any(), + }; + let Some(event_type) = known_member(py, "OldNameViewEventType", raw_type)? else { + return Ok(None); + }; + let drop_reason: Option> = match row.optional_text("drop_reason")? { + Some(raw) => known_member(py, "OldNameViewDropReason", raw)?, + None => None, + }; + let none = || py.None().into_bound(py); + let expires_at: Bound<'py, PyAny> = match row.get("expires_at") { + Some(raw) if !raw.is_none() => timestamp(raw)?, + _ => none(), + }; + let aliases: Vec<(String, String)> = + decoded_column_aliases(stored_text(row, "column_aliases")?.as_deref()); + let grants: Bound<'py, PyAny> = + match decoded_grants(stored_text(row, "grants_copied")?.as_deref()) { + Some(grants) => PyTuple::new(py, grants)?.into_any(), + None => none(), + }; + let fields: [(&str, Bound<'py, PyAny>); 16] = [ + ("event_id", row.text("event_id")?), + ("target_name", row.nullable(py, "target_name")?), + ("event_type", event_type), + ("migration_event_id", row.text("migration_event_id")?), + ("destination_model", row.text("destination_model")?), + ("old", row.relation(py, "old")?), + ("new", row.relation(py, "new")?), + ("run_id", row.text("run_id")?), + ("created_at", timestamp(&row.required("created_at")?)?), + ("view_retention", row.nullable(py, "view_retention")?), + ("archive_name", row.nullable(py, "archive_name")?), + ("column_aliases", PyTuple::new(py, aliases)?.into_any()), + ("expires_at", expires_at), + ("drop_reason", drop_reason.unwrap_or_else(none)), + ("grants_copied", grants), + ("view_sql", row.nullable(py, "view_sql")?), + ]; + call_with( + &imported(py, MIGRATION_MODELS, "OldNameViewEvent")?, + &fields, + ) + .map(Some) +} + +/// Stored rows of the selected `columns` as events; rows this version cannot read are skipped. +#[pyfunction(name = "decode_migration_event_rows")] +#[pyo3(signature = (*, kind, rows, columns))] +fn decode_rows<'py>( + kind: &str, + rows: &Bound<'py, PyAny>, + columns: Vec, +) -> PyResult> { + let py: Python<'py> = rows.py(); + let table: EventTable = layout(kind)?.table; + let mut events: Vec> = Vec::new(); + for row in rows.try_iter()? { + let values: Vec> = row?.try_iter()?.collect::>()?; + let stored = StoredRow { + columns: &columns, + values, + }; + let decoded: Option> = match table { + EventTable::Model => Some(model_event(py, &stored)?), + EventTable::Column => match column_event(py, &stored) { + Ok(event) => Some(event), + Err(error) if error.is_instance_of::(py) => None, + Err(error) => return Err(error), + }, + EventTable::OldName => old_name_event(py, &stored)?, + }; + events.extend(decoded); + } + PyTuple::new(py, events) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(create_sql, module)?)?; + module.add_function(wrap_pyfunction!(read_sql, module)?)?; + module.add_function(wrap_pyfunction!(write_sql, module)?)?; + module.add_function(wrap_pyfunction!(decode_rows, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs index 936e0ba0fa..1c29eebbd6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs @@ -1,4 +1,9 @@ +pub(crate) mod constants; pub(crate) mod events; pub(crate) mod freshness; +pub(crate) mod migration_objects; +pub(crate) mod migration_projections; +pub(crate) mod migration_rows; +pub(crate) mod models; pub(crate) mod registration; pub(crate) mod timestamps; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/models.rs new file mode 100644 index 0000000000..c126b8ecd9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/models.rs @@ -0,0 +1,20 @@ +//! The layout of one append-only migration state table as the bindings use it. + +/// Which migration state table a request reads or writes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum EventTable { + Model, + Column, + OldName, +} + +/// Name, columns, event relation attribute and messages of one migration state table. +#[derive(Clone, Copy, Debug)] +pub(crate) struct EventTableLayout { + pub table: EventTable, + pub name: &'static str, + pub columns: &'static [&'static str], + pub relation_attribute: &'static str, + pub state_message: &'static str, + pub event_message: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs index c06548943e..819aa21fb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs @@ -2,10 +2,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::state_rows::{events, freshness}; +use crate::bindings::_helpers::state_rows::{ + events, freshness, migration_projections, migration_rows, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { events::register(module)?; freshness::register(module)?; + migration_rows::register(module)?; + migration_projections::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-state/Cargo.toml b/crates/sqlbuild-state/Cargo.toml index b8436c7922..17f6e3272b 100644 --- a/crates/sqlbuild-state/Cargo.toml +++ b/crates/sqlbuild-state/Cargo.toml @@ -8,6 +8,7 @@ publish.workspace = true rust-version.workspace = true [dependencies] +serde_json.workspace = true sha2.workspace = true sqlbuild-core.workspace = true sqlbuild-cursor.workspace = true diff --git a/crates/sqlbuild-state/src/events/_helpers/mod.rs b/crates/sqlbuild-state/src/events/_helpers/mod.rs index 059db07285..9280126ad8 100644 --- a/crates/sqlbuild-state/src/events/_helpers/mod.rs +++ b/crates/sqlbuild-state/src/events/_helpers/mod.rs @@ -1,2 +1,4 @@ pub(crate) mod identity; pub(crate) mod literals; +pub(crate) mod python_text; +pub(crate) mod relations; diff --git a/crates/sqlbuild-state/src/events/_helpers/python_text.rs b/crates/sqlbuild-state/src/events/_helpers/python_text.rs new file mode 100644 index 0000000000..e198799b8c --- /dev/null +++ b/crates/sqlbuild-state/src/events/_helpers/python_text.rs @@ -0,0 +1,21 @@ +//! `str()` of a decoded JSON value, as Python prints the value `json.loads` returns. + +use serde_json::Value; +use sqlbuild_core::json::main::python_float_repr::python_float_repr; + +use crate::events::constants::{PYTHON_FALSE, PYTHON_NONE, PYTHON_TRUE}; + +/// Python's `str()` of a JSON scalar; containers keep their compact JSON text. +pub(crate) fn python_text(value: &Value) -> String { + match value { + Value::String(text) => text.clone(), + Value::Bool(true) => PYTHON_TRUE.to_owned(), + Value::Bool(false) => PYTHON_FALSE.to_owned(), + Value::Null => PYTHON_NONE.to_owned(), + Value::Number(number) => match number.as_f64() { + Some(float) if number.is_f64() => python_float_repr(float), + _ => number.to_string(), + }, + Value::Array(_) | Value::Object(_) => value.to_string(), + } +} diff --git a/crates/sqlbuild-state/src/events/_helpers/relations.rs b/crates/sqlbuild-state/src/events/_helpers/relations.rs new file mode 100644 index 0000000000..a91170fbfa --- /dev/null +++ b/crates/sqlbuild-state/src/events/_helpers/relations.rs @@ -0,0 +1,34 @@ +//! Case-insensitive relation identity, treating an unknown database as a wildcard. + +use crate::events::models::EventRelation; + +/// `MigrationRelation.matches`: equal schema and name, and databases equal unless one is unknown. +pub(crate) fn relations_match(left: &EventRelation, right: &EventRelation) -> bool { + let schema = |relation: &EventRelation| -> String { + relation + .schema + .as_deref() + .unwrap_or_default() + .to_lowercase() + }; + if schema(left) != schema(right) || left.name.to_lowercase() != right.name.to_lowercase() { + return false; + } + match (&left.database, &right.database) { + (Some(left), Some(right)) => left.to_lowercase() == right.to_lowercase(), + _ => true, + } +} + +/// The index of the greatest order among the selected items, keeping the first of equal ones. +pub(crate) fn newest_index<'a>( + orders: impl Iterator, +) -> Option { + let mut newest: Option<(usize, &crate::events::models::EventOrder)> = None; + for (index, order) in orders { + if newest.is_none_or(|(_, current)| order > current) { + newest = Some((index, order)); + } + } + newest.map(|(index, _)| index) +} diff --git a/crates/sqlbuild-state/src/events/constants.rs b/crates/sqlbuild-state/src/events/constants.rs index e6992b279b..f0cf958db1 100644 --- a/crates/sqlbuild-state/src/events/constants.rs +++ b/crates/sqlbuild-state/src/events/constants.rs @@ -10,3 +10,66 @@ pub(crate) const NULL_LITERAL: &str = "NULL"; pub(crate) const UTC_SUFFIX: &str = "+00:00"; pub const STRING_VALUE_REQUIRED: &str = "state string columns require str values"; pub const TIMESTAMP_VALUE_REQUIRED: &str = "state timestamp columns require datetime values"; +pub(crate) const ALIAS_ITEM_SEPARATOR: &str = ","; +pub(crate) const ALIAS_KEY_SEPARATOR: &str = ":"; +pub(crate) const PYTHON_TRUE: &str = "True"; +pub(crate) const PYTHON_FALSE: &str = "False"; +pub(crate) const PYTHON_NONE: &str = "None"; +pub const MIGRATION_TABLE_NAME: &str = "_sqlbuild_migrations"; +pub const COLUMN_MIGRATION_TABLE_NAME: &str = "_sqlbuild_column_migrations"; +pub const OLD_NAME_VIEW_TABLE_NAME: &str = "_sqlbuild_old_name_views"; +/// Event columns stored as text timestamps; every other event column is a string. +pub const TIMESTAMP_COLUMNS: [&str; 2] = ["expires_at", "created_at"]; +pub const MIGRATION_COLUMNS: [&str; 15] = [ + "event_id", + "target_name", + "origin_model", + "origin_database", + "origin_schema", + "origin_name", + "destination_model", + "destination_database", + "destination_schema", + "destination_name", + "origin_version_hash", + "discovery", + "decision", + "run_id", + "created_at", +]; +pub const COLUMN_MIGRATION_COLUMNS: [&str; 12] = [ + "event_id", + "target_name", + "model_name", + "relation_database", + "relation_schema", + "relation_name", + "origin_column", + "destination_column", + "discovery", + "decision", + "run_id", + "created_at", +]; +pub const OLD_NAME_VIEW_COLUMNS: [&str; 20] = [ + "event_id", + "target_name", + "event_type", + "migration_event_id", + "destination_model", + "old_database", + "old_schema", + "old_name", + "new_database", + "new_schema", + "new_name", + "view_retention", + "archive_name", + "column_aliases", + "grants_copied", + "view_sql", + "expires_at", + "drop_reason", + "run_id", + "created_at", +]; diff --git a/crates/sqlbuild-state/src/events/main/column_aliases_json.rs b/crates/sqlbuild-state/src/events/main/column_aliases_json.rs new file mode 100644 index 0000000000..3287a7d788 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/column_aliases_json.rs @@ -0,0 +1,31 @@ +//! `column_aliases_json`: the stored text of an old-name view's column aliases. + +use std::collections::BTreeMap; + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::events::constants::{ALIAS_ITEM_SEPARATOR, ALIAS_KEY_SEPARATOR}; + +/// Compact sorted ASCII JSON of `dict(aliases)`, or `None` when there are no aliases. +pub fn column_aliases_json(aliases: &[(String, String)]) -> Result, JsonEmitError> { + if aliases.is_empty() { + return Ok(None); + } + let mapping: BTreeMap<&str, &str> = aliases + .iter() + .map(|(old, new)| (old.as_str(), new.as_str())) + .collect(); + let value = JsonValue::Object( + mapping + .into_iter() + .map(|(old, new)| (old.to_owned(), JsonValue::String(new.to_owned()))) + .collect(), + ); + let options = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None).with_separators(ALIAS_ITEM_SEPARATOR, ALIAS_KEY_SEPARATOR) + }; + dumps(&value, &JsonDialect::Stdlib(options)).map(Some) +} diff --git a/crates/sqlbuild-state/src/events/main/decoded_column_aliases.rs b/crates/sqlbuild-state/src/events/main/decoded_column_aliases.rs new file mode 100644 index 0000000000..9ba2e6cd1d --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/decoded_column_aliases.rs @@ -0,0 +1,22 @@ +//! `decoded_column_aliases`: stored column alias text back to sorted pairs. + +use serde_json::Value; + +use crate::events::_helpers::python_text::python_text; + +/// Sorted `(old, new)` pairs of a stored JSON object; empty, invalid or non-object text reads as none. +#[must_use] +pub fn decoded_column_aliases(raw: Option<&str>) -> Vec<(String, String)> { + let Some(raw) = raw.filter(|raw| !raw.is_empty()) else { + return Vec::new(); + }; + let Ok(Value::Object(mapping)) = serde_json::from_str::(raw) else { + return Vec::new(); + }; + let mut pairs: Vec<(String, String)> = mapping + .iter() + .map(|(old, new)| (old.clone(), python_text(new))) + .collect(); + pairs.sort(); + pairs +} diff --git a/crates/sqlbuild-state/src/events/main/decoded_grants.rs b/crates/sqlbuild-state/src/events/main/decoded_grants.rs new file mode 100644 index 0000000000..0c69ecd53c --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/decoded_grants.rs @@ -0,0 +1,14 @@ +//! `decoded_grants`: stored grant text back to the copied statements. + +use serde_json::Value; + +use crate::events::_helpers::python_text::python_text; + +/// The statements of a stored JSON array; absent, invalid or non-array text reads as unknown. +#[must_use] +pub fn decoded_grants(raw: Option<&str>) -> Option> { + match serde_json::from_str::(raw?) { + Ok(Value::Array(statements)) => Some(statements.iter().map(python_text).collect()), + _ => None, + } +} diff --git a/crates/sqlbuild-state/src/events/main/event_cells.rs b/crates/sqlbuild-state/src/events/main/event_cells.rs new file mode 100644 index 0000000000..0efb609b79 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/event_cells.rs @@ -0,0 +1,22 @@ +//! `event_cells`: one event row's values with their declared state column types. + +use crate::events::constants::TIMESTAMP_COLUMNS; +use crate::events::models::{StateCell, StateColumnType, StateValue}; + +/// Pair each column with its value, typing `expires_at` and `created_at` as text timestamps. +#[must_use] +pub fn event_cells(columns: &[&str], values: Vec) -> Vec { + columns + .iter() + .zip(values) + .map(|(column, value)| StateCell { + column: (*column).to_owned(), + column_type: if TIMESTAMP_COLUMNS.contains(column) { + StateColumnType::TextTimestamp + } else { + StateColumnType::String + }, + value, + }) + .collect() +} diff --git a/crates/sqlbuild-state/src/events/main/event_read_sql.rs b/crates/sqlbuild-state/src/events/main/event_read_sql.rs new file mode 100644 index 0000000000..8cfb6ae19f --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/event_read_sql.rs @@ -0,0 +1,10 @@ +//! `event_read_sql`: every stored event of one state table in history order. + +/// `SELECT FROM ORDER BY created_at, event_id`. +#[must_use] +pub fn event_read_sql(table: &str, columns: &[&str]) -> String { + format!( + "SELECT {} FROM {table} ORDER BY created_at, event_id", + columns.join(", ") + ) +} diff --git a/crates/sqlbuild-state/src/events/main/grants_json.rs b/crates/sqlbuild-state/src/events/main/grants_json.rs new file mode 100644 index 0000000000..b3964032ee --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/grants_json.rs @@ -0,0 +1,14 @@ +//! `grants_json`: the stored text of the grant statements copied to an old-name view. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +/// `json.dumps(list(grants))`, or `None` when grants were not recorded. +pub fn grants_json(grants: Option<&[String]>) -> Result, JsonEmitError> { + let Some(grants) = grants else { + return Ok(None); + }; + let value = JsonValue::Array(grants.iter().cloned().map(JsonValue::String).collect()); + dumps(&value, &JsonDialect::Stdlib(StdlibJsonOptions::new(None))).map(Some) +} diff --git a/crates/sqlbuild-state/src/events/main/mod.rs b/crates/sqlbuild-state/src/events/main/mod.rs index 3870be1589..0d0be3fda3 100644 --- a/crates/sqlbuild-state/src/events/main/mod.rs +++ b/crates/sqlbuild-state/src/events/main/mod.rs @@ -1,5 +1,15 @@ +pub mod column_aliases_json; pub mod column_migration_event_id; +pub mod decoded_column_aliases; +pub mod decoded_grants; +pub mod event_cells; pub mod event_exists_sql; pub mod event_insert_sql; +pub mod event_read_sql; +pub mod grants_json; pub mod migration_event_id; +pub mod newest_column_event; +pub mod newest_move_event; +pub mod old_name_histories; pub mod old_name_view_event_id; +pub mod readable_columns; diff --git a/crates/sqlbuild-state/src/events/main/newest_column_event.rs b/crates/sqlbuild-state/src/events/main/newest_column_event.rs new file mode 100644 index 0000000000..888d381ff2 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/newest_column_event.rs @@ -0,0 +1,28 @@ +//! `newest_column_event`: the latest recorded rename of either column of a pair. + +use crate::events::_helpers::relations::{newest_index, relations_match}; +use crate::events::models::{ColumnEventKey, EventRelation}; + +/// The index of the latest rename on `relation` naming either column; the first wins ties. +#[must_use] +pub fn newest_column_event( + events: &[ColumnEventKey], + relation: &EventRelation, + (origin_column, destination_column): (&str, &str), +) -> Option { + let columns: [String; 2] = [ + origin_column.to_lowercase(), + destination_column.to_lowercase(), + ]; + newest_index( + events + .iter() + .enumerate() + .filter(|(_, event)| { + relations_match(&event.relation, relation) + && (columns.contains(&event.origin_column.to_lowercase()) + || columns.contains(&event.destination_column.to_lowercase())) + }) + .map(|(index, event)| (index, &event.order)), + ) +} diff --git a/crates/sqlbuild-state/src/events/main/newest_move_event.rs b/crates/sqlbuild-state/src/events/main/newest_move_event.rs new file mode 100644 index 0000000000..f90d6cbd73 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/newest_move_event.rs @@ -0,0 +1,19 @@ +//! `newest_move_event`: the latest recorded move naming one relation. + +use crate::events::_helpers::relations::{newest_index, relations_match}; +use crate::events::models::{EventRelation, MoveEventKey}; + +/// The index of the latest move naming `relation` as origin or destination; the first wins ties. +#[must_use] +pub fn newest_move_event(events: &[MoveEventKey], relation: &EventRelation) -> Option { + newest_index( + events + .iter() + .enumerate() + .filter(|(_, event)| { + relations_match(&event.origin, relation) + || relations_match(&event.destination, relation) + }) + .map(|(index, event)| (index, &event.order)), + ) +} diff --git a/crates/sqlbuild-state/src/events/main/old_name_histories.rs b/crates/sqlbuild-state/src/events/main/old_name_histories.rs new file mode 100644 index 0000000000..8523358f0c --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/old_name_histories.rs @@ -0,0 +1,41 @@ +//! `old_name_histories`: one history per recorded move whose old name needs a compatibility view. + +use std::collections::HashMap; + +use crate::events::models::{EventOrder, OldNameFactKey, OldNameHistoryIndexes, OldNameStep}; + +/// Histories in move order, keeping the earliest fact of each step; moves without a requirement are skipped. +#[must_use] +pub fn old_name_histories( + moves: &[EventOrder], + facts: &[OldNameFactKey], +) -> Vec { + let mut fact_order: Vec = (0..facts.len()).collect(); + fact_order.sort_by(|left, right| facts[*left].order.cmp(&facts[*right].order)); + let mut by_move: HashMap<(&str, OldNameStep), usize> = HashMap::new(); + for index in fact_order { + let fact: &OldNameFactKey = &facts[index]; + by_move + .entry((fact.migration_event_id.as_str(), fact.step)) + .or_insert(index); + } + let mut move_order: Vec = (0..moves.len()).collect(); + move_order.sort_by(|left, right| moves[*left].cmp(&moves[*right])); + move_order + .into_iter() + .filter_map(|moved| { + let step = |step: OldNameStep| { + by_move + .get(&(moves[moved].event_id.as_str(), step)) + .copied() + }; + Some(OldNameHistoryIndexes { + moved, + required: step(OldNameStep::Required)?, + archived: step(OldNameStep::OriginArchived), + created: step(OldNameStep::ViewCreated), + dropped: step(OldNameStep::ViewDropped), + }) + }) + .collect() +} diff --git a/crates/sqlbuild-state/src/events/main/readable_columns.rs b/crates/sqlbuild-state/src/events/main/readable_columns.rs new file mode 100644 index 0000000000..248ce6e4e2 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/readable_columns.rs @@ -0,0 +1,14 @@ +//! `readable_columns`: the known columns a stored state table has, in their declared order. + +use std::collections::HashSet; + +/// The `known` columns present in `stored` (compared case-insensitively); later additions may be absent. +#[must_use] +pub fn readable_columns<'a>(known: &[&'a str], stored: &[String]) -> Vec<&'a str> { + let stored: HashSet = stored.iter().map(|column| column.to_lowercase()).collect(); + known + .iter() + .copied() + .filter(|column| stored.contains(*column)) + .collect() +} diff --git a/crates/sqlbuild-state/src/events/models.rs b/crates/sqlbuild-state/src/events/models.rs index 9f850af603..3103778446 100644 --- a/crates/sqlbuild-state/src/events/models.rs +++ b/crates/sqlbuild-state/src/events/models.rs @@ -35,3 +35,54 @@ pub struct StateCell { pub column_type: StateColumnType, pub value: StateValue, } + +/// The order of an event in its history: UTC creation time, then event ID. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct EventOrder { + pub created_at: LocalDateTime, + pub event_id: String, +} + +/// The fields of a recorded relation move that projections read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MoveEventKey { + pub order: EventOrder, + pub origin: EventRelation, + pub destination: EventRelation, +} + +/// The fields of a recorded column rename that projections read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ColumnEventKey { + pub order: EventOrder, + pub relation: EventRelation, + pub origin_column: String, + pub destination_column: String, +} + +/// One step of an old-name view's lifecycle. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum OldNameStep { + Required, + OriginArchived, + ViewCreated, + ViewDropped, +} + +/// The fields of one old-name fact that the history projection reads. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OldNameFactKey { + pub order: EventOrder, + pub migration_event_id: String, + pub step: OldNameStep, +} + +/// One projected old-name history as indexes into the moves and facts it was built from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct OldNameHistoryIndexes { + pub moved: usize, + pub required: usize, + pub archived: Option, + pub created: Option, + pub dropped: Option, +} diff --git a/crates/sqlbuild-state/src/events/tests.rs b/crates/sqlbuild-state/src/events/tests.rs index b55f12f730..7bd9f08251 100644 --- a/crates/sqlbuild-state/src/events/tests.rs +++ b/crates/sqlbuild-state/src/events/tests.rs @@ -1,6 +1,10 @@ +#[path = "tests/test_codecs.rs"] +mod codecs; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_insert_sql.rs"] mod insert_sql; +#[path = "tests/test_projections.rs"] +mod projections; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-state/src/events/tests/helpers.rs b/crates/sqlbuild-state/src/events/tests/helpers.rs index 3ad5209a10..fd12989882 100644 --- a/crates/sqlbuild-state/src/events/tests/helpers.rs +++ b/crates/sqlbuild-state/src/events/tests/helpers.rs @@ -1,4 +1,9 @@ -use crate::events::models::{StateCell, StateColumnType, StateValue}; +use sqlbuild_cursor::algebra::models::{CivilDate, LocalDateTime}; + +use crate::events::models::{ + ColumnEventKey, EventOrder, EventRelation, MoveEventKey, OldNameFactKey, OldNameStep, + StateCell, StateColumnType, StateValue, +}; pub(crate) fn cell(column: &str, column_type: StateColumnType, value: StateValue) -> StateCell { StateCell { @@ -7,3 +12,64 @@ pub(crate) fn cell(column: &str, column_type: StateColumnType, value: StateValue value, } } + +pub(crate) fn relation(database: Option<&str>, schema: &str, name: &str) -> EventRelation { + EventRelation { + database: database.map(str::to_owned), + schema: Some(schema.to_owned()), + name: name.to_owned(), + } +} + +pub(crate) fn order(hour: u32, event_id: &str) -> EventOrder { + EventOrder { + created_at: LocalDateTime { + date: CivilDate { + year: 2026, + month: 1, + day: 1, + }, + hour, + minute: 0, + second: 0, + microsecond: 0, + }, + event_id: event_id.to_owned(), + } +} + +pub(crate) fn moved(order: EventOrder, (origin, destination): (&str, &str)) -> MoveEventKey { + MoveEventKey { + order, + origin: relation(None, "main", origin), + destination: relation(None, "main", destination), + } +} + +pub(crate) fn renamed(order: EventOrder, (origin, destination): (&str, &str)) -> ColumnEventKey { + ColumnEventKey { + order, + relation: relation(None, "main", "orders"), + origin_column: origin.to_owned(), + destination_column: destination.to_owned(), + } +} + +pub(crate) fn fact( + order: EventOrder, + migration_event_id: &str, + step: OldNameStep, +) -> OldNameFactKey { + OldNameFactKey { + order, + migration_event_id: migration_event_id.to_owned(), + step, + } +} + +pub(crate) fn pairs(items: &[(&str, &str)]) -> Vec<(String, String)> { + items + .iter() + .map(|(old, new)| ((*old).to_owned(), (*new).to_owned())) + .collect() +} diff --git a/crates/sqlbuild-state/src/events/tests/test_codecs.rs b/crates/sqlbuild-state/src/events/tests/test_codecs.rs new file mode 100644 index 0000000000..3d52c6484e --- /dev/null +++ b/crates/sqlbuild-state/src/events/tests/test_codecs.rs @@ -0,0 +1,159 @@ +use crate::events::constants::OLD_NAME_VIEW_COLUMNS; +use crate::events::main::column_aliases_json::column_aliases_json; +use crate::events::main::decoded_column_aliases::decoded_column_aliases; +use crate::events::main::decoded_grants::decoded_grants; +use crate::events::main::event_read_sql::event_read_sql; +use crate::events::main::grants_json::grants_json; +use crate::events::main::readable_columns::readable_columns; +use crate::events::tests::helpers::pairs; +use crate::events::tests::test_types::{ + AliasesTestCase, DecodedAliasesTestCase, DecodedGrantsTestCase, GrantsTestCase, ReadSqlTestCase, +}; + +#[test] +fn given_aliases_when_encoding_then_matches_python_compact_sorted_json() { + let test_cases = [ + AliasesTestCase { + description: "sorted keys, last duplicate wins and non-ASCII is escaped", + aliases: pairs(&[("z", "\u{e4}"), ("a", "b"), ("a", "c")]), + expected_json: Some("{\"a\":\"c\",\"z\":\"\\u00e4\"}".to_owned()), + }, + AliasesTestCase { + description: "no aliases store null", + aliases: vec![], + expected_json: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + column_aliases_json(&test_case.aliases).ok(), + Some(test_case.expected_json), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_grants_when_encoding_then_matches_python_default_json() { + let test_cases = [ + GrantsTestCase { + description: "quotes kept and non-ASCII escaped with default separators", + grants: Some(vec!["GRANT 'x'".to_owned(), "\u{e9}".to_owned()]), + expected_json: Some("[\"GRANT 'x'\", \"\\u00e9\"]".to_owned()), + }, + GrantsTestCase { + description: "unrecorded grants store null", + grants: None, + expected_json: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + grants_json(test_case.grants.as_deref()).ok(), + Some(test_case.expected_json), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_stored_alias_text_when_decoding_then_reads_sorted_pairs_or_none() { + let test_cases = [ + DecodedAliasesTestCase { + description: "object values are printed as Python would", + raw: Some("{\"k\": 1, \"j\": true, \"i\": null, \"h\": 1.5, \"g\": \"x\"}"), + expected_aliases: pairs(&[ + ("g", "x"), + ("h", "1.5"), + ("i", "None"), + ("j", "True"), + ("k", "1"), + ]), + }, + DecodedAliasesTestCase { + description: "invalid JSON", + raw: Some("{not json"), + expected_aliases: vec![], + }, + DecodedAliasesTestCase { + description: "array is not a mapping", + raw: Some("[\"a\"]"), + expected_aliases: vec![], + }, + DecodedAliasesTestCase { + description: "empty text", + raw: Some(""), + expected_aliases: vec![], + }, + ]; + + for test_case in test_cases { + assert_eq!( + decoded_column_aliases(test_case.raw), + test_case.expected_aliases, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_stored_grant_text_when_decoding_then_reads_statements_or_unknown() { + let test_cases = [ + DecodedGrantsTestCase { + description: "array items are printed as Python would", + raw: Some("[\"GRANT\", 3, null]"), + expected_grants: Some(vec!["GRANT".to_owned(), "3".to_owned(), "None".to_owned()]), + }, + DecodedGrantsTestCase { + description: "object is not a statement list", + raw: Some("{\"a\": 1}"), + expected_grants: None, + }, + DecodedGrantsTestCase { + description: "absent column", + raw: None, + expected_grants: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + decoded_grants(test_case.raw), + test_case.expected_grants, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_stored_columns_when_reading_old_name_facts_then_selects_known_columns_in_order() { + let test_cases = [ReadSqlTestCase { + description: "unknown and missing columns are left out", + stored: vec![ + "EVENT_ID".to_owned(), + "zzz".to_owned(), + "created_at".to_owned(), + "event_type".to_owned(), + "view_sql".to_owned(), + ], + expected_sql: "SELECT event_id, event_type, view_sql, created_at FROM s.t ORDER BY created_at, event_id".to_owned(), + }]; + + for test_case in test_cases { + assert_eq!( + event_read_sql( + "s.t", + &readable_columns(&OLD_NAME_VIEW_COLUMNS, &test_case.stored) + ), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/events/tests/test_projections.rs b/crates/sqlbuild-state/src/events/tests/test_projections.rs new file mode 100644 index 0000000000..33bc1d3a9c --- /dev/null +++ b/crates/sqlbuild-state/src/events/tests/test_projections.rs @@ -0,0 +1,138 @@ +use crate::events::main::newest_column_event::newest_column_event; +use crate::events::main::newest_move_event::newest_move_event; +use crate::events::main::old_name_histories::old_name_histories; +use crate::events::models::{OldNameHistoryIndexes, OldNameStep}; +use crate::events::tests::helpers::{fact, moved, order, relation, renamed}; +use crate::events::tests::test_types::{HistoryTestCase, NewestColumnTestCase, NewestMoveTestCase}; + +#[test] +fn given_moves_when_projecting_newest_mention_then_latest_first_tied_event_wins() { + let test_cases = [ + NewestMoveTestCase { + description: "later event wins by time", + events: vec![ + moved(order(2, "a"), ("orders", "orders_v2")), + moved(order(1, "z"), ("orders_v2", "orders")), + ], + relation: relation(None, "MAIN", "Orders"), + expected_index: Some(0), + }, + NewestMoveTestCase { + description: "equal times order by event ID", + events: vec![ + moved(order(1, "b"), ("orders", "x")), + moved(order(1, "a"), ("x", "orders")), + ], + relation: relation(Some("db"), "main", "orders"), + expected_index: Some(0), + }, + NewestMoveTestCase { + description: "identical orders keep the first", + events: vec![ + moved(order(1, "a"), ("orders", "x")), + moved(order(1, "a"), ("orders", "y")), + ], + relation: relation(None, "main", "orders"), + expected_index: Some(0), + }, + NewestMoveTestCase { + description: "unrelated relation", + events: vec![moved(order(1, "a"), ("orders", "x"))], + relation: relation(None, "main", "customers"), + expected_index: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + newest_move_event(&test_case.events, &test_case.relation), + test_case.expected_index, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_column_renames_when_projecting_newest_mention_then_either_column_matches() { + let test_cases = [ + NewestColumnTestCase { + description: "destination column of a later rename matches case-insensitively", + events: vec![ + renamed(order(1, "a"), ("amount", "total")), + renamed(order(2, "b"), ("TOTAL", "net")), + ], + columns: ("x", "Total"), + expected_index: Some(1), + }, + NewestColumnTestCase { + description: "no rename names either column", + events: vec![renamed(order(1, "a"), ("amount", "total"))], + columns: ("x", "y"), + expected_index: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + newest_column_event( + &test_case.events, + &relation(None, "main", "orders"), + test_case.columns + ), + test_case.expected_index, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_moves_and_facts_when_projecting_histories_then_earliest_fact_of_each_step_is_kept() { + let test_cases = [ + HistoryTestCase { + description: "moves sort by time and keep the earliest fact per step", + moves: vec![order(3, "m2"), order(1, "m1"), order(1, "m0")], + facts: vec![ + fact(order(2, "f3"), "m1", OldNameStep::Required), + fact(order(2, "f2"), "m1", OldNameStep::Required), + fact(order(1, "f9"), "m1", OldNameStep::ViewDropped), + fact(order(1, "f1"), "m2", OldNameStep::Required), + fact(order(1, "f4"), "m2", OldNameStep::OriginArchived), + fact(order(1, "f5"), "m2", OldNameStep::ViewCreated), + fact(order(1, "f6"), "m0", OldNameStep::ViewCreated), + ], + expected_histories: vec![ + OldNameHistoryIndexes { + moved: 1, + required: 1, + archived: None, + created: None, + dropped: Some(2), + }, + OldNameHistoryIndexes { + moved: 0, + required: 3, + archived: Some(4), + created: Some(5), + dropped: None, + }, + ], + }, + HistoryTestCase { + description: "no moves", + moves: vec![], + facts: vec![fact(order(1, "f1"), "m1", OldNameStep::Required)], + expected_histories: vec![], + }, + ]; + + for test_case in test_cases { + assert_eq!( + old_name_histories(&test_case.moves, &test_case.facts), + test_case.expected_histories, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/events/tests/test_types.rs b/crates/sqlbuild-state/src/events/tests/test_types.rs index 46f300b779..e059578495 100644 --- a/crates/sqlbuild-state/src/events/tests/test_types.rs +++ b/crates/sqlbuild-state/src/events/tests/test_types.rs @@ -1,8 +1,62 @@ use crate::events::errors::StateSqlError; -use crate::events::models::StateCell; +use crate::events::models::{ + ColumnEventKey, EventOrder, EventRelation, MoveEventKey, OldNameFactKey, OldNameHistoryIndexes, + StateCell, +}; pub(crate) struct InsertTestCase { pub(crate) description: &'static str, pub(crate) cells: Vec, pub(crate) expected_sql: Result, } + +pub(crate) struct NewestMoveTestCase { + pub(crate) description: &'static str, + pub(crate) events: Vec, + pub(crate) relation: EventRelation, + pub(crate) expected_index: Option, +} + +pub(crate) struct NewestColumnTestCase { + pub(crate) description: &'static str, + pub(crate) events: Vec, + pub(crate) columns: (&'static str, &'static str), + pub(crate) expected_index: Option, +} + +pub(crate) struct HistoryTestCase { + pub(crate) description: &'static str, + pub(crate) moves: Vec, + pub(crate) facts: Vec, + pub(crate) expected_histories: Vec, +} + +pub(crate) struct AliasesTestCase { + pub(crate) description: &'static str, + pub(crate) aliases: Vec<(String, String)>, + pub(crate) expected_json: Option, +} + +pub(crate) struct DecodedAliasesTestCase { + pub(crate) description: &'static str, + pub(crate) raw: Option<&'static str>, + pub(crate) expected_aliases: Vec<(String, String)>, +} + +pub(crate) struct DecodedGrantsTestCase { + pub(crate) description: &'static str, + pub(crate) raw: Option<&'static str>, + pub(crate) expected_grants: Option>, +} + +pub(crate) struct ReadSqlTestCase { + pub(crate) description: &'static str, + pub(crate) stored: Vec, + pub(crate) expected_sql: String, +} + +pub(crate) struct GrantsTestCase { + pub(crate) description: &'static str, + pub(crate) grants: Option>, + pub(crate) expected_json: Option, +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index daa3467dd1..b4444c6552 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5,7 +5,7 @@ from contextlib import AbstractContextManager from contextvars import Token from datetime import date, datetime from pathlib import Path, PurePath -from typing import Any, TextIO, TypedDict +from typing import Any, Literal, TextIO, TypedDict, overload from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter @@ -182,7 +182,13 @@ from sqlbuild.compiler.lineage.models import ( ModelColumnLineage, ) from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability -from sqlbuild.compiler.migrations.models import MigrationRelation +from sqlbuild.compiler.migrations.models import ( + ColumnMigrationEvent, + MigrationEvent, + MigrationRelation, + OldNameViewEvent, + OldNameViewHistory, +) from sqlbuild.compiler.migrations.types import ( MigrationDecision, MigrationPromotion, @@ -4429,14 +4435,56 @@ def deterministic_column_migration_event_id( def deterministic_old_name_view_event_id( *, event_type: OldNameViewEventType, migration_event_id: str ) -> str: ... -def render_event_exists_sql(*, table: str, event_id: str) -> str: ... -def render_event_insert_sql( +def render_migration_state_create_sql( *, - table: str, - columns: tuple[str, ...], - column_types: Mapping[str, StateSqlValueType], - values: tuple[object | None, ...], + kind: Literal["model", "column", "old_name"], + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], ) -> str: ... +def render_migration_event_read_sql( + *, + kind: Literal["model", "column", "old_name"], + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + stored_columns: tuple[str, ...] | None = None, +) -> tuple[str, tuple[str, ...]]: ... +def render_migration_event_write_sql( + *, + kind: Literal["model", "column", "old_name"], + event: MigrationEvent | ColumnMigrationEvent | OldNameViewEvent, + render_qualified_name: Callable[..., str | None], +) -> tuple[str, str]: ... +@overload +def decode_migration_event_rows( + *, kind: Literal["model"], rows: Iterable[Sequence[object]], columns: tuple[str, ...] +) -> tuple[MigrationEvent, ...]: ... +@overload +def decode_migration_event_rows( + *, kind: Literal["column"], rows: Iterable[Sequence[object]], columns: tuple[str, ...] +) -> tuple[ColumnMigrationEvent, ...]: ... +@overload +def decode_migration_event_rows( + *, kind: Literal["old_name"], rows: Iterable[Sequence[object]], columns: tuple[str, ...] +) -> tuple[OldNameViewEvent, ...]: ... +def newest_migration_event_mentioning( + *, events: Iterable[MigrationEvent], relation: MigrationRelation +) -> MigrationEvent | None: ... +def newest_column_migration_event_mentioning( + *, + events: Iterable[ColumnMigrationEvent], + relation: MigrationRelation, + origin_column: str, + destination_column: str, +) -> ColumnMigrationEvent | None: ... +def project_old_name_views( + *, moves: Iterable[MigrationEvent], facts: Iterable[OldNameViewEvent] +) -> tuple[OldNameViewHistory, ...]: ... +def resolve_old_name_view_retention( + *, config_values: Mapping[str, object], project_retention: str | None +) -> str | None: ... def source_freshness_data_version_hash( *, source_name: str, diff --git a/src/sqlbuild/compiler/migrations/_helpers/column_sql.py b/src/sqlbuild/compiler/migrations/_helpers/column_sql.py deleted file mode 100644 index 4cacd717c9..0000000000 --- a/src/sqlbuild/compiler/migrations/_helpers/column_sql.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Portable SQL for append-only column migration events.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild._native import render_event_exists_sql, render_event_insert_sql -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( - render_state_table_create_sql, -) -from sqlbuild.compiler.migrations._helpers.sql import ( - decode_event_timestamp, - optional_text, -) -from sqlbuild.compiler.migrations.constants import ( - COLUMN_MIGRATION_COLUMN_TYPES, - COLUMN_MIGRATION_COLUMNS, - COLUMN_MIGRATION_TABLE_NAME, -) -from sqlbuild.compiler.migrations.exceptions import MigrationStateError -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation -from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDiscovery - - -def qualified_column_migration_table( - *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] -) -> str: - table: str | None = render_qualified_name( - database=database, schema=schema, name=COLUMN_MIGRATION_TABLE_NAME - ) - if table is None: - raise MigrationStateError("column migration state requires a target schema") - return table - - -def build_column_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - return render_state_table_create_sql( - qualified_name=qualified_column_migration_table( - database=database, schema=schema, render_qualified_name=render_qualified_name - ), - columns=COLUMN_MIGRATION_COLUMNS, - column_types=COLUMN_MIGRATION_COLUMN_TYPES, - required_columns=frozenset(), - render_framework_type=render_framework_type, - ) - - -def build_column_existing_event_sql( - *, event: ColumnMigrationEvent, render_qualified_name: Callable[..., str | None] -) -> str: - return render_event_exists_sql( - table=_event_table(event=event, render_qualified_name=render_qualified_name), - event_id=event.event_id, - ) - - -def build_column_insert_sql( - *, event: ColumnMigrationEvent, render_qualified_name: Callable[..., str | None] -) -> str: - return render_event_insert_sql( - table=_event_table(event=event, render_qualified_name=render_qualified_name), - columns=COLUMN_MIGRATION_COLUMNS, - column_types=COLUMN_MIGRATION_COLUMN_TYPES, - values=( - event.event_id, - event.target_name, - event.model_name, - event.relation.database, - event.relation.schema, - event.relation.name, - event.origin_column, - event.destination_column, - event.discovery.value, - event.decision.value, - event.run_id, - event.created_at, - ), - ) - - -def build_column_read_sql( - *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] -) -> str: - table: str = qualified_column_migration_table( - database=database, schema=schema, render_qualified_name=render_qualified_name - ) - return ( - f"SELECT {', '.join(COLUMN_MIGRATION_COLUMNS)} FROM {table} ORDER BY created_at, event_id" - ) - - -def decode_column_event_row(row: tuple[Any, ...]) -> ColumnMigrationEvent: - values: dict[str, Any] = dict(zip(COLUMN_MIGRATION_COLUMNS, row, strict=False)) - return ColumnMigrationEvent( - event_id=str(values["event_id"]), - target_name=optional_text(values["target_name"]), - model_name=str(values["model_name"]), - relation=MigrationRelation( - database=optional_text(values["relation_database"]), - schema=optional_text(values["relation_schema"]), - name=str(values["relation_name"]), - ), - origin_column=str(values["origin_column"]), - destination_column=str(values["destination_column"]), - discovery=MigrationDiscovery(str(values["discovery"])), - decision=ColumnMigrationDecision(str(values["decision"])), - run_id=str(values["run_id"]), - created_at=decode_event_timestamp(values["created_at"]), - ) - - -def _event_table( - *, event: ColumnMigrationEvent, render_qualified_name: Callable[..., str | None] -) -> str: - if event.relation.schema is None: - raise MigrationStateError("column migration events require a relation schema") - return qualified_column_migration_table( - database=event.relation.database, - schema=event.relation.schema, - render_qualified_name=render_qualified_name, - ) diff --git a/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py b/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py deleted file mode 100644 index 0f9e9b4887..0000000000 --- a/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py +++ /dev/null @@ -1,204 +0,0 @@ -"""Portable SQL for append-only old-name view facts.""" - -from __future__ import annotations - -import json -from collections.abc import Callable, Iterable -from typing import Any - -from sqlbuild._native import render_event_exists_sql, render_event_insert_sql -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( - render_state_table_create_sql, -) -from sqlbuild.compiler.migrations._helpers.sql import ( - decode_event_timestamp, - optional_text, -) -from sqlbuild.compiler.migrations.constants import ( - OLD_NAME_VIEW_COLUMN_TYPES, - OLD_NAME_VIEW_COLUMNS, - OLD_NAME_VIEW_TABLE_NAME, -) -from sqlbuild.compiler.migrations.exceptions import MigrationStateError -from sqlbuild.compiler.migrations.models import MigrationRelation, OldNameViewEvent -from sqlbuild.compiler.migrations.types import OldNameViewDropReason, OldNameViewEventType - - -def qualified_old_name_view_table( - *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] -) -> str: - table: str | None = render_qualified_name( - database=database, schema=schema, name=OLD_NAME_VIEW_TABLE_NAME - ) - if table is None: - raise MigrationStateError("old-name view state requires a target schema") - return table - - -def build_old_name_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - return render_state_table_create_sql( - qualified_name=qualified_old_name_view_table( - database=database, schema=schema, render_qualified_name=render_qualified_name - ), - columns=OLD_NAME_VIEW_COLUMNS, - column_types=OLD_NAME_VIEW_COLUMN_TYPES, - required_columns=frozenset(), - render_framework_type=render_framework_type, - ) - - -def build_old_name_existing_event_sql( - *, event: OldNameViewEvent, render_qualified_name: Callable[..., str | None] -) -> str: - return render_event_exists_sql( - table=_event_table(event=event, render_qualified_name=render_qualified_name), - event_id=event.event_id, - ) - - -def build_old_name_insert_sql( - *, event: OldNameViewEvent, render_qualified_name: Callable[..., str | None] -) -> str: - return render_event_insert_sql( - table=_event_table(event=event, render_qualified_name=render_qualified_name), - columns=OLD_NAME_VIEW_COLUMNS, - column_types=OLD_NAME_VIEW_COLUMN_TYPES, - values=( - event.event_id, - event.target_name, - event.event_type.value, - event.migration_event_id, - event.destination_model, - event.old.database, - event.old.schema, - event.old.name, - event.new.database, - event.new.schema, - event.new.name, - event.view_retention, - event.archive_name, - _encode_aliases(event.column_aliases) if event.column_aliases else None, - None if event.grants_copied is None else json.dumps(list(event.grants_copied)), - event.view_sql, - event.expires_at, - None if event.drop_reason is None else event.drop_reason.value, - event.run_id, - event.created_at, - ), - ) - - -def readable_old_name_columns(stored_columns: Iterable[str]) -> tuple[str, ...]: - """Return the known columns a stored table has; columns added later may be absent.""" - - stored: frozenset[str] = frozenset(column.lower() for column in stored_columns) - return tuple(column for column in OLD_NAME_VIEW_COLUMNS if column in stored) - - -def build_old_name_read_sql( - *, - database: str | None, - schema: str, - columns: tuple[str, ...], - render_qualified_name: Callable[..., str | None], -) -> str: - table: str = qualified_old_name_view_table( - database=database, schema=schema, render_qualified_name=render_qualified_name - ) - return f"SELECT {', '.join(columns)} FROM {table} ORDER BY created_at, event_id" - - -def decode_old_name_event_row( - *, row: tuple[Any, ...], columns: tuple[str, ...] = OLD_NAME_VIEW_COLUMNS -) -> OldNameViewEvent | None: - """Decode one row of ``columns``, reading absent ones as NULL; skip unknown event types.""" - - values: dict[str, Any] = dict(zip(columns, row, strict=False)) - try: - event_type: OldNameViewEventType = OldNameViewEventType(str(values.get("event_type"))) - except ValueError: - return None - raw_reason: str | None = optional_text(values.get("drop_reason")) - raw_expires: object | None = values.get("expires_at") - return OldNameViewEvent( - event_id=str(values["event_id"]), - target_name=optional_text(values.get("target_name")), - event_type=event_type, - migration_event_id=str(values["migration_event_id"]), - destination_model=str(values["destination_model"]), - old=MigrationRelation( - database=optional_text(values.get("old_database")), - schema=optional_text(values.get("old_schema")), - name=str(values["old_name"]), - ), - new=MigrationRelation( - database=optional_text(values.get("new_database")), - schema=optional_text(values.get("new_schema")), - name=str(values["new_name"]), - ), - run_id=str(values["run_id"]), - created_at=decode_event_timestamp(values["created_at"]), - view_retention=optional_text(values.get("view_retention")), - archive_name=optional_text(values.get("archive_name")), - column_aliases=_decode_aliases(optional_text(values.get("column_aliases"))), - expires_at=None if raw_expires is None else decode_event_timestamp(raw_expires), - drop_reason=_decode_reason(raw_reason), - grants_copied=_decode_grants(optional_text(values.get("grants_copied"))), - view_sql=optional_text(values.get("view_sql")), - ) - - -def _decode_reason(raw: str | None) -> OldNameViewDropReason | None: - if raw is None: - return None - try: - return OldNameViewDropReason(raw) - except ValueError: - return None - - -def _decode_grants(raw: str | None) -> tuple[str, ...] | None: - if raw is None: - return None - try: - payload: object = json.loads(raw) - except json.JSONDecodeError: - return None - if not isinstance(payload, list): - return None - return tuple(str(statement) for statement in payload) - - -def _encode_aliases(aliases: tuple[tuple[str, str], ...]) -> str: - return json.dumps(dict(aliases), sort_keys=True, separators=(",", ":")) - - -def _decode_aliases(raw: str | None) -> tuple[tuple[str, str], ...]: - if not raw: - return () - try: - payload: object = json.loads(raw) - except json.JSONDecodeError: - return () - if not isinstance(payload, dict): - return () - return tuple(sorted((str(old), str(new)) for old, new in payload.items())) - - -def _event_table( - *, event: OldNameViewEvent, render_qualified_name: Callable[..., str | None] -) -> str: - if event.new.schema is None: - raise MigrationStateError("old-name view facts require a destination schema") - return qualified_old_name_view_table( - database=event.new.database, - schema=event.new.schema, - render_qualified_name=render_qualified_name, - ) diff --git a/src/sqlbuild/compiler/migrations/_helpers/sql.py b/src/sqlbuild/compiler/migrations/_helpers/sql.py deleted file mode 100644 index 0cf1616b6d..0000000000 --- a/src/sqlbuild/compiler/migrations/_helpers/sql.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Portable SQL for append-only model migration events.""" - -from __future__ import annotations - -from collections.abc import Callable -from datetime import UTC, datetime -from typing import Any - -from sqlbuild._native import render_event_exists_sql, render_event_insert_sql -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( - render_state_table_create_sql, -) -from sqlbuild.compiler.migrations.constants import ( - MIGRATION_COLUMN_TYPES, - MIGRATION_COLUMNS, - MIGRATION_TABLE_NAME, -) -from sqlbuild.compiler.migrations.exceptions import MigrationStateError -from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation -from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery - - -def qualified_migration_table( - *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] -) -> str: - table: str | None = render_qualified_name( - database=database, schema=schema, name=MIGRATION_TABLE_NAME - ) - if table is None: - raise MigrationStateError("model migration state requires a target schema") - return table - - -def build_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - return render_state_table_create_sql( - qualified_name=qualified_migration_table( - database=database, schema=schema, render_qualified_name=render_qualified_name - ), - columns=MIGRATION_COLUMNS, - column_types=MIGRATION_COLUMN_TYPES, - required_columns=frozenset(), - render_framework_type=render_framework_type, - ) - - -def build_existing_event_sql( - *, event: MigrationEvent, render_qualified_name: Callable[..., str | None] -) -> str: - table: str = _event_table(event=event, render_qualified_name=render_qualified_name) - return render_event_exists_sql(table=table, event_id=event.event_id) - - -def build_insert_sql( - *, event: MigrationEvent, render_qualified_name: Callable[..., str | None] -) -> str: - table: str = _event_table(event=event, render_qualified_name=render_qualified_name) - return render_event_insert_sql( - table=table, - columns=MIGRATION_COLUMNS, - column_types=MIGRATION_COLUMN_TYPES, - values=_event_values(event), - ) - - -def _event_table(*, event: MigrationEvent, render_qualified_name: Callable[..., str | None]) -> str: - if event.destination.schema is None: - raise MigrationStateError("model migration events require a destination schema") - return qualified_migration_table( - database=event.destination.database, - schema=event.destination.schema, - render_qualified_name=render_qualified_name, - ) - - -def build_read_sql( - *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] -) -> str: - table: str = qualified_migration_table( - database=database, schema=schema, render_qualified_name=render_qualified_name - ) - return f"SELECT {', '.join(MIGRATION_COLUMNS)} FROM {table} ORDER BY created_at, event_id" - - -def decode_event_row(row: tuple[Any, ...]) -> MigrationEvent: - values: dict[str, Any] = dict(zip(MIGRATION_COLUMNS, row, strict=False)) - created_at: datetime = decode_event_timestamp(values["created_at"]) - return MigrationEvent( - event_id=str(values["event_id"]), - target_name=optional_text(values["target_name"]), - origin_model=optional_text(values["origin_model"]), - origin=MigrationRelation( - database=optional_text(values["origin_database"]), - schema=optional_text(values["origin_schema"]), - name=str(values["origin_name"]), - ), - destination_model=str(values["destination_model"]), - destination=MigrationRelation( - database=optional_text(values["destination_database"]), - schema=optional_text(values["destination_schema"]), - name=str(values["destination_name"]), - ), - origin_version_hash=str(values["origin_version_hash"] or ""), - discovery=MigrationDiscovery(str(values["discovery"])), - decision=MigrationDecision(str(values["decision"])), - run_id=str(values["run_id"]), - created_at=created_at, - ) - - -def _event_values(event: MigrationEvent) -> tuple[object | None, ...]: - return ( - event.event_id, - event.target_name, - event.origin_model, - event.origin.database, - event.origin.schema, - event.origin.name, - event.destination_model, - event.destination.database, - event.destination.schema, - event.destination.name, - event.origin_version_hash, - event.discovery.value, - event.decision.value, - event.run_id, - event.created_at, - ) - - -def decode_event_timestamp(raw: object) -> datetime: - """Decode a stored event timestamp as an aware UTC datetime.""" - - created_at: datetime = raw if isinstance(raw, datetime) else datetime.fromisoformat(str(raw)) - if created_at.tzinfo is None: - created_at = created_at.replace(tzinfo=UTC) - return created_at - - -def optional_text(value: object | None) -> str | None: - """Return a stored nullable text value as a string or None.""" - - return None if value is None else str(value) diff --git a/src/sqlbuild/compiler/migrations/constants.py b/src/sqlbuild/compiler/migrations/constants.py index 07c1e30dcb..51b0f7a8b9 100644 --- a/src/sqlbuild/compiler/migrations/constants.py +++ b/src/sqlbuild/compiler/migrations/constants.py @@ -2,85 +2,10 @@ from __future__ import annotations -from sqlbuild.sql_values.types import StateSqlValueType - MIGRATION_TABLE_NAME: str = "_sqlbuild_migrations" MIGRATION_WRITE_ATTEMPTS: int = 3 MIGRATION_WRITE_RETRY_BASE_SECONDS: float = 0.05 -MIGRATION_COLUMNS: tuple[str, ...] = ( - "event_id", - "target_name", - "origin_model", - "origin_database", - "origin_schema", - "origin_name", - "destination_model", - "destination_database", - "destination_schema", - "destination_name", - "origin_version_hash", - "discovery", - "decision", - "run_id", - "created_at", -) - -MIGRATION_COLUMN_TYPES: dict[str, StateSqlValueType] = { - **{column: StateSqlValueType.STRING for column in MIGRATION_COLUMNS}, - "created_at": StateSqlValueType.TEXT_TIMESTAMP, -} - COLUMN_MIGRATION_TABLE_NAME: str = "_sqlbuild_column_migrations" -COLUMN_MIGRATION_COLUMNS: tuple[str, ...] = ( - "event_id", - "target_name", - "model_name", - "relation_database", - "relation_schema", - "relation_name", - "origin_column", - "destination_column", - "discovery", - "decision", - "run_id", - "created_at", -) - -COLUMN_MIGRATION_COLUMN_TYPES: dict[str, StateSqlValueType] = { - **{column: StateSqlValueType.STRING for column in COLUMN_MIGRATION_COLUMNS}, - "created_at": StateSqlValueType.TEXT_TIMESTAMP, -} - OLD_NAME_VIEW_TABLE_NAME: str = "_sqlbuild_old_name_views" -OLD_NAME_VIEW_CONFIG_KEY: str = "old_name_view" - -OLD_NAME_VIEW_COLUMNS: tuple[str, ...] = ( - "event_id", - "target_name", - "event_type", - "migration_event_id", - "destination_model", - "old_database", - "old_schema", - "old_name", - "new_database", - "new_schema", - "new_name", - "view_retention", - "archive_name", - "column_aliases", - "grants_copied", - "view_sql", - "expires_at", - "drop_reason", - "run_id", - "created_at", -) - -OLD_NAME_VIEW_COLUMN_TYPES: dict[str, StateSqlValueType] = { - **{column: StateSqlValueType.STRING for column in OLD_NAME_VIEW_COLUMNS}, - "expires_at": StateSqlValueType.TEXT_TIMESTAMP, - "created_at": StateSqlValueType.TEXT_TIMESTAMP, -} diff --git a/src/sqlbuild/compiler/migrations/main/_newest_column_event.py b/src/sqlbuild/compiler/migrations/main/_newest_column_event.py deleted file mode 100644 index 6158873f46..0000000000 --- a/src/sqlbuild/compiler/migrations/main/_newest_column_event.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Project the newest column migration event that mentions a column pair.""" - -from __future__ import annotations - -from collections.abc import Iterable - -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation - - -def newest_column_migration_event_mentioning( - *, - events: Iterable[ColumnMigrationEvent], - relation: MigrationRelation, - origin_column: str, - destination_column: str, -) -> ColumnMigrationEvent | None: - """Return the latest event on the relation that renamed either column.""" - - columns: frozenset[str] = frozenset({origin_column.lower(), destination_column.lower()}) - newest: ColumnMigrationEvent | None = None - event: ColumnMigrationEvent - for event in events: - if not event.mentions(relation=relation, columns=columns): - continue - if newest is None or (event.created_at, event.event_id) > ( - newest.created_at, - newest.event_id, - ): - newest = event - return newest diff --git a/src/sqlbuild/compiler/migrations/main/_newest_event.py b/src/sqlbuild/compiler/migrations/main/_newest_event.py deleted file mode 100644 index 888f0d2a9e..0000000000 --- a/src/sqlbuild/compiler/migrations/main/_newest_event.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Project the newest migration event that mentions one relation.""" - -from __future__ import annotations - -from collections.abc import Iterable - -from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation - - -def newest_migration_event_mentioning( - *, events: Iterable[MigrationEvent], relation: MigrationRelation -) -> MigrationEvent | None: - """Return the latest event naming the relation as origin or destination.""" - - newest: MigrationEvent | None = None - event: MigrationEvent - for event in events: - if not event.mentions(relation): - continue - if newest is None or (event.created_at, event.event_id) > ( - newest.created_at, - newest.event_id, - ): - newest = event - return newest diff --git a/src/sqlbuild/compiler/migrations/main/_project_old_name_views.py b/src/sqlbuild/compiler/migrations/main/_project_old_name_views.py deleted file mode 100644 index c77a46db97..0000000000 --- a/src/sqlbuild/compiler/migrations/main/_project_old_name_views.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Project old-name view histories from recorded moves and old-name facts.""" - -from __future__ import annotations - -from collections.abc import Iterable - -from sqlbuild.compiler.migrations.models import ( - MigrationEvent, - OldNameViewEvent, - OldNameViewHistory, -) -from sqlbuild.compiler.migrations.types import OldNameViewEventType - - -def project_old_name_views( - *, moves: Iterable[MigrationEvent], facts: Iterable[OldNameViewEvent] -) -> tuple[OldNameViewHistory, ...]: - """Return one history per recorded move whose old name needs a compatibility view.""" - - by_move: dict[str, dict[OldNameViewEventType, OldNameViewEvent]] = {} - fact: OldNameViewEvent - for fact in sorted(facts, key=lambda item: (item.created_at, item.event_id)): - _ = by_move.setdefault(fact.migration_event_id, {}).setdefault(fact.event_type, fact) - histories: list[OldNameViewHistory] = [] - move: MigrationEvent - for move in sorted(moves, key=lambda item: (item.created_at, item.event_id)): - recorded: dict[OldNameViewEventType, OldNameViewEvent] = by_move.get(move.event_id, {}) - required: OldNameViewEvent | None = recorded.get(OldNameViewEventType.REQUIRED) - if required is None: - continue - histories.append( - OldNameViewHistory( - move=move, - required=required, - archived=recorded.get(OldNameViewEventType.ORIGIN_ARCHIVED), - created=recorded.get(OldNameViewEventType.VIEW_CREATED), - dropped=recorded.get(OldNameViewEventType.VIEW_DROPPED), - ) - ) - return tuple(histories) diff --git a/src/sqlbuild/compiler/migrations/main/_read_column_events.py b/src/sqlbuild/compiler/migrations/main/_read_column_events.py index a8b0b045f8..64de56e0e7 100644 --- a/src/sqlbuild/compiler/migrations/main/_read_column_events.py +++ b/src/sqlbuild/compiler/migrations/main/_read_column_events.py @@ -5,11 +5,8 @@ from collections.abc import Callable from typing import Any +from sqlbuild._native import decode_migration_event_rows, render_migration_event_read_sql from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations._helpers.column_sql import ( - build_column_read_sql, - decode_column_event_row, -) from sqlbuild.compiler.migrations.exceptions import MigrationStateError from sqlbuild.compiler.migrations.models import ColumnMigrationEvent @@ -24,8 +21,10 @@ def read_column_migration_events( ) -> tuple[ColumnMigrationEvent, ...]: """Read every column migration event this version understands from one schema.""" - sql: str = build_column_read_sql( - database=database, schema=schema, render_qualified_name=render_qualified_name + sql: str + columns: tuple[str, ...] + sql, columns = render_migration_event_read_sql( + kind="column", database=database, schema=schema, render_qualified_name=render_qualified_name ) try: rows: list[tuple[Any, ...]] = execute(connection=connection, sql=sql).fetchall() @@ -33,11 +32,4 @@ def read_column_migration_events( raise MigrationStateError( f"Unable to read column migration events from schema '{schema}': {error}" ) from error - events: list[ColumnMigrationEvent] = [] - row: tuple[Any, ...] - for row in rows: - try: - events.append(decode_column_event_row(tuple(row))) - except ValueError: - continue - return tuple(events) + return decode_migration_event_rows(kind="column", rows=rows, columns=columns) diff --git a/src/sqlbuild/compiler/migrations/main/_read_events.py b/src/sqlbuild/compiler/migrations/main/_read_events.py index af847b5aff..d45f8b0aae 100644 --- a/src/sqlbuild/compiler/migrations/main/_read_events.py +++ b/src/sqlbuild/compiler/migrations/main/_read_events.py @@ -5,8 +5,8 @@ from collections.abc import Callable from typing import Any +from sqlbuild._native import decode_migration_event_rows, render_migration_event_read_sql from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations._helpers.sql import build_read_sql, decode_event_row from sqlbuild.compiler.migrations.exceptions import MigrationStateError from sqlbuild.compiler.migrations.models import MigrationEvent @@ -21,8 +21,10 @@ def read_migration_events( ) -> tuple[MigrationEvent, ...]: """Read every migration event recorded in one schema's state table.""" - sql: str = build_read_sql( - database=database, schema=schema, render_qualified_name=render_qualified_name + sql: str + columns: tuple[str, ...] + sql, columns = render_migration_event_read_sql( + kind="model", database=database, schema=schema, render_qualified_name=render_qualified_name ) try: rows: list[tuple[Any, ...]] = execute(connection=connection, sql=sql).fetchall() @@ -30,4 +32,4 @@ def read_migration_events( raise MigrationStateError( f"Unable to read model migration events from schema '{schema}': {error}" ) from error - return tuple(decode_event_row(tuple(row)) for row in rows) + return decode_migration_event_rows(kind="model", rows=rows, columns=columns) diff --git a/src/sqlbuild/compiler/migrations/main/_resolve_old_name_view_retention.py b/src/sqlbuild/compiler/migrations/main/_resolve_old_name_view_retention.py deleted file mode 100644 index f65d17b304..0000000000 --- a/src/sqlbuild/compiler/migrations/main/_resolve_old_name_view_retention.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Resolve how long a migrated model keeps a compatibility view at its old name.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.compiler.migrations.constants import OLD_NAME_VIEW_CONFIG_KEY - - -def resolve_old_name_view_retention( - *, config_values: Mapping[str, object], project_retention: str | None -) -> str | None: - """Return the model's retention text, the project's when unset, or None when disabled.""" - - if OLD_NAME_VIEW_CONFIG_KEY not in config_values: - return project_retention - value: object = config_values[OLD_NAME_VIEW_CONFIG_KEY] - if value is False: - return None - return str(value).strip() diff --git a/src/sqlbuild/compiler/migrations/main/column_create_table_sql.py b/src/sqlbuild/compiler/migrations/main/column_create_table_sql.py index 37a22d0e4d..465b227cfc 100644 --- a/src/sqlbuild/compiler/migrations/main/column_create_table_sql.py +++ b/src/sqlbuild/compiler/migrations/main/column_create_table_sql.py @@ -4,8 +4,8 @@ from collections.abc import Callable +from sqlbuild._native import render_migration_state_create_sql from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.compiler.migrations._helpers.column_sql import build_column_create_table_sql def build_column_migration_state_create_table_sql( @@ -17,7 +17,8 @@ def build_column_migration_state_create_table_sql( ) -> str: """Build DDL that creates the column migration event table when it is missing.""" - return build_column_create_table_sql( + return render_migration_state_create_sql( + kind="column", database=database, schema=schema, render_qualified_name=render_qualified_name, diff --git a/src/sqlbuild/compiler/migrations/main/create_table_sql.py b/src/sqlbuild/compiler/migrations/main/create_table_sql.py index 75f9c3eea5..502ead0b50 100644 --- a/src/sqlbuild/compiler/migrations/main/create_table_sql.py +++ b/src/sqlbuild/compiler/migrations/main/create_table_sql.py @@ -4,8 +4,8 @@ from collections.abc import Callable +from sqlbuild._native import render_migration_state_create_sql from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.compiler.migrations._helpers.sql import build_create_table_sql def build_migration_state_create_table_sql( @@ -17,7 +17,8 @@ def build_migration_state_create_table_sql( ) -> str: """Build DDL that creates the model migration event table when it is missing.""" - return build_create_table_sql( + return render_migration_state_create_sql( + kind="model", database=database, schema=schema, render_qualified_name=render_qualified_name, diff --git a/src/sqlbuild/compiler/migrations/main/old_name_view_create_table_sql.py b/src/sqlbuild/compiler/migrations/main/old_name_view_create_table_sql.py index c489c71eee..3a82a9055c 100644 --- a/src/sqlbuild/compiler/migrations/main/old_name_view_create_table_sql.py +++ b/src/sqlbuild/compiler/migrations/main/old_name_view_create_table_sql.py @@ -4,8 +4,8 @@ from collections.abc import Callable +from sqlbuild._native import render_migration_state_create_sql from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.compiler.migrations._helpers.old_name_sql import build_old_name_create_table_sql def build_old_name_view_state_create_table_sql( @@ -17,7 +17,8 @@ def build_old_name_view_state_create_table_sql( ) -> str: """Build DDL that creates the old-name view fact table when it is missing.""" - return build_old_name_create_table_sql( + return render_migration_state_create_sql( + kind="old_name", database=database, schema=schema, render_qualified_name=render_qualified_name, diff --git a/src/sqlbuild/compiler/migrations/main/read_old_name_view_events.py b/src/sqlbuild/compiler/migrations/main/read_old_name_view_events.py index a08d199876..ffdec54df3 100644 --- a/src/sqlbuild/compiler/migrations/main/read_old_name_view_events.py +++ b/src/sqlbuild/compiler/migrations/main/read_old_name_view_events.py @@ -5,12 +5,8 @@ from collections.abc import Callable, Iterable from typing import Any +from sqlbuild._native import decode_migration_event_rows, render_migration_event_read_sql from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations._helpers.old_name_sql import ( - build_old_name_read_sql, - decode_old_name_event_row, - readable_old_name_columns, -) from sqlbuild.compiler.migrations.exceptions import MigrationStateError from sqlbuild.compiler.migrations.models import OldNameViewEvent @@ -26,12 +22,14 @@ def read_old_name_view_events( ) -> tuple[OldNameViewEvent, ...]: """Read every old-name fact this version understands from the columns the table has.""" - columns: tuple[str, ...] = readable_old_name_columns(stored_columns) - sql: str = build_old_name_read_sql( + sql: str + columns: tuple[str, ...] + sql, columns = render_migration_event_read_sql( + kind="old_name", database=database, schema=schema, - columns=columns, render_qualified_name=render_qualified_name, + stored_columns=tuple(stored_columns), ) try: rows: list[tuple[Any, ...]] = execute(connection=connection, sql=sql).fetchall() @@ -39,10 +37,4 @@ def read_old_name_view_events( raise MigrationStateError( f"Unable to read old-name view facts from schema '{schema}': {error}" ) from error - events: list[OldNameViewEvent] = [] - row: tuple[Any, ...] - for row in rows: - event: OldNameViewEvent | None = decode_old_name_event_row(row=tuple(row), columns=columns) - if event is not None: - events.append(event) - return tuple(events) + return decode_migration_event_rows(kind="old_name", rows=rows, columns=columns) diff --git a/src/sqlbuild/compiler/migrations/main/read_old_name_view_histories.py b/src/sqlbuild/compiler/migrations/main/read_old_name_view_histories.py index da9607a15c..9de86cae2e 100644 --- a/src/sqlbuild/compiler/migrations/main/read_old_name_view_histories.py +++ b/src/sqlbuild/compiler/migrations/main/read_old_name_view_histories.py @@ -5,8 +5,8 @@ from collections.abc import Callable, Iterable from typing import Any +from sqlbuild._native import project_old_name_views from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations.main._project_old_name_views import project_old_name_views from sqlbuild.compiler.migrations.main._read_events import read_migration_events from sqlbuild.compiler.migrations.main.read_old_name_view_events import ( read_old_name_view_events, diff --git a/src/sqlbuild/compiler/migrations/main/write_column_event.py b/src/sqlbuild/compiler/migrations/main/write_column_event.py index 2847fd7cf7..6e50ddb55f 100644 --- a/src/sqlbuild/compiler/migrations/main/write_column_event.py +++ b/src/sqlbuild/compiler/migrations/main/write_column_event.py @@ -5,11 +5,8 @@ from collections.abc import Callable from typing import Any +from sqlbuild._native import render_migration_event_write_sql from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations._helpers.column_sql import ( - build_column_existing_event_sql, - build_column_insert_sql, -) from sqlbuild.compiler.migrations._helpers.writes import append_event_row from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS from sqlbuild.compiler.migrations.models import ColumnMigrationEvent @@ -26,15 +23,16 @@ def write_column_migration_event( ) -> None: """Create the state table from adapter DDL when given, then insert the event if absent.""" + existing_sql: str + insert_sql: str + existing_sql, insert_sql = render_migration_event_write_sql( + kind="column", event=event, render_qualified_name=render_qualified_name + ) _ = append_event_row( connection=connection, execute=execute, - existing_sql=build_column_existing_event_sql( - event=event, render_qualified_name=render_qualified_name - ), - insert_sql=build_column_insert_sql( - event=event, render_qualified_name=render_qualified_name - ), + existing_sql=existing_sql, + insert_sql=insert_sql, create_table_sql=create_table_sql, attempts=attempts, subject=f"{event.model_name}.{event.destination_column}", diff --git a/src/sqlbuild/compiler/migrations/main/write_event.py b/src/sqlbuild/compiler/migrations/main/write_event.py index a34abb2ce8..1e005d1aa2 100644 --- a/src/sqlbuild/compiler/migrations/main/write_event.py +++ b/src/sqlbuild/compiler/migrations/main/write_event.py @@ -5,8 +5,8 @@ from collections.abc import Callable from typing import Any +from sqlbuild._native import render_migration_event_write_sql from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations._helpers.sql import build_existing_event_sql, build_insert_sql from sqlbuild.compiler.migrations._helpers.writes import append_event_row from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS from sqlbuild.compiler.migrations.models import MigrationEvent @@ -23,13 +23,16 @@ def write_migration_event( ) -> None: """Create the state table from adapter DDL when given, then insert the event if absent.""" + existing_sql: str + insert_sql: str + existing_sql, insert_sql = render_migration_event_write_sql( + kind="model", event=event, render_qualified_name=render_qualified_name + ) _ = append_event_row( connection=connection, execute=execute, - existing_sql=build_existing_event_sql( - event=event, render_qualified_name=render_qualified_name - ), - insert_sql=build_insert_sql(event=event, render_qualified_name=render_qualified_name), + existing_sql=existing_sql, + insert_sql=insert_sql, create_table_sql=create_table_sql, attempts=attempts, subject=event.destination_model, diff --git a/src/sqlbuild/compiler/migrations/main/write_old_name_view_event.py b/src/sqlbuild/compiler/migrations/main/write_old_name_view_event.py index 8c143f2e1f..508c428b4d 100644 --- a/src/sqlbuild/compiler/migrations/main/write_old_name_view_event.py +++ b/src/sqlbuild/compiler/migrations/main/write_old_name_view_event.py @@ -5,11 +5,8 @@ from collections.abc import Callable from typing import Any +from sqlbuild._native import render_migration_event_write_sql from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations._helpers.old_name_sql import ( - build_old_name_existing_event_sql, - build_old_name_insert_sql, -) from sqlbuild.compiler.migrations._helpers.writes import append_event_row from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS from sqlbuild.compiler.migrations.models import OldNameViewEvent @@ -26,15 +23,16 @@ def write_old_name_view_event( ) -> None: """Create the state table from adapter DDL when given, then insert the fact if absent.""" + existing_sql: str + insert_sql: str + existing_sql, insert_sql = render_migration_event_write_sql( + kind="old_name", event=event, render_qualified_name=render_qualified_name + ) _ = append_event_row( connection=connection, execute=execute, - existing_sql=build_old_name_existing_event_sql( - event=event, render_qualified_name=render_qualified_name - ), - insert_sql=build_old_name_insert_sql( - event=event, render_qualified_name=render_qualified_name - ), + existing_sql=existing_sql, + insert_sql=insert_sql, create_table_sql=create_table_sql, attempts=attempts, subject=f"{event.destination_model} old name {event.old.name}", diff --git a/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py b/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py index 25ce1f4011..8a8aaefbeb 100644 --- a/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py +++ b/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py @@ -4,7 +4,7 @@ from typing import Any -from sqlbuild._native import read_latest_fingerprints +from sqlbuild._native import project_old_name_views, read_latest_fingerprints from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapter.relations.main.get_columns_for_inspection import get_columns_for_inspection @@ -19,7 +19,6 @@ MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, ) -from sqlbuild.compiler.migrations.main._project_old_name_views import project_old_name_views from sqlbuild.compiler.migrations.main._read_column_events import read_column_migration_events from sqlbuild.compiler.migrations.main._read_events import read_migration_events from sqlbuild.compiler.migrations.main.read_old_name_view_events import ( diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py index e7d28eb393..28e1acf3db 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from sqlbuild.compiler.migrations.models import MigrationRelation -from sqlbuild.compiler.migrations.types import OldNameViewEventType +from sqlbuild.compiler.migrations.types import OldNameViewDropReason, OldNameViewEventType @dataclass(frozen=True) @@ -100,3 +100,21 @@ class PinnedOldNameEventIdTestCase: event_type: OldNameViewEventType migration_event_id: str expected_event_id: str + + +@dataclass(frozen=True) +class StoredOldNameRowTestCase: + description: str + column_aliases_sql: str + grants_copied_sql: str + drop_reason_sql: str + expected_aliases: tuple[tuple[str, str], ...] + expected_grants: tuple[str, ...] | None + expected_drop_reason: OldNameViewDropReason | None + + +@dataclass(frozen=True) +class StoredColumnRowTestCase: + description: str + decisions: tuple[str, ...] + expected_decisions: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py index cd49f9f219..a3f62a72fa 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py @@ -5,10 +5,8 @@ import duckdb import pytest +from sqlbuild._native import newest_column_migration_event_mentioning from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.main._newest_column_event import ( - newest_column_migration_event_mentioning, -) from sqlbuild.compiler.migrations.main._read_column_events import read_column_migration_events from sqlbuild.compiler.migrations.main.write_column_event import write_column_migration_event from sqlbuild.compiler.migrations.models import ColumnMigrationEvent diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py index 87d21f00ee..3eff4da42f 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py @@ -5,8 +5,8 @@ import duckdb import pytest +from sqlbuild._native import newest_migration_event_mentioning from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.main._newest_event import newest_migration_event_mentioning from sqlbuild.compiler.migrations.main._read_events import read_migration_events from sqlbuild.compiler.migrations.main.write_event import write_migration_event from sqlbuild.compiler.migrations.models import MigrationEvent diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py index 4e7638cbf7..bb8e0bb2ec 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py @@ -7,12 +7,14 @@ import duckdb import pytest +from sqlbuild._native import project_old_name_views from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.constants import OLD_NAME_VIEW_COLUMNS -from sqlbuild.compiler.migrations.main._project_old_name_views import project_old_name_views from sqlbuild.compiler.migrations.main.read_old_name_view_events import ( read_old_name_view_events, ) +from sqlbuild.compiler.migrations.main.stored_old_name_view_columns import ( + stored_old_name_view_columns, +) from sqlbuild.compiler.migrations.main.write_event import write_migration_event from sqlbuild.compiler.migrations.main.write_old_name_view_event import write_old_name_view_event from sqlbuild.compiler.migrations.models import ( @@ -90,7 +92,9 @@ def test_given_old_name_facts_when_writing_and_reading_then_rows_are_idempotent( execute=adapter.execute, database=None, schema="main", - stored_columns=OLD_NAME_VIEW_COLUMNS, + stored_columns=stored_old_name_view_columns( + adapter=adapter, connection=connection, database=None, schema="main" + ), render_qualified_name=adapter.render_qualified_name, ) diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py new file mode 100644 index 0000000000..5f76ea0717 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py @@ -0,0 +1,135 @@ +"""Unit coverage for decoding hand-written persisted migration state rows.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import duckdb +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.migrations.main._read_column_events import read_column_migration_events +from sqlbuild.compiler.migrations.main.read_old_name_view_events import ( + read_old_name_view_events, +) +from sqlbuild.compiler.migrations.main.stored_old_name_view_columns import ( + stored_old_name_view_columns, +) +from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, OldNameViewEvent +from sqlbuild.compiler.migrations.types import OldNameViewDropReason +from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( + StoredColumnRowTestCase, + StoredOldNameRowTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + StoredOldNameRowTestCase( + description="stored aliases are sorted and grants keep their order", + column_aliases_sql="""'{"b":"total","a":"amount"}'""", + grants_copied_sql="""'["GRANT SELECT ON main.orders TO reporting"]'""", + drop_reason_sql="'early'", + expected_aliases=(("a", "amount"), ("b", "total")), + expected_grants=("GRANT SELECT ON main.orders TO reporting",), + expected_drop_reason=OldNameViewDropReason.EARLY, + ), + StoredOldNameRowTestCase( + description="unreadable JSON and an unknown drop reason read as absent", + column_aliases_sql="'{not json'", + grants_copied_sql="""'{"a": 1}'""", + drop_reason_sql="'from_a_future_version'", + expected_aliases=(), + expected_grants=None, + expected_drop_reason=None, + ), + StoredOldNameRowTestCase( + description="null columns read as no aliases and unknown grants", + column_aliases_sql="NULL", + grants_copied_sql="NULL", + drop_reason_sql="NULL", + expected_aliases=(), + expected_grants=None, + expected_drop_reason=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_persisted_old_name_row_when_reading_then_codecs_decode_stored_text( + test_case: StoredOldNameRowTestCase, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: duckdb.DuckDBPyConnection = duckdb.connect(":memory:") + _ = connection.execute( + adapter.render_create_old_name_view_state_table_sql(database=None, schema="main") + ) + _ = connection.execute( + "INSERT INTO main._sqlbuild_old_name_views (event_id, event_type, migration_event_id, " + "destination_model, old_schema, old_name, new_schema, new_name, column_aliases, " + "grants_copied, drop_reason, run_id, created_at) VALUES ('fact', 'view_dropped', 'move', " + "'daily_orders', 'main', 'orders', 'main', 'daily_orders', " + f"{test_case.column_aliases_sql}, {test_case.grants_copied_sql}, " + f"{test_case.drop_reason_sql}, 'run-1', '2026-01-02T03:04:05+00:00')" + ) + + stored: tuple[OldNameViewEvent, ...] = read_old_name_view_events( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + stored_columns=stored_old_name_view_columns( + adapter=adapter, connection=connection, database=None, schema="main" + ), + render_qualified_name=adapter.render_qualified_name, + ) + + assert len(stored) == 1 + assert stored[0].column_aliases == test_case.expected_aliases + assert stored[0].grants_copied == test_case.expected_grants + assert stored[0].drop_reason == test_case.expected_drop_reason + assert stored[0].created_at == datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC) + + +@pytest.mark.parametrize( + "test_case", + [ + StoredColumnRowTestCase( + description="rows with a decision from a newer version are skipped", + decisions=("rename", "decided_by_a_future_version", "record"), + expected_decisions=("rename", "record"), + ), + ], + ids=lambda case: case.description, +) +def test_given_persisted_column_rows_when_reading_then_unknown_rows_are_skipped_in_order( + test_case: StoredColumnRowTestCase, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: duckdb.DuckDBPyConnection = duckdb.connect(":memory:") + _ = connection.execute( + adapter.render_create_column_migration_state_table_sql(database=None, schema="main") + ) + index: int + decision: str + for index, decision in enumerate(test_case.decisions): + _ = connection.execute( + "INSERT INTO main._sqlbuild_column_migrations (event_id, model_name, relation_schema, " + "relation_name, origin_column, destination_column, discovery, decision, run_id, " + f"created_at) VALUES ('event-{index}', 'orders', 'main', 'orders', 'amount', " + f"'total', 'manual', '{decision}', 'run-{index}', '2026-01-0{index + 1}')" + ) + + stored: tuple[ColumnMigrationEvent, ...] = read_column_migration_events( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + ) + + assert tuple(event.decision.value for event in stored) == test_case.expected_decisions + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 9308b1c6f3a495931b2182db52e9c5e8bd5999ad Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:20:48 +0100 Subject: [PATCH 15/22] chore: update Python boundary after migration event port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 72f5982736..00794ad6d4 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 30318 +remaining_lines = 29633 [[stays]] path = "executor" From 544a4e9eea65f51289540c43272b78239471fb34 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 02:09:08 +0100 Subject: [PATCH 16/22] test: compare planner dates relative to the run date --- .../_helpers/capturing/canonical.py | 23 +++++++++++++++++-- scripts/planner_differential/constants.py | 3 +++ .../_helpers/capturing/test_canonical.py | 15 ++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/scripts/planner_differential/_helpers/capturing/canonical.py b/scripts/planner_differential/_helpers/capturing/canonical.py index 574825e90f..927fb20359 100644 --- a/scripts/planner_differential/_helpers/capturing/canonical.py +++ b/scripts/planner_differential/_helpers/capturing/canonical.py @@ -9,12 +9,15 @@ from scripts.compiler_differential.classes.capture_file import expand_capture_text from scripts.planner_differential.constants import ( COMPACT_TIMESTAMP_PATTERN, + DATE_PATTERN, ELAPSED_TIME_MASK, ELAPSED_TIME_PATTERN, ISO_TIMESTAMP_PATTERN, MAPPING_MARKER, RECENT_TIMESTAMP_DAYS, RECENT_TIMESTAMP_MASK, + RUN_RELATIVE_FUTURE_DAYS, + RUN_RELATIVE_PAST_DAYS, VOLATILE_KEY_SUFFIXES, VOLATILE_KEYS, VOLATILE_VALUE_MASK, @@ -24,7 +27,7 @@ def mask_text(*, text: str, masks: TextMasks) -> str: - """Mask invocation ids, run paths and timestamps near the run date.""" + """Mask invocation ids, run paths, timestamps near the run date and run-relative dates.""" masked: str = mask_capture_noise(text) for prefix, replacement in masks.replacements: @@ -32,9 +35,12 @@ def mask_text(*, text: str, masks: TextMasks) -> str: masked = ISO_TIMESTAMP_PATTERN.sub( lambda match: _recent_or_kept(match=match, today=masks.today, iso=True), masked ) - return COMPACT_TIMESTAMP_PATTERN.sub( + masked = COMPACT_TIMESTAMP_PATTERN.sub( lambda match: _recent_or_kept(match=match, today=masks.today, iso=False), masked ) + return DATE_PATTERN.sub( + lambda match: _run_relative_date(match=match, today=masks.today), masked + ) def mask_console_text(*, text: str, masks: TextMasks) -> str: @@ -93,3 +99,16 @@ def _recent_or_kept(*, match: re.Match[str], today: date, iso: bool) -> str: return match.group(0) recent: bool = day >= today - timedelta(days=RECENT_TIMESTAMP_DAYS) return RECENT_TIMESTAMP_MASK if recent else match.group(0) + + +def _run_relative_date(*, match: re.Match[str], today: date) -> str: + """Write dates the planner derives from the run date, such as replay ranges, as offsets.""" + + try: + day: date = date.fromisoformat(match.group(1)) + except ValueError: + return match.group(0) + offset: int = (day - today).days + if -RUN_RELATIVE_PAST_DAYS <= offset <= RUN_RELATIVE_FUTURE_DAYS: + return f"" + return match.group(0) diff --git a/scripts/planner_differential/constants.py b/scripts/planner_differential/constants.py index 2ee124ef40..58d1c11d62 100644 --- a/scripts/planner_differential/constants.py +++ b/scripts/planner_differential/constants.py @@ -117,6 +117,9 @@ ) COMPACT_TIMESTAMP_PATTERN: re.Pattern[str] = re.compile(r"(?__migration_origin__orders", ), + MaskTextTestCase( + description="replay_range_date_relative_to_the_run_date", + raw="range 2026-10-11 → 2026-10-14", + expected_text="range → ", + ), + MaskTextTestCase( + description="retention_date_relative_to_the_run_date", + raw="live until 2026-11-13", + expected_text="live until ", + ), + MaskTextTestCase( + description="fixed_data_date_is_kept", + raw="partition 2026-04-02", + expected_text="partition 2026-04-02", + ), ], ids=lambda case: case.description, ) From a2ae7ae41629556c119cb17a3750f2812943d562 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 02:07:34 +0100 Subject: [PATCH 17/22] refactor: validate SQL-test fixtures natively --- .../src/planner_fixtures/constants.rs | 3 + .../main/collection_mismatch.rs | 19 ++ .../src/planner_fixtures/main/fixture_line.rs | 16 ++ .../src/planner_fixtures/main/mod.rs | 2 + .../src/planner_fixtures/tests.rs | 2 + .../src/planner_fixtures/tests/test_types.rs | 16 ++ .../planner_fixtures/tests/test_validation.rs | 92 ++++++++ .../planner_strategy/fixture_completion.rs | 2 +- .../_helpers/sql_test_glue/constants.rs | 8 +- .../sql_test_glue/fixture_validation.rs | 204 ++++++++++++++++++ .../bindings/_helpers/sql_test_glue/mod.rs | 1 + .../_helpers/sql_test_glue/plan_entries.rs | 13 +- .../_helpers/sql_test_glue/plan_fixtures.rs | 66 +++--- .../_helpers/sql_tests/fixture_validation.py | 191 ---------------- src/sqlbuild/compiler/planner/exceptions.py | 8 - 15 files changed, 394 insertions(+), 249 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/main/collection_mismatch.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_line.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/tests/test_validation.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/constants.rs b/crates/sqlbuild-analysis/src/planner_fixtures/constants.rs index 5f69d5e0eb..fdf2ad4716 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/constants.rs @@ -9,3 +9,6 @@ pub(crate) const MARKER_PATTERNS: [&str; 4] = [ pub(crate) const EMPTY_PATTERN: &str = r"(?i)^[\s\x1c-\x1f]*SELECT[\s\x1c-\x1f]+\*[\s\x1c-\x1f]+FROM[\s\x1c-\x1f]+__empty_fixture[\s\x1c-\x1f]*\([\s\x1c-\x1f]*\)[\s\x1c-\x1f]*$"; pub(crate) const READ_CACHE_CAPACITY: usize = 4096; +pub(crate) const FIXTURE_WHITESPACE: &str = r"[\s\x1c-\x1f]"; +pub(crate) const COLLECTION_TYPES: [&str; 6] = + ["ARRAY", "LIST", "MAP", "OBJECT", "STRUCT", "VARIANT"]; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/collection_mismatch.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/collection_mismatch.rs new file mode 100644 index 0000000000..6361eeb994 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/collection_mismatch.rs @@ -0,0 +1,19 @@ +//! Static collection-versus-scalar fixture type conflicts. + +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::planner_fixtures::constants::COLLECTION_TYPES; + +/// Whether exactly one of two known fixture types is a collection type. +pub fn collection_mismatch(expected: Option<&str>, inferred: Option<&str>) -> bool { + let (Some(expected), Some(inferred)) = (expected, inferred) else { + return false; + }; + is_collection(expected) != is_collection(inferred) +} + +fn is_collection(sql_type: &str) -> bool { + let upper = sql_type.to_uppercase(); + let base = python_strip(upper.split('(').next().unwrap_or_default()); + COLLECTION_TYPES.contains(&base) +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_line.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_line.rs new file mode 100644 index 0000000000..12915ac28c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_line.rs @@ -0,0 +1,16 @@ +//! Source line of one SQL-test fixture CTE definition. + +use regex::Regex; + +use crate::planner_fixtures::constants::FIXTURE_WHITESPACE; + +/// Return the one-based line that starts a fixture CTE definition, or line one when absent. +pub fn fixture_line(contents: &str, cte_name: &str) -> Result { + let pattern = Regex::new(&format!( + r"(?im)^{FIXTURE_WHITESPACE}*{}{FIXTURE_WHITESPACE}+AS{FIXTURE_WHITESPACE}*\(", + regex::escape(cte_name) + ))?; + Ok(pattern.find(contents).map_or(1, |found| { + contents[..found.start()].matches('\n').count() + 1 + })) +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs index 9d0320f464..61b86c8338 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs @@ -1,6 +1,8 @@ pub mod align_columns; +pub mod collection_mismatch; pub mod empty_fixture_error; pub mod empty_fixture_marker; +pub mod fixture_line; pub mod fixture_schema; pub mod merge_metadata; pub mod propagate_types; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs index 9ea0942c44..3684bc64a0 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs @@ -10,3 +10,5 @@ mod test_reads; mod test_schema; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_validation.rs"] +mod test_validation; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs index 62c6427f7e..feade452bc 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs @@ -42,3 +42,19 @@ pub(super) struct FixtureMarkerTestCase { pub sql: &'static str, pub expected_sql: &'static str, } + +#[derive(Debug)] +pub(crate) struct FixtureLineTestCase { + pub(crate) description: &'static str, + pub(crate) contents: &'static str, + pub(crate) cte_name: &'static str, + pub(crate) expected_line: usize, +} + +#[derive(Debug)] +pub(crate) struct CollectionMismatchTestCase { + pub(crate) description: &'static str, + pub(crate) expected_type: Option<&'static str>, + pub(crate) inferred_type: Option<&'static str>, + pub(crate) expected_mismatch: bool, +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_validation.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_validation.rs new file mode 100644 index 0000000000..a1df4390c6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_validation.rs @@ -0,0 +1,92 @@ +use crate::planner_fixtures::main::{ + collection_mismatch::collection_mismatch, fixture_line::fixture_line, +}; +use crate::planner_fixtures::tests::test_types::{CollectionMismatchTestCase, FixtureLineTestCase}; + +#[test] +fn given_fixture_cte_when_locating_then_reports_python_compatible_line() { + let test_cases = [ + FixtureLineTestCase { + description: "definition line", + contents: "TEST();\n\nWITH\n__source__orders AS (SELECT 1)\n", + cte_name: "__source__orders", + expected_line: 4, + }, + FixtureLineTestCase { + description: "leading blank line starts the match", + contents: "TEST();\nWITH\n\n__source__orders AS (SELECT 1)\n", + cte_name: "__source__orders", + expected_line: 3, + }, + FixtureLineTestCase { + description: "case-insensitive keyword across lines", + contents: "WITH\n__expected__orders\n as\n (SELECT 1)\n", + cte_name: "__expected__orders", + expected_line: 2, + }, + FixtureLineTestCase { + description: "information separator counts as whitespace", + contents: "WITH\n\u{1f}__ref__orders\u{1c}AS(SELECT 1)\n", + cte_name: "__ref__orders", + expected_line: 2, + }, + FixtureLineTestCase { + description: "missing definition falls back to line one", + contents: "WITH /* x */ __seed__codes AS (SELECT 1)\n", + cte_name: "__seed__codes", + expected_line: 1, + }, + FixtureLineTestCase { + description: "name is matched literally", + contents: "WITH\n__ref__aXb AS (SELECT 1)\n__ref__a.b AS (SELECT 1)\n", + cte_name: "__ref__a.b", + expected_line: 3, + }, + ]; + for test_case in test_cases { + assert_eq!( + fixture_line(test_case.contents, test_case.cte_name).ok(), + Some(test_case.expected_line), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_fixture_types_when_comparing_then_flags_collection_scalar_conflicts() { + let test_cases = [ + CollectionMismatchTestCase { + description: "array against scalar", + expected_type: Some("ARRAY"), + inferred_type: Some("VARCHAR"), + expected_mismatch: true, + }, + CollectionMismatchTestCase { + description: "parameterized collections agree", + expected_type: Some("struct(a INTEGER)"), + inferred_type: Some(" MAP (VARCHAR, INTEGER)"), + expected_mismatch: false, + }, + CollectionMismatchTestCase { + description: "scalars agree", + expected_type: Some("INTEGER"), + inferred_type: Some("NUMBER(38, 0)"), + expected_mismatch: false, + }, + CollectionMismatchTestCase { + description: "unknown type never conflicts", + expected_type: None, + inferred_type: Some("ARRAY"), + expected_mismatch: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + collection_mismatch(test_case.expected_type, test_case.inferred_type), + test_case.expected_mismatch, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs index 1f7562c3f7..6833326347 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs @@ -20,7 +20,7 @@ use crate::bindings::_helpers::planner_strategy::fixture_values::{ #[pyfunction] #[pyo3(signature = (*, project, adapter, request, planning_context = None))] -fn build_relation_fixture_completion<'py>( +pub(crate) fn build_relation_fixture_completion<'py>( project: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, request: (Bound<'py, PyAny>, Bound<'py, PyAny>), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs index 8c1e8c3318..b78aa93e68 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs @@ -7,8 +7,12 @@ pub(super) const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; pub(super) const PLAN_MODELS: &str = "sqlbuild.compiler.planner.models"; pub(super) const NATIVE_PLANNING: &str = "sqlbuild._native"; -pub(super) const FIXTURE_VALIDATION: &str = - "sqlbuild.compiler.planner._helpers.sql_tests.fixture_validation"; +pub(super) const FIXTURE_PREFIXES: [(&str, &str); 4] = [ + ("model", "__ref__"), + ("source", "__source__"), + ("seed", "__seed__"), + ("sql_test", "__expected__"), +]; pub(super) const CURSOR_ALGEBRA_EXCEPTIONS: &str = "sqlbuild.cursor_algebra.exceptions"; pub(super) const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; pub(super) const INTEGER_CURSOR_TYPE: &str = "integer"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs new file mode 100644 index 0000000000..6915959b3b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs @@ -0,0 +1,204 @@ +//! Static SQL-test fixture shape validation over native fixture completion. + +use std::collections::BTreeSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_analysis::planner_fixtures::main::{ + collection_mismatch::collection_mismatch, fixture_line::fixture_line, +}; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::planner_strategy::fixture_completion::build_relation_fixture_completion; +use crate::bindings::_helpers::planner_strategy::fixture_context::build_relation_fixture_context; +use crate::bindings::_helpers::planner_strategy::fixture_sql::{ + complete_empty_fixture_sql, complete_typed_null_columns, +}; +use crate::bindings::_helpers::sql_test_glue::constants::{COMPILE_TYPES, FIXTURE_PREFIXES}; + +/// Report every statically provable fixture defect or return the completed fixture SQL groups. +pub(super) fn validated_fixtures<'py>( + test: &Bound<'py, PyAny>, + request: (&Bound<'py, PyAny>, [Bound<'py, PyDict>; 4]), + inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + context: &Bound<'py, PyAny>, +) -> PyResult; 4], Vec>> { + let py = test.py(); + let (chain, [refs, sources, seeds, expected]) = request; + let (project, adapter) = inputs; + let resource_types = imported(py, COMPILE_TYPES, "CompiledResourceType")?; + let model = resource_types.getattr("MODEL")?; + let kinds = [ + model.clone(), + resource_types.getattr("SOURCE")?, + resource_types.getattr("SEED")?, + ]; + let groups = PyTuple::new( + py, + [ + PyTuple::new(py, [kinds[0].clone(), refs.clone().into_any()])?, + PyTuple::new(py, [kinds[1].clone(), sources.clone().into_any()])?, + PyTuple::new(py, [kinds[2].clone(), seeds.clone().into_any()])?, + ], + )?; + let context = if context.is_truthy()? { + context.clone() + } else { + build_relation_fixture_context(project, None)? + }; + let completion = build_relation_fixture_completion( + project, + adapter, + (chain.clone(), groups.into_any()), + Some(context.clone()), + )?; + let mut errors: BTreeSet = BTreeSet::new(); + for diagnostic in completion.getattr("diagnostics")?.try_iter()? { + let diagnostic = diagnostic?; + let key = diagnostic.getattr("key")?; + errors.insert(format!( + "{}: {}", + fixture_location( + test, + &key.get_item(0)?, + &key.get_item(1)?.extract::()? + )?, + diagnostic.getattr("message")?.extract::()? + )); + } + let expected_types = completion.getattr("expected_types")?; + let inferred_by_fixture = completion.getattr("inferred_by_fixture")?; + let completed_expected = expected.copy()?; + let sql_test = resource_types.getattr("SQL_TEST")?; + let type_groups = [ + (&kinds[0], &refs), + (&kinds[1], &sources), + (&kinds[2], &seeds), + (&sql_test, &expected), + ]; + for (kind, fixtures) in type_groups { + let expected_output = kind.is(&sql_test); + for (key_name, sql) in fixtures.iter() { + let name: String = key_name.extract()?; + let location = || fixture_location(test, kind, &name); + let inferred = if expected_output { + let model_key = PyTuple::new(py, [&model, &key_name])?; + let relation = context + .getattr("relations")? + .call_method1("get", (&model_key,))?; + let (completed_sql, empty_error) = + complete_empty_fixture_sql(&sql.extract::()?, &relation, adapter)?; + if let Some(error) = empty_error { + errors.insert(format!("{}: expected output '{name}' {error}", location()?)); + } + completed_expected.set_item(&name, &completed_sql)?; + let kwargs = PyDict::new(py); + kwargs.set_item("query_sql", &completed_sql)?; + kwargs.set_item( + "inference_profile", + adapter.call_method0("expression_inference_profile")?, + )?; + let inference = context.getattr("fixture_inferences")?.call_method( + "infer", + (), + Some(&kwargs), + )?; + if inference.is_none() { + None + } else { + let types = expected_types + .call_method1("get", (&model_key, PyDict::new(py)))? + .cast_into::()?; + completed_expected.set_item( + &name, + complete_typed_null_columns(&completed_sql, &inference, &types, adapter)?, + )?; + Some(inference.getattr("columns")?) + } + } else { + let key = PyTuple::new(py, [kind, &key_name])?; + Some(inferred_by_fixture.call_method1("get", (key,))?) + .filter(|value| !value.is_none()) + }; + let Some(columns) = inferred else { + continue; + }; + let type_kind = if expected_output { &model } else { kind }; + let type_key = PyTuple::new(py, [type_kind, &key_name])?; + let types = expected_types.call_method1("get", (type_key, PyDict::new(py)))?; + for column in columns.try_iter()? { + let column = column?; + let column_name = column.getattr("name")?; + let expected_type: Option = types + .call_method1("get", (column_name.call_method0("casefold")?,))? + .extract()?; + let inferred_type: Option = column.getattr("type")?.extract()?; + if !collection_mismatch(expected_type.as_deref(), inferred_type.as_deref()) { + continue; + } + let label = if expected_output { + "expected output".to_owned() + } else { + format!("mock {}", kind.getattr("value")?.extract::()?) + }; + errors.insert(format!( + "{}: {label} '{name}' column '{}' has incompatible type {}; resource type is {}. Use an explicit CAST, ARRAY_CONSTRUCT, or PARSE_JSON as appropriate", + location()?, + column_name.extract::()?, + inferred_type.unwrap_or_default(), + expected_type.unwrap_or_default(), + )); + } + } + } + if !errors.is_empty() { + let test_name: String = test.getattr("name")?.extract()?; + return Ok(Err(errors + .into_iter() + .map(|error| format!("SQL test '{test_name}': {error}")) + .collect())); + } + let fixture_sql = completion.getattr("fixture_sql_by_key")?; + let completed = + |kind: &Bound<'py, PyAny>, fixtures: &Bound<'py, PyDict>| -> PyResult> { + let result = PyDict::new(py); + for (name, sql) in fixtures.iter() { + let key = PyTuple::new(py, [kind.clone(), name.clone()])?; + result.set_item(&name, fixture_sql.call_method1("get", (key, sql))?)?; + } + Ok(result) + }; + Ok(Ok([ + completed(&kinds[0], &refs)?, + completed(&kinds[1], &sources)?, + completed(&kinds[2], &seeds)?, + completed_expected, + ])) +} + +/// Return the test file and line that defines one fixture or expected-output CTE. +pub(super) fn fixture_location( + test: &Bound<'_, PyAny>, + resource_type: &Bound<'_, PyAny>, + name: &str, +) -> PyResult { + let value: String = resource_type.getattr("value")?.extract()?; + let prefix = FIXTURE_PREFIXES + .iter() + .find_map(|(kind, prefix)| (*kind == value).then_some(*prefix)) + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(value))?; + let test_file = test.getattr("test_file")?; + let line = fixture_line( + &test_file.getattr("contents")?.extract::()?, + &format!("{prefix}{name}"), + ) + .map_err(value_error)?; + Ok(format!( + "{}:{line}", + test_file + .getattr("relative_path")? + .call_method0("as_posix")? + .extract::()? + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index 7fe7199aaf..e1a4f4766c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod authored_ctes; mod compiled_tests; mod constants; mod cursor_window; +mod fixture_validation; mod plan_assembly; mod plan_driver; mod plan_entries; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs index 398090d144..4f39f527d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs @@ -162,18 +162,15 @@ pub(super) fn expected_diagnostics( if message.is_none() { continue; } - let location = call_with( - &imported(py, super::constants::FIXTURE_VALIDATION, "fixture_location")?, - &[ - ("test", test.clone()), - ("resource_type", resource_types.getattr("SQL_TEST")?), - ("name", name), - ], + let location = super::fixture_validation::fixture_location( + test, + &resource_types.getattr("SQL_TEST")?, + &name.extract::()?, )?; result.push(format!( "SQL test '{}': {}: {}", test.getattr("name")?.extract::()?, - location.extract::()?, + location, message.extract::()? )); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs index ca492e61ad..df8734947d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs @@ -1,11 +1,12 @@ -//! Complete SQL-test fixtures through the existing fixture completion boundary. +//! Complete SQL-test fixtures through the native fixture validation boundary. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::sql_test_glue::authored_ctes::prefixed_ctes; -use crate::bindings::_helpers::sql_test_glue::constants::{FIXTURE_VALIDATION, NATIVE_PLANNING}; +use crate::bindings::_helpers::sql_test_glue::constants::NATIVE_PLANNING; +use crate::bindings::_helpers::sql_test_glue::fixture_validation::validated_fixtures; use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; pub(super) fn validated_test<'py>( @@ -13,31 +14,28 @@ pub(super) fn validated_test<'py>( chain: &Bound<'py, PyAny>, inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), context: &Bound<'py, PyAny>, -) -> PyResult> { +) -> PyResult, Vec>> { let py = test.py(); let payload = test.getattr("payload")?; let ctes = payload.getattr("authored_ctes")?; let expected = payload.getattr("expected_ctes")?; - let completed = call_with( - &imported(py, FIXTURE_VALIDATION, "build_validated_test_fixtures")?, - &[ - ("test", test.clone()), - ("project", inputs.0.clone()), - ("adapter", inputs.1.clone()), - ("ordered_model_names", chain.clone()), - ("mock_refs", prefixed_ctes(&ctes, "__ref__")?.into_any()), - ( - "mock_sources", - prefixed_ctes(&ctes, "__source__")?.into_any(), - ), - ("mock_seeds", prefixed_ctes(&ctes, "__seed__")?.into_any()), - ( - "expected_outputs", - prefixed_ctes(&expected, "__expected__")?.into_any(), - ), - ("planning_context", context.clone()), - ], - )?; + let completed = match validated_fixtures( + test, + ( + chain, + [ + prefixed_ctes(&ctes, "__ref__")?, + prefixed_ctes(&ctes, "__source__")?, + prefixed_ctes(&ctes, "__seed__")?, + prefixed_ctes(&expected, "__expected__")?, + ], + ), + inputs, + context, + )? { + Ok(completed) => completed, + Err(diagnostics) => return Ok(Err(diagnostics)), + }; let replace = imported(py, "dataclasses", "replace")?; let mut authored: Vec> = Vec::new(); for cte in ctes.try_iter()? { @@ -46,9 +44,7 @@ pub(super) fn validated_test<'py>( let mut replacement = cte.clone(); for (index, prefix) in ["__ref__", "__source__", "__seed__"].iter().enumerate() { if let Some(name) = name.strip_prefix(prefix) { - let sql = completed - .get_item(index)? - .call_method1("get", (name, cte.getattr("sql_body")?))?; + let sql = completed[index].call_method1("get", (name, cte.getattr("sql_body")?))?; let kwargs = PyDict::new(py); kwargs.set_item("sql_body", sql)?; replacement = replace.call((&cte,), Some(&kwargs))?; @@ -65,9 +61,7 @@ pub(super) fn validated_test<'py>( let kwargs = PyDict::new(py); kwargs.set_item( "sql_body", - completed - .get_item(3)? - .call_method1("get", (name, cte.getattr("sql_body")?))?, + completed[3].call_method1("get", (name, cte.getattr("sql_body")?))?, )?; expected_ctes.push(replace.call((&cte,), Some(&kwargs))?); } @@ -77,7 +71,7 @@ pub(super) fn validated_test<'py>( let payload = replace.call((payload,), Some(&kwargs))?; let kwargs = PyDict::new(py); kwargs.set_item("payload", payload)?; - replace.call((test,), Some(&kwargs)) + replace.call((test,), Some(&kwargs)).map(Ok) } pub(super) fn validate<'py>( @@ -110,21 +104,15 @@ pub(super) fn validate<'py>( "zip() argument lengths differ", )); } - let exception = imported( - py, - "sqlbuild.compiler.planner.exceptions", - "SqlTestFixtureValidationError", - )?; let mut validated = tests.to_vec(); let mut diagnostics: std::collections::BTreeMap> = std::collections::BTreeMap::new(); for (index, chain) in indexes.into_iter().zip(chains) { - match validated_test(&tests[index], &chain, inputs, context) { + match validated_test(&tests[index], &chain, inputs, context)? { Ok(test) => validated[index] = test, - Err(error) if error.value(py).is_instance(&exception)? => { - diagnostics.insert(index, error.value(py).getattr("diagnostics")?); + Err(errors) => { + diagnostics.insert(index, PyTuple::new(py, errors)?.into_any()); } - Err(error) => return Err(error), } } Ok((validated, diagnostics)) diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py deleted file mode 100644 index 03c5108d87..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py +++ /dev/null @@ -1,191 +0,0 @@ -"""Static SQL-test fixture shape validation.""" - -from __future__ import annotations - -import re - -from sqlbuild._native import ( - build_relation_fixture_completion, - build_relation_fixture_context, - complete_empty_fixture_sql, - complete_typed_null_columns, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( - CompiledSqlTest, - FixtureColumnInference, - InferredColumn, -) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.planner.exceptions import SqlTestFixtureValidationError -from sqlbuild.compiler.planner.models import ( - RelationFixtureCompletion, - RelationFixtureDiagnostic, - RelationFixturePlanningContext, -) - -_COLLECTION_TYPES: frozenset[str] = frozenset( - {"ARRAY", "LIST", "MAP", "OBJECT", "STRUCT", "VARIANT"} -) - - -def build_validated_test_fixtures( - *, - test: CompiledSqlTest, - project: CompiledProjectView, - adapter: BaseAdapter, - ordered_model_names: tuple[str, ...], - mock_refs: dict[str, str], - mock_sources: dict[str, str], - mock_seeds: dict[str, str], - expected_outputs: dict[str, str], - planning_context: RelationFixturePlanningContext | None = None, -) -> tuple[dict[str, str], dict[str, str], dict[str, str], dict[str, str]]: - """Report all statically provable missing columns and collection-type mismatches.""" - - fixture_groups: tuple[tuple[CompiledResourceType, dict[str, str]], ...] = ( - (CompiledResourceType.MODEL, mock_refs), - (CompiledResourceType.SOURCE, mock_sources), - (CompiledResourceType.SEED, mock_seeds), - ) - fixture_context: RelationFixturePlanningContext = ( - planning_context or build_relation_fixture_context(project=project) - ) - completion: RelationFixtureCompletion = build_relation_fixture_completion( - project=project, - adapter=adapter, - request=(ordered_model_names, fixture_groups), - planning_context=fixture_context, - ) - errors: list[str] = [ - _located_diagnostic(test=test, diagnostic=diagnostic) - for diagnostic in completion.diagnostics - ] - completed_expected_outputs: dict[str, str] = dict(expected_outputs) - type_fixture_groups: tuple[tuple[CompiledResourceType, dict[str, str]], ...] = ( - *fixture_groups, - (CompiledResourceType.SQL_TEST, expected_outputs), - ) - for resource_type, fixtures in type_fixture_groups: - for name, sql in fixtures.items(): - inferred_columns: tuple[InferredColumn, ...] | None - if resource_type == CompiledResourceType.SQL_TEST: - completed_sql, empty_error = complete_empty_fixture_sql( - sql=sql, - relation=fixture_context.relations.get((CompiledResourceType.MODEL, name)), - adapter=adapter, - ) - if empty_error is not None: - errors.append( - f"{fixture_location(test=test, resource_type=resource_type, name=name)}: " - f"expected output '{name}' {empty_error}" - ) - completed_expected_outputs[name] = completed_sql - inference: FixtureColumnInference | None = fixture_context.fixture_inferences.infer( - query_sql=completed_sql, - inference_profile=adapter.expression_inference_profile(), - ) - inferred_columns = inference.columns if inference is not None else None - if inference is not None: - completed_expected_outputs[name] = complete_typed_null_columns( - sql=completed_sql, - inference=inference, - expected_types=completion.expected_types.get( - (CompiledResourceType.MODEL, name), {} - ), - adapter=adapter, - ) - else: - inferred_columns = completion.inferred_by_fixture.get((resource_type, name)) - if inferred_columns is None: - continue - type_key: tuple[CompiledResourceType, str] = ( - CompiledResourceType.MODEL - if resource_type == CompiledResourceType.SQL_TEST - else resource_type, - name, - ) - for column in inferred_columns: - expected_type: str | None = completion.expected_types.get(type_key, {}).get( - column.name.casefold() - ) - if not _types_are_incompatible(expected=expected_type, inferred=column.type): - continue - fixture_label: str = ( - "expected output" - if resource_type == CompiledResourceType.SQL_TEST - else f"mock {resource_type.value}" - ) - errors.append( - f"{fixture_location(test=test, resource_type=resource_type, name=name)}: " - f"{fixture_label} '{name}' column '{column.name}' has incompatible type " - f"{column.type}; resource type is {expected_type}. Use an explicit CAST, " - "ARRAY_CONSTRUCT, or PARSE_JSON as appropriate" - ) - if errors: - raise SqlTestFixtureValidationError( - tuple(f"SQL test '{test.name}': {diagnostic}" for diagnostic in sorted(set(errors))) - ) - return ( - _completed_fixture_group( - resource_type=CompiledResourceType.MODEL, - fixtures=mock_refs, - completion=completion, - ), - _completed_fixture_group( - resource_type=CompiledResourceType.SOURCE, - fixtures=mock_sources, - completion=completion, - ), - _completed_fixture_group( - resource_type=CompiledResourceType.SEED, - fixtures=mock_seeds, - completion=completion, - ), - completed_expected_outputs, - ) - - -def _located_diagnostic(*, test: CompiledSqlTest, diagnostic: RelationFixtureDiagnostic) -> str: - return ( - f"{fixture_location(test=test, resource_type=diagnostic.key[0], name=diagnostic.key[1])}: " - f"{diagnostic.message}" - ) - - -def _completed_fixture_group( - *, - resource_type: CompiledResourceType, - fixtures: dict[str, str], - completion: RelationFixtureCompletion, -) -> dict[str, str]: - return { - name: completion.fixture_sql_by_key.get((resource_type, name), sql) - for name, sql in fixtures.items() - } - - -def fixture_location( - *, test: CompiledSqlTest, resource_type: CompiledResourceType, name: str -) -> str: - """Return the test file and line that defines one fixture or expected-output CTE.""" - - prefix: str = { - CompiledResourceType.MODEL: "__ref__", - CompiledResourceType.SOURCE: "__source__", - CompiledResourceType.SEED: "__seed__", - CompiledResourceType.SQL_TEST: "__expected__", - }[resource_type] - match: re.Match[str] | None = re.search( - rf"(?im)^\s*{re.escape(prefix + name)}\s+AS\s*\(", test.test_file.contents - ) - line: int = 1 if match is None else test.test_file.contents.count("\n", 0, match.start()) + 1 - return f"{test.test_file.relative_path.as_posix()}:{line}" - - -def _types_are_incompatible(*, expected: str | None, inferred: str | None) -> bool: - if expected is None or inferred is None: - return False - expected_base: str = expected.upper().split("(", maxsplit=1)[0].strip() - inferred_base: str = inferred.upper().split("(", maxsplit=1)[0].strip() - return (expected_base in _COLLECTION_TYPES) != (inferred_base in _COLLECTION_TYPES) diff --git a/src/sqlbuild/compiler/planner/exceptions.py b/src/sqlbuild/compiler/planner/exceptions.py index 9b3763810f..fb72dc94a5 100644 --- a/src/sqlbuild/compiler/planner/exceptions.py +++ b/src/sqlbuild/compiler/planner/exceptions.py @@ -23,14 +23,6 @@ class NativeSqlTestPlanningError(PlannerInputError): """Raised when the native SQL-test planning boundary returns invalid data.""" -class SqlTestFixtureValidationError(PlannerInputError): - """Raised with every invalid fixture diagnostic for one SQL test.""" - - def __init__(self, diagnostics: tuple[str, ...]) -> None: - self.diagnostics = diagnostics - super().__init__("\n- ".join(diagnostics)) - - class FutureCursorSafetyError(PlannerInputError): """Raised when an effective cursor exceeds the configured future limit.""" From 43530d2ae9b603e992765554b10ddf16bee0c372 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 01:49:47 +0100 Subject: [PATCH 18/22] refactor: host migration state reads, writes and inspection natively Move the planner's migration state inspection and the shared read and append orchestration of migration, column migration and old-name view events into one native owner. Adapter execute and metadata calls remain Python callbacks; event IDs, idempotent appends, retries and absent column tolerance are unchanged. Delete the replaced Python wrappers and the create-table SQL facades. --- .../planner_migrations/column_planning.rs | 6 +- .../_helpers/planner_migrations/constants.rs | 11 + .../_helpers/planner_migrations/mod.rs | 1 + .../planner_migrations/model_planning.rs | 6 +- .../planner_migrations/old_name_planning.rs | 6 +- .../planner_migrations/registration.rs | 3 +- .../planner_migrations/state_inspection.rs | 578 ++++++++++++++++++ .../bindings/_helpers/state_rows/constants.rs | 7 + .../_helpers/state_rows/migration_io.rs | 326 ++++++++++ .../_helpers/state_rows/migration_objects.rs | 3 + .../_helpers/state_rows/migration_rows.rs | 51 +- .../src/bindings/_helpers/state_rows/mod.rs | 1 + .../bindings/_helpers/state_rows/models.rs | 11 + .../_helpers/state_rows/registration.rs | 3 +- .../src/events/_helpers/relations.rs | 6 +- crates/sqlbuild-state/src/events/main/mod.rs | 2 + .../src/events/main/newest_fingerprint.rs | 38 ++ .../src/events/main/recorded_renames.rs | 27 + crates/sqlbuild-state/src/events/models.rs | 23 + crates/sqlbuild-state/src/events/tests.rs | 2 + .../src/events/tests/helpers.rs | 31 +- .../src/events/tests/test_lookups.rs | 107 ++++ .../src/events/tests/test_types.rs | 22 +- src/sqlbuild/_native.pyi | 89 ++- .../adapter/contract/classes/base_adapter.py | 21 +- .../contract/classes/duckdb_backed_adapter.py | 21 +- .../bigquery/classes/bigquery_adapter.py | 21 +- .../databricks/classes/databricks_adapter.py | 21 +- .../postgres/classes/postgres_adapter.py | 21 +- .../snowflake/classes/snowflake_adapter.py | 21 +- .../sqlserver/classes/sqlserver_adapter.py | 21 +- .../compiler/migrations/_helpers/__init__.py | 1 - .../compiler/migrations/_helpers/writes.py | 43 -- src/sqlbuild/compiler/migrations/constants.py | 1 - .../migrations/main/_read_column_events.py | 35 -- .../compiler/migrations/main/_read_events.py | 35 -- .../main/column_create_table_sql.py | 26 - .../migrations/main/create_table_sql.py | 26 - .../main/old_name_view_create_table_sql.py | 26 - .../main/read_old_name_view_events.py | 40 -- .../main/read_old_name_view_histories.py | 44 -- .../main/read_recorded_column_renames.py | 41 -- .../main/stored_old_name_view_columns.py | 21 - .../migrations/main/write_column_event.py | 39 -- .../compiler/migrations/main/write_event.py | 39 -- .../main/write_old_name_view_event.py | 39 -- .../classes/migration_state_inspection.py | 357 ----------- .../executor/migrations/_helpers/columns.py | 6 +- .../migrations/_helpers/old_name_facts.py | 20 +- .../migrations/_helpers/old_name_views.py | 5 +- .../executor/migrations/_helpers/promotion.py | 6 +- .../old_name_views/_helpers/janitor.py | 17 +- .../main/column_migrations/helpers.py | 2 +- .../commands/main/model_migrations/helpers.py | 2 +- .../commands/main/old_name_views/helpers.py | 2 +- .../_helpers/test_module_paths.py | 1 + .../sqlserver/test_migration_state_sql.py | 9 +- .../main/test_column_migration_events.py | 14 +- .../migrations/main/test_migration_events.py | 14 +- .../main/test_old_name_view_events.py | 29 +- .../migrations/main/test_stored_event_rows.py | 14 +- .../migrations/_helpers/test_columns.py | 5 +- 62 files changed, 1417 insertions(+), 1049 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs create mode 100644 crates/sqlbuild-state/src/events/main/newest_fingerprint.rs create mode 100644 crates/sqlbuild-state/src/events/main/recorded_renames.rs create mode 100644 crates/sqlbuild-state/src/events/tests/test_lookups.rs delete mode 100644 src/sqlbuild/compiler/migrations/_helpers/__init__.py delete mode 100644 src/sqlbuild/compiler/migrations/_helpers/writes.py delete mode 100644 src/sqlbuild/compiler/migrations/main/_read_column_events.py delete mode 100644 src/sqlbuild/compiler/migrations/main/_read_events.py delete mode 100644 src/sqlbuild/compiler/migrations/main/column_create_table_sql.py delete mode 100644 src/sqlbuild/compiler/migrations/main/create_table_sql.py delete mode 100644 src/sqlbuild/compiler/migrations/main/old_name_view_create_table_sql.py delete mode 100644 src/sqlbuild/compiler/migrations/main/read_old_name_view_events.py delete mode 100644 src/sqlbuild/compiler/migrations/main/read_old_name_view_histories.py delete mode 100644 src/sqlbuild/compiler/migrations/main/read_recorded_column_renames.py delete mode 100644 src/sqlbuild/compiler/migrations/main/stored_old_name_view_columns.py delete mode 100644 src/sqlbuild/compiler/migrations/main/write_column_event.py delete mode 100644 src/sqlbuild/compiler/migrations/main/write_event.py delete mode 100644 src/sqlbuild/compiler/migrations/main/write_old_name_view_event.py delete mode 100644 src/sqlbuild/compiler/planner/classes/migration_state_inspection.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index 6753114e68..200d78f652 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -48,11 +48,7 @@ fn plan_column_migrations<'py>( &[("runtime", runtime.clone())], )?; let state = call_with( - &imported( - py, - "sqlbuild.compiler.planner.classes.migration_state_inspection", - "MigrationStateInspection", - )?, + &imported(py, "sqlbuild._native", "MigrationStateInspection")?, &[ ("adapter", adapter), ("connection", runtime.getattr("connection")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs index ab2b48f728..b11f9801d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs @@ -4,3 +4,14 @@ pub(super) const AUTOMATIC: &str = "automatic"; pub(super) const DONE: &str = "done"; pub(super) const PENDING_ARCHIVE: &str = "pending_archive"; pub(super) const PENDING_VIEW: &str = "pending_view"; +pub(super) const NATIVE_MODULE: &str = "sqlbuild._native"; +pub(super) const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; +pub(super) const NODE_TYPE_MODEL: &str = "model"; +pub(super) const LIST_RELATIONS_FOR_INSPECTION: (&str, &str) = ( + "sqlbuild.adapter.relations.main.list_relations_for_inspection", + "list_relations_for_inspection", +); +pub(super) const GET_COLUMNS_FOR_INSPECTION: (&str, &str) = ( + "sqlbuild.adapter.relations.main.get_columns_for_inspection", + "get_columns_for_inspection", +); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index 328b408540..1d488c57e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -30,3 +30,4 @@ mod planning_execution; mod planning_origin; mod projections; pub(crate) mod registration; +mod state_inspection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index 2a2da8815b..23c5718b55 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -132,11 +132,7 @@ fn plan_model_migrations<'py>( &[("runtime", runtime.clone())], )?; let state = call_with( - &imported( - py, - "sqlbuild.compiler.planner.classes.migration_state_inspection", - "MigrationStateInspection", - )?, + &imported(py, "sqlbuild._native", "MigrationStateInspection")?, &[ ("adapter", runtime.getattr("adapter")?), ("connection", runtime.getattr("connection")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs index 90ee923e10..aff9f276f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -108,11 +108,7 @@ fn plan_old_name_views<'py>( } } let state = call_with( - &imported( - py, - "sqlbuild.compiler.planner.classes.migration_state_inspection", - "MigrationStateInspection", - )?, + &imported(py, "sqlbuild._native", "MigrationStateInspection")?, &[ ("adapter", runtime.getattr("adapter")?), ("connection", runtime.getattr("connection")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs index 6516b29d70..2718ea8c91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs @@ -1,6 +1,6 @@ //! Register pure migration helpers. -use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::discovery::register(module)?; @@ -15,5 +15,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::planning_origin::register(module)?; super::planning_execution::register(module)?; super::projections::register(module)?; + module.add_class::()?; super::fingerprint::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs new file mode 100644 index 0000000000..05ea22662c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs @@ -0,0 +1,578 @@ +//! Read-only warehouse evidence gathered once for all model migrations in one plan. + +use std::collections::{HashMap, HashSet}; + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyRef, PyRefMut, PyResult, Python, pyclass, pymethods, +}; +use pyo3::types::{PyBool, PyDict, PyString, PyTuple, PyTupleMethods}; +use sqlbuild_state::events::constants::{ + COLUMN_MIGRATION_TABLE_NAME, MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, +}; +use sqlbuild_state::events::main::newest_fingerprint::newest_fingerprint; +use sqlbuild_state::events::models::{EventRelation, FingerprintKey, FingerprintOrder}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_migrations::constants::{ + FINGERPRINT_TABLE_NAME, GET_COLUMNS_FOR_INSPECTION, LIST_RELATIONS_FOR_INSPECTION, + NATIVE_MODULE, NODE_TYPE_MODEL, +}; +use crate::bindings::_helpers::state_rows::constants::{COLUMN_KIND, MODEL_KIND, OLD_NAME_KIND}; +use crate::bindings::_helpers::state_rows::migration_io::{read_events, stored_old_name_columns}; +use crate::bindings::_helpers::state_rows::migration_objects::layout; +use crate::bindings::_helpers::state_rows::migration_projections::old_name_views; +use crate::bindings::_helpers::state_rows::models::StateAccess; + +/// A lowercased schema and relation name. +type RelationKey = (String, String); + +/// Relations, fingerprints and migration events for one planning database. +#[pyclass(module = "sqlbuild._native", name = "MigrationStateInspection")] +pub(crate) struct MigrationStateInspection { + adapter: Py, + connection: Py, + database: Py, + known_fingerprints: Py, + read_schemas: HashSet, + /// Model fingerprints per lowercased schema, in first-read order. + fingerprints: Vec<(String, Vec>)>, + events: Vec>, + event_schemas: HashSet, + relations: HashMap>, + columns: HashMap>, + column_event_schemas: HashSet, + column_events: Vec>, + old_name_schemas: HashSet, + old_name_moves: Vec>, + old_name_facts: Vec>, +} + +fn lowered(value: &Bound<'_, PyAny>) -> PyResult { + value.call_method0("lower")?.extract() +} + +fn schema_or_empty<'py>(value: Bound<'py, PyAny>) -> Bound<'py, PyAny> { + if value.is_none() { + PyString::new(value.py(), "").into_any() + } else { + value + } +} + +/// `((schema or "").lower(), name.lower())` of a relation or location. +fn relation_key(value: &Bound<'_, PyAny>) -> PyResult { + Ok(( + lowered(&schema_or_empty(value.getattr("schema")?))?, + lowered(&value.getattr("name")?)?, + )) +} + +/// The requested schemas not inspected yet, sorted, each with its lowercased key. +fn pending<'py>( + schemas: &Bound<'py, PyAny>, + inspected: &HashSet, +) -> PyResult, String)>> { + let mut pending: Vec<(String, Bound<'py, PyAny>, String)> = Vec::new(); + for schema in schemas.try_iter()? { + let schema: Bound<'py, PyAny> = schema?; + let key: String = lowered(&schema)?; + if !inspected.contains(&key) { + pending.push((schema.extract()?, schema, key)); + } + } + pending.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(pending + .into_iter() + .map(|(_, schema, key)| (schema, key)) + .collect()) +} + +fn tuple_of<'py>(py: Python<'py>, items: &[Py]) -> PyResult> { + PyTuple::new(py, items.iter().map(|item| item.bind(py))) +} + +fn fingerprint_key(fingerprint: &Bound<'_, PyAny>) -> PyResult { + let target_name: Bound<'_, PyAny> = fingerprint.getattr("target_name")?; + let target: Option = if target_name.is_none() { + None + } else { + Some(EventRelation { + database: fingerprint.getattr("target_database")?.extract()?, + schema: fingerprint.getattr("target_schema")?.extract()?, + name: target_name.extract()?, + }) + }; + Ok(FingerprintKey { + order: FingerprintOrder { + timestamp: fingerprint + .getattr("ts")? + .call_method0("isoformat")? + .extract()?, + run_id: fingerprint.getattr("run_id")?.extract()?, + }, + node_name: fingerprint.getattr("node_name")?.extract()?, + target, + }) +} + +fn newest<'py>( + py: Python<'py>, + candidates: &[Py], + (target, model_name): (Option<&EventRelation>, Option<&str>), +) -> PyResult>> { + let keys: Vec = candidates + .iter() + .map(|candidate| fingerprint_key(candidate.bind(py))) + .collect::>()?; + Ok(newest_fingerprint(&keys, target, model_name) + .map(|index| candidates[index].bind(py).clone())) +} + +impl MigrationStateInspection { + fn access<'py>(&self, py: Python<'py>) -> PyResult> { + let adapter: &Bound<'py, PyAny> = self.adapter.bind(py); + Ok(StateAccess { + connection: self.connection.bind(py).clone(), + execute: adapter.getattr("execute")?, + render_qualified_name: adapter.getattr("render_qualified_name")?, + }) + } + + fn list_relations<'py>( + &self, + py: Python<'py>, + schemas: Vec>, + names: Bound<'py, PyAny>, + ) -> PyResult>> { + let (module, name) = LIST_RELATIONS_FOR_INSPECTION; + call_with( + &imported(py, module, name)?, + &[ + ("adapter", self.adapter.bind(py).clone()), + ("connection", self.connection.bind(py).clone()), + ("database", self.database.bind(py).clone()), + ("schemas", PyTuple::new(py, schemas)?.into_any()), + ("names", names), + ], + )? + .try_iter()? + .collect() + } + + fn model_fingerprints<'py>( + &self, + py: Python<'py>, + schema: &Bound<'py, PyAny>, + key: &str, + ) -> PyResult>> { + let known: Bound<'py, PyAny> = self + .known_fingerprints + .bind(py) + .call_method1("get", (key,))?; + let latest: Bound<'py, PyAny> = if known.is_none() { + let adapter: &Bound<'py, PyAny> = self.adapter.bind(py); + let fingerprint_set: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "read_latest_fingerprints")?, + &[ + ("connection", self.connection.bind(py).clone()), + ("execute", adapter.getattr("execute")?), + ("table_exists", PyBool::new(py, true).to_owned().into_any()), + ("database", self.database.bind(py).clone()), + ("schema", schema.clone()), + ( + "render_qualified_name", + adapter.getattr("render_qualified_name")?, + ), + ( + "render_read_latest_sql", + adapter.getattr("render_read_latest_fingerprints_sql")?, + ), + ], + )?; + let by_identity: Bound<'py, PyAny> = + fingerprint_set.getattr("fingerprints_by_identity")?; + if by_identity.is_truthy()? { + by_identity.call_method0("values")? + } else { + PyTuple::empty(py).into_any() + } + } else { + known + }; + let mut models: Vec> = Vec::new(); + for fingerprint in latest.try_iter()? { + let fingerprint: Bound<'py, PyAny> = fingerprint?; + if fingerprint.getattr("node_type")?.eq(NODE_TYPE_MODEL)? { + models.push(fingerprint.unbind()); + } + } + Ok(models) + } +} + +#[pymethods] +impl MigrationStateInspection { + #[new] + #[pyo3(signature = (*, adapter, connection, database, known_fingerprints = None))] + fn new( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + database: &Bound<'_, PyAny>, + known_fingerprints: Option<&Bound<'_, PyAny>>, + ) -> PyResult { + let py: Python<'_> = adapter.py(); + let known: Bound<'_, PyAny> = match known_fingerprints { + Some(known) + if !known.is_none() && known.getattr("unfiltered_database")?.eq(database)? => + { + known.getattr("unfiltered_schemas")? + } + _ => PyDict::new(py).into_any(), + }; + Ok(Self { + adapter: adapter.clone().unbind(), + connection: connection.clone().unbind(), + database: database.clone().unbind(), + known_fingerprints: known.unbind(), + read_schemas: HashSet::new(), + fingerprints: Vec::new(), + events: Vec::new(), + event_schemas: HashSet::new(), + relations: HashMap::new(), + columns: HashMap::new(), + column_event_schemas: HashSet::new(), + column_events: Vec::new(), + old_name_schemas: HashSet::new(), + old_name_moves: Vec::new(), + old_name_facts: Vec::new(), + }) + } + + /// The database whose namespaces this inspection reads. + #[getter] + fn database(&self, py: Python<'_>) -> Py { + self.database.clone_ref(py) + } + + /// Every migration event read so far. + #[getter] + fn events<'py>(&self, py: Python<'py>) -> PyResult> { + tuple_of(py, &self.events) + } + + /// Every column migration event read so far. + #[getter] + fn column_events<'py>(&self, py: Python<'py>) -> PyResult> { + tuple_of(py, &self.column_events) + } + + /// The old-name history of every move read by `inspect_old_name_views`. + #[getter] + fn old_name_histories<'py>(&self, py: Python<'py>) -> PyResult> { + old_name_views( + tuple_of(py, &self.old_name_moves)?.as_any(), + tuple_of(py, &self.old_name_facts)?.as_any(), + ) + } + + /// Read the column migration state table once for every schema not inspected yet. + #[pyo3(signature = (*, schemas))] + fn inspect_column_events(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let pending = pending(schemas, &slf.borrow().column_event_schemas)?; + if pending.is_empty() { + return Ok(()); + } + slf.borrow_mut() + .column_event_schemas + .extend(pending.iter().map(|(_, key)| key.clone())); + let listed: Vec> = slf.borrow().list_relations( + py, + pending.iter().map(|(schema, _)| schema.clone()).collect(), + PyTuple::new(py, [COLUMN_MIGRATION_TABLE_NAME])?.into_any(), + )?; + let mut present: HashSet = HashSet::new(); + for relation in &listed { + if lowered(&relation.getattr("name")?)? == COLUMN_MIGRATION_TABLE_NAME { + present.insert(lowered(&schema_or_empty(relation.getattr("schema")?))?); + } + } + for (schema, key) in pending { + if !present.contains(&key) { + continue; + } + let (access, database) = { + let this: PyRef<'_, Self> = slf.borrow(); + (this.access(py)?, this.database.bind(py).clone()) + }; + let events = read_events(&access, &layout(COLUMN_KIND)?, (database, schema), None)?; + slf.borrow_mut() + .column_events + .extend(events.iter().map(Bound::unbind)); + } + Ok(()) + } + + /// Read moves and old-name facts once for every schema not inspected yet. + #[pyo3(signature = (*, schemas))] + fn inspect_old_name_views(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let pending = pending(schemas, &slf.borrow().old_name_schemas)?; + if pending.is_empty() { + return Ok(()); + } + slf.borrow_mut() + .old_name_schemas + .extend(pending.iter().map(|(_, key)| key.clone())); + let listed: Vec> = slf.borrow().list_relations( + py, + pending.iter().map(|(schema, _)| schema.clone()).collect(), + PyTuple::new(py, [MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME])?.into_any(), + )?; + let state_tables: HashSet = + listed.iter().map(relation_key).collect::>()?; + for (schema, key) in pending { + if !state_tables.contains(&(key.clone(), OLD_NAME_VIEW_TABLE_NAME.to_owned())) + || !state_tables.contains(&(key, MIGRATION_TABLE_NAME.to_owned())) + { + continue; + } + let (access, adapter, database) = { + let this: PyRef<'_, Self> = slf.borrow(); + ( + this.access(py)?, + this.adapter.bind(py).clone(), + this.database.bind(py).clone(), + ) + }; + let stored: Vec = stored_old_name_columns( + &adapter, + &access.connection, + (database.clone(), schema.clone()), + )?; + let facts = read_events( + &access, + &layout(OLD_NAME_KIND)?, + (database.clone(), schema.clone()), + Some(&stored), + )?; + slf.borrow_mut() + .old_name_facts + .extend(facts.iter().map(Bound::unbind)); + let moves = read_events(&access, &layout(MODEL_KIND)?, (database, schema), None)?; + slf.borrow_mut() + .old_name_moves + .extend(moves.iter().map(Bound::unbind)); + } + Ok(()) + } + + /// Read migration events once from schemas known to hold the migration table. + #[pyo3(signature = (*, schemas))] + fn inspect_migration_events(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let mut sorted: Vec<(String, Bound<'_, PyAny>)> = Vec::new(); + for schema in schemas.try_iter()? { + let schema: Bound<'_, PyAny> = schema?; + sorted.push((schema.extract()?, schema)); + } + sorted.sort_by(|left, right| left.0.cmp(&right.0)); + for (_, schema) in sorted { + let key: String = lowered(&schema)?; + if !slf.borrow_mut().event_schemas.insert(key) { + continue; + } + let (access, database) = { + let this: PyRef<'_, Self> = slf.borrow(); + (this.access(py)?, this.database.bind(py).clone()) + }; + let events = read_events(&access, &layout(MODEL_KIND)?, (database, schema), None)?; + slf.borrow_mut() + .events + .extend(events.iter().map(Bound::unbind)); + } + Ok(()) + } + + /// Read state tables once for every schema not inspected yet. + #[pyo3(signature = (*, schemas))] + fn inspect_schemas(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let pending = pending(schemas, &slf.borrow().read_schemas)?; + if pending.is_empty() { + return Ok(()); + } + slf.borrow_mut() + .read_schemas + .extend(pending.iter().map(|(_, key)| key.clone())); + let listed: Vec> = slf.borrow().list_relations( + py, + pending.iter().map(|(schema, _)| schema.clone()).collect(), + PyTuple::new(py, [MIGRATION_TABLE_NAME, FINGERPRINT_TABLE_NAME])?.into_any(), + )?; + let state_tables: HashSet = + listed.iter().map(relation_key).collect::>()?; + for (schema, key) in pending { + if state_tables.contains(&(key.clone(), MIGRATION_TABLE_NAME.to_owned())) { + let single = PyTuple::new(py, [schema.clone()])?; + Self::inspect_migration_events(slf, single.as_any())?; + } + if state_tables.contains(&(key.clone(), FINGERPRINT_TABLE_NAME.to_owned())) { + let models: Vec> = slf.borrow().model_fingerprints(py, &schema, &key)?; + let mut this: PyRefMut<'_, Self> = slf.borrow_mut(); + match this + .fingerprints + .iter_mut() + .find(|(stored, _)| *stored == key) + { + Some((_, stored)) => *stored = models, + None => this.fingerprints.push((key, models)), + } + } + } + Ok(()) + } + + /// List candidate relations and their columns with one metadata read. + #[pyo3(signature = (*, locations))] + fn inspect_relations(slf: &Bound<'_, Self>, locations: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let mut wanted: HashSet = HashSet::new(); + let mut schemas: Vec = Vec::new(); + let mut names: Vec = Vec::new(); + for location in locations.try_iter()? { + let location: Bound<'_, PyAny> = location?; + if location.getattr("schema")?.is_none() { + continue; + } + let key: RelationKey = relation_key(&location)?; + if slf.borrow().relations.contains_key(&key) { + continue; + } + wanted.insert(key); + schemas.push(location.getattr("schema")?.extract()?); + names.push(location.getattr("name")?.extract()?); + } + if wanted.is_empty() { + return Ok(()); + } + schemas.sort(); + schemas.dedup(); + names.sort(); + names.dedup(); + let listed: Vec> = slf.borrow().list_relations( + py, + schemas + .iter() + .map(|schema| PyString::new(py, schema).into_any()) + .collect(), + PyTuple::new(py, &names)?.into_any(), + )?; + let mut matched: Vec<(RelationKey, Bound<'_, PyAny>)> = Vec::new(); + for relation in listed { + let key: RelationKey = relation_key(&relation)?; + if wanted.contains(&key) { + matched.push((key, relation)); + } + } + let columns: Bound<'_, PyAny> = if matched.is_empty() { + PyDict::new(py).into_any() + } else { + let (module, name) = GET_COLUMNS_FOR_INSPECTION; + let this: PyRef<'_, Self> = slf.borrow(); + let adapter: Bound<'_, PyAny> = this.adapter.bind(py).clone(); + let connection: Bound<'_, PyAny> = this.connection.bind(py).clone(); + drop(this); + call_with( + &imported(py, module, name)?, + &[ + ("adapter", adapter), + ("connection", connection), + ( + "relations", + PyTuple::new(py, matched.iter().map(|(_, relation)| relation))?.into_any(), + ), + ], + )? + }; + let empty: Bound<'_, PyTuple> = PyTuple::empty(py); + for (key, relation) in matched { + let relation_columns: Bound<'_, PyAny> = + columns.call_method1("get", (relation.getattr("identity")?, &empty))?; + let mut this: PyRefMut<'_, Self> = slf.borrow_mut(); + this.relations.insert(key.clone(), relation.unbind()); + this.columns.insert(key, relation_columns.unbind()); + } + Ok(()) + } + + /// The listed relation for a location. + fn relation(&self, py: Python<'_>, location: &Bound<'_, PyAny>) -> PyResult> { + Ok(match self.relations.get(&relation_key(location)?) { + Some(relation) => relation.clone_ref(py), + None => py.None(), + }) + } + + /// Listed columns for a location. + fn relation_columns(&self, py: Python<'_>, location: &Bound<'_, PyAny>) -> PyResult> { + Ok(match self.columns.get(&relation_key(location)?) { + Some(columns) => columns.clone_ref(py), + None => PyTuple::empty(py).into_any().unbind(), + }) + } + + /// The latest model fingerprint that built the location or named model. + #[pyo3(signature = (*, location, model_name))] + fn model_fingerprint<'py>( + &self, + py: Python<'py>, + location: &Bound<'py, PyAny>, + model_name: Option, + ) -> PyResult>> { + let key: String = lowered(&schema_or_empty(location.getattr("schema")?))?; + let Some((_, candidates)) = self.fingerprints.iter().find(|(stored, _)| *stored == key) + else { + return Ok(None); + }; + let target = EventRelation { + database: location.getattr("database")?.extract()?, + schema: location.getattr("schema")?.extract()?, + name: location.getattr("name")?.extract()?, + }; + newest(py, candidates, (Some(&target), model_name.as_deref())) + } + + /// The newest model fingerprint with this node name in any inspected schema. + #[pyo3(signature = (*, model_name))] + fn named_fingerprint<'py>( + &self, + py: Python<'py>, + model_name: &str, + ) -> PyResult>> { + let mut all: Vec> = Vec::new(); + for (_, fingerprints) in &self.fingerprints { + all.extend(fingerprints.iter().map(|item| item.clone_ref(py))); + } + newest(py, &all, (None, Some(model_name))) + } + + /// Latest model fingerprints whose names are no longer in the project. + #[pyo3(signature = (*, project_model_names))] + fn orphan_fingerprints<'py>( + &self, + py: Python<'py>, + project_model_names: &Bound<'py, PyAny>, + ) -> PyResult> { + let mut orphans: Vec> = Vec::new(); + for (_, fingerprints) in &self.fingerprints { + for fingerprint in fingerprints { + let fingerprint: &Bound<'py, PyAny> = fingerprint.bind(py); + if !project_model_names.contains(fingerprint.getattr("node_name")?)? { + orphans.push(fingerprint.clone()); + } + } + } + PyTuple::new(py, orphans) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs index 5e23a58eda..79a3b60291 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs @@ -10,3 +10,10 @@ pub(crate) const OLD_NAME_VIEW_CONFIG_KEY: &str = "old_name_view"; pub(crate) const MODEL_KIND: &str = "model"; pub(crate) const COLUMN_KIND: &str = "column"; pub(crate) const OLD_NAME_KIND: &str = "old_name"; +pub(crate) const MIGRATION_LOGGER: &str = "sqlbuild.migrations"; +pub(crate) const LOGGING_MODULE: &str = "logging"; +pub(crate) const TIME_MODULE: &str = "time"; +pub(crate) const MIGRATION_WRITE_ATTEMPTS: i64 = 3; +pub(crate) const MIGRATION_WRITE_RETRY_BASE_SECONDS: f64 = 0.05; +pub(crate) const MIGRATION_WRITE_RETRY_MESSAGE: &str = + "migration event write attempt %s/%s failed for '%s'; retrying: %s"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs new file mode 100644 index 0000000000..732a2326b3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs @@ -0,0 +1,326 @@ +//! Read and append migration state events through the adapter's execute callable. + +use pyo3::exceptions::PyException; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyString, PyTuple, PyTupleMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_state::events::constants::OLD_NAME_VIEW_TABLE_NAME; +use sqlbuild_state::events::main::recorded_renames::recorded_renames; +use sqlbuild_state::events::models::RecordedRenameKey; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::state_rows::constants::{ + COLUMN_KIND, LOGGING_MODULE, MIGRATION_LOGGER, MIGRATION_WRITE_ATTEMPTS, + MIGRATION_WRITE_RETRY_BASE_SECONDS, MIGRATION_WRITE_RETRY_MESSAGE, MODEL_KIND, OLD_NAME_KIND, + TIME_MODULE, +}; +use crate::bindings::_helpers::state_rows::events::relation as event_relation; +use crate::bindings::_helpers::state_rows::migration_objects::{event_order, layout, state_error}; +use crate::bindings::_helpers::state_rows::migration_projections::old_name_views; +use crate::bindings::_helpers::state_rows::migration_rows::{ + decoded_events, read_statement, write_statements, +}; +use crate::bindings::_helpers::state_rows::models::{EventTable, EventTableLayout, StateAccess}; +use crate::bindings::_helpers::state_rows::timestamps::presumed_utc; + +fn execute<'py>(access: &StateAccess<'py>, sql: &str) -> PyResult> { + let py: Python<'py> = access.execute.py(); + call_with( + &access.execute, + &[ + ("connection", access.connection.clone()), + ("sql", PyString::new(py, sql).into_any()), + ], + ) +} + +fn unreadable(layout: &EventTableLayout, schema: &Bound<'_, PyAny>, error: PyErr) -> PyErr { + let py: Python<'_> = schema.py(); + let describe = |value: &Bound<'_, PyAny>| -> String { + value.str().map(|text| text.to_string()).unwrap_or_default() + }; + let message: String = format!( + "Unable to read {} from schema '{}': {}", + layout.read_subject, + describe(schema), + describe(error.value(py).as_any()), + ); + let wrapped: PyErr = state_error(py, &message); + wrapped.set_cause(py, Some(error)); + wrapped +} + +/// Every event of `layout`'s table this version reads from one schema, from `stored` columns when known. +pub(crate) fn read_events<'py>( + access: &StateAccess<'py>, + layout: &EventTableLayout, + (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), + stored: Option<&[String]>, +) -> PyResult> { + let py: Python<'py> = schema.py(); + let (sql, columns): (String, Vec) = read_statement( + layout, + &access.render_qualified_name, + (database, schema.clone()), + stored, + )?; + let rows: Bound<'py, PyAny> = + match execute(access, &sql).and_then(|cursor| cursor.call_method0("fetchall")) { + Ok(rows) => rows, + Err(error) if error.is_instance_of::(py) => { + return Err(unreadable(layout, &schema, error)); + } + Err(error) => return Err(error), + }; + decoded_events(layout.table, &rows, &columns) +} + +/// The column names of one schema's old-name view state table, as the adapter lists them. +pub(crate) fn stored_old_name_columns<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let columns: Bound<'py, PyAny> = call_with( + &adapter.getattr("get_columns")?, + &[ + ("connection", connection.clone()), + ("database", database), + ("schema", schema), + ( + "name", + PyString::new(py, OLD_NAME_VIEW_TABLE_NAME).into_any(), + ), + ], + )?; + columns + .try_iter()? + .map(|column| column?.getattr("name")?.extract()) + .collect() +} + +fn subject(table: EventTable, event: &Bound<'_, PyAny>) -> PyResult { + let text = |value: Bound<'_, PyAny>| -> PyResult { Ok(value.str()?.to_string()) }; + Ok(match table { + EventTable::Model => text(event.getattr("destination_model")?)?, + EventTable::Column => format!( + "{}.{}", + text(event.getattr("model_name")?)?, + text(event.getattr("destination_column")?)? + ), + EventTable::OldName => format!( + "{} old name {}", + text(event.getattr("destination_model")?)?, + text(event.getattr("old")?.getattr("name")?)? + ), + }) +} + +fn append_once( + access: &StateAccess<'_>, + create_table_sql: Option<&str>, + (existing_sql, insert_sql): (&str, &str), +) -> PyResult<()> { + if let Some(create_table_sql) = create_table_sql { + execute(access, create_table_sql)?; + } + if !execute(access, existing_sql)? + .call_method0("fetchall")? + .is_truthy()? + { + execute(access, insert_sql)?; + } + Ok(()) +} + +fn retry_pause( + py: Python<'_>, + (attempt, attempts): (i64, i64), + subject: &str, + error: &PyErr, +) -> PyResult<()> { + let logger: Bound<'_, PyAny> = py + .import(LOGGING_MODULE)? + .call_method1("getLogger", (MIGRATION_LOGGER,))?; + logger.call_method1( + "warning", + ( + MIGRATION_WRITE_RETRY_MESSAGE, + attempt + 1, + attempts, + subject, + error.value(py), + ), + )?; + let step: f64 = f64::from(i32::try_from(attempt + 1).unwrap_or(i32::MAX)); + py.import(TIME_MODULE)? + .call_method1("sleep", (MIGRATION_WRITE_RETRY_BASE_SECONDS * step,))?; + Ok(()) +} + +/// Create the state table when given DDL, then insert the event unless its ID is recorded. +fn append_event( + access: &StateAccess<'_>, + (kind, event): (&str, &Bound<'_, PyAny>), + create_table_sql: Option<&str>, + attempts: i64, +) -> PyResult<()> { + let py: Python<'_> = event.py(); + let layout: EventTableLayout = layout(kind)?; + let (existing_sql, insert_sql): (String, String) = + write_statements(&layout, event, &access.render_qualified_name)?; + let subject: String = subject(layout.table, event)?; + for attempt in 0..attempts { + let Err(error) = append_once(access, create_table_sql, (&existing_sql, &insert_sql)) else { + return Ok(()); + }; + if attempt + 1 == attempts || !error.is_instance_of::(py) { + return Err(error); + } + retry_pause(py, (attempt, attempts), &subject, &error)?; + } + Ok(()) +} + +fn access<'py>(request: &Request<'_, 'py>) -> PyResult> { + Ok(StateAccess { + connection: request.get("connection")?, + execute: request.get("execute")?, + render_qualified_name: request.get("render_qualified_name")?, + }) +} + +fn stored_names(stored: Option>) -> PyResult>> { + match stored { + Some(stored) if !stored.is_none() => stored + .try_iter()? + .map(|column| column?.extract()) + .collect::>>() + .map(Some), + _ => Ok(None), + } +} + +/// Every event of one migration state table this version reads from one schema. +#[pyfunction(name = "read_migration_state_events")] +#[pyo3(signature = (**request))] +fn read_state_events<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("read_migration_state_events", request)?; + let stored: Option> = stored_names(request.optional("stored_columns")?)?; + read_events( + &access(&request)?, + &layout(&request.get("kind")?.extract::()?)?, + (request.get("database")?, request.get("schema")?), + stored.as_deref(), + ) +} + +/// The old-name view history of every recorded move in a schema that holds both state tables. +#[pyfunction(name = "read_old_name_view_histories")] +#[pyo3(signature = (**request))] +fn read_histories<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("read_old_name_view_histories", request)?; + let access: StateAccess<'py> = access(&request)?; + let location = || -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + Ok((request.get("database")?, request.get("schema")?)) + }; + let stored: Option> = stored_names(Some(request.get("stored_columns")?))?; + let moves: Bound<'py, PyTuple> = read_events(&access, &layout(MODEL_KIND)?, location()?, None)?; + let facts: Bound<'py, PyTuple> = read_events( + &access, + &layout(OLD_NAME_KIND)?, + location()?, + stored.as_deref(), + )?; + old_name_views(moves.as_any(), facts.as_any()) +} + +/// Renames recorded on the relation at or after `since`, oldest first. +#[pyfunction(name = "read_recorded_column_renames")] +#[pyo3(signature = (**request))] +fn read_renames<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("read_recorded_column_renames", request)?; + let py: Python<'py> = request.py(); + let relation: Bound<'py, PyAny> = request.get("relation")?; + let since = presumed_utc(&request.get("since")?)?; + let schema: Bound<'py, PyAny> = relation.getattr("schema")?; + let schema: Bound<'py, PyAny> = if schema.is_none() { + PyString::new(py, "").into_any() + } else { + schema + }; + let events: Bound<'py, PyTuple> = read_events( + &access(&request)?, + &layout(COLUMN_KIND)?, + (relation.getattr("database")?, schema), + None, + )?; + let keys: Vec = events + .iter() + .map(|event| { + Ok(RecordedRenameKey { + order: event_order(&event)?, + relation: event_relation(&event.getattr("relation")?)?, + records_event: event + .getattr("decision")? + .getattr("records_event")? + .is_truthy()?, + }) + }) + .collect::>()?; + let target = event_relation(&relation)?; + let selected: Vec> = recorded_renames(&keys, &target, &since) + .into_iter() + .map(|index| events.get_item(index)) + .collect::>()?; + PyTuple::new(py, selected) +} + +/// The column names of one schema's old-name view state table. +#[pyfunction(name = "stored_old_name_view_columns")] +#[pyo3(signature = (*, adapter, connection, database, schema))] +fn stored_columns_binding<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + database: Bound<'py, PyAny>, + schema: Bound<'py, PyAny>, +) -> PyResult> { + PyTuple::new( + adapter.py(), + stored_old_name_columns(adapter, connection, (database, schema))?, + ) +} + +/// Create the state table from adapter DDL when given, then append the event unless recorded. +#[pyfunction(name = "write_migration_state_event")] +#[pyo3(signature = (**request))] +fn write_state_event(request: Option<&Bound<'_, PyDict>>) -> PyResult<()> { + let request = Request::new("write_migration_state_event", request)?; + let create_table_sql: Option = request.get("create_table_sql")?.extract()?; + let attempts: i64 = match request.optional("attempts")? { + Some(attempts) => attempts.extract()?, + None => MIGRATION_WRITE_ATTEMPTS, + }; + append_event( + &access(&request)?, + ( + &request.get("kind")?.extract::()?, + &request.get("event")?, + ), + create_table_sql.as_deref(), + attempts, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(read_state_events, module)?)?; + module.add_function(wrap_pyfunction!(read_histories, module)?)?; + module.add_function(wrap_pyfunction!(read_renames, module)?)?; + module.add_function(wrap_pyfunction!(stored_columns_binding, module)?)?; + module.add_function(wrap_pyfunction!(write_state_event, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs index a0fff30ca8..43cec94da4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs @@ -26,6 +26,7 @@ pub(crate) fn layout(kind: &str) -> PyResult { relation_attribute: "destination", state_message: "model migration state requires a target schema", event_message: "model migration events require a destination schema", + read_subject: "model migration events", }), COLUMN_KIND => Ok(EventTableLayout { table: EventTable::Column, @@ -34,6 +35,7 @@ pub(crate) fn layout(kind: &str) -> PyResult { relation_attribute: "relation", state_message: "column migration state requires a target schema", event_message: "column migration events require a relation schema", + read_subject: "column migration events", }), OLD_NAME_KIND => Ok(EventTableLayout { table: EventTable::OldName, @@ -42,6 +44,7 @@ pub(crate) fn layout(kind: &str) -> PyResult { relation_attribute: "new", state_message: "old-name view state requires a target schema", event_message: "old-name view facts require a destination schema", + read_subject: "old-name view facts", }), _ => Err(PyValueError::new_err(format!( "unknown migration state table kind: {kind}" diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs index 279c3c9e54..04c014d2d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs @@ -66,27 +66,20 @@ fn create_sql<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult(request: Option<&Bound<'py, PyDict>>) -> PyResult<(String, Bound<'py, PyTuple>)> { - let request = Request::new("render_migration_event_read_sql", request)?; - let layout: EventTableLayout = layout(&request.get("kind")?.extract::()?)?; - let stored: Option> = match request.optional("stored_columns")? { - Some(stored) if !stored.is_none() => Some(stored.extract()?), - _ => None, - }; - let columns: Vec<&str> = match &stored { +pub(crate) fn read_statement<'py>( + layout: &EventTableLayout, + render_qualified_name: &Bound<'py, PyAny>, + location: (Bound<'py, PyAny>, Bound<'py, PyAny>), + stored: Option<&[String]>, +) -> PyResult<(String, Vec)> { + let columns: Vec<&str> = match stored { Some(stored) => readable_columns(layout.columns, stored), None => layout.columns.to_vec(), }; - let table: String = qualified_table( - &layout, - &request.get("render_qualified_name")?, - (request.get("database")?, request.get("schema")?), - )?; + let table: String = qualified_table(layout, render_qualified_name, location)?; Ok(( event_read_sql(&table, &columns), - PyTuple::new(request.py(), columns)?, + columns.into_iter().map(str::to_owned).collect(), )) } @@ -170,22 +163,19 @@ fn json_error(error: sqlbuild_core::json::errors::JsonEmitError) -> PyErr { } /// The idempotency lookup and the insert that append one event to its schema's state table. -#[pyfunction(name = "render_migration_event_write_sql")] -#[pyo3(signature = (*, kind, event, render_qualified_name))] -fn write_sql<'py>( - kind: &str, +pub(crate) fn write_statements<'py>( + layout: &EventTableLayout, event: &Bound<'py, PyAny>, render_qualified_name: &Bound<'py, PyAny>, ) -> PyResult<(String, String)> { let py: Python<'py> = event.py(); - let layout: EventTableLayout = layout(kind)?; let relation: Bound<'py, PyAny> = event.getattr(layout.relation_attribute)?; let schema: Bound<'py, PyAny> = relation.getattr("schema")?; if schema.is_none() { return Err(state_error(py, layout.event_message)); } let table: String = qualified_table( - &layout, + layout, render_qualified_name, (relation.getattr("database")?, schema), )?; @@ -398,22 +388,16 @@ fn old_name_event<'py>( } /// Stored rows of the selected `columns` as events; rows this version cannot read are skipped. -#[pyfunction(name = "decode_migration_event_rows")] -#[pyo3(signature = (*, kind, rows, columns))] -fn decode_rows<'py>( - kind: &str, +pub(crate) fn decoded_events<'py>( + table: EventTable, rows: &Bound<'py, PyAny>, - columns: Vec, + columns: &[String], ) -> PyResult> { let py: Python<'py> = rows.py(); - let table: EventTable = layout(kind)?.table; let mut events: Vec> = Vec::new(); for row in rows.try_iter()? { let values: Vec> = row?.try_iter()?.collect::>()?; - let stored = StoredRow { - columns: &columns, - values, - }; + let stored = StoredRow { columns, values }; let decoded: Option> = match table { EventTable::Model => Some(model_event(py, &stored)?), EventTable::Column => match column_event(py, &stored) { @@ -430,8 +414,5 @@ fn decode_rows<'py>( pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(create_sql, module)?)?; - module.add_function(wrap_pyfunction!(read_sql, module)?)?; - module.add_function(wrap_pyfunction!(write_sql, module)?)?; - module.add_function(wrap_pyfunction!(decode_rows, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs index 1c29eebbd6..a130979144 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod constants; pub(crate) mod events; pub(crate) mod freshness; +pub(crate) mod migration_io; pub(crate) mod migration_objects; pub(crate) mod migration_projections; pub(crate) mod migration_rows; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/models.rs index c126b8ecd9..b69ffba5c7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/models.rs @@ -1,5 +1,7 @@ //! The layout of one append-only migration state table as the bindings use it. +use pyo3::prelude::{Bound, PyAny}; + /// Which migration state table a request reads or writes. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum EventTable { @@ -17,4 +19,13 @@ pub(crate) struct EventTableLayout { pub relation_attribute: &'static str, pub state_message: &'static str, pub event_message: &'static str, + pub read_subject: &'static str, +} + +/// How migration state reaches the warehouse: the connection, execute callable and name renderer. +#[derive(Debug)] +pub(crate) struct StateAccess<'py> { + pub connection: Bound<'py, PyAny>, + pub execute: Bound<'py, PyAny>, + pub render_qualified_name: Bound<'py, PyAny>, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs index 819aa21fb8..df9f69c0f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs @@ -3,13 +3,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::state_rows::{ - events, freshness, migration_projections, migration_rows, + events, freshness, migration_io, migration_projections, migration_rows, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { events::register(module)?; freshness::register(module)?; migration_rows::register(module)?; + migration_io::register(module)?; migration_projections::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-state/src/events/_helpers/relations.rs b/crates/sqlbuild-state/src/events/_helpers/relations.rs index a91170fbfa..0589b429be 100644 --- a/crates/sqlbuild-state/src/events/_helpers/relations.rs +++ b/crates/sqlbuild-state/src/events/_helpers/relations.rs @@ -21,10 +21,10 @@ pub(crate) fn relations_match(left: &EventRelation, right: &EventRelation) -> bo } /// The index of the greatest order among the selected items, keeping the first of equal ones. -pub(crate) fn newest_index<'a>( - orders: impl Iterator, +pub(crate) fn newest_index<'a, T: Ord + 'a>( + orders: impl Iterator, ) -> Option { - let mut newest: Option<(usize, &crate::events::models::EventOrder)> = None; + let mut newest: Option<(usize, &T)> = None; for (index, order) in orders { if newest.is_none_or(|(_, current)| order > current) { newest = Some((index, order)); diff --git a/crates/sqlbuild-state/src/events/main/mod.rs b/crates/sqlbuild-state/src/events/main/mod.rs index 0d0be3fda3..9b543bf5fb 100644 --- a/crates/sqlbuild-state/src/events/main/mod.rs +++ b/crates/sqlbuild-state/src/events/main/mod.rs @@ -9,7 +9,9 @@ pub mod event_read_sql; pub mod grants_json; pub mod migration_event_id; pub mod newest_column_event; +pub mod newest_fingerprint; pub mod newest_move_event; pub mod old_name_histories; pub mod old_name_view_event_id; pub mod readable_columns; +pub mod recorded_renames; diff --git a/crates/sqlbuild-state/src/events/main/newest_fingerprint.rs b/crates/sqlbuild-state/src/events/main/newest_fingerprint.rs new file mode 100644 index 0000000000..11332aaf4d --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/newest_fingerprint.rs @@ -0,0 +1,38 @@ +//! `newest_fingerprint`: the latest model fingerprint that built a relation or carries a name. + +use crate::events::_helpers::relations::{newest_index, relations_match}; +use crate::events::models::{EventRelation, FingerprintKey}; + +/// The newest fingerprint that built `target`, else the newest named `model_name`; the first wins ties. +#[must_use] +pub fn newest_fingerprint( + fingerprints: &[FingerprintKey], + target: Option<&EventRelation>, + model_name: Option<&str>, +) -> Option { + if let Some(target) = target { + let built: Option = newest_index( + fingerprints + .iter() + .enumerate() + .filter(|(_, fingerprint)| { + fingerprint + .target + .as_ref() + .is_some_and(|built| relations_match(target, built)) + }) + .map(|(index, fingerprint)| (index, &fingerprint.order)), + ); + if built.is_some() { + return built; + } + } + let model_name: &str = model_name?; + newest_index( + fingerprints + .iter() + .enumerate() + .filter(|(_, fingerprint)| fingerprint.node_name == model_name) + .map(|(index, fingerprint)| (index, &fingerprint.order)), + ) +} diff --git a/crates/sqlbuild-state/src/events/main/recorded_renames.rs b/crates/sqlbuild-state/src/events/main/recorded_renames.rs new file mode 100644 index 0000000000..e35245b2a0 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/recorded_renames.rs @@ -0,0 +1,27 @@ +//! `recorded_renames`: the renames recorded on one relation since a point in time. + +use sqlbuild_cursor::algebra::models::LocalDateTime; + +use crate::events::_helpers::relations::relations_match; +use crate::events::models::{EventRelation, RecordedRenameKey}; + +/// Indexes of recorded renames on `relation` created at or after `since`, oldest first. +#[must_use] +pub fn recorded_renames( + events: &[RecordedRenameKey], + relation: &EventRelation, + since: &LocalDateTime, +) -> Vec { + let mut indexes: Vec = events + .iter() + .enumerate() + .filter(|(_, event)| { + relations_match(relation, &event.relation) + && event.records_event + && event.order.created_at >= *since + }) + .map(|(index, _)| index) + .collect(); + indexes.sort_by(|left, right| events[*left].order.cmp(&events[*right].order)); + indexes +} diff --git a/crates/sqlbuild-state/src/events/models.rs b/crates/sqlbuild-state/src/events/models.rs index 3103778446..f806f49e0f 100644 --- a/crates/sqlbuild-state/src/events/models.rs +++ b/crates/sqlbuild-state/src/events/models.rs @@ -86,3 +86,26 @@ pub struct OldNameHistoryIndexes { pub created: Option, pub dropped: Option, } + +/// The fields of a recorded column rename that the since-filter reads. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RecordedRenameKey { + pub order: EventOrder, + pub relation: EventRelation, + pub records_event: bool, +} + +/// The order of a stored fingerprint: its ISO timestamp text, then its run ID. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct FingerprintOrder { + pub timestamp: String, + pub run_id: String, +} + +/// The fields of a stored model fingerprint that the newest-fingerprint lookup reads. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FingerprintKey { + pub order: FingerprintOrder, + pub node_name: String, + pub target: Option, +} diff --git a/crates/sqlbuild-state/src/events/tests.rs b/crates/sqlbuild-state/src/events/tests.rs index 7bd9f08251..abf04f2330 100644 --- a/crates/sqlbuild-state/src/events/tests.rs +++ b/crates/sqlbuild-state/src/events/tests.rs @@ -4,6 +4,8 @@ mod codecs; mod helpers; #[path = "tests/test_insert_sql.rs"] mod insert_sql; +#[path = "tests/test_lookups.rs"] +mod lookups; #[path = "tests/test_projections.rs"] mod projections; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-state/src/events/tests/helpers.rs b/crates/sqlbuild-state/src/events/tests/helpers.rs index fd12989882..47e1632627 100644 --- a/crates/sqlbuild-state/src/events/tests/helpers.rs +++ b/crates/sqlbuild-state/src/events/tests/helpers.rs @@ -1,8 +1,8 @@ use sqlbuild_cursor::algebra::models::{CivilDate, LocalDateTime}; use crate::events::models::{ - ColumnEventKey, EventOrder, EventRelation, MoveEventKey, OldNameFactKey, OldNameStep, - StateCell, StateColumnType, StateValue, + ColumnEventKey, EventOrder, EventRelation, FingerprintKey, FingerprintOrder, MoveEventKey, + OldNameFactKey, OldNameStep, RecordedRenameKey, StateCell, StateColumnType, StateValue, }; pub(crate) fn cell(column: &str, column_type: StateColumnType, value: StateValue) -> StateCell { @@ -73,3 +73,30 @@ pub(crate) fn pairs(items: &[(&str, &str)]) -> Vec<(String, String)> { .map(|(old, new)| ((*old).to_owned(), (*new).to_owned())) .collect() } + +pub(crate) fn recorded( + order: EventOrder, + relation: EventRelation, + records_event: bool, +) -> RecordedRenameKey { + RecordedRenameKey { + order, + relation, + records_event, + } +} + +pub(crate) fn fingerprint( + (timestamp, run_id): (&str, &str), + node_name: &str, + target: Option, +) -> FingerprintKey { + FingerprintKey { + order: FingerprintOrder { + timestamp: timestamp.to_owned(), + run_id: run_id.to_owned(), + }, + node_name: node_name.to_owned(), + target, + } +} diff --git a/crates/sqlbuild-state/src/events/tests/test_lookups.rs b/crates/sqlbuild-state/src/events/tests/test_lookups.rs new file mode 100644 index 0000000000..8c1b670195 --- /dev/null +++ b/crates/sqlbuild-state/src/events/tests/test_lookups.rs @@ -0,0 +1,107 @@ +use crate::events::main::newest_fingerprint::newest_fingerprint; +use crate::events::main::recorded_renames::recorded_renames; +use crate::events::tests::helpers::{fingerprint, order, recorded, relation}; +use crate::events::tests::test_types::{NewestFingerprintTestCase, RecordedRenamesTestCase}; + +#[test] +fn given_column_renames_when_reading_since_then_recorded_matches_return_oldest_first() { + let test_cases = [ + RecordedRenamesTestCase { + description: "renames at or after the cutoff sort by time then event ID", + events: vec![ + recorded(order(5, "b"), relation(None, "main", "orders"), true), + recorded(order(3, "z"), relation(Some("db"), "MAIN", "Orders"), true), + recorded(order(5, "a"), relation(None, "main", "orders"), true), + recorded(order(2, "c"), relation(None, "main", "orders"), true), + ], + relation: relation(None, "main", "orders"), + since: order(3, "").created_at, + expected_indexes: vec![1, 2, 0], + }, + RecordedRenamesTestCase { + description: "unrecorded decisions and other relations are skipped", + events: vec![ + recorded(order(4, "a"), relation(None, "main", "orders"), false), + recorded(order(4, "b"), relation(None, "main", "customers"), true), + recorded(order(4, "c"), relation(Some("one"), "main", "orders"), true), + ], + relation: relation(Some("two"), "main", "orders"), + since: order(1, "").created_at, + expected_indexes: vec![], + }, + ]; + + for test_case in test_cases { + assert_eq!( + recorded_renames(&test_case.events, &test_case.relation, &test_case.since), + test_case.expected_indexes, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_fingerprints_when_finding_newest_then_built_relation_wins_before_name() { + let test_cases = [ + NewestFingerprintTestCase { + description: "a fingerprint that built the relation wins over a newer named one", + fingerprints: vec![ + fingerprint(("2026-01-03", "r3"), "orders", None), + fingerprint( + ("2026-01-01", "r1"), + "legacy", + Some(relation(None, "MAIN", "ORDERS")), + ), + ], + target: Some(relation(Some("db"), "main", "orders")), + model_name: Some("orders"), + expected_index: Some(1), + }, + NewestFingerprintTestCase { + description: "without a built match the newest named fingerprint wins", + fingerprints: vec![ + fingerprint(("2026-01-01", "r1"), "orders", None), + fingerprint(("2026-01-02", "r0"), "orders", None), + fingerprint(("2026-01-02", "r0"), "orders", None), + ], + target: Some(relation(None, "main", "orders")), + model_name: Some("orders"), + expected_index: Some(1), + }, + NewestFingerprintTestCase { + description: "no built match and no model name", + fingerprints: vec![fingerprint(("2026-01-01", "r1"), "orders", None)], + target: Some(relation(None, "main", "orders")), + model_name: None, + expected_index: None, + }, + NewestFingerprintTestCase { + description: "a name lookup alone ignores built relations", + fingerprints: vec![ + fingerprint( + ("2026-01-05", "r5"), + "customers", + Some(relation(None, "main", "orders")), + ), + fingerprint(("2026-01-01", "r1"), "orders", None), + ], + target: None, + model_name: Some("orders"), + expected_index: Some(1), + }, + ]; + + for test_case in test_cases { + assert_eq!( + newest_fingerprint( + &test_case.fingerprints, + test_case.target.as_ref(), + test_case.model_name, + ), + test_case.expected_index, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/events/tests/test_types.rs b/crates/sqlbuild-state/src/events/tests/test_types.rs index e059578495..e40b5bc07c 100644 --- a/crates/sqlbuild-state/src/events/tests/test_types.rs +++ b/crates/sqlbuild-state/src/events/tests/test_types.rs @@ -1,7 +1,9 @@ use crate::events::errors::StateSqlError; +use sqlbuild_cursor::algebra::models::LocalDateTime; + use crate::events::models::{ - ColumnEventKey, EventOrder, EventRelation, MoveEventKey, OldNameFactKey, OldNameHistoryIndexes, - StateCell, + ColumnEventKey, EventOrder, EventRelation, FingerprintKey, MoveEventKey, OldNameFactKey, + OldNameHistoryIndexes, RecordedRenameKey, StateCell, }; pub(crate) struct InsertTestCase { @@ -60,3 +62,19 @@ pub(crate) struct GrantsTestCase { pub(crate) grants: Option>, pub(crate) expected_json: Option, } + +pub(crate) struct RecordedRenamesTestCase { + pub(crate) description: &'static str, + pub(crate) events: Vec, + pub(crate) relation: EventRelation, + pub(crate) since: LocalDateTime, + pub(crate) expected_indexes: Vec, +} + +pub(crate) struct NewestFingerprintTestCase { + pub(crate) description: &'static str, + pub(crate) fingerprints: Vec, + pub(crate) target: Option, + pub(crate) model_name: Option<&'static str>, + pub(crate) expected_index: Option, +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b4444c6552..1d74e1c8ad 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5,7 +5,7 @@ from contextlib import AbstractContextManager from contextvars import Token from datetime import date, datetime from pathlib import Path, PurePath -from typing import Any, Literal, TextIO, TypedDict, overload +from typing import Any, Literal, TextIO, TypedDict from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter @@ -207,9 +207,6 @@ from sqlbuild.compiler.pipeline.models import ( StaticCommandContext, ) from sqlbuild.compiler.planner.classes.migration_fingerprint_cache import MigrationFingerprintCache -from sqlbuild.compiler.planner.classes.migration_state_inspection import ( - MigrationStateInspection, -) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( AuditPlanEntry, @@ -277,6 +274,7 @@ from sqlbuild.compiler.planner.models import ( SqlTestPlanResult, SqlTestSelection, UnitTestSelectorSplit, + WarehouseFingerprints, WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import ( @@ -4443,32 +4441,81 @@ def render_migration_state_create_sql( render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], ) -> str: ... -def render_migration_event_read_sql( +def read_migration_state_events( *, kind: Literal["model", "column", "old_name"], + connection: Any, + execute: AdapterExecute[Any, Any], database: str | None, schema: str, render_qualified_name: Callable[..., str | None], - stored_columns: tuple[str, ...] | None = None, -) -> tuple[str, tuple[str, ...]]: ... -def render_migration_event_write_sql( + stored_columns: Iterable[str] | None = None, +) -> tuple[Any, ...]: ... +def read_old_name_view_histories( + *, + connection: Any, + execute: AdapterExecute[Any, Any], + database: str | None, + schema: str, + stored_columns: Iterable[str], + render_qualified_name: Callable[..., str | None], +) -> tuple[OldNameViewHistory, ...]: ... +def read_recorded_column_renames( + *, + connection: Any, + execute: AdapterExecute[Any, Any], + relation: MigrationRelation, + since: datetime, + render_qualified_name: Callable[..., str | None], +) -> tuple[ColumnMigrationEvent, ...]: ... +def stored_old_name_view_columns( + *, adapter: BaseAdapter, connection: Any, database: str | None, schema: str +) -> tuple[str, ...]: ... +def write_migration_state_event( *, kind: Literal["model", "column", "old_name"], + connection: Any, + execute: AdapterExecute[Any, Any], event: MigrationEvent | ColumnMigrationEvent | OldNameViewEvent, render_qualified_name: Callable[..., str | None], -) -> tuple[str, str]: ... -@overload -def decode_migration_event_rows( - *, kind: Literal["model"], rows: Iterable[Sequence[object]], columns: tuple[str, ...] -) -> tuple[MigrationEvent, ...]: ... -@overload -def decode_migration_event_rows( - *, kind: Literal["column"], rows: Iterable[Sequence[object]], columns: tuple[str, ...] -) -> tuple[ColumnMigrationEvent, ...]: ... -@overload -def decode_migration_event_rows( - *, kind: Literal["old_name"], rows: Iterable[Sequence[object]], columns: tuple[str, ...] -) -> tuple[OldNameViewEvent, ...]: ... + create_table_sql: str | None, + attempts: int = 3, +) -> None: ... + +class MigrationStateInspection: + """Relations, fingerprints, and migration events for one planning database.""" + + def __init__( + self, + *, + adapter: BaseAdapter, + connection: Any, + database: str | None, + known_fingerprints: WarehouseFingerprints | None = None, + ) -> None: ... + @property + def database(self) -> str | None: ... + @property + def events(self) -> tuple[MigrationEvent, ...]: ... + @property + def column_events(self) -> tuple[ColumnMigrationEvent, ...]: ... + @property + def old_name_histories(self) -> tuple[OldNameViewHistory, ...]: ... + def inspect_column_events(self, *, schemas: set[str]) -> None: ... + def inspect_old_name_views(self, *, schemas: set[str]) -> None: ... + def inspect_migration_events(self, *, schemas: set[str]) -> None: ... + def inspect_schemas(self, *, schemas: set[str]) -> None: ... + def inspect_relations(self, *, locations: tuple[CompiledRelationLocation, ...]) -> None: ... + def relation(self, location: CompiledRelationLocation) -> RelationInfo | None: ... + def relation_columns(self, location: CompiledRelationLocation) -> tuple[ColumnInfo, ...]: ... + def model_fingerprint( + self, *, location: CompiledRelationLocation, model_name: str | None + ) -> Fingerprint | None: ... + def named_fingerprint(self, *, model_name: str) -> Fingerprint | None: ... + def orphan_fingerprints( + self, *, project_model_names: frozenset[str] + ) -> tuple[Fingerprint, ...]: ... + def newest_migration_event_mentioning( *, events: Iterable[MigrationEvent], relation: MigrationRelation ) -> MigrationEvent | None: ... diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 43980ea07a..9578d2fc0b 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -2295,11 +2295,10 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: """Render DDL that creates the model migration event table when it is missing.""" - from sqlbuild.compiler.migrations.main.create_table_sql import ( - build_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="model", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -2311,11 +2310,10 @@ def render_create_old_name_view_state_table_sql( ) -> str: """Render DDL that creates the old-name view fact table when it is missing.""" - from sqlbuild.compiler.migrations.main.old_name_view_create_table_sql import ( - build_old_name_view_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_old_name_view_state_create_table_sql( + return render_migration_state_create_sql( + kind="old_name", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -2327,11 +2325,10 @@ def render_create_column_migration_state_table_sql( ) -> str: """Render DDL that creates the column migration event table when it is missing.""" - from sqlbuild.compiler.migrations.main.column_create_table_sql import ( - build_column_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_column_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="column", database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 2f85f55996..cf6a389d47 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -909,11 +909,10 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: ) def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.migrations.main.create_table_sql import ( - build_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="model", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -923,11 +922,10 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem def render_create_old_name_view_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.old_name_view_create_table_sql import ( - build_old_name_view_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_old_name_view_state_create_table_sql( + return render_migration_state_create_sql( + kind="old_name", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -937,11 +935,10 @@ def render_create_old_name_view_state_table_sql( def render_create_column_migration_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.column_create_table_sql import ( - build_column_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_column_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="column", database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 67557b5a56..8184bbba30 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -363,11 +363,10 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: ) def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.migrations.main.create_table_sql import ( - build_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="model", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -377,11 +376,10 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem def render_create_old_name_view_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.old_name_view_create_table_sql import ( - build_old_name_view_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_old_name_view_state_create_table_sql( + return render_migration_state_create_sql( + kind="old_name", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -391,11 +389,10 @@ def render_create_old_name_view_state_table_sql( def render_create_column_migration_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.column_create_table_sql import ( - build_column_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_column_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="column", database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 7e295f05f7..478eaf5c33 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -459,11 +459,10 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: ) def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.migrations.main.create_table_sql import ( - build_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="model", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -473,11 +472,10 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem def render_create_old_name_view_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.old_name_view_create_table_sql import ( - build_old_name_view_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_old_name_view_state_create_table_sql( + return render_migration_state_create_sql( + kind="old_name", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -487,11 +485,10 @@ def render_create_old_name_view_state_table_sql( def render_create_column_migration_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.column_create_table_sql import ( - build_column_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_column_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="column", database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 035c742a9a..8eb92916f0 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -1728,11 +1728,10 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: ) def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.migrations.main.create_table_sql import ( - build_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="model", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -1742,11 +1741,10 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem def render_create_old_name_view_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.old_name_view_create_table_sql import ( - build_old_name_view_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_old_name_view_state_create_table_sql( + return render_migration_state_create_sql( + kind="old_name", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -1756,11 +1754,10 @@ def render_create_old_name_view_state_table_sql( def render_create_column_migration_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.column_create_table_sql import ( - build_column_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_column_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="column", database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 0c0324a0fe..826a015573 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -679,12 +679,11 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: ) def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.migrations.main.create_table_sql import ( - build_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql return self._with_state_table_retention( - create_sql=build_migration_state_create_table_sql( + create_sql=render_migration_state_create_sql( + kind="model", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -697,12 +696,11 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem def render_create_old_name_view_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.old_name_view_create_table_sql import ( - build_old_name_view_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql return self._with_state_table_retention( - create_sql=build_old_name_view_state_create_table_sql( + create_sql=render_migration_state_create_sql( + kind="old_name", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -715,12 +713,11 @@ def render_create_old_name_view_state_table_sql( def render_create_column_migration_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.column_create_table_sql import ( - build_column_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql return self._with_state_table_retention( - create_sql=build_column_migration_state_create_table_sql( + create_sql=render_migration_state_create_sql( + kind="column", database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 3c47f98943..2dcddfb1e7 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -657,12 +657,11 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: ) def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild._native import render_migration_state_create_sql from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME - from sqlbuild.compiler.migrations.main.create_table_sql import ( - build_migration_state_create_table_sql, - ) - create_sql: str = build_migration_state_create_table_sql( + create_sql: str = render_migration_state_create_sql( + kind="model", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -678,12 +677,11 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem def render_create_old_name_view_state_table_sql( self, *, database: str | None, schema: str ) -> str: + from sqlbuild._native import render_migration_state_create_sql from sqlbuild.compiler.migrations.constants import OLD_NAME_VIEW_TABLE_NAME - from sqlbuild.compiler.migrations.main.old_name_view_create_table_sql import ( - build_old_name_view_state_create_table_sql, - ) - create_sql: str = build_old_name_view_state_create_table_sql( + create_sql: str = render_migration_state_create_sql( + kind="old_name", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -699,12 +697,11 @@ def render_create_old_name_view_state_table_sql( def render_create_column_migration_state_table_sql( self, *, database: str | None, schema: str ) -> str: + from sqlbuild._native import render_migration_state_create_sql from sqlbuild.compiler.migrations.constants import COLUMN_MIGRATION_TABLE_NAME - from sqlbuild.compiler.migrations.main.column_create_table_sql import ( - build_column_migration_state_create_table_sql, - ) - create_sql: str = build_column_migration_state_create_table_sql( + create_sql: str = render_migration_state_create_sql( + kind="column", database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/compiler/migrations/_helpers/__init__.py b/src/sqlbuild/compiler/migrations/_helpers/__init__.py deleted file mode 100644 index 114aa7085f..0000000000 --- a/src/sqlbuild/compiler/migrations/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Model migration state helper package.""" diff --git a/src/sqlbuild/compiler/migrations/_helpers/writes.py b/src/sqlbuild/compiler/migrations/_helpers/writes.py deleted file mode 100644 index e01b7c8b72..0000000000 --- a/src/sqlbuild/compiler/migrations/_helpers/writes.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Idempotent append of one migration event row with bounded retries.""" - -from __future__ import annotations - -import logging -import time -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_RETRY_BASE_SECONDS - - -def append_event_row( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - existing_sql: str, - insert_sql: str, - create_table_sql: str | None, - attempts: int, - subject: str, -) -> None: - """Create the state table when given, then insert the row unless its event ID exists.""" - - attempt: int - for attempt in range(attempts): - try: - if create_table_sql is not None: - _ = execute(connection=connection, sql=create_table_sql) - if not execute(connection=connection, sql=existing_sql).fetchall(): - _ = execute(connection=connection, sql=insert_sql) - return - except Exception as error: - if attempt + 1 == attempts: - raise - logging.getLogger("sqlbuild.migrations").warning( - "migration event write attempt %s/%s failed for '%s'; retrying: %s", - attempt + 1, - attempts, - subject, - error, - ) - time.sleep(MIGRATION_WRITE_RETRY_BASE_SECONDS * (attempt + 1)) diff --git a/src/sqlbuild/compiler/migrations/constants.py b/src/sqlbuild/compiler/migrations/constants.py index 51b0f7a8b9..07187e2a3c 100644 --- a/src/sqlbuild/compiler/migrations/constants.py +++ b/src/sqlbuild/compiler/migrations/constants.py @@ -4,7 +4,6 @@ MIGRATION_TABLE_NAME: str = "_sqlbuild_migrations" MIGRATION_WRITE_ATTEMPTS: int = 3 -MIGRATION_WRITE_RETRY_BASE_SECONDS: float = 0.05 COLUMN_MIGRATION_TABLE_NAME: str = "_sqlbuild_column_migrations" diff --git a/src/sqlbuild/compiler/migrations/main/_read_column_events.py b/src/sqlbuild/compiler/migrations/main/_read_column_events.py deleted file mode 100644 index 64de56e0e7..0000000000 --- a/src/sqlbuild/compiler/migrations/main/_read_column_events.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Read append-only column migration events from one schema.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild._native import decode_migration_event_rows, render_migration_event_read_sql -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations.exceptions import MigrationStateError -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent - - -def read_column_migration_events( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], -) -> tuple[ColumnMigrationEvent, ...]: - """Read every column migration event this version understands from one schema.""" - - sql: str - columns: tuple[str, ...] - sql, columns = render_migration_event_read_sql( - kind="column", database=database, schema=schema, render_qualified_name=render_qualified_name - ) - try: - rows: list[tuple[Any, ...]] = execute(connection=connection, sql=sql).fetchall() - except Exception as error: - raise MigrationStateError( - f"Unable to read column migration events from schema '{schema}': {error}" - ) from error - return decode_migration_event_rows(kind="column", rows=rows, columns=columns) diff --git a/src/sqlbuild/compiler/migrations/main/_read_events.py b/src/sqlbuild/compiler/migrations/main/_read_events.py deleted file mode 100644 index d45f8b0aae..0000000000 --- a/src/sqlbuild/compiler/migrations/main/_read_events.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Read append-only model migration events from one schema.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild._native import decode_migration_event_rows, render_migration_event_read_sql -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations.exceptions import MigrationStateError -from sqlbuild.compiler.migrations.models import MigrationEvent - - -def read_migration_events( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], -) -> tuple[MigrationEvent, ...]: - """Read every migration event recorded in one schema's state table.""" - - sql: str - columns: tuple[str, ...] - sql, columns = render_migration_event_read_sql( - kind="model", database=database, schema=schema, render_qualified_name=render_qualified_name - ) - try: - rows: list[tuple[Any, ...]] = execute(connection=connection, sql=sql).fetchall() - except Exception as error: - raise MigrationStateError( - f"Unable to read model migration events from schema '{schema}': {error}" - ) from error - return decode_migration_event_rows(kind="model", rows=rows, columns=columns) diff --git a/src/sqlbuild/compiler/migrations/main/column_create_table_sql.py b/src/sqlbuild/compiler/migrations/main/column_create_table_sql.py deleted file mode 100644 index 465b227cfc..0000000000 --- a/src/sqlbuild/compiler/migrations/main/column_create_table_sql.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Column migration state table SQL entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild._native import render_migration_state_create_sql -from sqlbuild.adapter.contract.types import FrameworkType - - -def build_column_migration_state_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - """Build DDL that creates the column migration event table when it is missing.""" - - return render_migration_state_create_sql( - kind="column", - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - render_framework_type=render_framework_type, - ) diff --git a/src/sqlbuild/compiler/migrations/main/create_table_sql.py b/src/sqlbuild/compiler/migrations/main/create_table_sql.py deleted file mode 100644 index 502ead0b50..0000000000 --- a/src/sqlbuild/compiler/migrations/main/create_table_sql.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Model migration state table SQL entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild._native import render_migration_state_create_sql -from sqlbuild.adapter.contract.types import FrameworkType - - -def build_migration_state_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - """Build DDL that creates the model migration event table when it is missing.""" - - return render_migration_state_create_sql( - kind="model", - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - render_framework_type=render_framework_type, - ) diff --git a/src/sqlbuild/compiler/migrations/main/old_name_view_create_table_sql.py b/src/sqlbuild/compiler/migrations/main/old_name_view_create_table_sql.py deleted file mode 100644 index 3a82a9055c..0000000000 --- a/src/sqlbuild/compiler/migrations/main/old_name_view_create_table_sql.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Old-name view state table SQL entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild._native import render_migration_state_create_sql -from sqlbuild.adapter.contract.types import FrameworkType - - -def build_old_name_view_state_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - """Build DDL that creates the old-name view fact table when it is missing.""" - - return render_migration_state_create_sql( - kind="old_name", - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - render_framework_type=render_framework_type, - ) diff --git a/src/sqlbuild/compiler/migrations/main/read_old_name_view_events.py b/src/sqlbuild/compiler/migrations/main/read_old_name_view_events.py deleted file mode 100644 index ffdec54df3..0000000000 --- a/src/sqlbuild/compiler/migrations/main/read_old_name_view_events.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Read append-only old-name view facts from one schema.""" - -from __future__ import annotations - -from collections.abc import Callable, Iterable -from typing import Any - -from sqlbuild._native import decode_migration_event_rows, render_migration_event_read_sql -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations.exceptions import MigrationStateError -from sqlbuild.compiler.migrations.models import OldNameViewEvent - - -def read_old_name_view_events( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - database: str | None, - schema: str, - stored_columns: Iterable[str], - render_qualified_name: Callable[..., str | None], -) -> tuple[OldNameViewEvent, ...]: - """Read every old-name fact this version understands from the columns the table has.""" - - sql: str - columns: tuple[str, ...] - sql, columns = render_migration_event_read_sql( - kind="old_name", - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - stored_columns=tuple(stored_columns), - ) - try: - rows: list[tuple[Any, ...]] = execute(connection=connection, sql=sql).fetchall() - except Exception as error: - raise MigrationStateError( - f"Unable to read old-name view facts from schema '{schema}': {error}" - ) from error - return decode_migration_event_rows(kind="old_name", rows=rows, columns=columns) diff --git a/src/sqlbuild/compiler/migrations/main/read_old_name_view_histories.py b/src/sqlbuild/compiler/migrations/main/read_old_name_view_histories.py deleted file mode 100644 index 9de86cae2e..0000000000 --- a/src/sqlbuild/compiler/migrations/main/read_old_name_view_histories.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Read the old-name view history of every recorded move in one schema.""" - -from __future__ import annotations - -from collections.abc import Callable, Iterable -from typing import Any - -from sqlbuild._native import project_old_name_views -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations.main._read_events import read_migration_events -from sqlbuild.compiler.migrations.main.read_old_name_view_events import ( - read_old_name_view_events, -) -from sqlbuild.compiler.migrations.models import OldNameViewHistory - - -def read_old_name_view_histories( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - database: str | None, - schema: str, - stored_columns: Iterable[str], - render_qualified_name: Callable[..., str | None], -) -> tuple[OldNameViewHistory, ...]: - """Read moves and old-name facts from a schema that holds both state tables.""" - - return project_old_name_views( - moves=read_migration_events( - connection=connection, - execute=execute, - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - ), - facts=read_old_name_view_events( - connection=connection, - execute=execute, - database=database, - schema=schema, - stored_columns=stored_columns, - render_qualified_name=render_qualified_name, - ), - ) diff --git a/src/sqlbuild/compiler/migrations/main/read_recorded_column_renames.py b/src/sqlbuild/compiler/migrations/main/read_recorded_column_renames.py deleted file mode 100644 index 74f4de2165..0000000000 --- a/src/sqlbuild/compiler/migrations/main/read_recorded_column_renames.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Read the column renames recorded on one relation since a point in time.""" - -from __future__ import annotations - -from collections.abc import Callable -from datetime import datetime -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations.main._read_column_events import read_column_migration_events -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation - - -def read_recorded_column_renames( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - relation: MigrationRelation, - since: datetime, - render_qualified_name: Callable[..., str | None], -) -> tuple[ColumnMigrationEvent, ...]: - """Return renames recorded on the relation at or after ``since``, oldest first.""" - - return tuple( - sorted( - ( - event - for event in read_column_migration_events( - connection=connection, - execute=execute, - database=relation.database, - schema=relation.schema or "", - render_qualified_name=render_qualified_name, - ) - if event.relation.matches(relation) - and event.decision.records_event - and event.created_at >= since - ), - key=lambda event: (event.created_at, event.event_id), - ) - ) diff --git a/src/sqlbuild/compiler/migrations/main/stored_old_name_view_columns.py b/src/sqlbuild/compiler/migrations/main/stored_old_name_view_columns.py deleted file mode 100644 index 3fd7053a88..0000000000 --- a/src/sqlbuild/compiler/migrations/main/stored_old_name_view_columns.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Inspect which columns a stored old-name view state table has.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.migrations.constants import OLD_NAME_VIEW_TABLE_NAME - - -def stored_old_name_view_columns( - *, adapter: BaseAdapter, connection: Any, database: str | None, schema: str -) -> tuple[str, ...]: - """Return the column names of one schema's old-name view state table.""" - - return tuple( - column.name - for column in adapter.get_columns( - connection=connection, database=database, schema=schema, name=OLD_NAME_VIEW_TABLE_NAME - ) - ) diff --git a/src/sqlbuild/compiler/migrations/main/write_column_event.py b/src/sqlbuild/compiler/migrations/main/write_column_event.py deleted file mode 100644 index 6e50ddb55f..0000000000 --- a/src/sqlbuild/compiler/migrations/main/write_column_event.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Append one column migration event idempotently.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild._native import render_migration_event_write_sql -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations._helpers.writes import append_event_row -from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent - - -def write_column_migration_event( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - event: ColumnMigrationEvent, - render_qualified_name: Callable[..., str | None], - create_table_sql: str | None, - attempts: int = MIGRATION_WRITE_ATTEMPTS, -) -> None: - """Create the state table from adapter DDL when given, then insert the event if absent.""" - - existing_sql: str - insert_sql: str - existing_sql, insert_sql = render_migration_event_write_sql( - kind="column", event=event, render_qualified_name=render_qualified_name - ) - _ = append_event_row( - connection=connection, - execute=execute, - existing_sql=existing_sql, - insert_sql=insert_sql, - create_table_sql=create_table_sql, - attempts=attempts, - subject=f"{event.model_name}.{event.destination_column}", - ) diff --git a/src/sqlbuild/compiler/migrations/main/write_event.py b/src/sqlbuild/compiler/migrations/main/write_event.py deleted file mode 100644 index 1e005d1aa2..0000000000 --- a/src/sqlbuild/compiler/migrations/main/write_event.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Append one model migration event idempotently.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild._native import render_migration_event_write_sql -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations._helpers.writes import append_event_row -from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.models import MigrationEvent - - -def write_migration_event( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - event: MigrationEvent, - render_qualified_name: Callable[..., str | None], - create_table_sql: str | None, - attempts: int = MIGRATION_WRITE_ATTEMPTS, -) -> None: - """Create the state table from adapter DDL when given, then insert the event if absent.""" - - existing_sql: str - insert_sql: str - existing_sql, insert_sql = render_migration_event_write_sql( - kind="model", event=event, render_qualified_name=render_qualified_name - ) - _ = append_event_row( - connection=connection, - execute=execute, - existing_sql=existing_sql, - insert_sql=insert_sql, - create_table_sql=create_table_sql, - attempts=attempts, - subject=event.destination_model, - ) diff --git a/src/sqlbuild/compiler/migrations/main/write_old_name_view_event.py b/src/sqlbuild/compiler/migrations/main/write_old_name_view_event.py deleted file mode 100644 index 508c428b4d..0000000000 --- a/src/sqlbuild/compiler/migrations/main/write_old_name_view_event.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Append one old-name view fact idempotently.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild._native import render_migration_event_write_sql -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations._helpers.writes import append_event_row -from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.models import OldNameViewEvent - - -def write_old_name_view_event( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - event: OldNameViewEvent, - render_qualified_name: Callable[..., str | None], - create_table_sql: str | None, - attempts: int = MIGRATION_WRITE_ATTEMPTS, -) -> None: - """Create the state table from adapter DDL when given, then insert the fact if absent.""" - - existing_sql: str - insert_sql: str - existing_sql, insert_sql = render_migration_event_write_sql( - kind="old_name", event=event, render_qualified_name=render_qualified_name - ) - _ = append_event_row( - connection=connection, - execute=execute, - existing_sql=existing_sql, - insert_sql=insert_sql, - create_table_sql=create_table_sql, - attempts=attempts, - subject=f"{event.destination_model} old name {event.old.name}", - ) diff --git a/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py b/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py deleted file mode 100644 index 8a8aaefbeb..0000000000 --- a/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py +++ /dev/null @@ -1,357 +0,0 @@ -"""Read-only warehouse evidence gathered once for all model migrations in one plan.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild._native import project_old_name_views, read_latest_fingerprints -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo -from sqlbuild.adapter.relations.main.get_columns_for_inspection import get_columns_for_inspection -from sqlbuild.adapter.relations.main.list_relations_for_inspection import ( - list_relations_for_inspection, -) -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME, NODE_TYPE_MODEL -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet -from sqlbuild.compiler.migrations.constants import ( - COLUMN_MIGRATION_TABLE_NAME, - MIGRATION_TABLE_NAME, - OLD_NAME_VIEW_TABLE_NAME, -) -from sqlbuild.compiler.migrations.main._read_column_events import read_column_migration_events -from sqlbuild.compiler.migrations.main._read_events import read_migration_events -from sqlbuild.compiler.migrations.main.read_old_name_view_events import ( - read_old_name_view_events, -) -from sqlbuild.compiler.migrations.main.stored_old_name_view_columns import ( - stored_old_name_view_columns, -) -from sqlbuild.compiler.migrations.models import ( - ColumnMigrationEvent, - MigrationEvent, - MigrationRelation, - OldNameViewEvent, - OldNameViewHistory, -) -from sqlbuild.compiler.planner.models import WarehouseFingerprints - - -class MigrationStateInspection: - """Relations, fingerprints, and migration events for one planning database.""" - - def __init__( - self, - *, - adapter: BaseAdapter, - connection: Any, - database: str | None, - known_fingerprints: WarehouseFingerprints | None = None, - ) -> None: - self._adapter: BaseAdapter = adapter - self._connection: Any = connection - self._database: str | None = database - self._known_fingerprints: dict[str, tuple[Fingerprint, ...]] = ( - known_fingerprints.unfiltered_schemas - if known_fingerprints is not None and known_fingerprints.unfiltered_database == database - else {} - ) - self._read_schemas: set[str] = set() - self._fingerprints: dict[str, tuple[Fingerprint, ...]] = {} - self._events: list[MigrationEvent] = [] - self._event_schemas: set[str] = set() - self._relations: dict[tuple[str, str], RelationInfo] = {} - self._columns: dict[tuple[str, str], tuple[ColumnInfo, ...]] = {} - self._column_event_schemas: set[str] = set() - self._column_events: list[ColumnMigrationEvent] = [] - self._old_name_schemas: set[str] = set() - self._old_name_moves: list[MigrationEvent] = [] - self._old_name_facts: list[OldNameViewEvent] = [] - - @property - def database(self) -> str | None: - """Return the database whose namespaces this inspection reads.""" - - return self._database - - @property - def events(self) -> tuple[MigrationEvent, ...]: - """Return every migration event read so far.""" - - return tuple(self._events) - - @property - def column_events(self) -> tuple[ColumnMigrationEvent, ...]: - """Return every column migration event read so far.""" - - return tuple(self._column_events) - - def inspect_column_events(self, *, schemas: set[str]) -> None: - """Read the column migration state table once for every schema not inspected yet.""" - - pending: tuple[str, ...] = tuple( - sorted(schema for schema in schemas if schema.lower() not in self._column_event_schemas) - ) - if not pending: - return - self._column_event_schemas.update(schema.lower() for schema in pending) - listed: tuple[RelationInfo, ...] = list_relations_for_inspection( - adapter=self._adapter, - connection=self._connection, - database=self._database, - schemas=pending, - names=(COLUMN_MIGRATION_TABLE_NAME,), - ) - present: set[str] = { - (relation.schema or "").lower() - for relation in listed - if relation.name.lower() == COLUMN_MIGRATION_TABLE_NAME - } - schema: str - for schema in pending: - if schema.lower() in present: - self._column_events.extend( - read_column_migration_events( - connection=self._connection, - execute=self._adapter.execute, - database=self._database, - schema=schema, - render_qualified_name=self._adapter.render_qualified_name, - ) - ) - - @property - def old_name_histories(self) -> tuple[OldNameViewHistory, ...]: - """Return the old-name history of every move read by ``inspect_old_name_views``.""" - - return project_old_name_views(moves=self._old_name_moves, facts=self._old_name_facts) - - def inspect_old_name_views(self, *, schemas: set[str]) -> None: - """Read moves and old-name facts once for every schema not inspected yet.""" - - pending: tuple[str, ...] = tuple( - sorted(schema for schema in schemas if schema.lower() not in self._old_name_schemas) - ) - if not pending: - return - self._old_name_schemas.update(schema.lower() for schema in pending) - listed: tuple[RelationInfo, ...] = list_relations_for_inspection( - adapter=self._adapter, - connection=self._connection, - database=self._database, - schemas=pending, - names=(MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME), - ) - state_tables: set[tuple[str, str]] = { - ((relation.schema or "").lower(), relation.name.lower()) for relation in listed - } - schema: str - for schema in pending: - if (schema.lower(), OLD_NAME_VIEW_TABLE_NAME) not in state_tables or ( - schema.lower(), - MIGRATION_TABLE_NAME, - ) not in state_tables: - continue - self._old_name_facts.extend( - read_old_name_view_events( - connection=self._connection, - execute=self._adapter.execute, - database=self._database, - schema=schema, - stored_columns=stored_old_name_view_columns( - adapter=self._adapter, - connection=self._connection, - database=self._database, - schema=schema, - ), - render_qualified_name=self._adapter.render_qualified_name, - ) - ) - self._old_name_moves.extend( - read_migration_events( - connection=self._connection, - execute=self._adapter.execute, - database=self._database, - schema=schema, - render_qualified_name=self._adapter.render_qualified_name, - ) - ) - - def inspect_migration_events(self, *, schemas: set[str]) -> None: - """Read migration events once from schemas known to hold the migration table.""" - - schema: str - for schema in sorted(schemas): - if schema.lower() in self._event_schemas: - continue - self._event_schemas.add(schema.lower()) - self._events.extend( - read_migration_events( - connection=self._connection, - execute=self._adapter.execute, - database=self._database, - schema=schema, - render_qualified_name=self._adapter.render_qualified_name, - ) - ) - - def inspect_schemas(self, *, schemas: set[str]) -> None: - """Read state tables once for every schema not inspected yet.""" - - pending: tuple[str, ...] = tuple( - sorted(schema for schema in schemas if schema.lower() not in self._read_schemas) - ) - if not pending: - return - self._read_schemas.update(schema.lower() for schema in pending) - listed: tuple[RelationInfo, ...] = list_relations_for_inspection( - adapter=self._adapter, - connection=self._connection, - database=self._database, - schemas=pending, - names=(MIGRATION_TABLE_NAME, FINGERPRINT_TABLE_NAME), - ) - state_tables: set[tuple[str, str]] = { - ((relation.schema or "").lower(), relation.name.lower()) for relation in listed - } - schema: str - for schema in pending: - if (schema.lower(), MIGRATION_TABLE_NAME) in state_tables: - self.inspect_migration_events(schemas={schema}) - if (schema.lower(), FINGERPRINT_TABLE_NAME) in state_tables: - self._fingerprints[schema.lower()] = self._read_model_fingerprints(schema=schema) - - def _read_model_fingerprints(self, *, schema: str) -> tuple[Fingerprint, ...]: - latest: tuple[Fingerprint, ...] | None = self._known_fingerprints.get(schema.lower()) - if latest is None: - fingerprint_set: FingerprintSet = read_latest_fingerprints( - connection=self._connection, - execute=self._adapter.execute, - table_exists=True, - database=self._database, - schema=schema, - render_qualified_name=self._adapter.render_qualified_name, - render_read_latest_sql=self._adapter.render_read_latest_fingerprints_sql, - ) - latest = tuple((fingerprint_set.fingerprints_by_identity or {}).values()) - return tuple( - fingerprint for fingerprint in latest if fingerprint.node_type == NODE_TYPE_MODEL - ) - - def inspect_relations(self, *, locations: tuple[CompiledRelationLocation, ...]) -> None: - """List candidate relations and their columns with one metadata read.""" - - pending: tuple[CompiledRelationLocation, ...] = tuple( - location - for location in locations - if location.schema is not None and _relation_key(location) not in self._relations - ) - if not pending: - return - listed: tuple[RelationInfo, ...] = list_relations_for_inspection( - adapter=self._adapter, - connection=self._connection, - database=self._database, - schemas=tuple(sorted({location.schema or "" for location in pending})), - names=tuple(sorted({location.name for location in pending})), - ) - wanted: set[tuple[str, str]] = {_relation_key(location) for location in pending} - matched: tuple[RelationInfo, ...] = tuple( - relation for relation in listed if _listed_key(relation) in wanted - ) - columns: dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]] = ( - get_columns_for_inspection( - adapter=self._adapter, connection=self._connection, relations=matched - ) - if matched - else {} - ) - relation: RelationInfo - for relation in matched: - self._relations[_listed_key(relation)] = relation - self._columns[_listed_key(relation)] = columns.get(relation.identity, ()) - - def relation(self, location: CompiledRelationLocation) -> RelationInfo | None: - """Return the listed relation for a location.""" - - return self._relations.get(_relation_key(location)) - - def relation_columns(self, location: CompiledRelationLocation) -> tuple[ColumnInfo, ...]: - """Return listed columns for a location.""" - - return self._columns.get(_relation_key(location), ()) - - def model_fingerprint( - self, *, location: CompiledRelationLocation, model_name: str | None - ) -> Fingerprint | None: - """Return the latest model fingerprint that built the location or named model.""" - - candidates: tuple[Fingerprint, ...] = self._fingerprints.get( - (location.schema or "").lower(), () - ) - target: MigrationRelation = MigrationRelation( - database=location.database, schema=location.schema, name=location.name - ) - by_relation: tuple[Fingerprint, ...] = tuple( - fingerprint - for fingerprint in candidates - if _fingerprint_built(fingerprint=fingerprint, target=target) - ) - if by_relation: - return max(by_relation, key=_fingerprint_order) - if model_name is None: - return None - named: tuple[Fingerprint, ...] = tuple( - fingerprint for fingerprint in candidates if fingerprint.node_name == model_name - ) - return max(named, key=_fingerprint_order) if named else None - - def named_fingerprint(self, *, model_name: str) -> Fingerprint | None: - """Return the newest model fingerprint with this node name in any inspected schema.""" - - matches: list[Fingerprint] = [] - fingerprints: tuple[Fingerprint, ...] - for fingerprints in self._fingerprints.values(): - matches.extend( - fingerprint for fingerprint in fingerprints if fingerprint.node_name == model_name - ) - return max(matches, key=_fingerprint_order) if matches else None - - def orphan_fingerprints( - self, *, project_model_names: frozenset[str] - ) -> tuple[Fingerprint, ...]: - """Return latest model fingerprints whose names are no longer in the project.""" - - orphans: list[Fingerprint] = [] - fingerprints: tuple[Fingerprint, ...] - for fingerprints in self._fingerprints.values(): - orphans.extend( - fingerprint - for fingerprint in fingerprints - if fingerprint.node_name not in project_model_names - ) - return tuple(orphans) - - -def _relation_key(location: CompiledRelationLocation) -> tuple[str, str]: - return ((location.schema or "").lower(), location.name.lower()) - - -def _listed_key(relation: RelationInfo) -> tuple[str, str]: - return ((relation.schema or "").lower(), relation.name.lower()) - - -def _fingerprint_built(*, fingerprint: Fingerprint, target: MigrationRelation) -> bool: - if fingerprint.target_name is None: - return False - return target.matches( - MigrationRelation( - database=fingerprint.target_database, - schema=fingerprint.target_schema, - name=fingerprint.target_name, - ) - ) - - -def _fingerprint_order(fingerprint: Fingerprint) -> tuple[object, str]: - return (fingerprint.ts.isoformat(), fingerprint.run_id) diff --git a/src/sqlbuild/executor/migrations/_helpers/columns.py b/src/sqlbuild/executor/migrations/_helpers/columns.py index 31231a63b4..195558d9c2 100644 --- a/src/sqlbuild/executor/migrations/_helpers/columns.py +++ b/src/sqlbuild/executor/migrations/_helpers/columns.py @@ -5,13 +5,12 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import deterministic_column_migration_event_id +from sqlbuild._native import deterministic_column_migration_event_id, write_migration_state_event from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location -from sqlbuild.compiler.migrations.main.write_column_event import write_column_migration_event from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation from sqlbuild.compiler.migrations.types import ColumnMigrationDecision from sqlbuild.compiler.planner.models import ColumnMigrationPlanEntry @@ -90,7 +89,8 @@ def _rename_and_record( event: ColumnMigrationEvent = column_migration_event( entry=entry, decision=decision, run_id=run_id ) - write_column_migration_event( + write_migration_state_event( + kind="column", connection=connection, execute=adapter.execute, event=event, diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py b/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py index 2ddafcdb9f..1ad02e77bf 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py @@ -5,17 +5,15 @@ from datetime import datetime from typing import Any -from sqlbuild._native import deterministic_old_name_view_event_id +from sqlbuild._native import ( + deterministic_old_name_view_event_id, + read_migration_state_events, + stored_old_name_view_columns, + write_migration_state_event, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.main.read_old_name_view_events import ( - read_old_name_view_events, -) from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location -from sqlbuild.compiler.migrations.main.stored_old_name_view_columns import ( - stored_old_name_view_columns, -) -from sqlbuild.compiler.migrations.main.write_old_name_view_event import write_old_name_view_event from sqlbuild.compiler.migrations.models import MigrationRelation, OldNameViewEvent from sqlbuild.compiler.migrations.types import OldNameViewEventType from sqlbuild.compiler.planner.models import OldNameViewPlanEntry @@ -67,7 +65,8 @@ def record_old_name_fact( ) -> None: """Append one old-name fact through the adapter's state-table rendering.""" - write_old_name_view_event( + write_migration_state_event( + kind="old_name", connection=connection, execute=adapter.execute, event=event, @@ -98,7 +97,8 @@ def recorded_old_name_facts( ) facts: dict[OldNameViewEventType, OldNameViewEvent] = {} event: OldNameViewEvent - for event in read_old_name_view_events( + for event in read_migration_state_events( + kind="old_name", connection=connection, execute=adapter.execute, database=relation.database, diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py index 45656d83b5..e8495b0dd5 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py @@ -8,7 +8,7 @@ from datetime import UTC, datetime, timedelta from typing import Any -from sqlbuild._native import deterministic_migration_event_id +from sqlbuild._native import deterministic_migration_event_id, read_recorded_column_renames from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.main.grants_for_columns import grants_for_columns @@ -20,9 +20,6 @@ from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.migrations.constants import COLUMN_MIGRATION_TABLE_NAME -from sqlbuild.compiler.migrations.main.read_recorded_column_renames import ( - read_recorded_column_renames, -) from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location from sqlbuild.compiler.migrations.models import ( ColumnMigrationEvent, diff --git a/src/sqlbuild/executor/migrations/_helpers/promotion.py b/src/sqlbuild/executor/migrations/_helpers/promotion.py index e3cd213f9c..42d2f1be39 100644 --- a/src/sqlbuild/executor/migrations/_helpers/promotion.py +++ b/src/sqlbuild/executor/migrations/_helpers/promotion.py @@ -5,12 +5,11 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import deterministic_migration_event_id +from sqlbuild._native import deterministic_migration_event_id, write_migration_state_event from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location -from sqlbuild.compiler.migrations.main.write_event import write_migration_event from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation, OldNameViewEvent from sqlbuild.compiler.migrations.types import MigrationDecision, OldNameViewEventType from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, OldNameViewPlanEntry @@ -89,7 +88,8 @@ def record_event( ) -> None: """Append one migration event through the adapter's state-table rendering.""" - write_migration_event( + write_migration_state_event( + kind="model", connection=connection, execute=adapter.execute, event=event, diff --git a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py index e33c45671b..3abcd272a3 100644 --- a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py +++ b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py @@ -6,7 +6,12 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import deterministic_old_name_view_event_id +from sqlbuild._native import ( + deterministic_old_name_view_event_id, + read_old_name_view_histories, + stored_old_name_view_columns, + write_migration_state_event, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import RelationInfo @@ -20,13 +25,6 @@ MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, ) -from sqlbuild.compiler.migrations.main.read_old_name_view_histories import ( - read_old_name_view_histories, -) -from sqlbuild.compiler.migrations.main.stored_old_name_view_columns import ( - stored_old_name_view_columns, -) -from sqlbuild.compiler.migrations.main.write_old_name_view_event import write_old_name_view_event from sqlbuild.compiler.migrations.models import OldNameViewEvent, OldNameViewHistory from sqlbuild.compiler.migrations.types import ( OldNameViewDropReason, @@ -165,7 +163,8 @@ def _record_drop( *, adapter: BaseAdapter, connection: Any, view: JanitorOldNameView, run_id: str ) -> None: required: OldNameViewEvent = view.history.required - write_old_name_view_event( + write_migration_state_event( + kind="old_name", connection=connection, execute=adapter.execute, event=OldNameViewEvent( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py index 47bee99870..7500593196 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py @@ -378,6 +378,6 @@ def fail_non_transactional_record(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(DuckDbAdapter, "supports_transactional_ddl", lambda self: False) monkeypatch.setattr( - "sqlbuild.executor.migrations._helpers.columns.write_column_migration_event", + "sqlbuild.executor.migrations._helpers.columns.write_migration_state_event", _raise_interruption, ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 4c8eb6891d..2286abfaee 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -513,7 +513,7 @@ def fail_record(monkeypatch: pytest.MonkeyPatch) -> None: """Crash after the clone but before the migration event is recorded.""" monkeypatch.setattr( - "sqlbuild.executor.migrations._helpers.promotion.write_migration_event", + "sqlbuild.executor.migrations._helpers.promotion.write_migration_state_event", _raise_interruption, ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py index 5b95c8f51f..0b236fa5de 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py @@ -197,7 +197,7 @@ def fail_janitor_drop_fact(monkeypatch: pytest.MonkeyPatch) -> None: """Crash after the janitor drops a view but before the drop is recorded.""" monkeypatch.setattr( - "sqlbuild.executor.old_name_views._helpers.janitor.write_old_name_view_event", + "sqlbuild.executor.old_name_views._helpers.janitor.write_migration_state_event", _raise_interruption, ) diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py index 57fb768f0a..fe0ec8fa83 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -117,6 +117,7 @@ def test_given_native_source_when_checking_module_paths_then_reports_errors( "sqlbuild.executor.runtime", "sqlbuild.fingerprints", "sqlbuild.lineage", + "sqlbuild.migrations", "sqlbuild.planner", "sqlbuild.planner.changes", ), diff --git a/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py b/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py index 302b428eac..f9fda0f342 100644 --- a/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py +++ b/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py @@ -6,9 +6,8 @@ import pytest +from sqlbuild._native import read_migration_state_events, write_migration_state_event from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.migrations.main._read_events import read_migration_events -from sqlbuild.compiler.migrations.main.write_event import write_migration_event from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery from tests.unit.src.sqlbuild.adapters.sqlserver._test_types import ( @@ -92,7 +91,8 @@ def test_given_sqlserver_when_writing_migration_event_then_statements_are_tsql( rows=test_case.existing_rows ) - write_migration_event( + write_migration_state_event( + kind="model", connection=connection, execute=adapter.execute, event=_EVENT, @@ -151,7 +151,8 @@ def test_given_sqlserver_datetime2_rows_when_reading_events_then_decodes_utc_eve ) ) - events: tuple[MigrationEvent, ...] = read_migration_events( + events: tuple[MigrationEvent, ...] = read_migration_state_events( + kind="model", connection=connection, execute=adapter.execute, database=None, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py index a3f62a72fa..916b3c6873 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py @@ -5,10 +5,12 @@ import duckdb import pytest -from sqlbuild._native import newest_column_migration_event_mentioning +from sqlbuild._native import ( + newest_column_migration_event_mentioning, + read_migration_state_events, + write_migration_state_event, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.main._read_column_events import read_column_migration_events -from sqlbuild.compiler.migrations.main.write_column_event import write_column_migration_event from sqlbuild.compiler.migrations.models import ColumnMigrationEvent from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( ColumnEventStorageTestCase, @@ -48,7 +50,8 @@ def test_given_column_events_when_writing_and_reading_then_rows_are_idempotent( ) for _ in range(test_case.write_count): - write_column_migration_event( + write_migration_state_event( + kind="column", connection=connection, execute=adapter.execute, event=event, @@ -64,7 +67,8 @@ def test_given_column_events_when_writing_and_reading_then_rows_are_idempotent( f"('future', 'fct_orders', 'fct_orders', 'tax', 'levy', 'manual', " f"'{foreign_decision}', 'run-9', TIMESTAMP '2026-01-09')" ) - stored: tuple[ColumnMigrationEvent, ...] = read_column_migration_events( + stored: tuple[ColumnMigrationEvent, ...] = read_migration_state_events( + kind="column", connection=connection, execute=adapter.execute, database=None, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py index 3eff4da42f..7c1e77718d 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py @@ -5,10 +5,12 @@ import duckdb import pytest -from sqlbuild._native import newest_migration_event_mentioning +from sqlbuild._native import ( + newest_migration_event_mentioning, + read_migration_state_events, + write_migration_state_event, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.main._read_events import read_migration_events -from sqlbuild.compiler.migrations.main.write_event import write_migration_event from sqlbuild.compiler.migrations.models import MigrationEvent from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( NewestEventTestCase, @@ -87,7 +89,8 @@ def test_given_repeated_event_write_when_reading_then_event_is_stored_once( ) for _ in range(test_case.write_count): - write_migration_event( + write_migration_state_event( + kind="model", connection=connection, execute=adapter.execute, event=event, @@ -96,7 +99,8 @@ def test_given_repeated_event_write_when_reading_then_event_is_stored_once( database=None, schema="main" ), ) - stored: tuple[MigrationEvent, ...] = read_migration_events( + stored: tuple[MigrationEvent, ...] = read_migration_state_events( + kind="model", connection=connection, execute=adapter.execute, database=None, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py index bb8e0bb2ec..eb25c85b74 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py @@ -7,25 +7,20 @@ import duckdb import pytest -from sqlbuild._native import project_old_name_views -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.main.read_old_name_view_events import ( - read_old_name_view_events, -) -from sqlbuild.compiler.migrations.main.stored_old_name_view_columns import ( +from sqlbuild._native import ( + MigrationStateInspection, + project_old_name_views, + read_migration_state_events, stored_old_name_view_columns, + write_migration_state_event, ) -from sqlbuild.compiler.migrations.main.write_event import write_migration_event -from sqlbuild.compiler.migrations.main.write_old_name_view_event import write_old_name_view_event +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.migrations.models import ( MigrationEvent, OldNameViewEvent, OldNameViewHistory, ) from sqlbuild.compiler.migrations.types import OldNameViewEventType -from sqlbuild.compiler.planner.classes.migration_state_inspection import ( - MigrationStateInspection, -) from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( OldNameViewAbsentColumnsTestCase, OldNameViewProjectionTestCase, @@ -71,7 +66,8 @@ def test_given_old_name_facts_when_writing_and_reading_then_rows_are_idempotent( ) for _ in range(test_case.write_count): - write_old_name_view_event( + write_migration_state_event( + kind="old_name", connection=connection, execute=adapter.execute, event=fact, @@ -87,7 +83,8 @@ def test_given_old_name_facts_when_writing_and_reading_then_rows_are_idempotent( f"VALUES ('future', '{foreign}', 'm', 'daily_revenue', 'revenue', 'daily_revenue', " "'run-9', TIMESTAMP '2026-01-09')" ) - stored: tuple[OldNameViewEvent, ...] = read_old_name_view_events( + stored: tuple[OldNameViewEvent, ...] = read_migration_state_events( + kind="old_name", connection=connection, execute=adapter.execute, database=None, @@ -203,7 +200,8 @@ def test_given_state_table_without_optional_columns_when_planning_then_absent_co adapter: DuckDbAdapter = DuckDbAdapter() connection: duckdb.DuckDBPyConnection = duckdb.connect(":memory:") move: MigrationEvent = migration_event(origin="revenue", destination="daily_revenue", day=1) - write_migration_event( + write_migration_state_event( + kind="model", connection=connection, execute=adapter.execute, event=move, @@ -218,7 +216,8 @@ def test_given_state_table_without_optional_columns_when_planning_then_absent_co OldNameViewEventType.ORIGIN_ARCHIVED, OldNameViewEventType.VIEW_CREATED, ): - write_old_name_view_event( + write_migration_state_event( + kind="old_name", connection=connection, execute=adapter.execute, event=old_name_fact( diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py index 5f76ea0717..0b18656d46 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py @@ -7,14 +7,8 @@ import duckdb import pytest +from sqlbuild._native import read_migration_state_events, stored_old_name_view_columns from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.main._read_column_events import read_column_migration_events -from sqlbuild.compiler.migrations.main.read_old_name_view_events import ( - read_old_name_view_events, -) -from sqlbuild.compiler.migrations.main.stored_old_name_view_columns import ( - stored_old_name_view_columns, -) from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, OldNameViewEvent from sqlbuild.compiler.migrations.types import OldNameViewDropReason from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( @@ -73,7 +67,8 @@ def test_given_persisted_old_name_row_when_reading_then_codecs_decode_stored_tex f"{test_case.drop_reason_sql}, 'run-1', '2026-01-02T03:04:05+00:00')" ) - stored: tuple[OldNameViewEvent, ...] = read_old_name_view_events( + stored: tuple[OldNameViewEvent, ...] = read_migration_state_events( + kind="old_name", connection=connection, execute=adapter.execute, database=None, @@ -120,7 +115,8 @@ def test_given_persisted_column_rows_when_reading_then_unknown_rows_are_skipped_ f"'total', 'manual', '{decision}', 'run-{index}', '2026-01-0{index + 1}')" ) - stored: tuple[ColumnMigrationEvent, ...] = read_column_migration_events( + stored: tuple[ColumnMigrationEvent, ...] = read_migration_state_events( + kind="column", connection=connection, execute=adapter.execute, database=None, diff --git a/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py b/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py index f1bdb26f8f..5df4303783 100644 --- a/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py +++ b/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py @@ -5,8 +5,8 @@ import duckdb import pytest +from sqlbuild._native import read_migration_state_events from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.main._read_column_events import read_column_migration_events from sqlbuild.compiler.migrations.models import ColumnMigrationEvent from sqlbuild.executor.migrations._helpers.columns import apply_model_column_renames from tests.unit.src.sqlbuild.executor.migrations._helpers._test_types import ( @@ -57,7 +57,8 @@ def test_given_overlapping_build_between_renames_when_applying_then_reconciles_f ), run_id="run-1", ) - events: tuple[ColumnMigrationEvent, ...] = read_column_migration_events( + events: tuple[ColumnMigrationEvent, ...] = read_migration_state_events( + kind="column", connection=connection, execute=adapter.execute, database=None, From 5517cf7cf51c14743197860cec9b1440c02fd60c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:08:34 +0100 Subject: [PATCH 19/22] refactor: assemble model plan entries natively Move model plan entry assembly into native code: cursor input relations, cursor input column validation, runtime cursor producers, microbatch lookback defaults, seed column extraction, adapter-rendered logical DDL and the ModelPlanEntry construction. Python keeps cursor bound, hook and SQL resolution and calls the native assembly. --- .../src/planner_strategy/constants.rs | 10 + .../main/cursor_input_column.rs | 44 + .../main/microbatch_lookback.rs | 25 + .../src/planner_strategy/main/mod.rs | 3 + .../planner_strategy/main/table_type_ddl.rs | 13 + .../src/planner_strategy/models.rs | 27 + .../src/planner_strategy/tests.rs | 2 + .../src/planner_strategy/tests/helpers.rs | 29 +- .../tests/test_model_entry.rs | 130 +++ .../src/planner_strategy/tests/test_types.rs | 26 +- .../_helpers/model_config/config_values.rs | 14 +- .../_helpers/planner_identity/metadata.rs | 5 +- .../_helpers/planner_scenario/entries.rs | 2 +- .../_helpers/planner_strategy/constants.rs | 12 + .../planner_strategy/cursor_inputs.rs | 314 +++++++ .../_helpers/planner_strategy/lineage.rs | 2 +- .../bindings/_helpers/planner_strategy/mod.rs | 2 + .../planner_strategy/model_entries.rs | 560 +++++++++++++ .../_helpers/planner_strategy/registration.rs | 24 + src/sqlbuild/_native.pyi | 58 ++ .../_helpers/command_planning/static.py | 6 +- .../planner/_helpers/output/plan_entry.py | 773 ++---------------- .../planner/_helpers/output/plan_output.py | 2 +- .../_helpers/test_microbatch_lookback.py | 2 +- .../planner/_helpers/test_source_columns.py | 18 +- 25 files changed, 1352 insertions(+), 751 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/cursor_input_column.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/microbatch_lookback.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/table_type_ddl.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/tests/test_model_entry.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs diff --git a/crates/sqlbuild-analysis/src/planner_strategy/constants.rs b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs index 9029fe4ae4..2eabec0ab1 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs @@ -76,3 +76,13 @@ pub const DECIMAL_TYPE_NAMES: [&str; 3] = ["DECIMAL", "NUMERIC", "NUMBER"]; pub const POLYGLOT_CUSTOM_TYPE_NAME: &str = "CUSTOM"; /// The dialect cursor type classification parses unknown types with. pub const GENERIC_DIALECT: &str = "generic"; +/// Error code of an authored cursor input column its input does not expose. +pub const CURSOR_INPUT_COLUMN_ERROR_CODE: &str = "S302"; +/// Incremental strategies whose microbatches replay one batch by default. +pub const IDEMPOTENT_MICROBATCH_STRATEGIES: [&str; 2] = ["delete_insert", "merge"]; +/// The table type that drops the adapter's transient DDL keyword. +pub const PERMANENT_TABLE_TYPE: &str = "permanent"; +/// The transient table DDL a permanent table rewrites once. +pub const TRANSIENT_TABLE_DDL: &str = "TRANSIENT TABLE"; +/// The permanent table DDL replacing the transient keyword. +pub const PERMANENT_TABLE_DDL: &str = "TABLE"; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_input_column.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_input_column.rs new file mode 100644 index 0000000000..d64b814a1e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_input_column.rs @@ -0,0 +1,44 @@ +//! Whether an authored cursor input column is exposed by the input it names. + +use crate::planner_strategy::constants::CURSOR_INPUT_COLUMN_ERROR_CODE; +use crate::planner_strategy::models::{CursorInputColumnCheck, CursorInputColumnOwner}; + +/// The planner error message and code when the column is missing from the known columns, matched case-insensitively. +pub fn cursor_input_column_failure( + check: &CursorInputColumnCheck, +) -> Option<(String, &'static str)> { + let column: String = check.column.to_lowercase(); + if check + .known_columns + .iter() + .any(|known| known.to_lowercase() == column) + { + return None; + } + let (kind, exposer, label) = match check.owner { + CursorInputColumnOwner::ModelContract => { + ("model", "that model contract", "Known output columns") + } + CursorInputColumnOwner::SourceContract => ( + "source", + "that source contract", + "Declared contract columns", + ), + CursorInputColumnOwner::SourceWarehouse => { + ("source", "that source", "Known source columns") + } + }; + let known: String = if check.known_columns.is_empty() { + "none".to_owned() + } else { + check.known_columns.join(", ") + }; + Some(( + format!( + "model '{}': cursor_inputs references {kind} '{}' column '{}', but {exposer} does \ + not expose the column. {label}: {known}", + check.model_name, check.input_name, check.column + ), + CURSOR_INPUT_COLUMN_ERROR_CODE, + )) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/microbatch_lookback.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/microbatch_lookback.rs new file mode 100644 index 0000000000..15f2eb1d22 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/microbatch_lookback.rs @@ -0,0 +1,25 @@ +//! The explicit or default lookback of a microbatch model. + +use sqlbuild_core::plan_display::errors::UnknownCursorGrain; +use sqlbuild_core::plan_display::main::effective_microbatch_batch_size::effective_microbatch_batch_size; + +use crate::planner_strategy::constants::IDEMPOTENT_MICROBATCH_STRATEGIES; +use crate::planner_strategy::models::LookbackConfig; + +/// The authored lookback, else one batch at the replay grain for idempotent strategies. +pub fn microbatch_lookback(config: &LookbackConfig) -> Result, UnknownCursorGrain> { + if let Some(lookback) = &config.lookback { + return Ok(Some(lookback.clone())); + } + let idempotent: bool = config + .incremental_strategy + .as_deref() + .is_some_and(|strategy| IDEMPOTENT_MICROBATCH_STRATEGIES.contains(&strategy)); + let (true, Some(batch_size)) = (idempotent, &config.batch_size) else { + return Ok(None); + }; + match &config.grain { + Some(grain) => effective_microbatch_batch_size(batch_size, grain).map(Some), + None => Ok(Some(batch_size.clone())), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs index 533eb51d7b..b7532df463 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs @@ -4,12 +4,15 @@ pub mod capped_watermark_input; pub mod classify_cursor_sql_type; pub mod count_integer_batches; pub mod count_timeline_batches; +pub mod cursor_input_column; pub mod cursor_type_warning; pub mod effective_run_scope; pub mod lineage_reference; pub mod materialization_type; pub mod microbatch_limit_warning; +pub mod microbatch_lookback; pub mod model_plan_action; pub mod model_warnings; pub mod schema_actions; pub mod singular_attachment; +pub mod table_type_ddl; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/table_type_ddl.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/table_type_ddl.rs new file mode 100644 index 0000000000..ee63f64cca --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/table_type_ddl.rs @@ -0,0 +1,13 @@ +//! Logical table DDL for the model's table type. + +use crate::planner_strategy::constants::{ + PERMANENT_TABLE_DDL, PERMANENT_TABLE_TYPE, TRANSIENT_TABLE_DDL, +}; + +/// The adapter's DDL, with its first transient table keyword dropped for permanent tables. +pub fn table_type_ddl(ddl: &str, table_type: &str) -> String { + if table_type == PERMANENT_TABLE_TYPE { + return ddl.replacen(TRANSIENT_TABLE_DDL, PERMANENT_TABLE_DDL, 1); + } + ddl.to_owned() +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/models.rs b/crates/sqlbuild-analysis/src/planner_strategy/models.rs index 5553b77357..959f26c038 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/models.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/models.rs @@ -178,3 +178,30 @@ pub enum CappedBound { End(T), Start(T), } + +/// Which declaration a cursor input column is checked against. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CursorInputColumnOwner { + ModelContract, + SourceContract, + SourceWarehouse, +} + +/// One authored cursor input column and the columns its input is known to expose. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CursorInputColumnCheck { + pub model_name: String, + pub owner: CursorInputColumnOwner, + pub input_name: String, + pub column: String, + pub known_columns: Vec, +} + +/// The configuration a microbatch model's default lookback is resolved from. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LookbackConfig { + pub lookback: Option, + pub incremental_strategy: Option, + pub batch_size: Option, + pub grain: Option, +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests.rs index 30e1350d0c..9cc302c56a 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/tests.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_model_entry.rs"] +mod model_entry; #[path = "tests/test_planner_strategy.rs"] mod planner_strategy; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/helpers.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/helpers.rs index 6d2c4b6c41..1277e868ca 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/helpers.rs @@ -3,7 +3,8 @@ use std::collections::BTreeMap; use crate::planner_changes::models::{SchemaChangeKind, SchemaColumnSource, SchemaFindingRow}; use crate::planner_strategy::errors::LineageFailure; use crate::planner_strategy::models::{ - ChangeFacts, PlanChangeKind, ProducerFacts, ReferenceIdentity, WarningRow, WarningSeverity, + ChangeFacts, CursorInputColumnCheck, CursorInputColumnOwner, LookbackConfig, PlanChangeKind, + ProducerFacts, ReferenceIdentity, WarningRow, WarningSeverity, }; use crate::planner_strategy::tests::test_types::{Lineage, Steps}; use crate::planner_strategy::types::{ReferenceGraph, Timeline}; @@ -113,3 +114,29 @@ impl ReferenceGraph for Lineage { pub(super) fn failure_text(failure: LineageFailure<()>) -> String { format!("{failure:?}") } + +pub(super) fn column_check( + owner: CursorInputColumnOwner, + known: &[&str], +) -> CursorInputColumnCheck { + CursorInputColumnCheck { + model_name: "daily_orders".to_owned(), + owner, + input_name: "orders".to_owned(), + column: "Updated_At".to_owned(), + known_columns: known.iter().map(|name| (*name).to_owned()).collect(), + } +} + +pub(super) fn lookback_config( + lookback: Option<&str>, + strategy: Option<&str>, + grain: Option<&str>, +) -> LookbackConfig { + LookbackConfig { + lookback: lookback.map(str::to_owned), + incremental_strategy: strategy.map(str::to_owned), + batch_size: Some("effective".to_owned()), + grain: grain.map(str::to_owned), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_model_entry.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_model_entry.rs new file mode 100644 index 0000000000..9e9e202fe6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_model_entry.rs @@ -0,0 +1,130 @@ +use sqlbuild_core::plan_display::errors::UnknownCursorGrain; + +use crate::planner_strategy::main::cursor_input_column::cursor_input_column_failure; +use crate::planner_strategy::main::microbatch_lookback::microbatch_lookback; +use crate::planner_strategy::main::table_type_ddl::table_type_ddl; +use crate::planner_strategy::models::CursorInputColumnOwner; +use crate::planner_strategy::tests::helpers::{column_check, lookback_config}; +use crate::planner_strategy::tests::test_types::{ + CursorInputColumnTestCase, LookbackTestCase, TableTypeDdlTestCase, +}; + +#[test] +fn given_cursor_input_column_when_checking_exposure_then_reports_missing_columns() { + let test_cases = [ + CursorInputColumnTestCase { + description: "case-insensitive model contract match", + check: column_check(CursorInputColumnOwner::ModelContract, &["updated_at"]), + expected_failure: None, + }, + CursorInputColumnTestCase { + description: "model contract without the column", + check: column_check(CursorInputColumnOwner::ModelContract, &["order_id"]), + expected_failure: Some(( + "model 'daily_orders': cursor_inputs references model 'orders' column \ + 'Updated_At', but that model contract does not expose the column. Known output \ + columns: order_id" + .to_owned(), + "S302", + )), + }, + CursorInputColumnTestCase { + description: "source contract without columns", + check: column_check(CursorInputColumnOwner::SourceContract, &[]), + expected_failure: Some(( + "model 'daily_orders': cursor_inputs references source 'orders' column \ + 'Updated_At', but that source contract does not expose the column. Declared \ + contract columns: none" + .to_owned(), + "S302", + )), + }, + CursorInputColumnTestCase { + description: "warehouse source without the column", + check: column_check( + CursorInputColumnOwner::SourceWarehouse, + &["order_id", "status"], + ), + expected_failure: Some(( + "model 'daily_orders': cursor_inputs references source 'orders' column \ + 'Updated_At', but that source does not expose the column. Known source \ + columns: order_id, status" + .to_owned(), + "S302", + )), + }, + ]; + for test_case in test_cases { + assert_eq!( + cursor_input_column_failure(&test_case.check), + test_case.expected_failure, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_microbatch_config_when_resolving_lookback_then_matches_python_default() { + let test_cases = [ + LookbackTestCase { + description: "explicit lookback wins", + config: lookback_config(Some("2 days"), Some("append"), None), + expected_lookback: Ok(Some("2 days".to_owned())), + }, + LookbackTestCase { + description: "append has no default", + config: lookback_config(None, Some("append"), Some("day")), + expected_lookback: Ok(None), + }, + LookbackTestCase { + description: "merge without grain keeps the batch size", + config: lookback_config(None, Some("merge"), None), + expected_lookback: Ok(Some("effective".to_owned())), + }, + LookbackTestCase { + description: "delete insert resolves the effective grain", + config: lookback_config(None, Some("delete_insert"), Some("hour")), + expected_lookback: Ok(Some("1h".to_owned())), + }, + LookbackTestCase { + description: "unknown grain", + config: lookback_config(None, Some("merge"), Some("fortnight")), + expected_lookback: Err(UnknownCursorGrain("fortnight".to_owned())), + }, + ]; + for test_case in test_cases { + assert_eq!( + microbatch_lookback(&test_case.config), + test_case.expected_lookback, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_table_type_when_rendering_ddl_then_rewrites_only_permanent_tables() { + let test_cases = [ + TableTypeDdlTestCase { + description: "permanent rewrites the first keyword", + ddl: "CREATE TRANSIENT TABLE a AS SELECT 1; CREATE TRANSIENT TABLE b", + table_type: "permanent", + expected_ddl: "CREATE TABLE a AS SELECT 1; CREATE TRANSIENT TABLE b", + }, + TableTypeDdlTestCase { + description: "transient keeps the adapter DDL", + ddl: "CREATE TRANSIENT TABLE a AS SELECT 1", + table_type: "transient", + expected_ddl: "CREATE TRANSIENT TABLE a AS SELECT 1", + }, + ]; + for test_case in test_cases { + assert_eq!( + table_type_ddl(test_case.ddl, test_case.table_type), + test_case.expected_ddl, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs index e28e6bc03a..325815169d 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs @@ -1,8 +1,11 @@ use std::collections::BTreeSet; +use sqlbuild_core::plan_display::errors::UnknownCursorGrain; + use crate::planner_strategy::models::{ - AuditAttachmentKind, CapDirection, CappedBound, ChangeFacts, CursorType, MaterializationType, - PlanAction, PlanReason, SchemaActionRow, WarningRow, + AuditAttachmentKind, CapDirection, CappedBound, ChangeFacts, CursorInputColumnCheck, + CursorType, LookbackConfig, MaterializationType, PlanAction, PlanReason, SchemaActionRow, + WarningRow, }; pub(super) struct ActionTestCase { @@ -81,3 +84,22 @@ pub(super) struct Steps; pub(super) struct Lineage { pub(super) capped: BTreeSet, } + +pub(super) struct CursorInputColumnTestCase { + pub(super) description: &'static str, + pub(super) check: CursorInputColumnCheck, + pub(super) expected_failure: Option<(String, &'static str)>, +} + +pub(super) struct LookbackTestCase { + pub(super) description: &'static str, + pub(super) config: LookbackConfig, + pub(super) expected_lookback: Result, UnknownCursorGrain>, +} + +pub(super) struct TableTypeDdlTestCase { + pub(super) description: &'static str, + pub(super) ddl: &'static str, + pub(super) table_type: &'static str, + pub(super) expected_ddl: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs index 26f364aa21..46cab3f137 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs @@ -54,7 +54,10 @@ pub(crate) fn config_str<'py>( /// An authored boolean, or `None` when the key is absent or `None`. #[pyfunction(name = "get_config_bool")] #[pyo3(signature = (*, values, key))] -fn config_bool<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult>> { +pub(crate) fn config_bool<'py>( + values: &Bound<'py, PyAny>, + key: &str, +) -> PyResult>> { let Some(value) = authored(values, key)? else { return Ok(None); }; @@ -67,7 +70,7 @@ fn config_bool<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult( +pub(crate) fn config_string_tuple<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult>> { @@ -87,7 +90,10 @@ fn config_string_tuple<'py>( /// An authored integer, excluding booleans, or `None` when absent or `None`. #[pyfunction(name = "get_config_int")] #[pyo3(signature = (*, values, key))] -fn config_int<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult>> { +pub(crate) fn config_int<'py>( + values: &Bound<'py, PyAny>, + key: &str, +) -> PyResult>> { let Some(value) = authored(values, key)? else { return Ok(None); }; @@ -100,7 +106,7 @@ fn config_int<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult( +pub(crate) fn config_cursor_bound<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs index 15dcb30116..5ac2e6cc45 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs @@ -45,7 +45,10 @@ fn version_identity_config<'py>(config_values: &Bound<'py, PyAny>) -> PyResult(value: Option>, py: Python<'py>) -> PyResult> { +pub(crate) fn or_empty<'py>( + value: Option>, + py: Python<'py>, +) -> PyResult> { match value { Some(value) if value.is_truthy()? => Ok(value), _ => Ok(PyDict::new(py).into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs index 69ad9e77e2..b50be17c04 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs @@ -56,7 +56,7 @@ pub(super) fn seed_entries<'py>( "columns", imported( py, - "sqlbuild.compiler.planner._helpers.output.plan_entry", + "sqlbuild._native", "extract_seed_columns", )? .call1((&seed,))?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs index 350d9284d5..d46eb709b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs @@ -7,3 +7,15 @@ pub(crate) const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models pub(crate) const PLAN_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; pub(crate) const MODEL_ATTACHMENT: &str = "model"; pub(crate) const SOURCE_ATTACHMENT: &str = "source"; +pub(crate) const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +pub(crate) const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; +pub(crate) const CURSOR_TYPES_MODULE: &str = "sqlbuild.cursor_algebra.types"; +pub(crate) const INCREMENTAL_MATERIALIZATION: &str = "incremental"; +pub(crate) const MICROBATCH_MODE: &str = "microbatch"; +pub(crate) const CUSTOM_MATERIALIZATION: &str = "custom"; +pub(crate) const ENFORCED_CONTRACT: &str = "enforced"; +pub(crate) const TIMESTAMP_CURSOR: &str = "timestamp"; +pub(crate) const MODEL_REFERENCE: &str = "ref"; +pub(crate) const SEED_REFERENCE: &str = "seed"; +pub(crate) const SOURCE_REFERENCE: &str = "source"; +pub(crate) const DDL_SEPARATOR: &str = ";\n\n"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs new file mode 100644 index 0000000000..2a266b95c7 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs @@ -0,0 +1,314 @@ +//! Cursor input relations and authored cursor input column checks of model plan entries. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyBool, PyDict, PyFrozenSet, PyString, PyTuple}; + +use sqlbuild_analysis::planner_strategy::main::cursor_input_column::cursor_input_column_failure; +use sqlbuild_analysis::planner_strategy::models::{CursorInputColumnCheck, CursorInputColumnOwner}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cursor::scalars::parse_value; +use crate::bindings::_helpers::model_config::config_values::{config_cursor_bound, config_str}; +use crate::bindings::_helpers::planner_identity::metadata::or_empty; +use crate::bindings::_helpers::planner_strategy::constants::{ + ENFORCED_CONTRACT, INCREMENTAL_MATERIALIZATION, MODEL_REFERENCE, PLAN_MODELS_MODULE, + PLANNER_TYPES_MODULE, SEED_REFERENCE, SOURCE_REFERENCE, TIMESTAMP_CURSOR, +}; +use crate::bindings::_helpers::planner_strategy::lineage::resolve_lineage_reference; +use crate::bindings::_helpers::planner_strategy::values::{ + PLANNER_EXCEPTIONS_MODULE, input_error, text, +}; +use crate::bindings::_helpers::references::source_relations::render_source_relation; + +/// A model's authored config values. +pub(crate) fn model_values<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + model.getattr("config")?.getattr("values") +} + +/// Whether a model is incremental with a cursor column. +fn incremental_cursor_flag( + model: &Bound<'_, PyAny>, + cursor_column: &Bound<'_, PyAny>, +) -> PyResult { + let materialized: Option> = config_str(&model_values(model)?, "materialized")?; + Ok(!cursor_column.is_none() + && match materialized { + Some(materialized) => materialized.eq(INCREMENTAL_MATERIALIZATION)?, + None => false, + }) +} + +/// The names of a sequence of objects with a `name`. +fn names(items: &Bound<'_, PyAny>) -> PyResult> { + items + .try_iter()? + .map(|item| text(&item?.getattr("name")?)) + .collect() +} + +/// Raise the planner error of a cursor input column its input does not expose. +fn check_column(check: CursorInputColumnCheck, py: Python<'_>) -> PyResult<()> { + match cursor_input_column_failure(&check) { + Some((message, code)) => Err(input_error( + py, + (PLANNER_EXCEPTIONS_MODULE, "PlannerInputError"), + &message, + Some(code), + )), + None => Ok(()), + } +} + +/// Validate authored cursor input columns against contracts, then source warehouse metadata. +#[pyfunction(name = "validate_source_cursor_input_columns")] +#[pyo3(signature = (**request))] +pub(crate) fn reject_unexposed_cursor_input_columns<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult<()> { + let request = Request::new("validate_source_cursor_input_columns", request)?; + let py: Python<'py> = request.py(); + let model: Bound<'py, PyAny> = request.get("model")?; + if !incremental_cursor_flag(&model, &request.get("cursor_column")?)? { + return Ok(()); + } + let roles: Bound<'py, PyAny> = resolve_cursor_input_roles(&model)?; + let models_by_name = or_empty(request.optional("models_by_name")?, py)?; + let functions_by_name = or_empty(request.optional("functions_by_name")?, py)?; + let source_map = or_empty(request.optional("source_map")?, py)?; + let warehouse_columns: Bound<'py, PyAny> = request.get("source_warehouse_columns")?; + let model_name: String = text(&model.getattr("name")?)?; + for role in ["filter_inputs", "watermark_inputs"] { + for item in roles.getattr(role)?.call_method0("items")?.try_iter()? { + let (input_name, column): (String, String) = item?.extract()?; + let reference: Bound<'py, PyAny> = resolve_lineage_reference( + &model, + &input_name, + models_by_name.clone(), + functions_by_name.clone(), + )?; + let kind: String = text(&reference.getattr("ref_kind")?)?; + let name: String = text(&reference.getattr("ref_name")?)?; + let check = |owner, known_columns| CursorInputColumnCheck { + model_name: model_name.clone(), + owner, + input_name: name.clone(), + column: column.clone(), + known_columns, + }; + if kind == MODEL_REFERENCE { + let upstream: Bound<'py, PyAny> = models_by_name.call_method1("get", (&name,))?; + if upstream.is_none() + || !model_values(&upstream)? + .call_method1("get", ("contract",))? + .eq(ENFORCED_CONTRACT)? + { + continue; + } + let entry: Bound<'py, PyAny> = upstream.getattr("schema_entry")?; + let declared: Vec = if entry.is_none() { + Vec::new() + } else { + names(&entry.getattr("columns")?)? + }; + check_column(check(CursorInputColumnOwner::ModelContract, declared), py)?; + continue; + } + if kind != SOURCE_REFERENCE { + continue; + } + let source: Bound<'py, PyAny> = source_map.call_method1("get", (&name,))?; + if !source.is_none() && source.getattr("contract")?.eq(ENFORCED_CONTRACT)? { + let declared: Vec = names(&source.getattr("columns")?)?; + check_column(check(CursorInputColumnOwner::SourceContract, declared), py)?; + continue; + } + let known: Bound<'py, PyAny> = warehouse_columns.call_method1("get", (&name,))?; + if known.is_none() { + continue; + } + check_column( + check(CursorInputColumnOwner::SourceWarehouse, names(&known)?), + py, + )?; + } + } + Ok(()) +} + +/// The qualified relation one cursor input reference reads, when it resolves. +fn input_relation<'py>( + request: &Request<'_, 'py>, + (kind, name): (&str, &str), +) -> PyResult> { + let py: Python<'py> = request.py(); + let location: Bound<'py, PyAny> = match kind { + MODEL_REFERENCE => { + let model: Bound<'py, PyAny> = request + .get("model_locations")? + .call_method1("get", (name,))?; + if model.is_none() { + request + .get("seed_locations")? + .call_method1("get", (name,))? + } else { + model + } + } + SEED_REFERENCE => request + .get("seed_locations")? + .call_method1("get", (name,))?, + SOURCE_REFERENCE => { + let source: Bound<'py, PyAny> = + request.get("source_map")?.call_method1("get", (name,))?; + if source.is_none() { + return Ok(None); + } + let adapter: Bound<'py, PyAny> = request.get("adapter")?; + return render_source_relation(&source, Some(&adapter)).map(Some); + } + _ => py.None().into_bound(py), + }; + if location.is_none() { + return Ok(None); + } + location.getattr("qualified_name")?.extract() +} + +/// The resource kind a resolved cursor input reference reads. +fn input_kind<'py>( + py: Python<'py>, + (kind, model_backed): (&str, bool), +) -> PyResult> { + let value: &str = match kind { + MODEL_REFERENCE if model_backed => "model", + MODEL_REFERENCE | SEED_REFERENCE => "seed", + SOURCE_REFERENCE => "source", + _ => return Ok(py.None().into_bound(py)), + }; + imported(py, PLANNER_TYPES_MODULE, "GraphResourceKind")?.call1((value,)) +} + +/// A producer model's terminal cursor bound parsed with the consumer's cursor type. +fn terminal_bound<'py>( + producer: Option<&Bound<'py, PyAny>>, + (key, cursor_type): (&str, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = cursor_type.py(); + let Some(producer) = producer else { + return Ok(py.None().into_bound(py)); + }; + match config_cursor_bound(&model_values(producer)?, key)? { + Some(raw) => parse_value(&raw, cursor_type), + None => Ok(py.None().into_bound(py)), + } +} + +/// Cursor-bearing input relations of an incremental model, for runtime range discovery. +#[pyfunction] +#[pyo3(signature = (**request))] +pub(crate) fn build_cursor_input_relations<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("build_cursor_input_relations", request)?; + let py: Python<'py> = request.py(); + let model: Bound<'py, PyAny> = request.get("model")?; + if !incremental_cursor_flag(&model, &request.get("cursor_column")?)? { + return Ok(PyTuple::empty(py)); + } + let models_by_name: Bound<'py, PyAny> = request.get("models_by_name")?; + let functions_by_name = or_empty(request.optional("functions_by_name")?, py)?; + let model_locations: Bound<'py, PyAny> = request.get("model_locations")?; + let producers: Bound<'py, PyAny> = request.get("runtime_cursor_producer_names")?; + let cursor_type: Bound<'py, PyAny> = match config_str(&model_values(&model)?, "cursor_type")? { + Some(cursor_type) if cursor_type.is_truthy()? => cursor_type, + _ => PyString::new(py, TIMESTAMP_CURSOR).into_any(), + }; + let class: Bound<'py, PyAny> = imported(py, PLAN_MODELS_MODULE, "CursorInputRelation")?; + let mut relations: Vec> = Vec::new(); + let watermarks: Bound<'py, PyAny> = + resolve_cursor_input_roles(&model)?.getattr("watermark_inputs")?; + for item in watermarks.call_method0("items")?.try_iter()? { + let (input_name, column): (String, Bound<'py, PyAny>) = item?.extract()?; + let reference: Bound<'py, PyAny> = resolve_lineage_reference( + &model, + &input_name, + models_by_name.clone(), + functions_by_name.clone(), + )?; + let kind: String = text(&reference.getattr("ref_kind")?)?; + let name: Bound<'py, PyAny> = reference.getattr("ref_name")?; + let name_text: String = text(&name)?; + let Some(relation) = input_relation(&request, (&kind, &name_text))? else { + continue; + }; + let producer: Option> = if kind == MODEL_REFERENCE { + Some(models_by_name.call_method1("get", (&name,))?).filter(|model| !model.is_none()) + } else { + None + }; + let grain: Bound<'py, PyAny> = match &producer { + Some(producer) => config_str(&model_values(producer)?, "cursor_grain")? + .unwrap_or_else(|| py.None().into_bound(py)), + None => py.None().into_bound(py), + }; + let model_backed: bool = kind == MODEL_REFERENCE && model_locations.contains(&name)?; + relations.push(call_with( + &class, + &[ + ("relation", PyString::new(py, &relation).into_any()), + ("cursor_column", column), + ("cursor_grain", grain), + ( + "is_model_backed", + PyBool::new(py, model_backed).to_owned().into_any(), + ), + ( + "is_runtime_produced", + PyBool::new(py, producers.contains(&name)?) + .to_owned() + .into_any(), + ), + ("input_name", name.clone()), + ("input_kind", input_kind(py, (&kind, model_backed))?), + ( + "terminal_cursor_start", + terminal_bound(producer.as_ref(), ("cursor_start", &cursor_type))?, + ), + ( + "terminal_cursor_end", + terminal_bound(producer.as_ref(), ("cursor_end", &cursor_type))?, + ), + ], + )?); + } + PyTuple::new(py, relations) +} + +/// Selected models and seeds, and selected sources with loaders, whose relations this invocation produces. +#[pyfunction] +#[pyo3(signature = (*, scope, source_map))] +pub(crate) fn runtime_cursor_producer_names<'py>( + scope: &Bound<'py, PyAny>, + source_map: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = scope.py(); + let mut names: Vec> = Vec::new(); + for key in scope.getattr("selected_keys")?.try_iter()? { + let key: Bound<'py, PyAny> = key?; + let name: Bound<'py, PyAny> = key.getattr("name")?; + match text(&key.getattr("resource_type")?)?.as_str() { + "model" | "seed" => names.push(name), + "source" => { + let source: Bound<'py, PyAny> = source_map.call_method1("get", (&name,))?; + if !source.is_none() && !source.getattr("loader")?.is_none() { + names.push(name); + } + } + _ => {} + } + } + PyFrozenSet::new(py, names) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs index 47bd1cb509..1a7d6f905c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs @@ -116,7 +116,7 @@ fn lineage_error(py: Python<'_>, failure: LineageFailure) -> PyErr { /// The reference one named input resolves to through model and SQL function references. #[pyfunction] #[pyo3(signature = (*, model, input_name, models_by_name, functions_by_name))] -fn resolve_lineage_reference<'py>( +pub(crate) fn resolve_lineage_reference<'py>( model: &Bound<'py, PyAny>, input_name: &str, models_by_name: Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs index 1b18069a05..2c34e49ec2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod audit_entries; pub(crate) mod audits; pub(crate) mod constants; +pub(crate) mod cursor_inputs; pub(crate) mod cursor_types; pub(crate) mod fixture_completion; pub(crate) mod fixture_context; @@ -13,6 +14,7 @@ pub(crate) mod fixture_types; pub(crate) mod fixture_values; pub(crate) mod lineage; pub(crate) mod microbatches; +pub(crate) mod model_entries; pub(crate) mod models; pub(crate) mod registration; pub(crate) mod strategy; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs new file mode 100644 index 0000000000..7763ae7fc3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -0,0 +1,560 @@ +//! Assemble model plan dataclasses from resolved plan decisions and authored model config. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction}; + +use sqlbuild_analysis::planner_strategy::main::microbatch_lookback::microbatch_lookback; +use sqlbuild_analysis::planner_strategy::main::table_type_ddl::table_type_ddl; +use sqlbuild_analysis::planner_strategy::models::LookbackConfig; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; +use crate::bindings::_helpers::model_config::config_values::{ + config_bool, config_cursor_bound, config_int, config_str, config_string_tuple, +}; +use crate::bindings::_helpers::planner_strategy::constants::{ + ADAPTER_MODELS_MODULE, CURSOR_TYPES_MODULE, CUSTOM_MATERIALIZATION, DDL_SEPARATOR, + ENFORCED_CONTRACT, MICROBATCH_MODE, NATIVE_MODULE, PLAN_MODELS_MODULE, PLANNER_TYPES_MODULE, +}; +use crate::bindings::_helpers::planner_strategy::cursor_inputs::model_values; +use crate::bindings::_helpers::planner_strategy::microbatches::microbatch_limit_config; +use crate::bindings::_helpers::planner_strategy::values::{ + PLANNER_EXCEPTIONS_MODULE, input_error, optional_string, text, +}; + +/// Model config keys copied to the plan entry as optional strings, with their entry fields. +const STRING_FIELDS: [(&str, &str); 15] = [ + ("incremental_strategy", "incremental_strategy"), + ("incremental_mode", "incremental_mode"), + ("microbatch_strategy", "microbatch_strategy"), + ("cursor_watermark_mode", "cursor_watermark_mode"), + ("cursor_type", "cursor_type"), + ("cursor_grain", "cursor_grain"), + ("batch_size", "batch_size"), + ( + "unaccounted_partition_policy", + "unaccounted_partition_policy", + ), + ("snapshot_strategy", "snapshot_strategy"), + ("updated_at", "updated_at_column"), + ("observed_at", "observed_at_column"), + ("historical_input", "historical_input"), + ("valid_from_column", "valid_from_column"), + ("valid_to_column", "valid_to_column"), + ("initial_valid_from", "initial_valid_from"), +]; +/// Model config keys copied as optional strings that complete the snapshot settings. +const SNAPSHOT_POLICY_FIELDS: [&str; 2] = ["snapshot_full_refresh", "snapshot_schema_change"]; +/// Plan decisions passed through to the entry unchanged. +const PASSED_FIELDS: [&str; 16] = [ + "action", + "reason", + "materialization_type", + "resolved_sql", + "cursor_column", + "cursor_bounds", + "empty_cursor_inputs", + "waiting_on_empty_inputs", + "cursor_input_relations", + "microbatch_range", + "start_cursor_override", + "end_cursor_override", + "future_cursor_config", + "start_cursor_config", + "invocation_time", + "backfill", +]; +/// Change-detection facts copied to the entry. +const CHANGE_FIELDS: [&str; 8] = [ + "fingerprint_metadata_json", + "previous_metadata_json", + "fingerprint_version_hash", + "previous_version_hash", + "query_changed", + "config_changed", + "schema_findings", + "changed_functions", +]; + +fn none(py: Python<'_>) -> Bound<'_, PyAny> { + py.None().into_bound(py) +} + +fn or_none<'py>(py: Python<'py>, value: Option>) -> Bound<'py, PyAny> { + value.unwrap_or_else(|| none(py)) +} + +/// `ColumnInfo(name=..., type=...)`. +fn column_info<'py>( + class: &Bound<'py, PyAny>, + (name, column_type): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + call_with(class, &[("name", name), ("type", column_type)]) +} + +/// Declared schema columns as `ColumnInfo`: typed columns only, or every column with an empty default type. +fn schema_columns<'py>( + model: &Bound<'py, PyAny>, + typed_only: bool, +) -> PyResult> { + let py: Python<'py> = model.py(); + let entry: Bound<'py, PyAny> = model.getattr("schema_entry")?; + if entry.is_none() { + return Ok(PyTuple::empty(py)); + } + let class: Bound<'py, PyAny> = imported(py, ADAPTER_MODELS_MODULE, "ColumnInfo")?; + let mut columns: Vec> = Vec::new(); + for column in entry.getattr("columns")?.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let column_type: Bound<'py, PyAny> = column.getattr("type")?; + if typed_only && column_type.is_none() { + continue; + } + let column_type: Bound<'py, PyAny> = if typed_only || column_type.is_truthy()? { + column_type + } else { + PyString::new(py, "").into_any() + }; + columns.push(column_info(&class, (column.getattr("name")?, column_type))?); + } + PyTuple::new(py, columns) +} + +/// A string or list-of-strings config value as a tuple of strings. +fn string_list<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult> { + let py: Python<'py> = values.py(); + let raw: Bound<'py, PyAny> = values.call_method1("get", (key,))?; + if raw.is_instance_of::() { + return PyTuple::new(py, [raw]); + } + if raw.is_instance_of::() { + let items: Vec> = raw + .try_iter()? + .filter(|item| { + item.as_ref() + .map_or(true, |item| item.is_instance_of::()) + }) + .collect::>()?; + return PyTuple::new(py, items); + } + Ok(PyTuple::empty(py)) +} + +/// The model's `on_schema_change` policy, rejecting unknown values. +fn on_schema_change<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + let raw: Bound<'py, PyAny> = model_values(model)?.call_method1("get", ("on_schema_change",))?; + if raw.is_none() { + return Ok(raw); + } + if raw.is_instance_of::() { + match imported(py, PLANNER_TYPES_MODULE, "OnSchemaChange")?.call1((&raw,)) { + Ok(policy) => return Ok(policy), + Err(error) if !error.is_instance_of::(py) => return Err(error), + Err(_) => {} + } + } + Err(input_error( + py, + (PLANNER_EXCEPTIONS_MODULE, "PlannerInputError"), + &format!( + "model '{}': unknown on_schema_change '{}'", + text(&model.getattr("name")?)?, + raw.str()? + ), + None, + )) +} + +/// Whether the model's schema enforces declared column types. +fn type_enforcement<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + let entry: Bound<'py, PyAny> = model.getattr("schema_entry")?; + if !entry.is_none() { + let enforcement: Bound<'py, PyAny> = entry.getattr("type_enforcement")?; + if !enforcement.is_none() { + return Ok(enforcement); + } + } + Ok(PyBool::new(py, false).to_owned().into_any()) +} + +/// `CursorGrain(grain)`'s error for a grain outside the known grains. +fn unknown_grain(py: Python<'_>, grain: &str) -> PyErr { + imported(py, PLANNER_TYPES_MODULE, "CursorGrain") + .and_then(|class| class.call1((grain,))) + .err() + .unwrap_or_else(|| PyValueError::new_err("unknown cursor grain")) +} + +/// The explicit or default lookback of a microbatch model at the effective grain. +fn lookback_of( + model: &Bound<'_, PyAny>, + effective_grain: Option, +) -> PyResult> { + let values: Bound<'_, PyAny> = model_values(model)?; + let read = |key: &str| -> PyResult> { + config_str(&values, key)? + .map(|value| text(&value)) + .transpose() + }; + let config = LookbackConfig { + lookback: read("lookback")?, + incremental_strategy: read("incremental_strategy")?, + batch_size: read("batch_size")?, + grain: match effective_grain.filter(|grain| !grain.is_empty()) { + Some(grain) => Some(grain), + None => read("cursor_grain")?, + }, + }; + microbatch_lookback(&config).map_err(|error| unknown_grain(model.py(), &error.0)) +} + +/// Resolve explicit or default lookback for one microbatch model. +#[pyfunction] +#[pyo3(signature = (*, model, effective_grain = None))] +pub(crate) fn resolve_microbatch_lookback( + model: &Bound<'_, PyAny>, + effective_grain: Option, +) -> PyResult> { + lookback_of(model, effective_grain) +} + +/// Declared seed columns with types, as `ColumnInfo`. +#[pyfunction] +#[pyo3(signature = (seed))] +pub(crate) fn extract_seed_columns<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + schema_columns(seed, true) +} + +/// The adapter-rendered logical DDL of one plan action. +fn logical_ddl<'py>( + request: &Request<'_, 'py>, + (unique_key, merge_exclude): (&Bound<'py, PyTuple>, &Bound<'py, PyTuple>), + (warehouse_columns, cursor_bounds): (&Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult { + let py: Python<'py> = request.py(); + let adapter: Bound<'py, PyAny> = request.get("adapter")?; + let model: Bound<'py, PyAny> = request.get("model")?; + let destination: Bound<'py, PyAny> = model.getattr("destination")?; + let qualified: Bound<'py, PyAny> = destination.getattr("qualified_name")?; + let qualified: Bound<'py, PyAny> = if qualified.is_truthy()? { + qualified + } else { + destination.getattr("name")? + }; + let sql: Bound<'py, PyAny> = request.get("resolved_sql")?; + let mut pairs: Vec<(&str, Bound<'py, PyAny>)> = vec![("destination", qualified), ("sql", sql)]; + let cursor_column: Bound<'py, PyAny> = request.get("cursor_column")?; + let method: &str = match text(&request.get("action")?)?.as_str() { + "create_view" => "render_create_view_as", + "create_table" => "render_create_table_as", + "incremental_append" => "render_append", + "incremental_delete_insert" if !cursor_column.is_none() && !cursor_bounds.is_none() => { + let values: Bound<'py, PyAny> = model_values(&model)?; + let start = sentinel_to_token(&cursor_bounds.getattr("start")?)?; + let end = sentinel_to_token(&cursor_bounds.getattr("end")?)?; + pairs.push(("cursor_column", cursor_column)); + pairs.push(("cursor_start", PyString::new(py, &start).into_any())); + pairs.push(("cursor_end", PyString::new(py, &end).into_any())); + pairs.push(( + "cursor_type", + or_none(py, config_str(&values, "cursor_type")?), + )); + "render_delete_insert_cursor" + } + "incremental_delete_insert" => { + pairs.push(("unique_key", unique_key.clone().into_any())); + "render_delete_insert" + } + "incremental_merge" => { + let mut source_columns: Vec> = Vec::new(); + for column in warehouse_columns.try_iter()? { + source_columns.push(column?.getattr("name")?); + } + pairs.push(("unique_key", unique_key.clone().into_any())); + pairs.push(( + "source_columns", + PyTuple::new(py, source_columns)?.into_any(), + )); + pairs.push(("exclude_columns", merge_exclude.clone().into_any())); + "render_merge" + } + _ => return Ok(String::new()), + }; + let statements: Vec = call_with(&adapter.getattr(method)?, &pairs)? + .try_iter()? + .map(|statement| statement?.extract()) + .collect::>()?; + Ok(statements.join(DDL_SEPARATOR)) +} + +/// The custom materialization name, config and placeholders of a custom model. +fn custom_fields<'py>( + values: &Bound<'py, PyAny>, + custom: bool, + entry: &Bound<'py, PyDict>, +) -> PyResult<()> { + let py: Python<'py> = values.py(); + let config: Bound<'py, PyDict> = PyDict::new(py); + let placeholders: Bound<'py, PyDict> = PyDict::new(py); + let mut name: Bound<'py, PyAny> = none(py); + if custom { + let raw_name: Bound<'py, PyAny> = values.call_method1("get", ("materialized",))?; + if raw_name.is_instance_of::() { + name = raw_name; + } + if let Ok(raw) = values + .call_method1("get", ("config",))? + .downcast::() + { + for (key, value) in raw.iter() { + config.set_item(key.str()?, value)?; + } + } + if let Ok(raw) = values + .call_method1("get", ("placeholders",))? + .downcast::() + { + for (key, value) in raw.iter() { + placeholders.set_item(key.str()?, value.str()?)?; + } + } + } + entry.set_item("custom_materialization_name", name)?; + entry.set_item("custom_config", config)?; + entry.set_item("custom_placeholders", placeholders) +} + +/// The model's base warnings followed by its cursor type warning. +fn model_warnings<'py>( + request: &Request<'_, 'py>, + (schema_actions, policy): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (enforcement, warehouse_columns): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = request.py(); + let model: Bound<'py, PyAny> = request.get("model")?; + let name: Bound<'py, PyAny> = model.getattr("name")?; + let mut warnings: Vec> = call_with( + &imported(py, NATIVE_MODULE, "build_model_warnings")?, + &[ + ("model_name", name.clone()), + ("materialization_type", request.get("materialization_type")?), + ("change_result", request.get("change_result")?), + ("schema_actions", schema_actions.clone()), + ("on_schema_change", policy.clone()), + ("type_enforcement", enforcement.clone()), + ], + )? + .try_iter()? + .collect::>()?; + let cursor_warning: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "check_cursor_type_consistency")?, + &[ + ("model_name", name), + ("cursor_column", request.get("cursor_column")?), + ( + "cursor_type", + or_none(py, config_str(&model_values(&model)?, "cursor_type")?), + ), + ("warehouse_columns", warehouse_columns.clone()), + ("sql_analysis_enabled", request.get("sql_analysis_enabled")?), + ], + )?; + if !cursor_warning.is_none() { + warnings.push(cursor_warning); + } + PyTuple::new(py, warnings) +} + +/// The cursor bounds the logical DDL renders: placeholders for runtime-owned windows, else the batch range or cursor bounds. +fn ddl_cursor_bounds<'py>(request: &Request<'_, 'py>) -> PyResult> { + let py: Python<'py> = request.py(); + let microbatch_range: Bound<'py, PyAny> = request.get("microbatch_range")?; + Ok( + if request.get("runtime_owned_cursor_bounds")?.is_truthy()? + && !request.get("cursor_column")?.is_none() + { + let sentinel: Bound<'py, PyAny> = imported(py, CURSOR_TYPES_MODULE, "BoundSentinel")?; + call_with( + &imported(py, PLAN_MODELS_MODULE, "CursorBounds")?, + &[ + ("start", sentinel.getattr("START")?), + ("end", sentinel.getattr("END")?), + ], + )? + } else if microbatch_range.is_truthy()? { + microbatch_range.clone() + } else { + request.get("cursor_bounds")? + }, + ) +} + +/// Copy the model's authored strategy, cursor, batch and snapshot settings onto the entry. +fn config_fields<'py>( + request: &Request<'_, 'py>, + values: &Bound<'py, PyAny>, + entry: &Bound<'py, PyDict>, +) -> PyResult<()> { + let py: Python<'py> = request.py(); + let model: Bound<'py, PyAny> = request.get("model")?; + for (key, field) in STRING_FIELDS { + entry.set_item(field, or_none(py, config_str(values, key)?))?; + } + for key in SNAPSHOT_POLICY_FIELDS { + entry.set_item(key, or_none(py, config_str(values, key)?))?; + } + for key in ["cursor_start", "cursor_end"] { + entry.set_item(key, or_none(py, config_cursor_bound(values, key)?))?; + } + let microbatch: bool = match config_str(values, "incremental_mode")? { + Some(mode) => mode.eq(MICROBATCH_MODE)?, + None => false, + }; + let lookback: Option = if microbatch { + let grain: Option = optional_string(&request.get("effective_grain")?)?; + lookback_of(&model, grain)? + } else { + config_str(values, "lookback")? + .map(|value| text(&value)) + .transpose()? + }; + entry.set_item("lookback", lookback)?; + let concurrency: Bound<'py, PyAny> = match config_int(values, "batch_concurrency")? { + Some(concurrency) if concurrency.is_truthy()? => concurrency, + _ => 1_i64.into_pyobject(py)?.into_any(), + }; + entry.set_item("batch_concurrency", concurrency)?; + let (limit, limit_action) = microbatch_limit_config(values)?; + entry.set_item("microbatch_limit", or_none(py, limit))?; + entry.set_item( + "declared_microbatch_limit_action", + or_none(py, limit_action), + )?; + entry.set_item("check_columns", string_list(values, "check_columns")?)?; + let hard_deletes: Bound<'py, PyAny> = match config_bool(values, "invalidate_hard_deletes")? { + Some(flag) if flag.is_truthy()? => flag, + _ => PyBool::new(py, false).to_owned().into_any(), + }; + entry.set_item("invalidate_hard_deletes", hard_deletes)?; + Ok(()) +} + +/// Copy the model's declared columns and contract onto the entry. +fn contract_fields<'py>( + model: &Bound<'py, PyAny>, + values: &Bound<'py, PyAny>, + entry: &Bound<'py, PyDict>, +) -> PyResult<()> { + let py: Python<'py> = model.py(); + entry.set_item("declared_columns", schema_columns(model, true)?)?; + entry.set_item( + "contract_enforced", + values + .call_method1("get", ("contract",))? + .eq(ENFORCED_CONTRACT)?, + )?; + entry.set_item("contract_columns", schema_columns(model, false)?)?; + let schema_entry: Bound<'py, PyAny> = model.getattr("schema_entry")?; + entry.set_item( + "contract_dynamic_columns", + if schema_entry.is_none() { + PyTuple::empty(py).into_any() + } else { + schema_entry.getattr("dynamic_columns")? + }, + )?; + Ok(()) +} + +/// Build a model plan entry and its warnings from resolved plan decisions, keeping the Python dataclasses. +#[pyfunction] +#[pyo3(signature = (**request))] +pub(crate) fn assemble_model_plan_entry<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("assemble_model_plan_entry", request)?; + let py: Python<'py> = request.py(); + let model: Bound<'py, PyAny> = request.get("model")?; + let values: Bound<'py, PyAny> = model_values(&model)?; + let change: Bound<'py, PyAny> = request.get("change_result")?; + let snapshot: Bound<'py, PyAny> = request.get("snapshot")?; + let name: Bound<'py, PyAny> = model.getattr("name")?; + let policy: Bound<'py, PyAny> = on_schema_change(&model)?; + let schema_actions: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_schema_actions")?, + &[ + ("schema_findings", change.getattr("schema_findings")?), + ("on_schema_change", policy.clone()), + ], + )?; + let enforcement: Bound<'py, PyAny> = type_enforcement(&model)?; + let warehouse_columns: Bound<'py, PyAny> = snapshot + .getattr("existing_columns")? + .call_method1("get", (&name, PyTuple::empty(py)))?; + let warnings: Bound<'py, PyTuple> = model_warnings( + &request, + (&schema_actions, &policy), + (&enforcement, &warehouse_columns), + )?; + let unique_key: Bound<'py, PyTuple> = string_list(&values, "unique_key")?; + let merge_exclude: Bound<'py, PyTuple> = config_string_tuple(&values, "merge_exclude_columns")? + .unwrap_or_else(|| PyTuple::empty(py)); + let table_type: Bound<'py, PyAny> = model + .getattr("config")? + .getattr("table_type")? + .getattr("value")? + .getattr("value")?; + let ddl: String = logical_ddl( + &request, + (&unique_key, &merge_exclude), + (&warehouse_columns, ddl_cursor_bounds(&request)?), + )?; + let entry: Bound<'py, PyDict> = PyDict::new(py); + for field in ["key", "name", "relative_path", "destination"] { + entry.set_item(field, model.getattr(field)?)?; + } + entry.set_item("fingerprint_query_sql", model.getattr("query_sql")?)?; + entry.set_item("logical_ddl", table_type_ddl(&ddl, &text(&table_type)?))?; + for field in PASSED_FIELDS { + entry.set_item(field, request.get(field)?)?; + } + for field in CHANGE_FIELDS { + entry.set_item(field, change.getattr(field)?)?; + } + config_fields(&request, &values, &entry)?; + entry.set_item("unique_key", unique_key)?; + entry.set_item("merge_exclude_columns", merge_exclude)?; + entry.set_item("table_type", table_type)?; + entry.set_item("on_schema_change", policy)?; + entry.set_item("type_enforcement", enforcement)?; + contract_fields(&model, &values, &entry)?; + for field in ["pre_hooks", "post_hooks"] { + entry.set_item(field, request.get(field)?)?; + } + let fingerprint: Bound<'py, PyAny> = snapshot + .getattr("fingerprints")? + .getattr("models")? + .call_method1("get", (&name,))?; + entry.set_item( + "previous_query_sql", + if fingerprint.is_none() { + fingerprint + } else { + fingerprint.getattr("definition")? + }, + )?; + entry.set_item("schema_actions", schema_actions)?; + custom_fields( + &values, + text(&request.get("materialization_type")?)? == CUSTOM_MATERIALIZATION, + &entry, + )?; + let plan_entry: Bound<'py, PyAny> = + imported(py, PLAN_MODELS_MODULE, "ModelPlanEntry")?.call((), Some(&entry))?; + PyTuple::new(py, [plan_entry, warnings.into_any()]) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs index 3f85873151..acdffe6e60 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -18,6 +18,30 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::audit_entries::build_selected_audit_entries, module )?)?; + module.add_function(wrap_pyfunction!( + super::cursor_inputs::reject_unexposed_cursor_input_columns, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::cursor_inputs::build_cursor_input_relations, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::cursor_inputs::runtime_cursor_producer_names, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::model_entries::assemble_model_plan_entry, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::model_entries::resolve_microbatch_lookback, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::model_entries::extract_seed_columns, + module + )?)?; audits::register(module)?; cursor_types::register(module)?; fixture_completion::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 1d74e1c8ad..18b41a5538 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -219,6 +219,7 @@ from sqlbuild.compiler.planner.models import ( CursorInputEvidence, CursorOverridePair, CursorOverrides, + CursorInputRelation, DeferralInputs, DirectModelVersionIdentities, EmptyCursorInputDecision, @@ -449,6 +450,7 @@ from sqlbuild.spec.contracts.models import ( SourceEntry, SourceFreshnessAgePolicy, SourceLocation, + StartCursorsConfig, TargetConfig, ) from sqlbuild.spec.contracts.types import ( @@ -5022,6 +5024,62 @@ def build_model_warnings( on_schema_change: OnSchemaChange | None, type_enforcement: bool, ) -> tuple[PlanWarning, ...]: ... +def validate_source_cursor_input_columns( + *, + model: CompiledModelView, + cursor_column: str | None, + models_by_name: dict[str, CompiledModelView] | None = None, + functions_by_name: dict[str, CompiledFunctionView] | None = None, + source_map: dict[str, SourceEntry] | None = None, + source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]], +) -> None: ... +def build_cursor_input_relations( + *, + model: CompiledModelView, + adapter: BaseAdapter, + model_locations: dict[str, CompiledRelationLocation], + models_by_name: dict[str, CompiledModelView], + functions_by_name: dict[str, CompiledFunctionView] | None = None, + seed_locations: dict[str, CompiledRelationLocation], + source_map: dict[str, SourceEntry], + cursor_column: str | None, + runtime_cursor_producer_names: frozenset[str], +) -> tuple[CursorInputRelation, ...]: ... +def runtime_cursor_producer_names( + *, scope: PlannerScope, source_map: dict[str, SourceEntry] +) -> frozenset[str]: ... +def resolve_microbatch_lookback( + *, model: CompiledModelView, effective_grain: str | None = None +) -> str | None: ... +def extract_seed_columns(seed: CompiledSeedView) -> tuple[ColumnInfo, ...]: ... +def assemble_model_plan_entry( + *, + model: CompiledModelView, + adapter: BaseAdapter, + snapshot: WarehouseSnapshot, + change_result: ChangeDetectionResult, + action: PlanAction, + reason: PlanReason, + materialization_type: MaterializationType, + resolved_sql: str, + sql_analysis_enabled: bool, + pre_hooks: object, + post_hooks: object, + cursor_column: str | None, + cursor_input_relations: tuple[CursorInputRelation, ...], + runtime_owned_cursor_bounds: bool, + cursor_bounds: CursorBounds | None, + microbatch_range: CursorBounds | None, + empty_cursor_inputs: tuple[str, ...], + waiting_on_empty_inputs: bool, + backfill: BackfillResult, + start_cursor_override: str | None, + end_cursor_override: str | None, + future_cursor_config: FutureCursorsConfig | None, + start_cursor_config: StartCursorsConfig | None, + invocation_time: datetime | None, + effective_grain: str | None, +) -> tuple[ModelPlanEntry, tuple[PlanWarning, ...]]: ... def plan_audit( *, audit: CompiledAuditView, diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py index fe32ee724d..3b0738dc89 100644 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py +++ b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py @@ -6,16 +6,14 @@ build_compiled_function_fingerprint_sql, build_seed_identity, build_selected_audit_entries, + extract_seed_columns, get_materialization_type, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.output.plan_entry import ( - build_planner_relations_context, - extract_seed_columns, -) +from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context from sqlbuild.compiler.planner._helpers.output.plan_output import ( build_selected_test_entries, plan_function, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index 4e676c5eb5..fe0d90a141 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -8,56 +8,44 @@ from typing import Any from sqlbuild._native import ( + assemble_model_plan_entry, audit_is_selected, - build_model_warnings, + build_cursor_input_relations, build_source_load_map, cap_microbatch_bounds, - check_cursor_type_consistency, count_microbatches, detect_model_changes, - get_config_bool, get_config_cursor_bound, - get_config_int, get_config_str, - get_config_string_tuple, get_materialization_type, microbatch_guidance, microbatch_limit_warning, - render_source_relation, replaces_incremental_relation, resolve_bounded_cursor_override, - resolve_cursor_input_roles, resolve_effective_microbatch_batch_size, - resolve_lineage_reference, - resolve_microbatch_limit_config, + resolve_microbatch_lookback, resolve_model_plan_action, - resolve_schema_actions, + runtime_cursor_producer_names, source_binding_diagnostics, validate_capped_watermark_inputs, + validate_source_cursor_input_columns, ) -from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, CompilerDiagnostic, - CompileSqlReference, - CursorInputRoles, ) from sqlbuild.compiler.compile.types import ( CompiledAuditView, - CompiledFunctionView, CompiledModelView, CompiledProjectView, CompiledResourceType, - CompiledSeedView, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( check_full_rebuild_allowed, describe_full_rebuild_cause, @@ -119,41 +107,30 @@ PlannerScope, PlanWarning, ResolvedModelAction, - SchemaAction, WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import ( BackfillAction, - ContractPolicy, CursorGrain, CursorType, - GraphResourceKind, IncrementalMode, - IncrementalStrategy, MaterializationType, MicrobatchStrategy, - OnSchemaChange, PlanAction, PlanReason, WarningSeverity, ) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind +from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.cursor_algebra.models import TimestampValue from sqlbuild.cursor_algebra.types import BoundSentinel from sqlbuild.spec.contracts.models import ( FutureCursorsConfig, LocalConfig, ProjectConfig, - SchemaColumn, - SchemaDynamicColumnFamily, SourceEntry, StartCursorsConfig, ) -from sqlbuild.spec.contracts.types import MicrobatchLimitAction, TableType - -_IDEMPOTENT_MICROBATCH_STRATEGIES: frozenset[IncrementalStrategy] = frozenset( - (IncrementalStrategy.DELETE_INSERT, IncrementalStrategy.MERGE) -) +from sqlbuild.spec.contracts.types import MicrobatchLimitAction @dataclass(frozen=True) @@ -171,33 +148,6 @@ class _MicrobatchRangeInputs: replaces_relation: bool = False -@dataclass(frozen=True) -class _ModelPlanConfig: - incremental_strategy: str | None - incremental_mode: str | None - microbatch_strategy: str | None - cursor_watermark_mode: str | None - batch_size: str | None - snapshot_strategy: str | None - updated_at_column: str | None - check_columns: tuple[str, ...] - observed_at_column: str | None - historical_input: str | None - valid_from_column: str | None - valid_to_column: str | None - initial_valid_from: str | None - invalidate_hard_deletes: bool - snapshot_full_refresh: str | None - snapshot_schema_change: str | None - - -@dataclass(frozen=True) -class _CustomMaterializationConfig: - name: str | None - config: dict[str, object] - placeholders: dict[str, str] - - def build_planner_relations_context( *, project: CompiledProjectView, @@ -405,7 +355,7 @@ def build_plan_entries( end_cursor_override: str | None = inputs.end_cursor_override entries: list[ModelPlanEntry] = [] warnings: list[PlanWarning] = [] - runtime_cursor_producer_names: frozenset[str] = _runtime_cursor_producer_names( + runtime_producer_names: frozenset[str] = runtime_cursor_producer_names( scope=scope, source_map=relations.source_map, ) @@ -444,7 +394,7 @@ def build_plan_entries( source_map=relations.source_read_map, source_warehouse_columns=relations.source_warehouse_columns, star_exclude_keyword=relations.star_exclude_keyword, - runtime_cursor_producer_names=runtime_cursor_producer_names, + runtime_cursor_producer_names=runtime_producer_names, future_cursor_config=inputs.future_cursor_config, start_cursor_config=inputs.start_cursor_config, invocation_time=inputs.invocation_time, @@ -728,42 +678,12 @@ def plan_model_from_change( full_rebuild_cause=full_rebuild_cause, ) - on_schema_change: OnSchemaChange | None = _get_on_schema_change(model) - schema_actions: tuple[SchemaAction, ...] = resolve_schema_actions( - schema_findings=change_result.schema_findings, - on_schema_change=on_schema_change, - ) - - type_enforcement: bool = _get_type_enforcement(model) - declared_columns: tuple[ColumnInfo, ...] = _get_declared_columns(model) - contract_enforced: bool = model.config.values.get("contract") == ContractPolicy.ENFORCED - contract_columns: tuple[ColumnInfo, ...] = _get_contract_columns(model) - contract_dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = ( - model.schema_entry.dynamic_columns if model.schema_entry is not None else () - ) - unique_key: tuple[str, ...] = _get_unique_key(model) - warehouse_columns: tuple[ColumnInfo, ...] = snapshot.existing_columns.get(model.name, ()) - - warnings: tuple[PlanWarning, ...] = build_model_warnings( - model_name=model.name, - materialization_type=materialization_type, - change_result=change_result, - schema_actions=schema_actions, - on_schema_change=on_schema_change, - type_enforcement=type_enforcement, - ) - pre_hooks, post_hooks = resolve_model_hook_entries( model=model, adapter=adapter, context=context, external_sql_reference_resolver=external_sql_reference_resolver, ) - cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") - cursor_grain: str | None = get_config_str(values=model.config.values, key="cursor_grain") - cursor_start: str | None = get_config_cursor_bound( - values=model.config.values, key="cursor_start" - ) effective_future_cursor_config, effective_start_cursor_config = _resolve_cursor_safety_configs( model=model, context=context ) @@ -772,7 +692,7 @@ def plan_model_from_change( models_by_name=context.models_by_name, functions_by_name=context.functions_by_name, ) - cursor_input_relations: tuple[CursorInputRelation, ...] = _build_cursor_input_relations( + cursor_input_relations: tuple[CursorInputRelation, ...] = build_cursor_input_relations( model=model, adapter=adapter, model_locations=context.model_locations, @@ -783,8 +703,8 @@ def plan_model_from_change( cursor_column=cursor_column, runtime_cursor_producer_names=context.runtime_cursor_producer_names, ) - runtime_owned_cursor_bounds: bool = _has_runtime_owned_cursor_watermarks( - cursor_input_relations + runtime_owned_cursor_bounds: bool = any( + relation.is_runtime_owned for relation in cursor_input_relations ) and not (start_cursor_override is not None and end_cursor_override is not None) cursor_bounds: CursorBounds | None = _compute_plan_cursor_bounds( model=model, @@ -809,18 +729,6 @@ def plan_model_from_change( cursor_overrides=cursor_overrides, ) - cursor_type_warning: PlanWarning | None = check_cursor_type_consistency( - model_name=model.name, - cursor_column=cursor_column, - cursor_type=cursor_type, - warehouse_columns=warehouse_columns, - sql_analysis_enabled=sql_analysis_enabled, - ) - if cursor_type_warning is not None: - warnings = (*warnings, cursor_type_warning) - - plan_config: _ModelPlanConfig = _resolve_model_plan_config(model=model) - microbatch_range: CursorBounds | None = _compute_microbatch_range( model=model, inputs=_MicrobatchRangeInputs( @@ -838,139 +746,54 @@ def plan_model_from_change( ), ) - warnings = _append_cursor_safety_warnings( - warnings=warnings, - model_name=model.name, - bounds=microbatch_range or cursor_bounds, - ) - - fingerprint: Fingerprint | None = snapshot.fingerprints.models.get(model.name) - previous_query_sql: str | None = fingerprint.definition if fingerprint is not None else None - - custom: _CustomMaterializationConfig = _resolve_custom_materialization_config( - model=model, materialization_type=materialization_type - ) - - ddl_cursor_bounds: CursorBounds | None = ( - _build_runtime_placeholder_bounds() - if runtime_owned_cursor_bounds and cursor_column is not None - else (microbatch_range if microbatch_range else cursor_bounds) - ) - logical_ddl: str = _render_table_type_ddl( - ddl=_build_logical_ddl_from_adapter( - adapter=adapter, - action=action, - resolved_sql=resolved_sql, - destination=model.destination, - unique_key=unique_key, - merge_exclude_columns=get_config_string_tuple( - values=model.config.values, key="merge_exclude_columns" - ) - or (), - warehouse_columns=warehouse_columns, - cursor_column=cursor_column, - cursor_bounds=ddl_cursor_bounds, - cursor_type=cursor_type, - ), - table_type=model.config.table_type.value.value, - ) - - entry: ModelPlanEntry = ModelPlanEntry( - key=model.key, - name=model.name, - relative_path=model.relative_path, - materialization_type=materialization_type, + entry: ModelPlanEntry + warnings: tuple[PlanWarning, ...] + entry, warnings = assemble_model_plan_entry( + model=model, + adapter=adapter, + snapshot=snapshot, + change_result=change_result, action=action, reason=reason, - destination=model.destination, - fingerprint_query_sql=model.query_sql, + materialization_type=materialization_type, resolved_sql=resolved_sql, - logical_ddl=logical_ddl, - incremental_strategy=plan_config.incremental_strategy, - incremental_mode=plan_config.incremental_mode, - microbatch_strategy=plan_config.microbatch_strategy, - cursor_watermark_mode=plan_config.cursor_watermark_mode, + sql_analysis_enabled=sql_analysis_enabled, + pre_hooks=pre_hooks, + post_hooks=post_hooks, cursor_column=cursor_column, - cursor_type=cursor_type, - cursor_grain=cursor_grain, - cursor_start=cursor_start, - cursor_end=get_config_cursor_bound(values=model.config.values, key="cursor_end"), - lookback=( - resolve_microbatch_lookback( - model=model, - effective_grain=resolve_effective_timestamp_grain( - cursor_type=cursor_type, - downstream_grain=cursor_grain, - microbatch_strategy=plan_config.microbatch_strategy, - cursor_input_grains=tuple( - relation.cursor_grain for relation in cursor_input_relations - ), - ), - ) - if plan_config.incremental_mode == IncrementalMode.MICROBATCH - else get_config_str(values=model.config.values, key="lookback") - ), + cursor_input_relations=cursor_input_relations, + runtime_owned_cursor_bounds=runtime_owned_cursor_bounds, cursor_bounds=cursor_bounds, + microbatch_range=microbatch_range, empty_cursor_inputs=empty_cursor_inputs, waiting_on_empty_inputs=waiting_on_empty_inputs, - cursor_input_relations=cursor_input_relations, - batch_size=plan_config.batch_size, - batch_concurrency=get_config_int(values=model.config.values, key="batch_concurrency") or 1, - unaccounted_partition_policy=get_config_str( - values=model.config.values, key="unaccounted_partition_policy" - ), - microbatch_range=microbatch_range, - microbatch_limit=resolve_microbatch_limit_config(values=model.config.values)[0], - declared_microbatch_limit_action=resolve_microbatch_limit_config( - values=model.config.values - )[1], + backfill=backfill, start_cursor_override=start_cursor_override, end_cursor_override=end_cursor_override, future_cursor_config=effective_future_cursor_config, start_cursor_config=effective_start_cursor_config, invocation_time=context.invocation_time, - unique_key=unique_key, - merge_exclude_columns=get_config_string_tuple( - values=model.config.values, key="merge_exclude_columns" - ) - or (), - snapshot_strategy=plan_config.snapshot_strategy, - updated_at_column=plan_config.updated_at_column, - check_columns=plan_config.check_columns, - observed_at_column=plan_config.observed_at_column, - historical_input=plan_config.historical_input, - valid_from_column=plan_config.valid_from_column, - valid_to_column=plan_config.valid_to_column, - initial_valid_from=plan_config.initial_valid_from, - invalidate_hard_deletes=plan_config.invalidate_hard_deletes, - snapshot_full_refresh=plan_config.snapshot_full_refresh, - snapshot_schema_change=plan_config.snapshot_schema_change, - table_type=model.config.table_type.value.value, - on_schema_change=on_schema_change, - type_enforcement=type_enforcement, - declared_columns=declared_columns, - contract_enforced=contract_enforced, - contract_columns=contract_columns, - contract_dynamic_columns=contract_dynamic_columns, - pre_hooks=pre_hooks, - post_hooks=post_hooks, - previous_query_sql=previous_query_sql, - fingerprint_metadata_json=change_result.fingerprint_metadata_json, - previous_metadata_json=change_result.previous_metadata_json, - fingerprint_version_hash=change_result.fingerprint_version_hash, - previous_version_hash=change_result.previous_version_hash, - query_changed=change_result.query_changed, - config_changed=change_result.config_changed, - schema_actions=schema_actions, - schema_findings=change_result.schema_findings, - backfill=backfill, - changed_functions=change_result.changed_functions, - custom_materialization_name=custom.name, - custom_config=custom.config, - custom_placeholders=custom.placeholders, + effective_grain=( + resolve_effective_timestamp_grain( + cursor_type=get_config_str(values=model.config.values, key="cursor_type"), + downstream_grain=get_config_str(values=model.config.values, key="cursor_grain"), + microbatch_strategy=get_config_str( + values=model.config.values, key="microbatch_strategy" + ), + cursor_input_grains=tuple( + relation.cursor_grain for relation in cursor_input_relations + ), + ) + if get_config_str(values=model.config.values, key="incremental_mode") + == IncrementalMode.MICROBATCH + else None + ), + ) + return entry, _append_cursor_safety_warnings( + warnings=warnings, + model_name=model.name, + bounds=microbatch_range or cursor_bounds, ) - - return entry, warnings def resolve_cursor_overrides( @@ -1029,79 +852,6 @@ def resolve_cursor_overrides( return resolved_start, resolved_end -def _get_on_schema_change(model: CompiledModelView) -> OnSchemaChange | None: - """Extract on_schema_change from model config.""" - - raw: object | None = model.config.values.get("on_schema_change") - if raw is None: - return None - if isinstance(raw, str) and raw in set(OnSchemaChange): - return OnSchemaChange(raw) - raise PlannerInputError(f"model '{model.name}': unknown on_schema_change '{raw}'") - - -def extract_seed_columns(seed: CompiledSeedView) -> tuple[ColumnInfo, ...]: - """Extract declared columns from seed schema entry.""" - - columns: list[ColumnInfo] = [] - col: SchemaColumn - for col in seed.schema_entry.columns: - if col.type is not None: - columns.append(ColumnInfo(name=col.name, type=col.type)) - return tuple(columns) - - -def _get_type_enforcement(model: CompiledModelView) -> bool: - """Resolve whether type enforcement is active for a model.""" - - if model.schema_entry is not None and model.schema_entry.type_enforcement is not None: - return model.schema_entry.type_enforcement - return False - - -def _get_declared_columns(model: CompiledModelView) -> tuple[ColumnInfo, ...]: - """Extract declared columns with types from schema entry.""" - - if model.schema_entry is None: - return () - columns: list[ColumnInfo] = [] - col: SchemaColumn - for col in model.schema_entry.columns: - if col.type is not None: - columns.append(ColumnInfo(name=col.name, type=col.type)) - return tuple(columns) - - -def _get_contract_columns(model: CompiledModelView) -> tuple[ColumnInfo, ...]: - if model.schema_entry is None: - return () - return tuple( - ColumnInfo(name=col.name, type=col.type or "") for col in model.schema_entry.columns - ) - - -def _get_unique_key(model: CompiledModelView) -> tuple[str, ...]: - """Extract unique_key from model config as a normalized tuple.""" - - raw: object | None = model.config.values.get("unique_key") - if isinstance(raw, str): - return (raw,) - if isinstance(raw, list): - return tuple(k for k in raw if isinstance(k, str)) - return () - - -def _get_check_columns(model: CompiledModelView) -> tuple[str, ...]: - """Extract check_columns from model config as a normalized tuple.""" - - raw: object | None = model.config.values.get("check_columns") - if isinstance(raw, str): - return (raw,) - if isinstance(raw, list): - return tuple(column for column in raw if isinstance(column, str)) - return () - - def _resolve_cursor_safety_configs( *, model: CompiledModelView, context: ModelPlanContext ) -> tuple[FutureCursorsConfig | None, StartCursorsConfig | None]: @@ -1111,59 +861,6 @@ def _resolve_cursor_safety_configs( ) -def _resolve_model_plan_config(*, model: CompiledModelView) -> _ModelPlanConfig: - return _ModelPlanConfig( - incremental_strategy=get_config_str(values=model.config.values, key="incremental_strategy"), - incremental_mode=get_config_str(values=model.config.values, key="incremental_mode"), - microbatch_strategy=get_config_str(values=model.config.values, key="microbatch_strategy"), - cursor_watermark_mode=get_config_str( - values=model.config.values, key="cursor_watermark_mode" - ), - batch_size=get_config_str(values=model.config.values, key="batch_size"), - snapshot_strategy=get_config_str(values=model.config.values, key="snapshot_strategy"), - updated_at_column=get_config_str(values=model.config.values, key="updated_at"), - check_columns=_get_check_columns(model), - observed_at_column=get_config_str(values=model.config.values, key="observed_at"), - historical_input=get_config_str(values=model.config.values, key="historical_input"), - valid_from_column=get_config_str(values=model.config.values, key="valid_from_column"), - valid_to_column=get_config_str(values=model.config.values, key="valid_to_column"), - initial_valid_from=get_config_str(values=model.config.values, key="initial_valid_from"), - invalidate_hard_deletes=get_config_bool( - values=model.config.values, key="invalidate_hard_deletes" - ) - or False, - snapshot_full_refresh=get_config_str( - values=model.config.values, key="snapshot_full_refresh" - ), - snapshot_schema_change=get_config_str( - values=model.config.values, key="snapshot_schema_change" - ), - ) - - -def _resolve_custom_materialization_config( - *, model: CompiledModelView, materialization_type: MaterializationType -) -> _CustomMaterializationConfig: - if materialization_type != MaterializationType.CUSTOM: - return _CustomMaterializationConfig(name=None, config={}, placeholders={}) - raw_name: object | None = model.config.values.get("materialized") - raw_config: object | None = model.config.values.get("config") - raw_placeholders: object | None = model.config.values.get("placeholders") - return _CustomMaterializationConfig( - name=raw_name if isinstance(raw_name, str) else None, - config=( - {str(key): value for key, value in raw_config.items()} - if isinstance(raw_config, dict) - else {} - ), - placeholders=( - {str(key): str(value) for key, value in raw_placeholders.items()} - if isinstance(raw_placeholders, dict) - else {} - ), - ) - - def _compute_microbatch_range( *, model: CompiledModelView, inputs: _MicrobatchRangeInputs ) -> CursorBounds | None: @@ -1269,36 +966,6 @@ def _compute_microbatch_range( ) -def resolve_microbatch_lookback( - *, model: CompiledModelView, effective_grain: str | None = None -) -> str | None: - """Resolve explicit or default lookback for one microbatch model.""" - - lookback: str | None = get_config_str(values=model.config.values, key="lookback") - if lookback is not None: - return lookback - raw_strategy: str | None = get_config_str( - values=model.config.values, key="incremental_strategy" - ) - strategy: IncrementalStrategy | None = ( - IncrementalStrategy(raw_strategy) if raw_strategy is not None else None - ) - if strategy not in _IDEMPOTENT_MICROBATCH_STRATEGIES: - return None - batch_size: str | None = get_config_str(values=model.config.values, key="batch_size") - if batch_size is None: - return None - resolved_grain: str | None = effective_grain or get_config_str( - values=model.config.values, key="cursor_grain" - ) - if resolved_grain is None: - return batch_size - return resolve_effective_microbatch_batch_size( - batch_size=batch_size, - effective_grain=resolved_grain, - ) - - def _compute_plan_cursor_bounds( *, model: CompiledModelView, @@ -1488,352 +1155,6 @@ def _append_maximum_start_warning( ) -def _render_table_type_ddl(*, ddl: str, table_type: str) -> str: - """Render permanent Snowflake table DDL from the adapter's transient default.""" - - return ddl.replace("TRANSIENT TABLE", "TABLE", 1) if table_type == TableType.PERMANENT else ddl - - -def _build_logical_ddl_from_adapter( - *, - adapter: BaseAdapter, - action: PlanAction, - resolved_sql: str, - destination: CompiledRelationLocation, - unique_key: tuple[str, ...], - merge_exclude_columns: tuple[str, ...], - warehouse_columns: tuple[ColumnInfo, ...], - cursor_column: str | None = None, - cursor_bounds: CursorBounds | None = None, - cursor_type: str | None = None, -) -> str: - """Generate logical DDL using adapter render methods.""" - - qualified_name: str = destination.qualified_name or destination.name - - if action == PlanAction.CREATE_VIEW: - return ";\n\n".join( - adapter.render_create_view_as(destination=qualified_name, sql=resolved_sql) - ) - - if action == PlanAction.CREATE_TABLE: - return ";\n\n".join( - adapter.render_create_table_as(destination=qualified_name, sql=resolved_sql) - ) - - if action == PlanAction.INCREMENTAL_APPEND: - return ";\n\n".join(adapter.render_append(destination=qualified_name, sql=resolved_sql)) - - if action == PlanAction.INCREMENTAL_DELETE_INSERT: - if cursor_column is not None and cursor_bounds is not None: - return ";\n\n".join( - adapter.render_delete_insert_cursor( - destination=qualified_name, - sql=resolved_sql, - cursor_column=cursor_column, - cursor_start=sentinel_to_token(sentinel=cursor_bounds.start), - cursor_end=sentinel_to_token(sentinel=cursor_bounds.end), - cursor_type=cursor_type, - ) - ) - return ";\n\n".join( - adapter.render_delete_insert( - destination=qualified_name, - sql=resolved_sql, - unique_key=unique_key, - ) - ) - - if action == PlanAction.INCREMENTAL_MERGE: - source_columns: tuple[str, ...] = tuple(col.name for col in warehouse_columns) - return ";\n\n".join( - adapter.render_merge( - destination=qualified_name, - sql=resolved_sql, - unique_key=unique_key, - source_columns=source_columns, - exclude_columns=merge_exclude_columns, - ) - ) - - return "" - - -def _build_cursor_input_relations( - *, - model: CompiledModelView, - adapter: BaseAdapter, - model_locations: dict[str, CompiledRelationLocation], - models_by_name: dict[str, CompiledModelView], - functions_by_name: dict[str, CompiledFunctionView] | None = None, - seed_locations: dict[str, CompiledRelationLocation], - source_map: dict[str, SourceEntry], - cursor_column: str | None, - runtime_cursor_producer_names: frozenset[str], -) -> tuple[CursorInputRelation, ...]: - """Build cursor-bearing input relation metadata for runtime range discovery.""" - - materialized: str | None = get_config_str(values=model.config.values, key="materialized") - if materialized != MaterializationType.INCREMENTAL or cursor_column is None: - return () - - cursor_watermark_inputs: dict[str, str] = resolve_cursor_input_roles( - model=model - ).watermark_inputs - relations: list[CursorInputRelation] = [] - input_name: str - input_cursor_column: str - for input_name, input_cursor_column in cursor_watermark_inputs.items(): - ref: CompileSqlReference = resolve_lineage_reference( - model=model, - input_name=input_name, - models_by_name=models_by_name, - functions_by_name=functions_by_name or {}, - ) - producer_model: CompiledModelView | None = ( - models_by_name.get(ref.ref_name) if ref.ref_kind == SqlReferenceKind.REF else None - ) - relation: str | None = _resolve_cursor_input_relation( - ref=ref, - adapter=adapter, - model_locations=model_locations, - seed_locations=seed_locations, - source_map=source_map, - ) - if relation is not None: - relations.append( - CursorInputRelation( - relation=relation, - cursor_column=input_cursor_column, - cursor_grain=( - CursorGrain(input_grain) - if ( - input_grain := _resolve_cursor_input_grain( - ref=ref, models_by_name=models_by_name - ) - ) - is not None - else None - ), - is_model_backed=( - ref.ref_kind == SqlReferenceKind.REF and ref.ref_name in model_locations - ), - is_runtime_produced=ref.ref_name in runtime_cursor_producer_names, - input_name=ref.ref_name, - input_kind=_cursor_input_kind(ref=ref, model_locations=model_locations), - terminal_cursor_start=( - parse( - raw=get_config_cursor_bound( - values=producer_model.config.values, key="cursor_start" - ), - cursor_type=get_config_str( - values=model.config.values, key="cursor_type" - ) - or CursorType.TIMESTAMP, - ) - if producer_model is not None - and get_config_cursor_bound( - values=producer_model.config.values, key="cursor_start" - ) - is not None - else None - ), - terminal_cursor_end=( - parse( - raw=get_config_cursor_bound( - values=producer_model.config.values, key="cursor_end" - ), - cursor_type=get_config_str( - values=model.config.values, key="cursor_type" - ) - or CursorType.TIMESTAMP, - ) - if producer_model is not None - and get_config_cursor_bound( - values=producer_model.config.values, key="cursor_end" - ) - is not None - else None - ), - ) - ) - return tuple(relations) - - -def validate_source_cursor_input_columns( - *, - model: CompiledModelView, - cursor_column: str | None, - models_by_name: dict[str, CompiledModelView] | None = None, - functions_by_name: dict[str, CompiledFunctionView] | None = None, - source_map: dict[str, SourceEntry] | None = None, - source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]], -) -> None: - """Validate cursor input columns using contracts before source warehouse metadata.""" - - materialized: str | None = get_config_str(values=model.config.values, key="materialized") - if materialized != MaterializationType.INCREMENTAL or cursor_column is None: - return - - cursor_roles: CursorInputRoles = resolve_cursor_input_roles(model=model) - effective_models_by_name: dict[str, CompiledModelView] = models_by_name or {} - effective_source_map: dict[str, SourceEntry] = source_map or {} - authored_inputs: tuple[tuple[str, str], ...] = tuple( - cursor_roles.filter_inputs.items() - ) + tuple(cursor_roles.watermark_inputs.items()) - input_name: str - input_cursor_column: str - for input_name, input_cursor_column in authored_inputs: - ref: CompileSqlReference = resolve_lineage_reference( - model=model, - input_name=input_name, - models_by_name=effective_models_by_name, - functions_by_name=functions_by_name or {}, - ) - if ref.ref_kind == SqlReferenceKind.REF: - upstream_model: CompiledModelView | None = effective_models_by_name.get(ref.ref_name) - if upstream_model is None: - continue - has_enforced_contract: bool = ( - upstream_model.config.values.get("contract") == ContractPolicy.ENFORCED - ) - if not has_enforced_contract: - continue - declared_names: tuple[str, ...] = _model_declared_column_names(upstream_model) - if input_cursor_column.lower() in {name.lower() for name in declared_names}: - continue - declared_display: str = ", ".join(declared_names) or "none" - raise PlannerInputError( - f"model '{model.name}': {CURSOR_INPUTS_CONFIG_KEY} " - f"references model '{ref.ref_name}' " - f"column '{input_cursor_column}', but that model contract does not expose " - f"the column. Known output columns: {declared_display}", - code="S302", - ) - if ref.ref_kind != SqlReferenceKind.SOURCE: - continue - source_entry: SourceEntry | None = effective_source_map.get(ref.ref_name) - if source_entry is not None and source_entry.contract == ContractPolicy.ENFORCED: - declared_names = tuple(column.name for column in source_entry.columns) - if input_cursor_column.lower() in {name.lower() for name in declared_names}: - continue - declared_display = ", ".join(declared_names) or "none" - raise PlannerInputError( - f"model '{model.name}': {CURSOR_INPUTS_CONFIG_KEY} " - f"references source '{ref.ref_name}' " - f"column '{input_cursor_column}', but that source contract does not expose " - f"the column. Declared contract columns: {declared_display}", - code="S302", - ) - known_columns: tuple[ColumnInfo, ...] | None = source_warehouse_columns.get(ref.ref_name) - if known_columns is None: - continue - known_column_names: frozenset[str] = frozenset(col.name.lower() for col in known_columns) - if input_cursor_column.lower() in known_column_names: - continue - known_display: str = ", ".join(col.name for col in known_columns) or "none" - raise PlannerInputError( - f"model '{model.name}': {CURSOR_INPUTS_CONFIG_KEY} references source '{ref.ref_name}' " - f"column '{input_cursor_column}', but that source does not expose the column. " - f"Known source columns: {known_display}", - code="S302", - ) - - -def _model_declared_column_names(model: CompiledModelView) -> tuple[str, ...]: - if model.schema_entry is None: - return () - return tuple(column.name for column in model.schema_entry.columns) - - -def _has_runtime_owned_cursor_watermarks( - cursor_input_relations: tuple[CursorInputRelation, ...], -) -> bool: - """Return whether any cursor input is produced by this invocation.""" - - return any(relation.is_runtime_owned for relation in cursor_input_relations) - - -def _runtime_cursor_producer_names( - *, scope: PlannerScope, source_map: dict[str, SourceEntry] -) -> frozenset[str]: - """Return selected relations whose contents are produced during this invocation.""" - - names: set[str] = set() - key: CompiledObjectKey - for key in scope.selected_keys: - if key.resource_type in {CompiledResourceType.MODEL, CompiledResourceType.SEED}: - names.add(key.name) - elif key.resource_type == CompiledResourceType.SOURCE: - source: SourceEntry | None = source_map.get(key.name) - if source is not None and source.loader is not None: - names.add(key.name) - return frozenset(names) - - -def _build_runtime_placeholder_bounds() -> CursorBounds: - """Return placeholder cursor bounds for runtime-owned models.""" - - return CursorBounds(start=BoundSentinel.START, end=BoundSentinel.END) - - -def _resolve_cursor_input_relation( - *, - ref: CompileSqlReference, - adapter: BaseAdapter, - model_locations: dict[str, CompiledRelationLocation], - seed_locations: dict[str, CompiledRelationLocation], - source_map: dict[str, SourceEntry], -) -> str | None: - """Resolve one cursor input reference to a qualified relation name.""" - - if ref.ref_kind == SqlReferenceKind.REF: - target: CompiledRelationLocation | None = model_locations.get(ref.ref_name) - if target is None: - target = seed_locations.get(ref.ref_name) - return target.qualified_name if target is not None else None - if ref.ref_kind == SqlReferenceKind.SEED: - seed_target: CompiledRelationLocation | None = seed_locations.get(ref.ref_name) - return seed_target.qualified_name if seed_target is not None else None - if ref.ref_kind == SqlReferenceKind.SOURCE: - source: SourceEntry | None = source_map.get(ref.ref_name) - if source is None: - return None - return render_source_relation(entry=source, adapter=adapter) - return None - - -def _cursor_input_kind( - *, ref: CompileSqlReference, model_locations: dict[str, CompiledRelationLocation] -) -> GraphResourceKind | None: - """Return the resource kind a resolved cursor input reference reads.""" - - if ref.ref_kind == SqlReferenceKind.REF: - return ( - GraphResourceKind.MODEL if ref.ref_name in model_locations else GraphResourceKind.SEED - ) - if ref.ref_kind == SqlReferenceKind.SEED: - return GraphResourceKind.SEED - if ref.ref_kind == SqlReferenceKind.SOURCE: - return GraphResourceKind.SOURCE - return None - - -def _resolve_cursor_input_grain( - *, - ref: CompileSqlReference, - models_by_name: dict[str, CompiledModelView], -) -> str | None: - """Resolve timestamp grain metadata for a model-backed cursor input relation.""" - - if ref.ref_kind != SqlReferenceKind.REF: - return None - upstream_model: CompiledModelView | None = models_by_name.get(ref.ref_name) - if upstream_model is None: - return None - return get_config_str(values=upstream_model.config.values, key="cursor_grain") - - def scope_overlaps( *, scope_deps: tuple[CompiledObjectKey, ...], diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py index 6c7cb28659..850550e264 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py @@ -6,6 +6,7 @@ build_relation_fixture_context, build_selected_audit_entries, build_source_load_entries, + extract_seed_columns, plan_sql_tests, upstream_loader_dependency_names, ) @@ -28,7 +29,6 @@ from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.output.plan_entry import ( build_model_materializations, - extract_seed_columns, scope_overlaps, ) from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_function_sql diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py index 351663804b..1a38e56bda 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import resolve_microbatch_lookback from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner._helpers.output.plan_entry import resolve_microbatch_lookback from sqlbuild.compiler.planner.types import IncrementalStrategy from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( MicrobatchLookbackTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py index 2137d7e8b7..f89f17f262 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py @@ -6,6 +6,11 @@ import pytest +from sqlbuild._native import ( + build_cursor_input_relations, + runtime_cursor_producer_names, + validate_source_cursor_input_columns, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -19,12 +24,7 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner._helpers.output.plan_entry import ( - _build_cursor_input_relations, - _runtime_cursor_producer_names, - build_planner_relations_context, - validate_source_cursor_input_columns, -) +from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context from sqlbuild.compiler.planner._helpers.warehouse.source_columns import gather_source_columns from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( @@ -447,7 +447,7 @@ def test_given_view_filter_and_transitive_watermark_when_planning_then_relation_ ) view_model: CompiledModelView = models_by_name["orders_view"] - relations: tuple[CursorInputRelation, ...] = _build_cursor_input_relations( + relations: tuple[CursorInputRelation, ...] = build_cursor_input_relations( model=model, adapter=DuckDbAdapter(), model_locations={view_model.name: view_model.destination}, @@ -608,7 +608,7 @@ def test_given_table_function_and_transitive_watermark_when_planning_then_source source_map={"raw_orders": SourceEntry(name="raw_orders", schema="raw")}, source_warehouse_columns={"raw_orders": (ColumnInfo(name="loaded_at", type="TIMESTAMP"),)}, ) - relations: tuple[CursorInputRelation, ...] = _build_cursor_input_relations( + relations: tuple[CursorInputRelation, ...] = build_cursor_input_relations( model=model, adapter=DuckDbAdapter(), model_locations={name: upstream.destination for name, upstream in models_by_name.items()}, @@ -716,7 +716,7 @@ def test_given_selected_relations_when_classifying_runtime_watermarks_then_only_ execution_order=(), ) - names: frozenset[str] = _runtime_cursor_producer_names( + names: frozenset[str] = runtime_cursor_producer_names( scope=scope, source_map=test_case.source_map, ) From 73f81dc34ea5013009884ac52b6db5fa269bbd51 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 01:42:30 +0100 Subject: [PATCH 20/22] refactor: apply microbatch limits to plan entries natively Move configured microbatch limit precedence and per-entry limit application into native code, and resolve the effective timestamp grain for default lookbacks inside native model entry assembly. The Python limit helpers are deleted and their tests call the native functions. --- .../src/plan_display/constants.rs | 1 + .../main/effective_timestamp_grain.rs | 42 +++ .../src/plan_display/main/mod.rs | 1 + .../sqlbuild-core/src/plan_display/models.rs | 9 + .../src/plan_display/tests/helpers.rs | 19 +- .../plan_display/tests/test_plan_display.rs | 59 +++- .../src/plan_display/tests/test_types.rs | 8 +- .../_helpers/compiled_project/records.rs | 3 +- .../planner_strategy/microbatch_limits.rs | 266 ++++++++++++++++++ .../_helpers/planner_strategy/microbatches.rs | 50 +--- .../bindings/_helpers/planner_strategy/mod.rs | 1 + .../planner_strategy/model_entries.rs | 29 +- .../_helpers/planner_strategy/registration.rs | 8 + python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 23 +- .../planner/_helpers/output/plan_entry.py | 160 +---------- .../planner/_helpers/test_microbatch_limit.py | 16 +- 17 files changed, 459 insertions(+), 238 deletions(-) create mode 100644 crates/sqlbuild-core/src/plan_display/main/effective_timestamp_grain.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs diff --git a/crates/sqlbuild-core/src/plan_display/constants.rs b/crates/sqlbuild-core/src/plan_display/constants.rs index ad1ed55a95..0445540d36 100644 --- a/crates/sqlbuild-core/src/plan_display/constants.rs +++ b/crates/sqlbuild-core/src/plan_display/constants.rs @@ -32,3 +32,4 @@ pub(crate) const MIGRATION_RENAMED: &str = "renamed"; pub(crate) const MIGRATION_MANUAL: &str = "manual"; pub(crate) const MIGRATION_AUTOMATIC: &str = "automatic"; pub(crate) const MIGRATION_INCOMPATIBLE: &str = "incompatible"; +pub(crate) const WATERMARK_STRATEGY: &str = "watermark"; diff --git a/crates/sqlbuild-core/src/plan_display/main/effective_timestamp_grain.rs b/crates/sqlbuild-core/src/plan_display/main/effective_timestamp_grain.rs new file mode 100644 index 0000000000..7fa9269f4f --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/effective_timestamp_grain.rs @@ -0,0 +1,42 @@ +//! The coarsest timestamp grain participating in cursor replay. + +use crate::plan_display::constants::{ + DEFAULT_CURSOR_GRAIN, GRAIN_BATCH_SIZE, TIMESTAMP_CURSOR, WATERMARK_STRATEGY, +}; +use crate::plan_display::errors::UnknownCursorGrain; +use crate::plan_display::models::TimestampGrainFacts; + +/// The grain's position from finest to coarsest; an unknown grain is an error. +fn grain_rank(grain: &str) -> Result { + GRAIN_BATCH_SIZE + .iter() + .position(|(known, _)| *known == grain) + .ok_or_else(|| UnknownCursorGrain(grain.to_owned())) +} + +/// The declared grain or `second`, an empty grain counting as absent. +fn grain_or_default(grain: Option<&String>) -> String { + grain + .filter(|grain| !grain.is_empty()) + .map_or_else(|| DEFAULT_CURSOR_GRAIN.to_owned(), Clone::clone) +} + +/// `None` for non-timestamp cursors; the declared grain for watermark replay; else the coarsest declared or input grain. +pub fn effective_timestamp_grain( + facts: &TimestampGrainFacts, +) -> Result, UnknownCursorGrain> { + if facts.cursor_type.as_deref() != Some(TIMESTAMP_CURSOR) { + return Ok(None); + } + let mut effective: String = grain_or_default(facts.downstream_grain.as_ref()); + if facts.microbatch_strategy.as_deref() == Some(WATERMARK_STRATEGY) { + return Ok(Some(effective)); + } + for input_grain in &facts.input_grains { + let candidate: String = grain_or_default(input_grain.as_ref()); + if grain_rank(&candidate)? > grain_rank(&effective)? { + effective = candidate; + } + } + Ok(Some(effective)) +} diff --git a/crates/sqlbuild-core/src/plan_display/main/mod.rs b/crates/sqlbuild-core/src/plan_display/main/mod.rs index a1921ecd30..969c1db466 100644 --- a/crates/sqlbuild-core/src/plan_display/main/mod.rs +++ b/crates/sqlbuild-core/src/plan_display/main/mod.rs @@ -1,6 +1,7 @@ pub mod cursor_bound_display; pub mod effective_microbatch_batch_size; pub mod format_model_migrations; +pub mod effective_timestamp_grain; pub mod format_no_input_rows; pub mod format_waiting_on_empty_inputs; pub mod model_execution_annotation; diff --git a/crates/sqlbuild-core/src/plan_display/models.rs b/crates/sqlbuild-core/src/plan_display/models.rs index e1072e7392..ea79ac7945 100644 --- a/crates/sqlbuild-core/src/plan_display/models.rs +++ b/crates/sqlbuild-core/src/plan_display/models.rs @@ -23,3 +23,12 @@ pub struct ParsedBound { /// Hour, minute, second and microsecond are all zero. pub midnight: bool, } + +/// A cursor's type, strategy, declared grain and its inputs' grains, as the replay grain reads them. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TimestampGrainFacts { + pub cursor_type: Option, + pub microbatch_strategy: Option, + pub downstream_grain: Option, + pub input_grains: Vec>, +} diff --git a/crates/sqlbuild-core/src/plan_display/tests/helpers.rs b/crates/sqlbuild-core/src/plan_display/tests/helpers.rs index 047946bc2d..ffb4555267 100644 --- a/crates/sqlbuild-core/src/plan_display/tests/helpers.rs +++ b/crates/sqlbuild-core/src/plan_display/tests/helpers.rs @@ -1,4 +1,4 @@ -use crate::plan_display::models::ModelLabelFacts; +use crate::plan_display::models::{ModelLabelFacts, TimestampGrainFacts}; /// A planned model with only its materialization and action. pub(super) fn facts(materialization_type: &str, action: &str) -> ModelLabelFacts { @@ -8,3 +8,20 @@ pub(super) fn facts(materialization_type: &str, action: &str) -> ModelLabelFacts ..ModelLabelFacts::default() } } + +/// A timestamp cursor with the strategy, declared grain and input grains given. +pub(super) fn timestamp_grains( + microbatch_strategy: Option<&str>, + downstream_grain: Option<&str>, + input_grains: &[Option<&str>], +) -> TimestampGrainFacts { + TimestampGrainFacts { + cursor_type: Some("timestamp".to_owned()), + microbatch_strategy: microbatch_strategy.map(str::to_owned), + downstream_grain: downstream_grain.map(str::to_owned), + input_grains: input_grains + .iter() + .map(|grain| grain.map(str::to_owned)) + .collect(), + } +} diff --git a/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs b/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs index 0e8ed49570..5e41e9cb63 100644 --- a/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs +++ b/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs @@ -1,14 +1,15 @@ use crate::plan_display::main::cursor_bound_display::cursor_bound_display; use crate::plan_display::main::effective_microbatch_batch_size::effective_microbatch_batch_size; +use crate::plan_display::main::effective_timestamp_grain::effective_timestamp_grain; use crate::plan_display::main::format_no_input_rows::format_no_input_rows; use crate::plan_display::main::format_waiting_on_empty_inputs::format_waiting_on_empty_inputs; use crate::plan_display::main::model_execution_annotation::model_execution_annotation; use crate::plan_display::main::model_materialization_label::model_materialization_label; use crate::plan_display::main::model_resource_type::model_resource_type; -use crate::plan_display::models::{ModelLabelFacts, ParsedBound}; -use crate::plan_display::tests::helpers::facts; +use crate::plan_display::models::{ModelLabelFacts, ParsedBound, TimestampGrainFacts}; +use crate::plan_display::tests::helpers::{facts, timestamp_grains}; use crate::plan_display::tests::test_types::{ - BatchSizeTestCase, BoundTestCase, LabelTestCase, NamesTestCase, + BatchSizeTestCase, BoundTestCase, LabelTestCase, NamesTestCase, TimestampGrainTestCase, }; #[test] @@ -207,3 +208,55 @@ fn given_batch_size_when_resolving_effective_size_then_uses_grain_unit() { ); } } + +#[test] +fn given_cursor_grains_when_resolving_replay_grain_then_coarsest_timestamp_grain_wins() { + let test_cases = [ + TimestampGrainTestCase { + description: "an integer cursor has no grain", + facts: TimestampGrainFacts { + cursor_type: Some("integer".to_owned()), + downstream_grain: Some("day".to_owned()), + ..TimestampGrainFacts::default() + }, + expected_grain: Ok(None), + }, + TimestampGrainTestCase { + description: "no declared grain defaults to second", + facts: timestamp_grains(None, None, &[]), + expected_grain: Ok(Some("second")), + }, + TimestampGrainTestCase { + description: "a coarser input grain wins", + facts: timestamp_grains(None, Some("day"), &[Some("month"), None, Some("hour")]), + expected_grain: Ok(Some("month")), + }, + TimestampGrainTestCase { + description: "an empty declared grain counts as second", + facts: timestamp_grains(None, Some(""), &[Some("minute")]), + expected_grain: Ok(Some("minute")), + }, + TimestampGrainTestCase { + description: "watermark replay keeps the declared grain", + facts: timestamp_grains(Some("watermark"), Some("hour"), &[Some("year")]), + expected_grain: Ok(Some("hour")), + }, + TimestampGrainTestCase { + description: "an unknown input grain is an error", + facts: timestamp_grains(None, Some("day"), &[Some("week")]), + expected_grain: Err("week"), + }, + ]; + + for test_case in test_cases { + assert_eq!( + effective_timestamp_grain(&test_case.facts).map_err(|error| error.0), + test_case + .expected_grain + .map(|grain| grain.map(str::to_owned)) + .map_err(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/plan_display/tests/test_types.rs b/crates/sqlbuild-core/src/plan_display/tests/test_types.rs index 08cbf72a61..d924f3052d 100644 --- a/crates/sqlbuild-core/src/plan_display/tests/test_types.rs +++ b/crates/sqlbuild-core/src/plan_display/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::plan_display::models::{ModelLabelFacts, ParsedBound}; +use crate::plan_display::models::{ModelLabelFacts, ParsedBound, TimestampGrainFacts}; pub(super) struct LabelTestCase { pub(super) description: &'static str, @@ -29,3 +29,9 @@ pub(super) struct BatchSizeTestCase { pub(super) grain: &'static str, pub(super) expected_size: Result<&'static str, &'static str>, } + +pub(super) struct TimestampGrainTestCase { + pub(super) description: &'static str, + pub(super) facts: TimestampGrainFacts, + pub(super) expected_grain: Result, &'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs index 86cb146d1e..272d1aecb9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs @@ -20,7 +20,8 @@ const DECIMAL_KIND: &str = "decimal"; const COLLECTION_KINDS: [&str; 2] = ["list", "set"]; const SCALAR_KINDS: [&str; 5] = ["string", "integer", "boolean", "float", "null"]; -fn replace<'py>( +/// `dataclasses.replace(value, **changes)`. +pub(crate) fn replace<'py>( value: &Bound<'py, PyAny>, changes: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs new file mode 100644 index 0000000000..a5a0018752 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs @@ -0,0 +1,266 @@ +//! Microbatch limits applied to planned model entries in CLI, project and model precedence. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction}; + +use sqlbuild_analysis::planner_strategy::constants::MICROBATCH_LIMIT_WARNING_CODE; +use sqlbuild_analysis::planner_strategy::main::microbatch_limit_warning::microbatch_limit_warning; +use sqlbuild_core::plan_display::errors::UnknownCursorGrain; +use sqlbuild_core::plan_display::main::effective_timestamp_grain::effective_timestamp_grain; +use sqlbuild_core::plan_display::models::TimestampGrainFacts; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compiled_project::records::replace; +use crate::bindings::_helpers::plan_output::display::resolve_effective_microbatch_batch_size; +use crate::bindings::_helpers::planner_changes::results::planner_enum; +use crate::bindings::_helpers::planner_strategy::constants::{ + PLAN_MODELS_MODULE, PLANNER_TYPES_MODULE, +}; +use crate::bindings::_helpers::planner_strategy::microbatches::{ + CAP_ACTIONS, CONTRACT_TYPES_MODULE, capped_microbatch_bounds, count_microbatches, +}; +use crate::bindings::_helpers::planner_strategy::values::{ + COMPILE_EXCEPTIONS_MODULE, input_error, optional_string, text, +}; + +const ERROR_ACTION: &str = "error"; +const SKIP_ACTION: &str = "skip"; +const CREATE_TABLE_ACTION: &str = "create_table"; +const ROLLING_WINDOW_STRATEGY: &str = "rolling_window"; +const FULL_REFRESH_REASON: &str = "full_refresh"; + +/// A planned entry and the warning its limit raised. +type LimitedEntry<'py> = (Bound<'py, PyAny>, Option>); + +/// `CursorGrain(grain)`'s error for a grain outside the known grains. +pub(crate) fn unknown_grain(py: Python<'_>, error: UnknownCursorGrain) -> PyErr { + imported(py, PLANNER_TYPES_MODULE, "CursorGrain") + .and_then(|class| class.call1((error.0,))) + .err() + .unwrap_or_else(|| pyo3::exceptions::PyValueError::new_err("unknown cursor grain")) +} + +/// The coarsest timestamp grain of a cursor and the `cursor_grain` of its input relations. +pub(crate) fn replay_grain( + (cursor_type, microbatch_strategy, downstream_grain): ( + Option, + Option, + Option, + ), + relations: &Bound<'_, PyAny>, +) -> PyResult> { + let mut input_grains: Vec> = Vec::new(); + for relation in relations.try_iter()? { + input_grains.push(optional_string(&relation?.getattr("cursor_grain")?)?); + } + effective_timestamp_grain(&TimestampGrainFacts { + cursor_type, + microbatch_strategy, + downstream_grain, + input_grains, + }) + .map_err(|error| unknown_grain(relations.py(), error)) +} + +/// The entry's effective batch size and its batch count over the planned range. +fn batch_count(entry: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = entry.py(); + let range: Bound<'_, PyAny> = entry.getattr("microbatch_range")?; + let batch_size: Bound<'_, PyAny> = entry.getattr("batch_size")?; + let cursor_type: Bound<'_, PyAny> = entry.getattr("cursor_type")?; + if range.is_none() || batch_size.is_none() || cursor_type.is_none() { + return Ok(None); + } + let (batch_size, cursor_type): (String, String) = (text(&batch_size)?, text(&cursor_type)?); + let grain: Option = replay_grain( + ( + Some(cursor_type.clone()), + optional_string(&entry.getattr("microbatch_strategy")?)?, + optional_string(&entry.getattr("cursor_grain")?)?, + ), + &entry.getattr("cursor_input_relations")?, + )?; + let effective_size: String = match grain { + Some(grain) => resolve_effective_microbatch_batch_size(py, &batch_size, &grain)?, + None => batch_size, + }; + let creates: bool = entry.getattr("action")?.eq(CREATE_TABLE_ACTION)?; + let count: i128 = count_microbatches(&range, &effective_size, &cursor_type, creates)?; + Ok(Some((effective_size, count))) +} + +/// The entry with one batch limit applied, and the warning when a non-error limit is exceeded. +fn limited<'py>( + entry: &Bound<'py, PyAny>, + max_batches: Bound<'py, PyAny>, + action: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = entry.py(); + if entry.getattr("action")?.eq(SKIP_ACTION)? { + return Ok((entry.clone(), None)); + } + let action_text: String = text(&action)?; + let maximum: Option = if max_batches.is_none() { + None + } else { + Some(max_batches.extract()?) + }; + let counted: Option<(String, i128)> = batch_count(entry)?; + let name: Bound<'py, PyAny> = entry.getattr("name")?; + let cap: bool = CAP_ACTIONS.contains(&action_text.as_str()); + let warning: Option = match &counted { + Some((_, count)) => { + microbatch_limit_warning(&text(&name)?, (maximum, *count), (&action_text, cap)) + } + None => None, + }; + let mut range: Bound<'py, PyAny> = entry.getattr("microbatch_range")?; + if let (Some(_), Some((size, _)), Some(maximum), true) = + (&warning, &counted, maximum, cap && !range.is_none()) + { + let cursor_type: String = + optional_string(&entry.getattr("cursor_type")?)?.unwrap_or_default(); + range = capped_microbatch_bounds(range, (size, &cursor_type), maximum, &action_text)?; + } + let none: Bound<'py, PyAny> = py.None().into_bound(py); + let limited_entry: Bound<'py, PyAny> = replace( + entry, + &[ + ("microbatch_range", range), + ( + "microbatch_limit_count", + counted + .as_ref() + .map(|(_, count)| *count) + .into_pyobject(py)? + .into_any(), + ), + ( + "microbatch_limit_action", + if max_batches.is_none() { none } else { action }, + ), + ("microbatch_limit", max_batches), + ( + "microbatch_limit_warning", + warning.clone().into_pyobject(py)?.into_any(), + ), + ], + )?; + let Some(message) = warning else { + return Ok((limited_entry, None)); + }; + if action_text == ERROR_ACTION { + return Err(input_error( + py, + (COMPILE_EXCEPTIONS_MODULE, "CompileInputError"), + &message, + None, + )); + } + let plan_warning: Bound<'py, PyAny> = call_with( + &imported(py, PLAN_MODELS_MODULE, "PlanWarning")?, + &[ + ("model_name", name), + ("severity", planner_enum(py, "WarningSeverity", "warning")?), + ( + "code", + PyString::new(py, MICROBATCH_LIMIT_WARNING_CODE).into_any(), + ), + ("message", PyString::new(py, &message).into_any()), + ], + )?; + Ok((limited_entry, Some(plan_warning))) +} + +/// Apply one batch limit to a planned microbatch entry, failing when an `error` limit is exceeded. +#[pyfunction] +#[pyo3(signature = (*, entry, max_batches, action))] +pub(crate) fn apply_microbatch_limit<'py>( + entry: &Bound<'py, PyAny>, + max_batches: Bound<'py, PyAny>, + action: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = entry.py(); + let (limited_entry, warning) = limited(entry, max_batches, action)?; + PyTuple::new( + py, + [ + limited_entry, + warning.unwrap_or_else(|| py.None().into_bound(py)), + ], + ) +} + +/// `(entry, warnings)` with the limit outcome's warning, if any. +fn with_warnings<'py>( + (entry, warning): LimitedEntry<'py>, + mut warnings: Vec>, +) -> PyResult> { + let py: Python<'py> = entry.py(); + warnings.extend(warning); + PyTuple::new(py, [entry, PyTuple::new(py, warnings)?.into_any()]) +} + +/// The default limit: none for an incremental rolling window, else the model's limit or the configured ceiling. +fn default_limit<'py>( + entry: &Bound<'py, PyAny>, + configured: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = entry.py(); + if entry + .getattr("microbatch_strategy")? + .eq(ROLLING_WINDOW_STRATEGY)? + && !entry.getattr("reason")?.eq(FULL_REFRESH_REASON)? + && entry.getattr("start_cursor_override")?.is_none() + && entry.getattr("end_cursor_override")?.is_none() + { + return Ok(py.None().into_bound(py)); + } + let model_limit: Bound<'py, PyAny> = entry.getattr("microbatch_limit")?; + Ok(if model_limit.is_truthy()? { + model_limit + } else { + configured + }) +} + +/// Apply CLI, project, and model batch limits in their documented precedence order. +#[pyfunction] +#[pyo3(signature = (*, entry, inputs))] +pub(crate) fn apply_configured_microbatch_limits<'py>( + entry: &Bound<'py, PyAny>, + inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = entry.py(); + let configured: Bound<'py, PyAny> = inputs.getattr("max_microbatches")?; + let configured_action: Bound<'py, PyAny> = inputs.getattr("microbatch_limit_action")?; + if inputs + .getattr("max_microbatches_is_override")? + .is_truthy()? + { + let error: Bound<'py, PyAny> = + imported(py, CONTRACT_TYPES_MODULE, "MicrobatchLimitAction")?.getattr("ERROR")?; + return with_warnings(limited(entry, configured, error)?, Vec::new()); + } + let declared: Bound<'py, PyAny> = entry.getattr("declared_microbatch_limit_action")?; + if declared.is_none() { + let maximum: Bound<'py, PyAny> = default_limit(entry, configured)?; + return with_warnings(limited(entry, maximum, configured_action)?, Vec::new()); + } + let mut warnings: Vec> = Vec::new(); + let mut entry: Bound<'py, PyAny> = entry.clone(); + if !configured.is_none() { + let (_, safety_warning) = limited(&entry, configured.clone(), configured_action.clone())?; + entry = replace( + &entry, + &[ + ("microbatch_safety_limit", configured), + ("microbatch_safety_limit_action", configured_action), + ], + )?; + warnings.extend(safety_warning); + } + let model_limit: Bound<'py, PyAny> = entry.getattr("microbatch_limit")?; + with_warnings(limited(&entry, model_limit, declared)?, warnings) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs index 916e4d3de5..ea42c15438 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs @@ -18,16 +18,14 @@ use sqlbuild_analysis::planner_strategy::types::Timeline; use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; use sqlbuild_cursor::algebra::models::{DurationDigits, DurationSpan}; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::cursor::durations::shifted_moment; use crate::bindings::_helpers::plan_output::display::aligned_timestamp_bounds; use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; use crate::bindings::_helpers::planner_strategy::values::text; const CURSOR_MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; -const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +pub(crate) const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; const INTEGER_CURSOR: &str = "integer"; const TIMESTAMP_CURSOR: &str = "timestamp"; const MONTHS_PER_YEAR: i64 = 12; @@ -131,9 +129,7 @@ fn timeline_bounds<'py>( } /// Batches in one concrete cursor range. -#[pyfunction] -#[pyo3(signature = (*, bounds, batch_size, cursor_type, equal_bounds_are_batch = false))] -fn count_microbatches( +pub(crate) fn count_microbatches( bounds: &Bound<'_, PyAny>, batch_size: &str, cursor_type: &str, @@ -177,33 +173,15 @@ fn cursor_value<'py>( ) } -/// One concrete range capped to its earliest or latest batch windows. -#[pyfunction] -#[pyo3(signature = (**options))] -fn cap_microbatch_bounds<'py>( - py: Python<'py>, - options: Option>, +/// One concrete range capped to `max_batches` batch windows from the end the action keeps. +pub(crate) fn capped_microbatch_bounds<'py>( + bounds: Bound<'py, PyAny>, + (batch_size, cursor_type): (&str, &str), + max_batches: i128, + action: &str, ) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "cap_microbatch_bounds", - options, - ( - &[ - "bounds", - "batch_size", - "cursor_type", - "max_batches", - "action", - ], - &[], - ), - )?; - let bounds: Bound<'py, PyAny> = arguments.value("bounds")?; - let batch_size: String = text(&arguments.value("batch_size")?)?; - let cursor_type: String = text(&arguments.value("cursor_type")?)?; - let max_batches: i128 = arguments.value("max_batches")?.extract()?; - let direction: CapDirection = match text(&arguments.value("action")?)?.as_str() { + let py: Python<'py> = bounds.py(); + let direction: CapDirection = match action { CAP_FROM_START => CapDirection::FromStart, CAP_FROM_END => CapDirection::FromEnd, _ => return Ok(bounds), @@ -213,7 +191,7 @@ fn cap_microbatch_bounds<'py>( let Some(range) = integer_bounds(&bounds)? else { return Ok(bounds); }; - let Some(capped) = cap_integer_bounds(range, &batch_size, max_batches, direction) else { + let Some(capped) = cap_integer_bounds(range, batch_size, max_batches, direction) else { return Ok(bounds); }; match capped { @@ -232,7 +210,7 @@ fn cap_microbatch_bounds<'py>( if cursor_type != TIMESTAMP_CURSOR { return Ok(bounds); } - let Some((timeline, range)) = timeline_bounds(&bounds, &batch_size)? else { + let Some((timeline, range)) = timeline_bounds(&bounds, batch_size)? else { return Ok(bounds); }; match cap_timeline_bounds(&timeline, range, max_batches, direction)? { @@ -317,8 +295,6 @@ fn resolve_microbatch_limit_config<'py>( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(count_microbatches, module)?)?; - module.add_function(wrap_pyfunction!(cap_microbatch_bounds, module)?)?; module.add_function(wrap_pyfunction!(microbatch_limit_warning, module)?)?; module.add_function(wrap_pyfunction!(resolve_microbatch_limit_config, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs index 2c34e49ec2..ca7401805f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs @@ -13,6 +13,7 @@ pub(crate) mod fixture_sql; pub(crate) mod fixture_types; pub(crate) mod fixture_values; pub(crate) mod lineage; +pub(crate) mod microbatch_limits; pub(crate) mod microbatches; pub(crate) mod model_entries; pub(crate) mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index 7763ae7fc3..8ed737996f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -1,7 +1,7 @@ //! Assemble model plan dataclasses from resolved plan decisions and authored model config. use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction}; @@ -20,9 +20,10 @@ use crate::bindings::_helpers::planner_strategy::constants::{ ENFORCED_CONTRACT, MICROBATCH_MODE, NATIVE_MODULE, PLAN_MODELS_MODULE, PLANNER_TYPES_MODULE, }; use crate::bindings::_helpers::planner_strategy::cursor_inputs::model_values; +use crate::bindings::_helpers::planner_strategy::microbatch_limits::{replay_grain, unknown_grain}; use crate::bindings::_helpers::planner_strategy::microbatches::microbatch_limit_config; use crate::bindings::_helpers::planner_strategy::values::{ - PLANNER_EXCEPTIONS_MODULE, input_error, optional_string, text, + PLANNER_EXCEPTIONS_MODULE, input_error, text, }; /// Model config keys copied to the plan entry as optional strings, with their entry fields. @@ -182,14 +183,6 @@ fn type_enforcement<'py>(model: &Bound<'py, PyAny>) -> PyResult, grain: &str) -> PyErr { - imported(py, PLANNER_TYPES_MODULE, "CursorGrain") - .and_then(|class| class.call1((grain,))) - .err() - .unwrap_or_else(|| PyValueError::new_err("unknown cursor grain")) -} - /// The explicit or default lookback of a microbatch model at the effective grain. fn lookback_of( model: &Bound<'_, PyAny>, @@ -210,7 +203,7 @@ fn lookback_of( None => read("cursor_grain")?, }, }; - microbatch_lookback(&config).map_err(|error| unknown_grain(model.py(), &error.0)) + microbatch_lookback(&config).map_err(|error| unknown_grain(model.py(), error)) } /// Resolve explicit or default lookback for one microbatch model. @@ -416,7 +409,19 @@ fn config_fields<'py>( None => false, }; let lookback: Option = if microbatch { - let grain: Option = optional_string(&request.get("effective_grain")?)?; + let read = |key: &str| -> PyResult> { + config_str(values, key)? + .map(|value| text(&value)) + .transpose() + }; + let grain: Option = replay_grain( + ( + read("cursor_type")?, + read("microbatch_strategy")?, + read("cursor_grain")?, + ), + &request.get("cursor_input_relations")?, + )?; lookback_of(&model, grain)? } else { config_str(values, "lookback")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs index acdffe6e60..c5bdbebbc5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -42,6 +42,14 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::model_entries::extract_seed_columns, module )?)?; + module.add_function(wrap_pyfunction!( + super::microbatch_limits::apply_microbatch_limit, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::microbatch_limits::apply_configured_microbatch_limits, + module + )?)?; audits::register(module)?; cursor_types::register(module)?; fixture_completion::register(module)?; diff --git a/python-boundary.toml b/python-boundary.toml index 00794ad6d4..3449eeb06b 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 29633 +remaining_lines = 27784 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 18b41a5538..7a86e1981f 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -245,6 +245,7 @@ from sqlbuild.compiler.planner.models import ( ParsedScenarioArtifactName, ParsedSelector, PathSelector, + PlanEntryBuildInputs, PlannerChangeResults, PlannerOverrides, PlannerRelationsContext, @@ -5078,7 +5079,6 @@ def assemble_model_plan_entry( future_cursor_config: FutureCursorsConfig | None, start_cursor_config: StartCursorsConfig | None, invocation_time: datetime | None, - effective_grain: str | None, ) -> tuple[ModelPlanEntry, tuple[PlanWarning, ...]]: ... def plan_audit( *, @@ -5124,21 +5124,12 @@ def check_cursor_type_consistency( warehouse_columns: tuple[ColumnInfo, ...], sql_analysis_enabled: bool, ) -> PlanWarning | None: ... -def count_microbatches( - *, - bounds: CursorBounds, - batch_size: str, - cursor_type: str, - equal_bounds_are_batch: bool = False, -) -> int: ... -def cap_microbatch_bounds( - *, - bounds: CursorBounds, - batch_size: str, - cursor_type: str, - max_batches: int, - action: MicrobatchLimitAction, -) -> CursorBounds: ... +def apply_microbatch_limit( + *, entry: ModelPlanEntry, max_batches: int | None, action: MicrobatchLimitAction +) -> tuple[ModelPlanEntry, PlanWarning | None]: ... +def apply_configured_microbatch_limits( + *, entry: ModelPlanEntry, inputs: PlanEntryBuildInputs +) -> tuple[ModelPlanEntry, tuple[PlanWarning, ...]]: ... def microbatch_limit_warning( *, model_name: str, max_batches: int | None, batch_count: int, action: MicrobatchLimitAction ) -> str | None: ... diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index fe0d90a141..c398c92029 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -8,21 +8,18 @@ from typing import Any from sqlbuild._native import ( + apply_configured_microbatch_limits, assemble_model_plan_entry, audit_is_selected, build_cursor_input_relations, build_source_load_map, - cap_microbatch_bounds, - count_microbatches, detect_model_changes, get_config_cursor_bound, get_config_str, get_materialization_type, microbatch_guidance, - microbatch_limit_warning, replaces_incremental_relation, resolve_bounded_cursor_override, - resolve_effective_microbatch_batch_size, resolve_microbatch_lookback, resolve_model_plan_action, runtime_cursor_producer_names, @@ -130,7 +127,6 @@ SourceEntry, StartCursorsConfig, ) -from sqlbuild.spec.contracts.types import MicrobatchLimitAction @dataclass(frozen=True) @@ -431,7 +427,7 @@ def build_plan_entries( ), ) if entry.action != PlanAction.SKIP: - entry, limit_warnings = _apply_configured_microbatch_limits( + entry, limit_warnings = apply_configured_microbatch_limits( entry=entry, inputs=inputs ) warnings.extend(limit_warnings) @@ -477,143 +473,6 @@ def build_plan_entries( return PlannerModelEntryResults(entries=tuple(entries), warnings=tuple(warnings)) -def _apply_configured_microbatch_limits( - *, entry: ModelPlanEntry, inputs: PlanEntryBuildInputs -) -> tuple[ModelPlanEntry, tuple[PlanWarning, ...]]: - """Apply CLI, project, and model policies in their documented precedence order.""" - - if inputs.max_microbatches_is_override: - limited_entry, warning = _apply_microbatch_limit( - entry=entry, - max_batches=inputs.max_microbatches, - action=MicrobatchLimitAction.ERROR, - ) - return limited_entry, (() if warning is None else (warning,)) - - if entry.declared_microbatch_limit_action is not None: - warnings: list[PlanWarning] = [] - declared_action: MicrobatchLimitAction = entry.declared_microbatch_limit_action - if inputs.max_microbatches is not None: - _, safety_warning = _apply_microbatch_limit( - entry=entry, - max_batches=inputs.max_microbatches, - action=inputs.microbatch_limit_action, - ) - entry = replace( - entry, - microbatch_safety_limit=inputs.max_microbatches, - microbatch_safety_limit_action=inputs.microbatch_limit_action, - ) - if safety_warning is not None: - warnings.append(safety_warning) - entry, model_warning = _apply_microbatch_limit( - entry=entry, - max_batches=entry.microbatch_limit, - action=declared_action, - ) - if model_warning is not None: - warnings.append(model_warning) - return entry, tuple(warnings) - - max_batches: int | None = ( - None - if entry.microbatch_strategy == MicrobatchStrategy.ROLLING_WINDOW - and entry.reason != PlanReason.FULL_REFRESH - and entry.start_cursor_override is None - and entry.end_cursor_override is None - else entry.microbatch_limit or inputs.max_microbatches - ) - limited_entry, warning = _apply_microbatch_limit( - entry=entry, - max_batches=max_batches, - action=inputs.microbatch_limit_action, - ) - return limited_entry, (() if warning is None else (warning,)) - - -def _apply_microbatch_limit( - *, entry: ModelPlanEntry, max_batches: int | None, action: MicrobatchLimitAction -) -> tuple[ModelPlanEntry, PlanWarning | None]: - if entry.action == PlanAction.SKIP: - return entry, None - batch_count: int | None = None - effective_batch_size: str | None = None - if ( - entry.microbatch_range is not None - and entry.batch_size is not None - and entry.cursor_type is not None - ): - effective_grain: str | None = resolve_effective_timestamp_grain( - cursor_type=entry.cursor_type, - downstream_grain=entry.cursor_grain, - microbatch_strategy=entry.microbatch_strategy, - cursor_input_grains=tuple( - relation.cursor_grain for relation in entry.cursor_input_relations - ), - ) - effective_batch_size = ( - resolve_effective_microbatch_batch_size( - batch_size=entry.batch_size, - effective_grain=effective_grain, - ) - if effective_grain is not None - else entry.batch_size - ) - batch_count = count_microbatches( - bounds=entry.microbatch_range, - batch_size=effective_batch_size, - cursor_type=entry.cursor_type, - equal_bounds_are_batch=entry.action == PlanAction.CREATE_TABLE, - ) - warning: str | None = ( - None - if batch_count is None - else microbatch_limit_warning( - model_name=entry.name, - max_batches=max_batches, - batch_count=batch_count, - action=action, - ) - ) - limited_range: CursorBounds | None = entry.microbatch_range - if ( - warning is not None - and effective_batch_size is not None - and entry.microbatch_range is not None - and max_batches is not None - and action - in { - MicrobatchLimitAction.CAP_FROM_END, - MicrobatchLimitAction.CAP_FROM_START, - } - ): - limited_range = cap_microbatch_bounds( - bounds=entry.microbatch_range, - batch_size=effective_batch_size, - cursor_type=entry.cursor_type or "", - max_batches=max_batches, - action=action, - ) - limited_entry: ModelPlanEntry = replace( - entry, - microbatch_range=limited_range, - microbatch_limit=max_batches, - microbatch_limit_count=batch_count, - microbatch_limit_action=action if max_batches is not None else None, - microbatch_limit_warning=warning, - ) - if warning is None: - return limited_entry, None - if action == MicrobatchLimitAction.ERROR: - raise CompileInputError(warning) - return limited_entry, PlanWarning( - model_name=entry.name, - severity=WarningSeverity.WARNING, - code="MICROBATCH_LIMIT_EXCEEDED", - message=warning, - ) - - def plan_model_from_change( *, model: CompiledModelView, @@ -773,21 +632,6 @@ def plan_model_from_change( future_cursor_config=effective_future_cursor_config, start_cursor_config=effective_start_cursor_config, invocation_time=context.invocation_time, - effective_grain=( - resolve_effective_timestamp_grain( - cursor_type=get_config_str(values=model.config.values, key="cursor_type"), - downstream_grain=get_config_str(values=model.config.values, key="cursor_grain"), - microbatch_strategy=get_config_str( - values=model.config.values, key="microbatch_strategy" - ), - cursor_input_grains=tuple( - relation.cursor_grain for relation in cursor_input_relations - ), - ) - if get_config_str(values=model.config.values, key="incremental_mode") - == IncrementalMode.MICROBATCH - else None - ), ) return entry, _append_cursor_safety_warnings( warnings=warnings, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py index 64b45550b2..437ceae8ce 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py @@ -4,6 +4,10 @@ import pytest +from sqlbuild._native import ( + apply_configured_microbatch_limits, + apply_microbatch_limit, +) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( @@ -11,10 +15,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner._helpers.output.plan_entry import ( - _apply_configured_microbatch_limits, - _apply_microbatch_limit, -) from sqlbuild.compiler.planner.models import ( CursorBounds, CursorInputRelation, @@ -87,7 +87,7 @@ def test_given_nested_model_policy_when_applying_outer_limit_then_precedence_is_ declared_microbatch_limit_action=MicrobatchLimitAction.CAP_FROM_END, ) - limited_entry, warnings = _apply_configured_microbatch_limits( + limited_entry, warnings = apply_configured_microbatch_limits( entry=entry, inputs=PlanEntryBuildInputs( max_microbatches=test_case.configured_limit, @@ -153,7 +153,7 @@ def test_given_coarse_effective_grain_when_applying_limit_then_authored_batch_se microbatch_range=CursorBounds(start="2026-01-01", end="2026-04-01"), ) - limited_entry, warning = _apply_microbatch_limit( + limited_entry, warning = apply_microbatch_limit( entry=entry, max_batches=3, action=MicrobatchLimitAction.WARN, @@ -272,7 +272,7 @@ def test_given_planner_owned_range_when_applying_limit_then_expected_decision_is ) action: MicrobatchLimitAction = MicrobatchLimitAction(test_case.action) - limited_entry, warning = _apply_microbatch_limit( + limited_entry, warning = apply_microbatch_limit( entry=entry, max_batches=test_case.max_batches, action=action ) @@ -323,7 +323,7 @@ def test_given_excess_planner_owned_range_when_action_is_error_then_planning_fai ) with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - _apply_microbatch_limit( + apply_microbatch_limit( entry=entry, max_batches=test_case.max_batches, action=MicrobatchLimitAction(test_case.action), From ca0dd9f0ab88bfd9074349ca594d781eb6926123 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 02:38:15 +0100 Subject: [PATCH 21/22] style: format integrated plan display modules --- crates/sqlbuild-core/src/plan_display/main/mod.rs | 2 +- .../src/bindings/_helpers/planner_scenario/entries.rs | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/crates/sqlbuild-core/src/plan_display/main/mod.rs b/crates/sqlbuild-core/src/plan_display/main/mod.rs index 969c1db466..c9d5735ad4 100644 --- a/crates/sqlbuild-core/src/plan_display/main/mod.rs +++ b/crates/sqlbuild-core/src/plan_display/main/mod.rs @@ -1,7 +1,7 @@ pub mod cursor_bound_display; pub mod effective_microbatch_batch_size; -pub mod format_model_migrations; pub mod effective_timestamp_grain; +pub mod format_model_migrations; pub mod format_no_input_rows; pub mod format_waiting_on_empty_inputs; pub mod model_execution_annotation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs index b50be17c04..1cf94c2c32 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs @@ -54,12 +54,7 @@ pub(super) fn seed_entries<'py>( ), ( "columns", - imported( - py, - "sqlbuild._native", - "extract_seed_columns", - )? - .call1((&seed,))?, + imported(py, "sqlbuild._native", "extract_seed_columns")?.call1((&seed,))?, ), ( "csv_settings", From e493f06fecd1b4a270a200bd908ae459aceb8a2d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 02:44:45 +0100 Subject: [PATCH 22/22] style: sort integrated native stub imports --- src/sqlbuild/_native.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 7a86e1981f..20bb2d0946 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -217,9 +217,9 @@ from sqlbuild.compiler.planner.models import ( ColumnRenameHint, CursorBounds, CursorInputEvidence, + CursorInputRelation, CursorOverridePair, CursorOverrides, - CursorInputRelation, DeferralInputs, DirectModelVersionIdentities, EmptyCursorInputDecision,