From af675507b2900c680e3e334e6003cec7d0e4c1e1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:14:17 +0100 Subject: [PATCH 01/29] refactor: port migration compatibility and matching decisions to Rust --- .../_helpers/decisions/compatibility.rs | 114 +++++++++ .../_helpers/decisions/matching.rs | 120 +++++++++ .../_helpers/decisions/mod.rs | 2 + .../src/planner_migrations/_helpers/mod.rs | 12 +- .../_helpers/{ => proofs}/ast.rs | 0 .../_helpers/{ => proofs}/binding_proof.rs | 2 +- .../_helpers/{ => proofs}/clauses.rs | 2 +- .../_helpers/{ => proofs}/encoding.rs | 0 .../_helpers/{ => proofs}/fingerprint.rs | 6 +- .../_helpers/{ => proofs}/inputs.rs | 6 +- .../_helpers/{ => proofs}/local_names.rs | 4 +- .../planner_migrations/_helpers/proofs/mod.rs | 10 + .../_helpers/{ => proofs}/parse.rs | 0 .../_helpers/{ => proofs}/projection.rs | 4 +- .../{ => proofs}/removed_and_added.rs | 0 .../src/planner_migrations/constants.rs | 4 + .../main/compatibility_findings.rs | 14 ++ .../main/declared_column_aliases.rs | 13 + .../main/identical_renames.rs | 2 +- .../main/match_migrations.rs | 42 ++++ .../main/migration_fingerprint.rs | 4 +- .../main/migration_ref_names.rs | 2 +- .../src/planner_migrations/main/mod.rs | 7 + .../main/old_name_claim_message.rs | 24 ++ .../main/old_name_schemas.rs | 19 ++ .../main/old_name_skip_reason.rs | 23 ++ .../planner_migrations/main/query_shape.rs | 12 +- .../planner_migrations/main/rename_hints.rs | 2 +- .../main/renames_explain_change.rs | 4 +- .../main/stored_migration_fingerprint.rs | 13 + .../src/planner_migrations/models.rs | 31 +++ .../src/planner_migrations/tests.rs | 2 + .../tests/test_decisions.rs | 157 ++++++++++++ .../planner_migrations/tests/test_types.rs | 16 ++ .../planner_migrations/compatibility.rs | 231 ++++++++++++++++++ .../planner_migrations/fingerprint.rs | 49 +++- .../_helpers/planner_migrations/matching.rs | 166 +++++++++++++ .../_helpers/planner_migrations/mod.rs | 3 + .../_helpers/planner_migrations/old_names.rs | 227 +++++++++++++++++ .../planner_migrations/registration.rs | 3 + src/sqlbuild/_native.pyi | 70 ++++++ .../_helpers/migrations/compatibility.py | 174 ------------- .../planner/_helpers/migrations/discovery.py | 159 +----------- .../_helpers/migrations/entry_fingerprints.py | 48 ---- .../planner/_helpers/migrations/old_names.py | 114 +-------- .../planner/_helpers/migrations/planning.py | 14 +- .../_helpers/planning/output_assembly.py | 6 +- 47 files changed, 1418 insertions(+), 519 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/_helpers/decisions/compatibility.rs create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/_helpers/decisions/matching.rs create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/_helpers/decisions/mod.rs rename crates/sqlbuild-analysis/src/planner_migrations/_helpers/{ => proofs}/ast.rs (100%) rename crates/sqlbuild-analysis/src/planner_migrations/_helpers/{ => proofs}/binding_proof.rs (97%) rename crates/sqlbuild-analysis/src/planner_migrations/_helpers/{ => proofs}/clauses.rs (93%) rename crates/sqlbuild-analysis/src/planner_migrations/_helpers/{ => proofs}/encoding.rs (100%) rename crates/sqlbuild-analysis/src/planner_migrations/_helpers/{ => proofs}/fingerprint.rs (93%) rename crates/sqlbuild-analysis/src/planner_migrations/_helpers/{ => proofs}/inputs.rs (95%) rename crates/sqlbuild-analysis/src/planner_migrations/_helpers/{ => proofs}/local_names.rs (98%) create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/mod.rs rename crates/sqlbuild-analysis/src/planner_migrations/_helpers/{ => proofs}/parse.rs (100%) rename crates/sqlbuild-analysis/src/planner_migrations/_helpers/{ => proofs}/projection.rs (92%) rename crates/sqlbuild-analysis/src/planner_migrations/_helpers/{ => proofs}/removed_and_added.rs (100%) create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/main/compatibility_findings.rs create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/main/declared_column_aliases.rs create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/main/match_migrations.rs create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/main/old_name_claim_message.rs create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/main/old_name_schemas.rs create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/main/old_name_skip_reason.rs create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/tests/test_decisions.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/migrations/entry_fingerprints.py diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/decisions/compatibility.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/decisions/compatibility.rs new file mode 100644 index 0000000000..856496dc16 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/decisions/compatibility.rs @@ -0,0 +1,114 @@ +//! Origin column compatibility under incremental and snapshot policies. + +use crate::planner_migrations::constants::{ + COLUMN_ADDED, COLUMN_TYPE_CHANGED, POLICY_DENY, POLICY_FAIL, +}; +use crate::planner_migrations::models::CompatibilityFinding; +use crate::type_system::main::normalize_type::normalize_type; +use crate::type_system::models::TypeFamily; + +/// Return blocking findings in their original order. +pub(crate) fn compatibility_findings( + findings: &[CompatibilityFinding], + policy: (&str, bool), + dialect: &str, +) -> Result, String> { + let (policy, snapshot) = policy; + let mut blocking: Vec = Vec::new(); + for finding in findings { + let suffix = if snapshot { + if finding.kind == COLUMN_TYPE_CHANGED && !convertible(finding, dialect, true)? { + Some("snapshot type change") + } else if finding.kind == COLUMN_ADDED && policy == POLICY_DENY { + Some("snapshot_schema_change deny") + } else { + None + } + } else if policy == POLICY_FAIL { + Some("on_schema_change fail") + } else if finding.kind == COLUMN_TYPE_CHANGED && !convertible(finding, dialect, false)? { + Some("incompatible types") + } else { + None + }; + if let Some(suffix) = suffix { + blocking.push(format!("{} ({suffix})", describe(finding))); + } + } + Ok(blocking) +} + +fn describe(finding: &CompatibilityFinding) -> String { + let name = &finding.column_name; + match finding.kind.as_str() { + "column_added" => format!("column {name} is missing from the origin"), + "column_removed" => format!("origin column {name} is not produced by the model"), + _ => format!( + "column {name} is {} in the origin but the model produces {}", + finding.actual_type.as_deref().unwrap_or("None"), + finding.expected_type.as_deref().unwrap_or("None") + ), + } +} + +fn convertible( + finding: &CompatibilityFinding, + dialect: &str, + snapshot: bool, +) -> Result { + let (Some(expected), Some(actual)) = (&finding.expected_type, &finding.actual_type) else { + return Ok(true); + }; + let incoming = normalize_type(expected, dialect) + .map_err(|error| error.message())? + .normalized; + let target = normalize_type(actual, dialect) + .map_err(|error| error.message())? + .normalized; + if snapshot { + if incoming == target { + return Ok(true); + } + if incoming.family != TypeFamily::String || target.family != TypeFamily::String { + return Ok(false); + } + return Ok(match (&incoming.length, &target.length) { + (_, None) => true, + (Some(incoming), Some(target)) => integer_le(incoming.as_str(), target.as_str()), + (None, Some(_)) => false, + }); + } + Ok(incoming.family == target.family + || incoming.family == TypeFamily::Other + || target.family == TypeFamily::Other + || numeric(incoming.family) && numeric(target.family) + || temporal(incoming.family) && temporal(target.family)) +} + +fn integer_le(left: &str, right: &str) -> bool { + let left_negative = left.starts_with('-'); + let right_negative = right.starts_with('-'); + if left_negative != right_negative { + return left_negative; + } + let order = (left.len(), left).cmp(&(right.len(), right)); + if left_negative { + order.is_ge() + } else { + order.is_le() + } +} + +fn numeric(family: TypeFamily) -> bool { + matches!( + family, + TypeFamily::Integer | TypeFamily::Decimal | TypeFamily::Float + ) +} + +fn temporal(family: TypeFamily) -> bool { + matches!( + family, + TypeFamily::Timestamp | TypeFamily::Date | TypeFamily::Datetime + ) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/decisions/matching.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/decisions/matching.rs new file mode 100644 index 0000000000..04e5001194 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/decisions/matching.rs @@ -0,0 +1,120 @@ +//! Ordered options and explanations for contested migration origins. + +use crate::planner_migrations::models::{MigrationCandidate, MigrationMatchPass, MigrationMatches}; +use std::collections::{BTreeMap, BTreeSet}; + +fn match_options( + hashes: &[String], + indexed: (&[MigrationCandidate], &BTreeMap<&str, Vec>), + blocked: &BTreeSet, +) -> Vec { + let (candidates, by_fingerprint) = indexed; + let mut options: Vec = Vec::new(); + for hash in hashes { + for index in by_fingerprint + .get(hash.as_str()) + .into_iter() + .flatten() + .copied() + { + let candidate = &candidates[index]; + if blocked.contains(&candidate.name) { + continue; + } + if let Some(existing) = options + .iter_mut() + .find(|previous| candidates[**previous].name == candidate.name) + { + *existing = index; + } else { + options.push(index); + } + } + } + options +} + +pub(crate) fn contested_details( + contested: Vec<(String, Vec)>, +) -> (BTreeSet, BTreeMap) { + let mut blocked: BTreeSet = BTreeSet::new(); + let mut ambiguous: BTreeMap = BTreeMap::new(); + for (origin, mut names) in contested { + blocked.insert(origin.clone()); + names.sort(); + for name in &names { + let others: Vec = names + .iter() + .filter(|other| *other != name) + .map(|other| format!("'{other}'")) + .collect(); + ambiguous.insert( + name.clone(), + format!( + "earlier model '{origin}', which also matches {}", + others.join(", ") + ), + ); + } + } + (blocked, ambiguous) +} + +fn multiple_origins(options: &[usize], candidates: &[MigrationCandidate]) -> String { + let mut names: Vec<&str> = options + .iter() + .map(|index| candidates[*index].name.as_str()) + .collect(); + names.sort_unstable(); + format!("several earlier models ({})", names.join(", ")) +} + +pub(crate) fn match_pass( + inputs: (&[String], &[MigrationCandidate]), + seeds: &[BTreeMap], + blocked: &BTreeSet, + mut fingerprints: impl FnMut(&str, &[BTreeMap]) -> Result, E>, +) -> Result { + let (names, candidates) = inputs; + let index = candidate_index(candidates); + let mut variants = seeds.to_vec(); + let mut matches: Vec<(String, usize)> = Vec::new(); + let mut ambiguous: BTreeMap = BTreeMap::new(); + let mut claims: BTreeMap> = BTreeMap::new(); + for name in names { + let hashes = fingerprints(name, &variants)?; + let options = match_options(&hashes, (candidates, &index), blocked); + if options.len() > 1 { + ambiguous.insert(name.clone(), multiple_origins(&options, candidates)); + } else if let Some(index) = options.first() { + let origin = &candidates[*index].name; + for variant in &mut variants { + variant.insert(name.clone(), origin.clone()); + } + matches.push((name.clone(), *index)); + claims.entry(origin.clone()).or_default().push(name.clone()); + } + } + Ok(MigrationMatchPass { + result: MigrationMatches { + variants, + matches, + ambiguous, + }, + contested: claims + .into_iter() + .filter(|(_, names)| names.len() > 1) + .collect(), + }) +} + +fn candidate_index(candidates: &[MigrationCandidate]) -> BTreeMap<&str, Vec> { + let mut index: BTreeMap<&str, Vec> = BTreeMap::new(); + for (position, candidate) in candidates.iter().enumerate() { + index + .entry(&candidate.fingerprint) + .or_default() + .push(position); + } + index +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/decisions/mod.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/decisions/mod.rs new file mode 100644 index 0000000000..0d2fba9ca3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/decisions/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod compatibility; +pub(crate) mod matching; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/mod.rs index 9e34a2d27c..78b7d4239b 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/mod.rs @@ -1,10 +1,2 @@ -pub(crate) mod ast; -pub(crate) mod binding_proof; -pub(crate) mod clauses; -pub(crate) mod encoding; -pub(crate) mod fingerprint; -pub(crate) mod inputs; -pub(crate) mod local_names; -pub(crate) mod parse; -pub(crate) mod projection; -pub(crate) mod removed_and_added; +pub(crate) mod decisions; +pub(crate) mod proofs; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/ast.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/ast.rs similarity index 100% rename from crates/sqlbuild-analysis/src/planner_migrations/_helpers/ast.rs rename to crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/ast.rs diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/binding_proof.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/binding_proof.rs similarity index 97% rename from crates/sqlbuild-analysis/src/planner_migrations/_helpers/binding_proof.rs rename to crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/binding_proof.rs index 912ab4e69a..70c0cda8cf 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/binding_proof.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/binding_proof.rs @@ -1,6 +1,6 @@ //! Conservative alias resolution in unchanged and renamed projections. -use crate::planner_migrations::_helpers::ast::identifier; +use crate::planner_migrations::_helpers::proofs::ast::identifier; use crate::planner_migrations::models::QueryShape; use serde_json::Value; use std::collections::BTreeSet; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/clauses.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/clauses.rs similarity index 93% rename from crates/sqlbuild-analysis/src/planner_migrations/_helpers/clauses.rs rename to crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/clauses.rs index e605497b23..25f2f7249c 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/clauses.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/clauses.rs @@ -1,6 +1,6 @@ //! Non-projection clauses and positional references of a SELECT. -use crate::planner_migrations::_helpers::ast::{clause_items, items, references}; +use crate::planner_migrations::_helpers::proofs::ast::{clause_items, items, references}; use crate::planner_migrations::constants::NUMBER_LITERAL; use serde_json::Value; use std::collections::BTreeSet; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/encoding.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/encoding.rs similarity index 100% rename from crates/sqlbuild-analysis/src/planner_migrations/_helpers/encoding.rs rename to crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/encoding.rs diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/fingerprint.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/fingerprint.rs similarity index 93% rename from crates/sqlbuild-analysis/src/planner_migrations/_helpers/fingerprint.rs rename to crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/fingerprint.rs index ff72274bee..338624014a 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/fingerprint.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/fingerprint.rs @@ -2,9 +2,9 @@ use crate::planner_changes::main::version_identity_payload::version_identity_payload; use crate::planner_changes::models::IdentityPayload; -use crate::planner_migrations::_helpers::ast::strip_formatting; -use crate::planner_migrations::_helpers::local_names::{local_names, rewrite, unambiguous}; -use crate::planner_migrations::_helpers::parse::parse; +use crate::planner_migrations::_helpers::proofs::ast::strip_formatting; +use crate::planner_migrations::_helpers::proofs::local_names::{local_names, rewrite, unambiguous}; +use crate::planner_migrations::_helpers::proofs::parse::parse; use crate::planner_migrations::constants::{EXCLUDED_CONFIG, REF_MARKER}; use polyglot_sql::{Expression, dialects::Dialect}; use regex::Regex; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/inputs.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/inputs.rs similarity index 95% rename from crates/sqlbuild-analysis/src/planner_migrations/_helpers/inputs.rs rename to crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/inputs.rs index e548568c3a..703b1b0e20 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/inputs.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/inputs.rs @@ -1,7 +1,9 @@ //! Resolve only inputs whose columns can be proved from the query shape. -use crate::planner_migrations::_helpers::ast::{clause_items, identifier, identifiers, items}; -use crate::planner_migrations::_helpers::projection::projection; +use crate::planner_migrations::_helpers::proofs::ast::{ + clause_items, identifier, identifiers, items, +}; +use crate::planner_migrations::_helpers::proofs::projection::projection; use serde_json::Value; use std::collections::{BTreeMap, BTreeSet}; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/local_names.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/local_names.rs similarity index 98% rename from crates/sqlbuild-analysis/src/planner_migrations/_helpers/local_names.rs rename to crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/local_names.rs index d002926aae..d7ffe4c62e 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/local_names.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/local_names.rs @@ -1,6 +1,8 @@ //! CTE and relation-alias identity with conservative scope rejection. -use crate::planner_migrations::_helpers::ast::{identifier, items, matching_fields, opaque_field}; +use crate::planner_migrations::_helpers::proofs::ast::{ + identifier, items, matching_fields, opaque_field, +}; use crate::planner_migrations::constants::{TABLE_KEY, WITH_KEY}; use serde_json::Value; use std::collections::{BTreeMap, BTreeSet}; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/mod.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/mod.rs new file mode 100644 index 0000000000..9e34a2d27c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod ast; +pub(crate) mod binding_proof; +pub(crate) mod clauses; +pub(crate) mod encoding; +pub(crate) mod fingerprint; +pub(crate) mod inputs; +pub(crate) mod local_names; +pub(crate) mod parse; +pub(crate) mod projection; +pub(crate) mod removed_and_added; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/parse.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs similarity index 100% rename from crates/sqlbuild-analysis/src/planner_migrations/_helpers/parse.rs rename to crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/projection.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/projection.rs similarity index 92% rename from crates/sqlbuild-analysis/src/planner_migrations/_helpers/projection.rs rename to crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/projection.rs index e46bf22ec0..5abb886ef4 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/projection.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/projection.rs @@ -1,7 +1,7 @@ //! A named output and its expression proof. -use crate::planner_migrations::_helpers::ast::{canonical, identifier, references}; -use crate::planner_migrations::_helpers::encoding::encode; +use crate::planner_migrations::_helpers::proofs::ast::{canonical, identifier, references}; +use crate::planner_migrations::_helpers::proofs::encoding::encode; use crate::planner_migrations::models::QueryProjection; use serde_json::Value; use std::collections::BTreeSet; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/removed_and_added.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/removed_and_added.rs similarity index 100% rename from crates/sqlbuild-analysis/src/planner_migrations/_helpers/removed_and_added.rs rename to crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/removed_and_added.rs diff --git a/crates/sqlbuild-analysis/src/planner_migrations/constants.rs b/crates/sqlbuild-analysis/src/planner_migrations/constants.rs index 37daafa4a4..dd09a82c85 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/constants.rs @@ -42,3 +42,7 @@ pub(crate) const EXCLUDED_CONFIG: &[&str] = &[ "snapshot_full_refresh", "snapshot_schema_change", ]; +pub(crate) const COLUMN_ADDED: &str = "column_added"; +pub(crate) const COLUMN_TYPE_CHANGED: &str = "column_type_changed"; +pub(crate) const POLICY_DENY: &str = "deny"; +pub(crate) const POLICY_FAIL: &str = "fail"; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/compatibility_findings.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/compatibility_findings.rs new file mode 100644 index 0000000000..d38b2e919a --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/compatibility_findings.rs @@ -0,0 +1,14 @@ +//! Origin compatibility policy entry point. + +use crate::planner_migrations::models::CompatibilityFinding; + +/// Return blocking schema findings in their original order. +pub fn compatibility_findings( + findings: &[CompatibilityFinding], + policy: (&str, bool), + dialect: &str, +) -> Result, String> { + crate::planner_migrations::_helpers::decisions::compatibility::compatibility_findings( + findings, policy, dialect, + ) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/declared_column_aliases.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/declared_column_aliases.rs new file mode 100644 index 0000000000..b48560f907 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/declared_column_aliases.rs @@ -0,0 +1,13 @@ +//! Ordered aliases declared by table and view models. + +/// Strip origin names and sort aliases, retaining duplicates exactly as declared. +pub fn declared_column_aliases(columns: &[(String, Option)]) -> Vec<(String, String)> { + let mut aliases: Vec<(String, String)> = Vec::new(); + for (name, origin) in columns { + if let Some(origin) = origin { + aliases.push((origin.trim().to_owned(), name.clone())); + } + } + aliases.sort(); + aliases +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/identical_renames.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/identical_renames.rs index d19a3ad0a0..3c33fe06c4 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/main/identical_renames.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/identical_renames.rs @@ -2,7 +2,7 @@ use std::collections::BTreeSet; -use crate::planner_migrations::_helpers::removed_and_added::removed_and_added; +use crate::planner_migrations::_helpers::proofs::removed_and_added::removed_and_added; use crate::planner_migrations::models::Projection; use crate::planner_migrations::types::RenameProof; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/match_migrations.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/match_migrations.rs new file mode 100644 index 0000000000..bc1ac6cfab --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/match_migrations.rs @@ -0,0 +1,42 @@ +//! Conservative one-to-one migration matching with rename-map propagation. + +use crate::planner_migrations::_helpers::decisions::matching::{contested_details, match_pass}; +use crate::planner_migrations::models::{MigrationCandidate, MigrationMatches}; +use std::collections::{BTreeMap, BTreeSet}; + +/// Match ordered models, rejecting every origin claimed by several destinations. +pub fn match_migrations( + names: &[String], + seeds: &[BTreeMap], + candidates: &[MigrationCandidate], + mut fingerprints: impl FnMut(&str, &[BTreeMap]) -> Result, E>, +) -> Result { + let mut blocked: BTreeSet = BTreeSet::new(); + let mut ambiguous: BTreeMap = BTreeMap::new(); + if candidates.is_empty() { + return Ok(MigrationMatches { + variants: seeds.to_vec(), + matches: Vec::new(), + ambiguous, + }); + } + for _ in 0..=candidates.len() { + let pass = match_pass((names, candidates), seeds, &blocked, &mut fingerprints)?; + ambiguous.extend(pass.result.ambiguous); + if pass.contested.is_empty() { + return Ok(MigrationMatches { + variants: pass.result.variants, + matches: pass.result.matches, + ambiguous, + }); + } + let (origins, details) = contested_details(pass.contested); + blocked.extend(origins); + ambiguous.extend(details); + } + Ok(MigrationMatches { + variants: seeds.to_vec(), + matches: Vec::new(), + ambiguous, + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/migration_fingerprint.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/migration_fingerprint.rs index b9c48ce968..c862c924ef 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/main/migration_fingerprint.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/migration_fingerprint.rs @@ -1,7 +1,7 @@ //! Persisted rename identity of a model. -use crate::planner_migrations::_helpers::encoding::encode; -use crate::planner_migrations::_helpers::fingerprint::{identity_metadata, normalized}; +use crate::planner_migrations::_helpers::proofs::encoding::encode; +use crate::planner_migrations::_helpers::proofs::fingerprint::{identity_metadata, normalized}; use serde_json::json; use sha2::{Digest, Sha256}; use std::collections::BTreeMap; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/migration_ref_names.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/migration_ref_names.rs index 0c1e66a937..bbfdc547f3 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/main/migration_ref_names.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/migration_ref_names.rs @@ -1,6 +1,6 @@ //! Model names that participate in a migration identity. -use crate::planner_migrations::_helpers::fingerprint::metadata; +use crate::planner_migrations::_helpers::proofs::fingerprint::metadata; use crate::planner_migrations::constants::REF_MARKER; use regex::Regex; use std::collections::BTreeSet; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs index 5c524b7ad7..5bc0d79def 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs @@ -1,6 +1,13 @@ +pub mod compatibility_findings; +pub mod declared_column_aliases; pub mod identical_renames; +pub mod match_migrations; pub mod migration_fingerprint; pub mod migration_ref_names; +pub mod old_name_claim_message; +pub mod old_name_schemas; +pub mod old_name_skip_reason; pub mod query_shape; pub mod rename_hints; pub mod renames_explain_change; +pub mod stored_migration_fingerprint; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/old_name_claim_message.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/old_name_claim_message.rs new file mode 100644 index 0000000000..a48530d1da --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/old_name_claim_message.rs @@ -0,0 +1,24 @@ +//! Diagnostics for attempts to replace a recorded compatibility view. + +/// Describe the cleanup needed before a model can claim an old name. +pub fn old_name_claim_message( + model: &str, + relation: (&str, &str), + expiration: (Option<&str>, bool), +) -> String { + let (name, destination) = relation; + let (date, expired) = expiration; + if let Some(date) = date + && expired + { + return format!( + "model '{model}' would replace {name}, a compatibility view for model:{destination} that expired {date}; run `sqb janitor` to drop it first" + ); + } + let until = date + .map(|date| format!(" until {date}")) + .unwrap_or_default(); + format!( + "model '{model}' would replace {name}, a compatibility view for model:{destination}{until}; drop it early with `sqb janitor --drop-old-name-view {name}`" + ) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/old_name_schemas.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/old_name_schemas.rs new file mode 100644 index 0000000000..c730896158 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/old_name_schemas.rs @@ -0,0 +1,19 @@ +//! Select old-name state schemas without repeating known-empty inspection. + +use std::collections::BTreeSet; + +/// Keep known state schemas and schemas absent from an available catalog listing. +pub fn old_name_schemas( + needed: &BTreeSet, + state: &BTreeSet, + listed: Option<&BTreeSet>, +) -> BTreeSet { + needed + .iter() + .filter(|schema| { + let key = schema.to_lowercase(); + state.contains(&key) || listed.is_some_and(|listed| !listed.contains(&key)) + }) + .cloned() + .collect() +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/old_name_skip_reason.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/old_name_skip_reason.rs new file mode 100644 index 0000000000..a6e4cb4b9f --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/old_name_skip_reason.rs @@ -0,0 +1,23 @@ +//! Priority of reasons to omit old-name compatibility steps. + +/// Explain why a migration should not create its compatibility view. +pub fn old_name_skip_reason( + retention: bool, + reuser: Option<&str>, + compatibility_view: bool, + tracked: Option, +) -> Option { + if !retention { + return Some("old_name_view false".to_owned()); + } + if let Some(name) = reuser { + return Some(format!("name reused by model:{name}")); + } + if compatibility_view { + return Some("old name is a compatibility view".to_owned()); + } + if tracked == Some(false) { + return Some("not built by SQLBuild".to_owned()); + } + None +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/query_shape.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/query_shape.rs index 5d8dfac8b0..c91df4b1fb 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/main/query_shape.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/query_shape.rs @@ -1,11 +1,11 @@ //! Parse the conservative shape used for column migration proofs. -use crate::planner_migrations::_helpers::ast::{canonical, strip_formatting}; -use crate::planner_migrations::_helpers::clauses::{clause_references, positional}; -use crate::planner_migrations::_helpers::encoding::encode; -use crate::planner_migrations::_helpers::inputs::select_inputs; -use crate::planner_migrations::_helpers::parse::parse; -use crate::planner_migrations::_helpers::projection::named_projections; +use crate::planner_migrations::_helpers::proofs::ast::{canonical, strip_formatting}; +use crate::planner_migrations::_helpers::proofs::clauses::{clause_references, positional}; +use crate::planner_migrations::_helpers::proofs::encoding::encode; +use crate::planner_migrations::_helpers::proofs::inputs::select_inputs; +use crate::planner_migrations::_helpers::proofs::parse::parse; +use crate::planner_migrations::_helpers::proofs::projection::named_projections; use crate::planner_migrations::models::QueryShape; use serde_json::json; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/rename_hints.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/rename_hints.rs index 8ad01abc49..d6e2d736ef 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/main/rename_hints.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/rename_hints.rs @@ -2,7 +2,7 @@ use std::collections::BTreeSet; -use crate::planner_migrations::_helpers::removed_and_added::removed_and_added; +use crate::planner_migrations::_helpers::proofs::removed_and_added::removed_and_added; use crate::planner_migrations::models::{Projection, RenameHint}; /// Hints ordered by added projection, preferring identical expressions over shared references. diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/renames_explain_change.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/renames_explain_change.rs index 7bafbf7db4..65d0769650 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/main/renames_explain_change.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/renames_explain_change.rs @@ -1,7 +1,7 @@ //! Prove that output renames explain the entire query change. -use crate::planner_migrations::_helpers::binding_proof::{binding_proven, substitute}; -use crate::planner_migrations::_helpers::encoding::encode; +use crate::planner_migrations::_helpers::proofs::binding_proof::{binding_proven, substitute}; +use crate::planner_migrations::_helpers::proofs::encoding::encode; use crate::planner_migrations::models::QueryShape; use serde_json::Value; use std::collections::{BTreeMap, BTreeSet}; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs new file mode 100644 index 0000000000..d9eab548d5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs @@ -0,0 +1,13 @@ +//! Read persisted migration hashes without changing their representation. + +/// Read only string migration fingerprints from object metadata. +pub fn stored_migration_fingerprint(metadata: &str) -> Option { + let payload: serde_json::Value = match serde_json::from_str(metadata) { + Ok(payload) => payload, + Err(_) => return None, + }; + payload + .get("migration_fingerprint")? + .as_str() + .map(str::to_owned) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/models.rs b/crates/sqlbuild-analysis/src/planner_migrations/models.rs index 2649924a27..ad4e625547 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/models.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/models.rs @@ -2,6 +2,37 @@ use std::collections::BTreeSet; +/// One origin schema difference used by compatibility policy. +#[derive(Debug)] +pub struct CompatibilityFinding { + pub kind: String, + pub column_name: String, + pub expected_type: Option, + pub actual_type: Option, +} + +/// Ordered candidates for automatic migration matching. +#[derive(Debug)] +pub struct MigrationCandidate { + pub name: String, + pub fingerprint: String, +} + +/// Rename variants and unique candidate indexes after collision rejection. +#[derive(Debug)] +pub struct MigrationMatches { + pub variants: Vec>, + pub matches: Vec<(String, usize)>, + pub ambiguous: std::collections::BTreeMap, +} + +/// Matches and origin claims from one collision-elimination pass. +#[derive(Debug)] +pub struct MigrationMatchPass { + pub result: MigrationMatches, + pub contested: Vec<(String, Vec)>, +} + /// A named projection with canonical expression and reference names. #[derive(Debug)] pub struct Projection { diff --git a/crates/sqlbuild-analysis/src/planner_migrations/tests.rs b/crates/sqlbuild-analysis/src/planner_migrations/tests.rs index 44054f0428..2f5cf327e6 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/tests.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/tests.rs @@ -2,5 +2,7 @@ mod helpers; #[path = "tests/test_column_renames.rs"] mod test_column_renames; +#[path = "tests/test_decisions.rs"] +mod test_decisions; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/tests/test_decisions.rs b/crates/sqlbuild-analysis/src/planner_migrations/tests/test_decisions.rs new file mode 100644 index 0000000000..ffcacced79 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/tests/test_decisions.rs @@ -0,0 +1,157 @@ +use crate::planner_migrations::main::{compatibility_findings, match_migrations}; +use crate::planner_migrations::models::{CompatibilityFinding, MigrationCandidate}; +use crate::planner_migrations::tests::test_types::{CompatibilityTestCase, MigrationMatchTestCase}; +use std::collections::BTreeMap; + +#[test] +fn given_type_changes_when_checking_compatibility_then_history_policy_is_preserved() +-> Result<(), String> { + let test_cases = [ + CompatibilityTestCase { + expected_type: "INTEGER", + actual_type: "BIGINT", + snapshot: false, + policy: "append_new_columns", + description: "incremental numeric conversion", + expected_findings: &[], + }, + CompatibilityTestCase { + expected_type: "INTEGER", + actual_type: "VARCHAR", + description: "incompatible families", + snapshot: false, + policy: "append_new_columns", + expected_findings: &[ + "column amount is VARCHAR in the origin but the model produces INTEGER (incompatible types)", + ], + }, + CompatibilityTestCase { + expected_type: "INTEGER", + actual_type: "BIGINT", + snapshot: false, + policy: "fail", + description: "fail policy blocks convertible changes", + expected_findings: &[ + "column amount is BIGINT in the origin but the model produces INTEGER (on_schema_change fail)", + ], + }, + CompatibilityTestCase { + expected_type: "VARCHAR(10)", + description: "snapshot fits the existing string length", + actual_type: "VARCHAR(20)", + snapshot: true, + policy: "deny", + expected_findings: &[], + }, + CompatibilityTestCase { + expected_type: "VARCHAR(30)", + description: "snapshot exceeds the existing string length", + actual_type: "VARCHAR(20)", + snapshot: true, + policy: "deny", + expected_findings: &[ + "column amount is VARCHAR(20) in the origin but the model produces VARCHAR(30) (snapshot type change)", + ], + }, + ]; + for test_case in test_cases { + let findings = [CompatibilityFinding { + kind: "column_type_changed".to_owned(), + column_name: "amount".to_owned(), + expected_type: Some(test_case.expected_type.to_owned()), + actual_type: Some(test_case.actual_type.to_owned()), + }]; + assert_eq!( + compatibility_findings::compatibility_findings( + &findings, + (test_case.policy, test_case.snapshot), + "snowflake" + )?, + test_case.expected_findings, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_candidate_collisions_when_matching_then_only_unique_origins_survive() -> Result<(), String> +{ + let test_cases = [ + MigrationMatchTestCase { + hashes: &["a"], + candidates: &[], + description: "no origins", + expected_matches: &[], + expected_ambiguous: 0, + }, + MigrationMatchTestCase { + hashes: &["a", "b"], + description: "two unique origins", + candidates: &[("old_orders", "a"), ("old_customers", "b")], + expected_matches: &[(0, 0), (1, 1)], + expected_ambiguous: 0, + }, + MigrationMatchTestCase { + hashes: &["a", "a"], + description: "contested origin rejected for both models", + candidates: &[("old_orders", "a")], + expected_matches: &[], + expected_ambiguous: 2, + }, + MigrationMatchTestCase { + hashes: &["a"], + candidates: &[("old_orders", "a"), ("old_customers", "a")], + description: "ambiguous candidates rejected", + expected_matches: &[], + expected_ambiguous: 1, + }, + MigrationMatchTestCase { + hashes: &["a"], + candidates: &[("old_orders", "a"), ("old_orders", "a")], + description: "duplicate candidate names retain the last value", + expected_matches: &[(0, 1)], + expected_ambiguous: 0, + }, + ]; + for test_case in test_cases { + let names: Vec = (0..test_case.hashes.len()) + .map(|index| index.to_string()) + .collect(); + let candidates: Vec = test_case + .candidates + .iter() + .map(|(name, fingerprint)| MigrationCandidate { + name: (*name).to_owned(), + fingerprint: (*fingerprint).to_owned(), + }) + .collect(); + let result = match_migrations::match_migrations( + &names, + &[BTreeMap::new()], + &candidates, + |name, _| { + let index = name.parse::().map_err(|error| error.to_string())?; + Ok::<_, String>(vec![test_case.hashes[index].to_owned()]) + }, + )?; + let expected_matches: Vec<(String, usize)> = test_case + .expected_matches + .iter() + .map(|(model, candidate)| (model.to_string(), *candidate)) + .collect(); + assert_eq!( + result.matches, expected_matches, + "{}", + test_case.description + ); + assert_eq!( + result.ambiguous.len(), + test_case.expected_ambiguous, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_migrations/tests/test_types.rs index bba7f61a2c..147726e883 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/tests/test_types.rs @@ -7,3 +7,19 @@ pub(super) struct ProofTestCase { pub(super) expected_pairs: Result, &'static str>, pub(super) expected_calls: usize, } +pub(super) struct CompatibilityTestCase { + pub description: &'static str, + pub expected_type: &'static str, + pub actual_type: &'static str, + pub snapshot: bool, + pub policy: &'static str, + pub expected_findings: &'static [&'static str], +} + +pub(super) struct MigrationMatchTestCase { + pub description: &'static str, + pub hashes: &'static [&'static str], + pub candidates: &'static [(&'static str, &'static str)], + pub expected_matches: &'static [(usize, usize)], + pub expected_ambiguous: usize, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs new file mode 100644 index 0000000000..5babeb15e4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs @@ -0,0 +1,231 @@ +//! Convert existing planner contracts for origin compatibility checks. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_migrations::main::compatibility_findings::compatibility_findings; +use sqlbuild_analysis::planner_migrations::models::CompatibilityFinding; + +const MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; + +fn config(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { + model + .getattr("config")? + .getattr("values")? + .call_method1("get", (key,))? + .extract() +} + +fn blocking<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, + snapshot: bool, +) -> PyResult> { + let model = arguments.value("model")?; + let policy = if snapshot { + config(&model, "snapshot_schema_change")? + .filter(|value| !value.is_empty()) + .unwrap_or( + arguments + .value("snapshots_config")? + .getattr("schema_change")? + .extract()?, + ) + } else { + config(&model, "on_schema_change")? + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| "append_new_columns".to_owned()) + }; + imported( + py, + TYPES_MODULE, + if snapshot { + "SnapshotSchemaChangePolicy" + } else { + "OnSchemaChange" + }, + )? + .call1((&policy,))?; + let mut findings: Vec = Vec::new(); + for finding in arguments.value("findings")?.try_iter()? { + let finding = finding?; + findings.push(CompatibilityFinding { + kind: finding.getattr("kind")?.extract()?, + column_name: finding.getattr("column_name")?.extract()?, + expected_type: finding.getattr("expected_type")?.extract()?, + actual_type: finding.getattr("actual_type")?.extract()?, + }); + } + let dialect: Option = arguments.value("dialect")?.extract()?; + PyTuple::new( + py, + compatibility_findings( + &findings, + (&policy, snapshot), + dialect.as_deref().unwrap_or("generic"), + ) + .map_err(PyValueError::new_err)?, + ) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +fn migration_incremental_blocking_findings<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments = KeywordArguments::read( + py, + "migration_incremental_blocking_findings", + options, + (&["model", "findings", "dialect"], &[]), + )?; + blocking(py, &arguments, false) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +fn migration_snapshot_blocking_findings<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments = KeywordArguments::read( + py, + "migration_snapshot_blocking_findings", + options, + (&["model", "findings", "dialect", "snapshots_config"], &[]), + )?; + blocking(py, &arguments, true) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +fn check_migration_compatibility<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments = KeywordArguments::read( + py, + "check_migration_compatibility", + options, + ( + &[ + "model", + "origin_relation", + "origin_columns", + "sql_analysis_enabled", + "column_dialect", + "snapshots_config", + ], + &[], + ), + )?; + let model = arguments.value("model")?; + let relation = arguments.value("origin_relation")?; + let relation_type: String = relation.getattr("relation_type")?.extract()?; + if relation_type.to_uppercase().contains("VIEW") { + return result( + py, + "incompatible", + PyTuple::new( + py, + ["origin relation is a view; only tables carry migratable history"], + )?, + ); + } + let relations = PyDict::new(py); + relations.set_item(model.getattr("name")?, relation)?; + let columns = PyDict::new(py); + columns.set_item(model.getattr("name")?, arguments.value("origin_columns")?)?; + let snapshot = call_with( + &imported(py, MODELS_MODULE, "WarehouseSnapshot")?, + &[ + ("existing_relations", relations.into_any()), + ("existing_columns", columns.into_any()), + ("column_dialect", arguments.value("column_dialect")?), + ], + )?; + let change = call_with( + &imported(py, "sqlbuild._native", "detect_model_changes")?, + &[ + ("model", model.clone()), + ("snapshot", snapshot), + ( + "sql_analysis_enabled", + arguments.value("sql_analysis_enabled")?, + ), + ( + "query_change_tracking", + false.into_pyobject(py)?.to_owned().into_any(), + ), + ( + "full_refresh", + false.into_pyobject(py)?.to_owned().into_any(), + ), + ], + )?; + let schema = model.getattr("schema_entry")?; + let mut has_columns = model.getattr("inferred_columns")?.is_truthy()?; + if !schema.is_none() { + for column in schema.getattr("columns")?.try_iter()? { + has_columns |= !column?.getattr("type")?.is_none(); + } + } + if !has_columns { + return result(py, "not_checked", PyTuple::empty(py)); + } + let snapshot = config(&model, "materialized")?.as_deref() == Some("snapshot"); + let options = PyDict::new(py); + options.set_item("model", model)?; + options.set_item("findings", change.getattr("schema_findings")?)?; + options.set_item("dialect", arguments.value("column_dialect")?)?; + let findings = if snapshot { + options.set_item("snapshots_config", arguments.value("snapshots_config")?)?; + migration_snapshot_blocking_findings(py, Some(options))? + } else { + migration_incremental_blocking_findings(py, Some(options))? + }; + result( + py, + if findings.is_empty() { + "compatible" + } else { + "incompatible" + }, + findings, + ) +} + +fn result<'py>( + py: Python<'py>, + status: &str, + findings: Bound<'py, PyTuple>, +) -> PyResult> { + let status = imported( + py, + "sqlbuild.compiler.migrations.types", + "MigrationCompatibility", + )? + .call1((status,))?; + call_with( + &imported(py, MODELS_MODULE, "MigrationCompatibilityResult")?, + &[("status", status), ("findings", findings.into_any())], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(check_migration_compatibility, module)?)?; + module.add_function(wrap_pyfunction!( + migration_incremental_blocking_findings, + module + )?)?; + module.add_function(wrap_pyfunction!( + migration_snapshot_blocking_findings, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs index 3bc0aa1cf1..1a667a1e29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs @@ -3,7 +3,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyFrozenSet}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_migrations::main::{migration_fingerprint, migration_ref_names}; use std::collections::BTreeMap; @@ -64,6 +64,53 @@ fn migration_fingerprint_ref_names<'py>( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(with_migration_fingerprints, module)?)?; module.add_function(wrap_pyfunction!(build_migration_fingerprint, module)?)?; module.add_function(wrap_pyfunction!(migration_fingerprint_ref_names, module)?) } + +/// Attach persisted migration identities through the existing cache boundary. +#[pyfunction] +#[pyo3(signature = (**options))] +fn with_migration_fingerprints<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + let arguments = KeywordArguments::read( + py, + "with_migration_fingerprints", + options, + ( + &["entries", "models_by_name", "fingerprints", "dialect"], + &[], + ), + )?; + let cache = arguments.value("fingerprints")?; + let mut attached: Vec> = Vec::new(); + for entry in arguments.value("entries")?.try_iter()? { + let entry = entry?; + let model = arguments + .value("models_by_name")? + .call_method1("get", (entry.getattr("name")?,))?; + let metadata = entry.getattr("fingerprint_metadata_json")?; + if model.is_none() || metadata.is_none() || entry.getattr("action")?.eq("skip")? { + attached.push(entry); + continue; + } + let fingerprint = call_with( + &cache.getattr("fingerprint")?, + &[ + ("query_sql", model.getattr("query_sql")?), + ("metadata_json", metadata), + ("ref_identities", PyDict::new(py).into_any()), + ("dialect", arguments.value("dialect")?), + ], + )?; + let kwargs = PyDict::new(py); + kwargs.set_item("migration_fingerprint", fingerprint)?; + attached.push(imported(py, "dataclasses", "replace")?.call((entry,), Some(&kwargs))?); + } + cache.call_method0("persist")?; + PyTuple::new(py, attached) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs new file mode 100644 index 0000000000..a8c06036a3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs @@ -0,0 +1,166 @@ +//! Fingerprint cache callbacks around native one-to-one migration matching. + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_migrations::main::match_migrations::match_migrations; +use sqlbuild_analysis::planner_migrations::models::MigrationCandidate; +use std::collections::{BTreeMap, BTreeSet}; + +fn variants<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, + model: &Bound<'py, PyAny>, + maps: Bound<'py, PyAny>, +) -> PyResult> { + let name = model.getattr("name")?; + let metadata = arguments.value("metadata_jsons")?.get_item(name)?; + let cache = arguments.value("fingerprints")?.getattr("fingerprint")?; + let mut hashes: Vec = Vec::new(); + for map in maps.try_iter()? { + let hash: Option = call_with( + &cache, + &[ + ("query_sql", model.getattr("query_sql")?), + ("metadata_json", metadata.clone()), + ("ref_identities", map?), + ("dialect", arguments.value("dialect")?), + ], + )? + .extract()?; + if let Some(hash) = hash + && !hashes.contains(&hash) + { + hashes.push(hash); + } + } + let _ = py; + Ok(hashes) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +fn match_migration_fingerprints<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments = KeywordArguments::read( + py, + "match_migration_fingerprints", + options, + ( + &[ + "models", + "new_names", + "seed_variants", + "candidates", + "metadata_jsons", + "dialect", + "fingerprints", + ], + &[], + ), + )?; + let new_names: BTreeSet = arguments.value("new_names")?.extract()?; + let mut models: BTreeMap> = BTreeMap::new(); + let mut names: Vec = Vec::new(); + for model in arguments.value("models")?.try_iter()? { + let model = model?; + let name: String = model.getattr("name")?.extract()?; + if new_names.contains(&name) { + names.push(name.clone()); + } + models.insert(name, model); + } + let original: Vec> = arguments + .value("candidates")? + .try_iter()? + .collect::>()?; + let mut candidates: Vec = Vec::new(); + for candidate in &original { + candidates.push(MigrationCandidate { + name: candidate.getattr("node_name")?.extract()?, + fingerprint: stored_migration_fingerprint(candidate)?.unwrap_or_default(), + }); + } + let seeds: Vec> = arguments.value("seed_variants")?.extract()?; + let matched = match_migrations(&names, &seeds, &candidates, |name, maps| { + variants( + py, + &arguments, + &models[name], + PyTuple::new(py, maps)?.into_any(), + ) + })?; + let matches = PyDict::new(py); + for (name, index) in matched.matches { + matches.set_item(name, &original[index])?; + } + PyTuple::new( + py, + [ + PyTuple::new(py, matched.variants)?.into_any(), + matches.into_any(), + matched.ambiguous.into_pyobject(py)?.into_any(), + ], + ) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +fn migration_destination_fingerprints<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments = KeywordArguments::read( + py, + "migration_destination_fingerprints", + options, + ( + &[ + "requests", + "variants", + "metadata_jsons", + "dialect", + "fingerprints", + ], + &[], + ), + )?; + let result = PyDict::new(py); + for request in arguments.value("requests")?.try_iter()? { + let model = request?.getattr("model")?; + let name = model.getattr("name")?; + if arguments + .value("metadata_jsons")? + .call_method1("get", (&name,))? + .is_none() + { + continue; + } + result.set_item( + name, + PyTuple::new( + py, + variants(py, &arguments, &model, arguments.value("variants")?)?, + )?, + )?; + } + Ok(result) +} + +#[pyfunction] +fn stored_migration_fingerprint(fingerprint: &Bound<'_, PyAny>) -> PyResult> { + let metadata: String = fingerprint.getattr("metadata_json")?.extract()?; + Ok(sqlbuild_analysis::planner_migrations::main::stored_migration_fingerprint::stored_migration_fingerprint(&metadata)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(match_migration_fingerprints, module)?)?; + module.add_function(wrap_pyfunction!( + migration_destination_fingerprints, + module + )?)?; + module.add_function(wrap_pyfunction!(stored_migration_fingerprint, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index 72ecfd1d90..cb1301eb3f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -1,5 +1,8 @@ pub(crate) mod columns; +mod compatibility; mod errors; mod fingerprint; +mod matching; +mod old_names; mod projections; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs new file mode 100644 index 0000000000..fd047a8ba9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs @@ -0,0 +1,227 @@ +//! Existing old-name planner models at the native decision boundary. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PySet, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_migrations::main::{ + declared_column_aliases as aliases, old_name_claim_message as claim, old_name_schemas, + old_name_skip_reason, +}; + +const MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; + +#[pyfunction] +fn declared_column_aliases<'py>( + py: Python<'py>, + model: &Bound<'py, PyAny>, +) -> PyResult> { + let schema = model.getattr("schema_entry")?; + let mut columns: Vec<(String, Option)> = Vec::new(); + if !schema.is_none() { + for column in schema.getattr("columns")?.try_iter()? { + let column = column?; + columns.push(( + column.getattr("name")?.extract()?, + column.getattr("migrate_from")?.extract()?, + )); + } + } + PyTuple::new(py, aliases::declared_column_aliases(&columns)) +} + +#[pyfunction] +#[pyo3(signature = (*, snapshot, needed))] +fn old_name_schemas_to_read<'py>( + py: Python<'py>, + snapshot: &Bound<'py, PyAny>, + needed: &Bound<'py, PyAny>, +) -> PyResult> { + let listed: Option> = + snapshot.getattr("listed_state_schemas")?.extract()?; + PySet::new( + py, + old_name_schemas::old_name_schemas( + &needed.extract()?, + &snapshot.getattr("old_name_view_state_schemas")?.extract()?, + listed.as_ref(), + ), + ) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +fn old_name_claim_message<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments = KeywordArguments::read( + py, + "old_name_claim_message", + options, + (&["model_name", "view", "now"], &[]), + )?; + claim_message( + &arguments.value("model_name")?.extract::()?, + &arguments.value("view")?, + &arguments.value("now")?, + ) +} + +fn claim_message(model: &str, view: &Bound<'_, PyAny>, now: &Bound<'_, PyAny>) -> PyResult { + let old = view.getattr("old")?; + let qualified: Option = old.getattr("qualified_name")?.extract()?; + let name: String = qualified + .filter(|name| !name.is_empty()) + .unwrap_or(old.getattr("name")?.extract()?); + let expires = view.getattr("expires_at")?; + let date: Option = if expires.is_none() { + None + } else { + Some(expires.call_method1("strftime", ("%Y-%m-%d",))?.extract()?) + }; + let expired = !expires.is_none() && expires.le(now)?; + Ok(claim::old_name_claim_message( + model, + ( + &name, + &view.getattr("destination_model")?.extract::()?, + ), + (date.as_deref(), expired), + )) +} + +fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { + imported( + location.py(), + "sqlbuild.compiler.migrations.main.relation_for_location", + "migration_relation_for_location", + )? + .call1((location,)) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +fn old_name_claim_errors<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments = KeywordArguments::read( + py, + "old_name_claim_errors", + options, + (&["selected", "snapshot", "views", "now"], &[]), + )?; + let mut errors: Vec> = Vec::new(); + for model in arguments.value("selected")?.try_iter()? { + let model = model?; + let name = model.getattr("name")?; + let existing = arguments + .value("snapshot")? + .getattr("existing_relations")? + .call_method1("get", (&name,))?; + if existing.is_none() { + continue; + } + let kind = imported( + py, + "sqlbuild.adapter.type_system.main.normalize_relation_type", + "normalize_relation_type", + )? + .call1((existing.getattr("relation_type")?,))?; + if !kind.eq("view")? { + continue; + } + let destination = relation(&model.getattr("destination")?)?; + for view in arguments.value("views")?.try_iter()? { + let view = view?; + if view.getattr("destination_model")?.eq(&name)? + || !relation(&view.getattr("old")?)? + .call_method1("matches", (&destination,))? + .is_truthy()? + { + continue; + } + let message = + claim_message(&name.extract::()?, &view, &arguments.value("now")?)?; + errors.push(call_with( + &imported(py, MODELS_MODULE, "PlanWarning")?, + &[ + ("model_name", name.clone()), + ( + "severity", + imported(py, "sqlbuild.compiler.planner.types", "WarningSeverity")? + .getattr("ERROR")?, + ), + ("message", message.into_pyobject(py)?.into_any()), + ("code", "M114".into_pyobject(py)?.into_any()), + ], + )?); + break; + } + } + PyTuple::new(py, errors) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +fn old_name_view_skip_reason<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments = KeywordArguments::read( + py, + "old_name_view_skip_reason", + options, + ( + &["origin", "origin_tracked", "retention", "models", "views"], + &[], + ), + )?; + let retained = !arguments.value("retention")?.is_none(); + if !retained { + return Ok(old_name_skip_reason::old_name_skip_reason( + false, None, false, None, + )); + } + let old = relation(&arguments.value("origin")?)?; + let mut reuser: Option = None; + for model in arguments.value("models")?.try_iter()? { + let model = model?; + if relation(&model.getattr("destination")?)? + .call_method1("matches", (&old,))? + .is_truthy()? + { + reuser = Some(model.getattr("name")?.extract()?); + break; + } + } + let mut compatibility_view = false; + if reuser.is_none() { + for view in arguments.value("views")?.try_iter()? { + if relation(&view?.getattr("old")?)? + .call_method1("matches", (&old,))? + .is_truthy()? + { + compatibility_view = true; + break; + } + } + } + Ok(old_name_skip_reason::old_name_skip_reason( + true, + reuser.as_deref(), + compatibility_view, + arguments.value("origin_tracked")?.extract()?, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(declared_column_aliases, module)?)?; + module.add_function(wrap_pyfunction!(old_name_schemas_to_read, module)?)?; + module.add_function(wrap_pyfunction!(old_name_claim_message, module)?)?; + module.add_function(wrap_pyfunction!(old_name_claim_errors, module)?)?; + module.add_function(wrap_pyfunction!(old_name_view_skip_reason, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs index 0bcd6568d4..464cc9cd0c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs @@ -3,6 +3,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + super::compatibility::register(module)?; + super::matching::register(module)?; + super::old_names::register(module)?; super::columns::register(module)?; super::projections::register(module)?; super::fingerprint::register(module) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 172ebbb4f1..e7d0cc1095 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -195,6 +195,7 @@ from sqlbuild.compiler.pipeline.models import ( PythonRunPlanOutputs, StaticCommandContext, ) +from sqlbuild.compiler.planner.classes.migration_fingerprint_cache import MigrationFingerprintCache from sqlbuild.compiler.planner.classes.migration_state_inspection import ( MigrationStateInspection, ) @@ -208,8 +209,11 @@ from sqlbuild.compiler.planner.models import ( DirectModelVersionIdentities, FutureCursorSafetyEvidence, MaximumStartSafetyEvidence, + MigrationCompatibilityResult, MissingUpstream, + ModelMigrationRequest, ModelPlanEntry, + OldNameView, ParsedSelector, PathSelector, PlannerChangeResults, @@ -4300,6 +4304,72 @@ def build_migration_fingerprint( *, query_sql: str, metadata_json: str, ref_identities: Mapping[str, str], dialect: str | None ) -> str | None: ... def migration_fingerprint_ref_names(*, query_sql: str, metadata_json: str) -> frozenset[str]: ... +def check_migration_compatibility( + *, + model: CompiledModelView, + origin_relation: RelationInfo, + origin_columns: tuple[ColumnInfo, ...], + sql_analysis_enabled: bool, + column_dialect: str | None, + snapshots_config: SnapshotsConfig, +) -> MigrationCompatibilityResult: ... +def migration_incremental_blocking_findings( + *, + model: CompiledModelView, + findings: tuple[SchemaFinding, ...], + dialect: str | None, +) -> tuple[str, ...]: ... +def migration_snapshot_blocking_findings( + *, + model: CompiledModelView, + findings: tuple[SchemaFinding, ...], + dialect: str | None, + snapshots_config: SnapshotsConfig, +) -> tuple[str, ...]: ... +def match_migration_fingerprints( + *, + models: tuple[CompiledModelView, ...], + new_names: frozenset[str], + seed_variants: tuple[dict[str, str], ...], + candidates: tuple[Fingerprint, ...], + metadata_jsons: dict[str, str], + dialect: str | None, + fingerprints: MigrationFingerprintCache, +) -> tuple[tuple[dict[str, str], ...], dict[str, Fingerprint], dict[str, str]]: ... +def migration_destination_fingerprints( + *, + requests: tuple[ModelMigrationRequest, ...], + variants: tuple[dict[str, str], ...], + metadata_jsons: dict[str, str], + dialect: str | None, + fingerprints: MigrationFingerprintCache, +) -> dict[str, tuple[str, ...]]: ... +def stored_migration_fingerprint(fingerprint: Fingerprint) -> str | None: ... +def declared_column_aliases(model: CompiledModelView) -> tuple[tuple[str, str], ...]: ... +def old_name_schemas_to_read(*, snapshot: WarehouseSnapshot, needed: set[str]) -> set[str]: ... +def old_name_claim_message(*, model_name: str, view: OldNameView, now: datetime) -> str: ... +def old_name_claim_errors( + *, + selected: Iterable[CompiledModelView], + snapshot: WarehouseSnapshot, + views: tuple[OldNameView, ...], + now: datetime, +) -> tuple[PlanWarning, ...]: ... +def old_name_view_skip_reason( + *, + origin: CompiledRelationLocation, + origin_tracked: bool | None, + retention: str | None, + models: Iterable[CompiledModelView], + views: tuple[OldNameView, ...], +) -> str | None: ... +def with_migration_fingerprints( + *, + entries: tuple[ModelPlanEntry, ...], + models_by_name: Mapping[str, CompiledModelView], + fingerprints: MigrationFingerprintCache, + dialect: str | None, +) -> tuple[ModelPlanEntry, ...]: ... def identical_renames( *, previous: QueryShape, diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py b/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py deleted file mode 100644 index b60b5bbbc1..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py +++ /dev/null @@ -1,174 +0,0 @@ -"""Compatibility of a migration origin relation with its destination model.""" - -from __future__ import annotations - -from sqlbuild._native import detect_model_changes, get_config_str -from sqlbuild.adapter.contract.models import ColumnInfo, NormalizedType, RelationInfo -from sqlbuild.adapter.contract.types import TypeFamily -from sqlbuild.adapter.type_system.main.normalize_type import normalize_type -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.migrations.types import MigrationCompatibility -from sqlbuild.compiler.planner.constants import VIEW_RELATION_TYPE_MARKER -from sqlbuild.compiler.planner.models import ( - ChangeDetectionResult, - MigrationCompatibilityResult, - SchemaFinding, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import ( - MaterializationType, - OnSchemaChange, - SchemaChangeKind, - SnapshotSchemaChangePolicy, -) -from sqlbuild.spec.contracts.models import SnapshotsConfig - -_DEFAULT_ON_SCHEMA_CHANGE: OnSchemaChange = OnSchemaChange.APPEND_NEW_COLUMNS -_NUMERIC_FAMILIES: frozenset[TypeFamily] = frozenset( - {TypeFamily.INTEGER, TypeFamily.DECIMAL, TypeFamily.FLOAT} -) -_TEMPORAL_FAMILIES: frozenset[TypeFamily] = frozenset( - {TypeFamily.TIMESTAMP, TypeFamily.DATE, TypeFamily.DATETIME} -) - - -def check_migration_compatibility( - *, - model: CompiledModelView, - origin_relation: RelationInfo, - origin_columns: tuple[ColumnInfo, ...], - sql_analysis_enabled: bool, - column_dialect: str | None, - snapshots_config: SnapshotsConfig, -) -> MigrationCompatibilityResult: - """Check the origin exactly as a normal build would check an existing destination.""" - - if VIEW_RELATION_TYPE_MARKER in origin_relation.relation_type.upper(): - return MigrationCompatibilityResult( - status=MigrationCompatibility.INCOMPATIBLE, - findings=("origin relation is a view; only tables carry migratable history",), - ) - change: ChangeDetectionResult = detect_model_changes( - model=model, - snapshot=WarehouseSnapshot( - existing_relations={model.name: origin_relation}, - existing_columns={model.name: origin_columns}, - column_dialect=column_dialect, - ), - sql_analysis_enabled=sql_analysis_enabled, - query_change_tracking=False, - full_refresh=False, - ) - if not _has_expected_columns(model=model): - return MigrationCompatibilityResult(status=MigrationCompatibility.NOT_CHECKED) - materialized: str | None = get_config_str(values=model.config.values, key="materialized") - blocking: tuple[str, ...] = ( - _snapshot_blocking_findings( - model=model, - findings=change.schema_findings, - dialect=column_dialect, - snapshots_config=snapshots_config, - ) - if materialized == MaterializationType.SNAPSHOT - else _incremental_blocking_findings( - model=model, findings=change.schema_findings, dialect=column_dialect - ) - ) - if blocking: - return MigrationCompatibilityResult( - status=MigrationCompatibility.INCOMPATIBLE, findings=blocking - ) - return MigrationCompatibilityResult(status=MigrationCompatibility.COMPATIBLE) - - -def _has_expected_columns(*, model: CompiledModelView) -> bool: - if model.schema_entry is not None and any( - column.type is not None for column in model.schema_entry.columns - ): - return True - return bool(model.inferred_columns) - - -def _incremental_blocking_findings( - *, model: CompiledModelView, findings: tuple[SchemaFinding, ...], dialect: str | None -) -> tuple[str, ...]: - raw_policy: str | None = get_config_str(values=model.config.values, key="on_schema_change") - policy: OnSchemaChange = OnSchemaChange(raw_policy) if raw_policy else _DEFAULT_ON_SCHEMA_CHANGE - blocking: list[str] = [] - finding: SchemaFinding - for finding in findings: - if policy == OnSchemaChange.FAIL: - blocking.append(f"{_describe(finding)} (on_schema_change fail)") - continue - if finding.kind == SchemaChangeKind.COLUMN_TYPE_CHANGED and not _types_convertible( - expected=finding.expected_type, actual=finding.actual_type, dialect=dialect - ): - blocking.append(f"{_describe(finding)} (incompatible types)") - return tuple(blocking) - - -def _snapshot_blocking_findings( - *, - model: CompiledModelView, - findings: tuple[SchemaFinding, ...], - dialect: str | None, - snapshots_config: SnapshotsConfig, -) -> tuple[str, ...]: - raw_policy: str | None = get_config_str( - values=model.config.values, key="snapshot_schema_change" - ) - policy: SnapshotSchemaChangePolicy = SnapshotSchemaChangePolicy( - raw_policy or snapshots_config.schema_change - ) - blocking: list[str] = [] - finding: SchemaFinding - for finding in findings: - if finding.kind == SchemaChangeKind.COLUMN_TYPE_CHANGED and not _string_widening( - expected=finding.expected_type, actual=finding.actual_type, dialect=dialect - ): - blocking.append(f"{_describe(finding)} (snapshot type change)") - elif ( - finding.kind == SchemaChangeKind.COLUMN_ADDED - and policy == SnapshotSchemaChangePolicy.DENY - ): - blocking.append(f"{_describe(finding)} (snapshot_schema_change deny)") - return tuple(blocking) - - -def _types_convertible(*, expected: str | None, actual: str | None, dialect: str | None) -> bool: - if expected is None or actual is None: - return True - expected_family: TypeFamily = normalize_type(type_sql=expected, dialect=dialect).family - actual_family: TypeFamily = normalize_type(type_sql=actual, dialect=dialect).family - if TypeFamily.OTHER in {expected_family, actual_family}: - return True - if expected_family == actual_family: - return True - if {expected_family, actual_family} <= _NUMERIC_FAMILIES: - return True - return {expected_family, actual_family} <= _TEMPORAL_FAMILIES - - -def _string_widening(*, expected: str | None, actual: str | None, dialect: str | None) -> bool: - if expected is None or actual is None: - return True - target: NormalizedType = normalize_type(type_sql=actual, dialect=dialect) - incoming: NormalizedType = normalize_type(type_sql=expected, dialect=dialect) - if target == incoming: - return True - if target.family != TypeFamily.STRING or incoming.family != TypeFamily.STRING: - return False - if target.length is None: - return True - return incoming.length is not None and incoming.length <= target.length - - -def _describe(finding: SchemaFinding) -> str: - if finding.kind == SchemaChangeKind.COLUMN_ADDED: - return f"column {finding.column_name} is missing from the origin" - if finding.kind == SchemaChangeKind.COLUMN_REMOVED: - return f"origin column {finding.column_name} is not produced by the model" - return ( - f"column {finding.column_name} is {finding.actual_type} in the origin but the model " - f"produces {finding.expected_type}" - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py index d454e09865..daa83a0aad 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py @@ -3,13 +3,15 @@ from __future__ import annotations import json -from collections import defaultdict from typing import Any from sqlbuild._native import ( build_function_local_hashes, build_model_version_identity_metadata_json, get_config_str, + match_migration_fingerprints, + migration_destination_fingerprints, + stored_migration_fingerprint, ) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -29,7 +31,6 @@ from sqlbuild.compiler.planner.classes.migration_state_inspection import ( MigrationStateInspection, ) -from sqlbuild.compiler.planner.constants import MIGRATION_FINGERPRINT_METADATA_KEY from sqlbuild.compiler.planner.models import ( ModelMigrationDeclaration, ModelMigrationDiscovery, @@ -84,7 +85,7 @@ def discover_model_migrations( if not triggered: return ModelMigrationDiscovery( requests=manual_requests, - destination_fingerprints=_destination_fingerprints( + destination_fingerprints=migration_destination_fingerprints( requests=manual_requests, variants=seed_variants, metadata_jsons=metadata_jsons, @@ -109,7 +110,7 @@ def discover_model_migrations( variants: tuple[dict[str, str], ...] matches: dict[str, Fingerprint] ambiguous: dict[str, str] - variants, matches, ambiguous = _match( + variants, matches, ambiguous = match_migration_fingerprints( models=models, new_names=unbuilt - frozenset(resumed), seed_variants=seed_variants, @@ -127,7 +128,7 @@ def discover_model_migrations( return ModelMigrationDiscovery( requests=requests, warnings=_ambiguity_warnings(models=models, scope=scope, ambiguous=ambiguous), - destination_fingerprints=_destination_fingerprints( + destination_fingerprints=migration_destination_fingerprints( requests=requests, variants=variants, metadata_jsons=metadata_jsons, @@ -406,146 +407,6 @@ def _superseded(*, events: tuple[MigrationEvent, ...], location: CompiledRelatio return newest is not None and newest.origin.matches(relation) -def _match( - *, - models: tuple[CompiledModelView, ...], - new_names: frozenset[str], - seed_variants: tuple[dict[str, str], ...], - candidates: tuple[Fingerprint, ...], - metadata_jsons: dict[str, str], - dialect: str | None, - fingerprints: MigrationFingerprintCache, -) -> tuple[tuple[dict[str, str], ...], dict[str, Fingerprint], dict[str, str]]: - """Match unbuilt models to candidates under every rename-map variant.""" - - if not candidates: - return tuple(dict(seed) for seed in seed_variants), {}, {} - by_fingerprint: dict[str, list[Fingerprint]] = defaultdict(list) - candidate: Fingerprint - for candidate in candidates: - by_fingerprint[stored_migration_fingerprint(candidate) or ""].append(candidate) - blocked: set[str] = set() - ambiguous: dict[str, str] = {} - for _ in range(len(candidates) + 1): - variants: tuple[dict[str, str], ...] = tuple(dict(seed) for seed in seed_variants) - matches: dict[str, Fingerprint] = {} - claims: dict[str, list[str]] = defaultdict(list) - model: CompiledModelView - for model in models: - if model.name not in new_names: - continue - options: list[Fingerprint] = _match_options( - fingerprints=_variant_fingerprints( - model=model, - metadata_json=metadata_jsons[model.name], - variants=variants, - dialect=dialect, - fingerprints=fingerprints, - ), - by_fingerprint=by_fingerprint, - blocked=blocked, - ) - if len(options) > 1: - ambiguous[model.name] = ( - "several earlier models " - f"({', '.join(sorted(option.node_name for option in options))})" - ) - continue - if len(options) == 1: - variant: dict[str, str] - for variant in variants: - variant[model.name] = options[0].node_name - matches[model.name] = options[0] - claims[options[0].node_name].append(model.name) - contested: dict[str, list[str]] = { - origin: names for origin, names in claims.items() if len(names) > 1 - } - if not contested: - return variants, matches, ambiguous - blocked.update(contested) - ambiguous.update(_contested_details(contested)) - return tuple(dict(seed) for seed in seed_variants), {}, ambiguous - - -def _variant_fingerprints( - *, - model: CompiledModelView, - metadata_json: str, - variants: tuple[dict[str, str], ...], - dialect: str | None, - fingerprints: MigrationFingerprintCache, -) -> tuple[str, ...]: - """Return the distinct migration fingerprints of one model under each rename map.""" - - computed: list[str | None] = [ - fingerprints.fingerprint( - query_sql=model.query_sql, - metadata_json=metadata_json, - ref_identities=variant, - dialect=dialect, - ) - for variant in variants - ] - return tuple(dict.fromkeys(item for item in computed if item is not None)) - - -def _match_options( - *, - fingerprints: tuple[str, ...], - by_fingerprint: dict[str, list[Fingerprint]], - blocked: set[str], -) -> list[Fingerprint]: - options: dict[str, Fingerprint] = {} - fingerprint: str - for fingerprint in fingerprints: - options.update( - { - option.node_name: option - for option in by_fingerprint.get(fingerprint, []) - if option.node_name not in blocked - } - ) - return list(options.values()) - - -def _contested_details(contested: dict[str, list[str]]) -> dict[str, str]: - """Describe each model that shares its only matching earlier model with others.""" - - details: dict[str, str] = {} - origin: str - names: list[str] - for origin, names in contested.items(): - name: str - for name in names: - others: list[str] = [f"'{other}'" for other in sorted(names) if other != name] - details[name] = f"earlier model '{origin}', which also matches {', '.join(others)}" - return details - - -def _destination_fingerprints( - *, - requests: tuple[ModelMigrationRequest, ...], - variants: tuple[dict[str, str], ...], - metadata_jsons: dict[str, str], - dialect: str | None, - fingerprints: MigrationFingerprintCache, -) -> dict[str, tuple[str, ...]]: - destinations: dict[str, tuple[str, ...]] = {} - request: ModelMigrationRequest - for request in requests: - metadata_json: str | None = metadata_jsons.get(request.model.name) - if metadata_json is None: - continue - destinations[request.model.name] = _variant_fingerprints( - model=request.model, - metadata_json=metadata_json, - variants=variants, - dialect=dialect, - fingerprints=fingerprints, - ) - return destinations - - def _moves_history(*, model: CompiledModelView, origin: Fingerprint) -> bool: return _stores_history(model) and _stored_materialization(origin) in _HISTORY_MATERIALIZATIONS @@ -571,14 +432,6 @@ def _stores_history(model: CompiledModelView) -> bool: ) -def stored_migration_fingerprint(fingerprint: Fingerprint) -> str | None: - """Return the migration fingerprint recorded in a model fingerprint row, if any.""" - - payload: Any = _metadata_payload(fingerprint) - value: Any = payload.get(MIGRATION_FINGERPRINT_METADATA_KEY) if payload else None - return value if isinstance(value, str) else None - - def _stored_materialization(fingerprint: Fingerprint) -> str | None: payload: Any = _metadata_payload(fingerprint) config: Any = payload.get(_CONFIG_KEY) if payload else None diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/entry_fingerprints.py b/src/sqlbuild/compiler/planner/_helpers/migrations/entry_fingerprints.py deleted file mode 100644 index 665bcea14f..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/entry_fingerprints.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Attach the migration fingerprint each built model stores with its fingerprint row.""" - -from __future__ import annotations - -from collections.abc import Mapping -from dataclasses import replace - -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner.classes.migration_fingerprint_cache import ( - MigrationFingerprintCache, -) -from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.compiler.planner.types import PlanAction - - -def with_migration_fingerprints( - *, - entries: tuple[ModelPlanEntry, ...], - models_by_name: Mapping[str, CompiledModelView], - fingerprints: MigrationFingerprintCache, - dialect: str | None, -) -> tuple[ModelPlanEntry, ...]: - """Attach current-name migration fingerprints to the entries that will be built.""" - - attached: list[ModelPlanEntry] = [] - entry: ModelPlanEntry - for entry in entries: - model: CompiledModelView | None = models_by_name.get(entry.name) - if ( - model is None - or entry.fingerprint_metadata_json is None - or entry.action == PlanAction.SKIP - ): - attached.append(entry) - continue - attached.append( - replace( - entry, - migration_fingerprint=fingerprints.fingerprint( - query_sql=model.query_sql, - metadata_json=entry.fingerprint_metadata_json, - ref_identities={}, - dialect=dialect, - ), - ) - ) - fingerprints.persist() - return tuple(attached) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py b/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py index fc5badd57b..2823838847 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py @@ -6,10 +6,14 @@ from datetime import datetime from sqlbuild._native import ( + declared_column_aliases, get_config_str, hard_coded_relation_remedy, match_project_relation, + old_name_claim_errors, old_name_reference_message, + old_name_schemas_to_read, + old_name_view_skip_reason, ) from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import RelationType @@ -55,7 +59,7 @@ from sqlbuild.cursor_algebra.models import Duration from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.spec.contracts.models import MigrationsConfig, SchemaColumn +from sqlbuild.spec.contracts.models import MigrationsConfig _HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( {MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT} @@ -99,7 +103,7 @@ def plan_old_name_views( connection=runtime.connection, database=planning_database(runtime=runtime), ) - schemas: set[str] = _schemas_to_read( + schemas: set[str] = old_name_schemas_to_read( snapshot=snapshot, needed=_needed_schemas( runtime=runtime, @@ -147,7 +151,7 @@ def plan_old_name_views( entries=tuple(entries), views=views, warnings=( - *_claim_errors( + *old_name_claim_errors( selected=selected, snapshot=snapshot, views=views, now=runtime.invocation_time ), *_old_name_reference_errors(runtime=runtime, views=views, entries=tuple(entries)), @@ -242,24 +246,13 @@ def _skip_reason( ) -> str | None: """Return why the old name gets no steps; ``origin_tracked`` None means unknown.""" - if retention is None: - return "old_name_view false" - old: MigrationRelation = migration_relation_for_location(origin) - reuser: CompiledModelView | None = next( - ( - model - for model in self._models_by_name.values() - if migration_relation_for_location(model.destination).matches(old) - ), - None, + return old_name_view_skip_reason( + origin=origin, + origin_tracked=origin_tracked, + retention=retention, + models=self._models_by_name.values(), + views=self._views, ) - if reuser is not None: - return f"name reused by model:{reuser.name}" - if any(migration_relation_for_location(view.old).matches(old) for view in self._views): - return "old name is a compatibility view" - if origin_tracked is False: - return "not built by SQLBuild" - return None def _from_history( self, *, history: OldNameViewHistory, model: CompiledModelView @@ -335,21 +328,6 @@ def _planned_aliases(self, model: CompiledModelView) -> tuple[tuple[str, str], . return declared_column_aliases(model) -def declared_column_aliases(model: CompiledModelView) -> tuple[tuple[str, str], ...]: - """Return old-to-new column names a table or view model declares with migrate_from.""" - - columns: tuple[SchemaColumn, ...] = ( - model.schema_entry.columns if model.schema_entry is not None else () - ) - return tuple( - sorted( - (column.migrate_from.strip(), column.name) - for column in columns - if column.migrate_from is not None - ) - ) - - def _starts_move(entry: ModelMigrationPlanEntry) -> bool: if entry.blocks_build: return False @@ -415,18 +393,6 @@ def _needed_schemas( return schemas -def _schemas_to_read(*, snapshot: WarehouseSnapshot, needed: set[str]) -> set[str]: - """Skip schemas the warehouse snapshot already listed without finding old-name state.""" - - listed: frozenset[str] | None = snapshot.listed_state_schemas - return { - schema - for schema in needed - if schema.lower() in snapshot.old_name_view_state_schemas - or (listed is not None and schema.lower() not in listed) - } - - def _may_claim_view(*, model: CompiledModelView, snapshot: WarehouseSnapshot) -> bool: relation: RelationInfo | None = snapshot.existing_relations.get(model.name) return relation is not None and _is_view(relation) @@ -436,42 +402,6 @@ def _is_view(relation: RelationInfo) -> bool: return normalize_relation_type(relation.relation_type) == RelationType.VIEW -def _claim_errors( - *, - selected: Iterable[CompiledModelView], - snapshot: WarehouseSnapshot, - views: tuple[OldNameView, ...], - now: datetime, -) -> tuple[PlanWarning, ...]: - """Refuse a model whose destination is a compatibility view still recorded at that name.""" - - errors: list[PlanWarning] = [] - model: CompiledModelView - for model in selected: - if not _may_claim_view(model=model, snapshot=snapshot): - continue - destination: MigrationRelation = migration_relation_for_location(model.destination) - view: OldNameView | None = next( - ( - view - for view in views - if view.destination_model != model.name - and migration_relation_for_location(view.old).matches(destination) - ), - None, - ) - if view is not None: - errors.append( - PlanWarning( - model_name=model.name, - severity=WarningSeverity.ERROR, - message=claim_message(model_name=model.name, view=view, now=now), - code="M114", - ) - ) - return tuple(errors) - - def _old_name_reference_errors( *, runtime: PlannerRuntime, @@ -542,24 +472,6 @@ def _old_name_relation(*, old: CompiledRelationLocation, model_name: str) -> Pro ) -def claim_message(*, model_name: str, view: OldNameView, now: datetime) -> str: - """Describe why a model cannot take a name held by a compatibility view.""" - - name: str = view.old.qualified_name or view.old.name - if view.expires_at is not None and view.expires_at <= now: - return ( - f"model '{model_name}' would replace {name}, a compatibility view for " - f"model:{view.destination_model} that expired {view.expires_at:%Y-%m-%d}; run " - "`sqb janitor` to drop it first" - ) - until: str = "" if view.expires_at is None else f" until {view.expires_at:%Y-%m-%d}" - return ( - f"model '{model_name}' would replace {name}, a compatibility view for " - f"model:{view.destination_model}{until}; drop it early with " - f"`sqb janitor --drop-old-name-view {name}`" - ) - - def _recorded_view( *, runtime: PlannerRuntime, history: OldNameViewHistory, models: Iterable[CompiledModelView] ) -> OldNameView: diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index a0df66e45c..193e9791cc 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -6,7 +6,13 @@ from dataclasses import replace from datetime import UTC, datetime -from sqlbuild._native import compute_query_hash, get_config_str, plan_reference_renames +from sqlbuild._native import ( + check_migration_compatibility, + compute_query_hash, + get_config_str, + plan_reference_renames, + stored_migration_fingerprint, +) from sqlbuild.adapter.contract.models import ColumnInfo, MigrationStagePlan, RelationInfo from sqlbuild.adapter.contract.types import BuiltinAdapter, MigrationTransfer from sqlbuild.compiler.compile.constants import MIGRATE_FORCE_CONFIG_KEY, MIGRATE_FROM_CONFIG_KEY @@ -22,13 +28,8 @@ MigrationDiscovery, MigrationPromotion, ) -from sqlbuild.compiler.planner._helpers.migrations.compatibility import ( - MigrationCompatibilityResult, - check_migration_compatibility, -) from sqlbuild.compiler.planner._helpers.migrations.discovery import ( discover_model_migrations, - stored_migration_fingerprint, ) from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( effectively_full_refreshed_model_names, @@ -53,6 +54,7 @@ from sqlbuild.compiler.planner.models import ( CursorSnapshotScope, DeferralInputs, + MigrationCompatibilityResult, ModelCursorSnapshot, ModelMigrationDeclaration, ModelMigrationDiscovery, diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py b/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py index b103f79c02..209616597e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py @@ -4,11 +4,11 @@ from dataclasses import replace -from sqlbuild._native import build_stale_out_of_selection_warnings -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner._helpers.migrations.entry_fingerprints import ( +from sqlbuild._native import ( + build_stale_out_of_selection_warnings, with_migration_fingerprints, ) +from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner._helpers.output.plan_output import build_plan_output from sqlbuild.compiler.planner._helpers.planning.retention import plan_retention, plan_table_types from sqlbuild.compiler.planner._helpers.warehouse.source_freshness_warnings import ( From 4c31a8a6a3c6408a9b94395f761dad70ceb9ed70 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:52:24 +0100 Subject: [PATCH 02/29] chore: update Python boundary after migration decision port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 1a50270759..78fddcfcae 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 39461 +remaining_lines = 39006 [[stays]] path = "executor" From 70351714e29d41aa5a3306fa74725ebdae03881a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:38:36 +0100 Subject: [PATCH 03/29] refactor: resolve planner references and sources natively --- .../src/bindings/_helpers/mod.rs | 1 + .../_helpers/planner_resolution/mod.rs | 6 + .../_helpers/planner_resolution/refs.rs | 258 ++++++++++ .../planner_resolution/registration.rs | 8 + .../_helpers/planner_resolution/sources.rs | 342 ++++++++++++++ .../_helpers/planner_resolution/values.rs | 74 +++ .../src/bindings/main/register.rs | 1 + crates/sqlbuild-sqltext/src/lib.rs | 1 + .../src/resolution/_helpers/alias_tokens.rs | 24 + .../src/resolution/_helpers/mod.rs | 3 + .../src/resolution/constants.rs | 28 ++ .../src/resolution/main/cursor_subquery.rs | 14 + .../src/resolution/main/mod.rs | 7 + .../src/resolution/main/ref_alias.rs | 44 ++ .../src/resolution/main/reference_spans.rs | 60 +++ .../src/resolution/main/source_alias.rs | 36 ++ .../src/resolution/main/source_cast_plan.rs | 67 +++ crates/sqlbuild-sqltext/src/resolution/mod.rs | 7 + .../sqlbuild-sqltext/src/resolution/models.rs | 34 ++ .../sqlbuild-sqltext/src/resolution/types.rs | 4 + src/sqlbuild/_native.pyi | 48 ++ .../compiler/planner/_helpers/resolve/refs.py | 338 +------------- .../planner/_helpers/resolve/sources.py | 440 +----------------- .../planner/_helpers/resolve/_test_types.py | 9 + .../planner/_helpers/resolve/helpers.py | 20 + .../resolve/test_adapter_overrides.py | 58 +++ 26 files changed, 1168 insertions(+), 764 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs create mode 100644 crates/sqlbuild-sqltext/src/resolution/_helpers/alias_tokens.rs create mode 100644 crates/sqlbuild-sqltext/src/resolution/_helpers/mod.rs create mode 100644 crates/sqlbuild-sqltext/src/resolution/constants.rs create mode 100644 crates/sqlbuild-sqltext/src/resolution/main/cursor_subquery.rs create mode 100644 crates/sqlbuild-sqltext/src/resolution/main/mod.rs create mode 100644 crates/sqlbuild-sqltext/src/resolution/main/ref_alias.rs create mode 100644 crates/sqlbuild-sqltext/src/resolution/main/reference_spans.rs create mode 100644 crates/sqlbuild-sqltext/src/resolution/main/source_alias.rs create mode 100644 crates/sqlbuild-sqltext/src/resolution/main/source_cast_plan.rs create mode 100644 crates/sqlbuild-sqltext/src/resolution/mod.rs create mode 100644 crates/sqlbuild-sqltext/src/resolution/models.rs create mode 100644 crates/sqlbuild-sqltext/src/resolution/types.rs create mode 100644 tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index f7989db28e..b7f4e1e29d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -48,6 +48,7 @@ pub(crate) mod plan_output; pub(crate) mod planner_changes; pub(crate) mod planner_identity; pub(crate) mod planner_migrations; +pub(crate) mod planner_resolution; pub(crate) mod planner_scope; pub(crate) mod planner_selection; pub(crate) mod planner_strategy; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs new file mode 100644 index 0000000000..bae6e5ba20 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs @@ -0,0 +1,6 @@ +//! Planner reference resolution with Python adapter rendering callbacks. + +mod refs; +pub(crate) mod registration; +mod sources; +mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs new file mode 100644 index 0000000000..24d868c4ff --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs @@ -0,0 +1,258 @@ +//! Reference lookup and adapter callbacks for planner SQL. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_resolution::values::{ + NATIVE_MODULE, cursor_wrap, method, string, +}; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_sqltext::resolution::main::{ref_alias::ref_alias, reference_spans::reference_spans}; + +const RELATION_MODULE: &str = + "sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name"; + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn resolve_ref_references(py: Python<'_>, kwargs: Option>) -> PyResult { + let args = KeywordArguments::read( + py, + "resolve_ref_references", + kwargs, + ( + &[ + "query_sql", + "model_locations", + "seed_locations", + "cursor_bounds", + "cursor_filter_inputs", + "adapter", + "cursor_type", + "lower_bound_inclusive", + ], + &[], + ), + )?; + let original: String = args.value("query_sql")?.extract()?; + let mut sql = original.clone(); + for (kind, key) in [("ref", "model_locations"), ("seed", "seed_locations")] { + let locations = args.value(key)?; + let mut output = String::new(); + let mut last = 0; + for span in reference_spans(&sql, kind) { + output.push_str(&sql[last..span.start]); + let target = locations.call_method1("get", (&span.name,))?; + if target.is_none() { + output.push_str(&sql[span.start..span.end]); + } else { + let relation: String = call_with( + &imported( + py, + RELATION_MODULE, + "resolve_relation_location_qualified_name", + )?, + &[("adapter", args.value("adapter")?), ("location", target)], + )? + .extract()?; + let char_end = sql[..span.end].chars().count(); + let original_end = original + .char_indices() + .nth(char_end) + .map_or(original.len(), |(offset, _)| offset); + output.push_str(&cursor_wrap( + &args, + relation, + &span.name, + !ref_alias(&original, original_end), + )?); + } + last = span.end; + } + output.push_str(&sql[last..]); + sql = output; + } + Ok(sql) +} + +#[pyfunction] +#[pyo3(signature = (*, query_sql, external_sql_reference_resolver=None))] +fn resolve_dbt_ref_references( + query_sql: &str, + external_sql_reference_resolver: Option<&Bound<'_, PyAny>>, +) -> PyResult { + let Some(resolver) = external_sql_reference_resolver.filter(|value| !value.is_none()) else { + return Ok(query_sql.to_owned()); + }; + let py = resolver.py(); + let mut output = String::new(); + let mut last = 0; + for span in reference_spans(query_sql, "dbt_ref") { + output.push_str(&query_sql[last..span.start]); + let relation: Option = method( + resolver, + "resolve_reference", + &[ + ("ref_kind", string(py, "dbt_ref")), + ("ref_package", span.package.into_pyobject(py)?.into_any()), + ("ref_name", string(py, &span.name)), + ], + )? + .extract()?; + output.push_str( + relation + .as_deref() + .unwrap_or(&query_sql[span.start..span.end]), + ); + last = span.end; + } + output.push_str(&query_sql[last..]); + Ok(output) +} + +fn resolve_function_calls( + query_sql: &str, + locations: &Bound<'_, PyAny>, + adapter: &Bound<'_, PyAny>, + table: bool, +) -> PyResult { + let py = adapter.py(); + let kind = if table { "table_fn" } else { "udf" }; + let spans = reference_spans(query_sql, kind); + if spans.is_empty() { + return Ok(query_sql.to_owned()); + } + let scanner_type = imported(py, NATIVE_MODULE, "SqlCodeScanner")?; + let scan = scanner_type.call1((query_sql, py.None(), "SQL reference resolution"))?; + let mut output = String::new(); + let mut last = 0; + let mut code_index = 0; + for span in spans { + if table { + let start = query_sql[..span.start].chars().count(); + while code_index < start { + let end: Option = scan + .call_method1("non_code_end", (code_index,))? + .extract()?; + code_index = end.unwrap_or(code_index + 1); + } + if code_index > start { + continue; + } + code_index = query_sql[..span.end].chars().count(); + } + output.push_str(query_sql.get(last..span.start).unwrap_or_default()); + let target = locations.call_method1("get", (&span.name,))?; + let qualified = if target.is_none() { + py.None().into_bound(py) + } else { + target.getattr("qualified_name")? + }; + let suffix = + span.end + query_sql[span.end..].len() - query_sql[span.end..].trim_start().len(); + if qualified.is_none() || !query_sql[suffix..].starts_with('(') { + output.push_str(&query_sql[span.start..span.end]); + last = span.end; + continue; + } + let context = if table { + "SQL table function call" + } else { + "SQL UDF call" + }; + let scanner = scanner_type.call1((query_sql, py.None(), context))?; + let end: usize = scanner + .call_method1("matching_paren", (query_sql[..suffix].chars().count(),))? + .extract()?; + let end = query_sql + .char_indices() + .nth(end + 1) + .map_or(query_sql.len(), |(index, _)| index); + let hook = if table { + "render_table_function_call" + } else { + "render_udf_call" + }; + let rendered: String = method( + adapter, + hook, + &[ + ("target", qualified), + ("call_suffix_sql", string(py, &query_sql[suffix..end])), + ], + )? + .extract()?; + output.push_str(&rendered); + last = end; + } + if table { + let length = query_sql.chars().count(); + while code_index < length { + let end: Option = scan + .call_method1("non_code_end", (code_index,))? + .extract()?; + code_index = end.unwrap_or(code_index + 1); + } + } + output.push_str(&query_sql[last..]); + Ok(output) +} + +#[pyfunction] +#[pyo3(signature = (*, query_sql, function_locations, adapter))] +fn resolve_udf_references( + query_sql: &str, + function_locations: &Bound<'_, PyAny>, + adapter: &Bound<'_, PyAny>, +) -> PyResult { + resolve_function_calls(query_sql, function_locations, adapter, false) +} + +#[pyfunction] +#[pyo3(signature = (*, query_sql, function_locations, adapter))] +fn resolve_table_function_references( + query_sql: &str, + function_locations: &Bound<'_, PyAny>, + adapter: &Bound<'_, PyAny>, +) -> PyResult { + resolve_function_calls(query_sql, function_locations, adapter, true) +} + +#[pyfunction] +#[pyo3(signature = (*, model_locations, seed_locations, deferred_locations, selected_keys))] +fn apply_deferred_locations<'py>( + model_locations: &Bound<'py, PyDict>, + seed_locations: &Bound<'py, PyDict>, + deferred_locations: &Bound<'py, PyDict>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { + let selected = selected_keys + .try_iter()? + .map(|key| key?.getattr("name")?.extract::()) + .collect::>>()?; + let models = model_locations.copy()?; + let seeds = seed_locations.copy()?; + for (name, target) in deferred_locations.iter() { + if selected.contains(&name.extract::()?) { + continue; + } + if models.contains(&name)? { + models.set_item(&name, &target)?; + } + if seeds.contains(&name)? { + seeds.set_item(&name, &target)?; + } + } + Ok((models, seeds)) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_ref_references, module)?)?; + module.add_function(wrap_pyfunction!(resolve_dbt_ref_references, module)?)?; + module.add_function(wrap_pyfunction!(resolve_udf_references, module)?)?; + module.add_function(wrap_pyfunction!(resolve_table_function_references, module)?)?; + module.add_function(wrap_pyfunction!(apply_deferred_locations, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs new file mode 100644 index 0000000000..b58c9098f8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs @@ -0,0 +1,8 @@ +//! Register planner SQL resolution entry points. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + super::refs::register(module)?; + super::sources::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs new file mode 100644 index 0000000000..748fa0f48c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs @@ -0,0 +1,342 @@ +//! Source contract validation and adapter-controlled projection rendering. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_resolution::values::{cursor_wrap, method, string}; +use crate::bindings::_helpers::planner_strategy::values::{PLANNER_EXCEPTIONS_MODULE, input_error}; +use crate::bindings::_helpers::references::source_relations::render_source_relation; +use pyo3::IntoPyObject; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_sqltext::resolution::main::{ + reference_spans::reference_spans, source_alias::source_alias, +}; + +const TYPES_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; + +fn source_error(py: Python<'_>, message: &str, code: &str) -> PyErr { + input_error( + py, + (PLANNER_EXCEPTIONS_MODULE, "PlannerInputError"), + message, + Some(code), + ) +} + +fn rows(value: &Bound<'_, PyAny>) -> PyResult)>> { + value + .try_iter()? + .map(|row| { + let row = row?; + Ok(( + row.getattr("name")?.extract()?, + row.getattr("type")?.extract()?, + )) + }) + .collect() +} + +fn check_columns( + entry: &Bound<'_, PyAny>, + columns: &Bound<'_, PyAny>, + adapter: &Bound<'_, PyAny>, + relation: &str, +) -> PyResult<()> { + let py = entry.py(); + let declared = rows(&entry.getattr("columns")?)?; + let available = rows(columns)?; + let available: std::collections::HashMap> = available + .into_iter() + .map(|(name, kind)| (name.to_lowercase(), kind)) + .collect(); + let missing = declared + .iter() + .filter(|(name, _)| !available.contains_key(&name.to_lowercase())) + .map(|(name, _)| name.as_str()) + .collect::>(); + if !missing.is_empty() { + return Err(source_error( + py, + &format!( + "source {relation} declares columns not found in warehouse: {}", + missing.join(", ") + ), + "S401", + )); + } + if entry + .getattr("contract")? + .extract::>()? + .as_deref() + != Some("enforced") + { + return Ok(()); + } + for (name, kind) in declared { + let Some(kind) = kind else { + continue; + }; + let Some(Some(actual)) = available.get(&name.to_lowercase()) else { + continue; + }; + if actual.is_empty() { + continue; + } + let equal = call_with( + &imported(py, TYPES_MODULE, "types_equal")?, + &[ + ("left", string(py, actual)), + ("right", string(py, &kind)), + ("dialect", adapter.getattr("sql_analysis_dialect_name")?), + ], + )? + .is_truthy()?; + if !equal { + return Err(source_error( + py, + &format!( + "source {relation} column '{name}' has type {actual} but contract declares {kind}" + ), + "S404", + )); + } + } + Ok(()) +} + +fn cast_projection<'py>( + adapter: &Bound<'py, PyAny>, + name: &str, + declared: &str, + kind: &str, +) -> PyResult> { + let py = adapter.py(); + method( + adapter, + "render_source_expression_cast", + &[ + ( + "expression", + adapter.call_method1("render_identifier", (name,))?, + ), + ("target_type", string(py, kind)), + ( + "alias", + adapter.call_method1("render_identifier", (declared,))?, + ), + ], + ) +} + +fn cast_source( + entry: &Bound<'_, PyAny>, + columns: &Bound<'_, PyAny>, + adapter: &Bound<'_, PyAny>, + relation: String, +) -> PyResult { + let declared = rows(&entry.getattr("columns")?)?; + if !declared.iter().any(|(_, kind)| kind.is_some()) { + return Ok(relation); + } + let py = entry.py(); + let expression = !entry.getattr("expression")?.is_none(); + let source_name: String = entry.getattr("name")?.extract()?; + if columns.is_none() { + if expression { + return Err(source_error( + py, + &format!( + "source expression '{source_name}' type enforcement requires query output column metadata" + ), + "S402", + )); + } + return Ok(relation); + } + let available = rows(columns)?; + let plan = sqlbuild_sqltext::resolution::main::source_cast_plan::source_cast_plan( + &declared, &available, expression, + ); + let names: Vec<&str> = available.iter().map(|(name, _)| name.as_str()).collect(); + if expression { + let name_map: std::collections::HashMap = names + .iter() + .map(|name| (name.to_lowercase(), *name)) + .collect(); + let missing = declared + .iter() + .filter(|(name, _)| !name_map.contains_key(&name.to_lowercase())) + .map(|(name, _)| name.as_str()) + .collect::>(); + if !missing.is_empty() { + let available = if names.is_empty() { + "".to_owned() + } else { + names.join(", ") + }; + return Err(source_error( + py, + &format!( + "source expression '{source_name}' declares columns not found in query output: {}. Available query output columns: {available}", + missing.join(", ") + ), + "S403", + )); + } + let mut projections: Vec> = Vec::new(); + for projection in &plan.projections { + projections.push(if let Some((declared, kind)) = &projection.cast { + cast_projection(adapter, &projection.name, declared, kind)? + } else { + adapter.call_method1("render_identifier", (&projection.name,))? + }); + } + return method( + adapter, + "render_source_expression_cast_subquery", + &[ + ("source_relation", string(py, &relation)), + ("projections", PyTuple::new(py, projections)?.into_any()), + ], + )? + .extract(); + } + if plan.projections.is_empty() { + return Ok(relation); + } + let mut projections: Vec> = Vec::new(); + for projection in &plan.projections { + if let Some((name, kind)) = &projection.cast { + projections.push(cast_projection(adapter, &projection.name, name, kind)?); + } + } + method( + adapter, + "_render_source_relation_cast_subquery_with_columns", + &[ + ("source_relation", string(py, &relation)), + ( + "cast_projections", + PyTuple::new(py, projections)?.into_any(), + ), + ( + "cast_column_names", + PyTuple::new( + py, + plan.projections.iter().map(|projection| &projection.name), + )? + .into_any(), + ), + ( + "warehouse_column_names", + PyTuple::new(py, names)?.into_any(), + ), + ( + "all_columns_cast", + plan.all_columns_cast + .into_pyobject(py)? + .to_owned() + .into_any(), + ), + ], + )? + .extract() +} + +#[pyfunction] +#[pyo3(signature = (*, adapter, source_entry))] +fn render_source_read_relation( + adapter: &Bound<'_, PyAny>, + source_entry: &Bound<'_, PyAny>, +) -> PyResult { + render_source_relation(source_entry, Some(adapter)) +} + +fn resolve_one(args: &KeywordArguments<'_>, name: &str, original: &str) -> PyResult { + let entry = args.value("source_map")?.call_method1("get", (name,))?; + if entry.is_none() { + return Ok(original.to_owned()); + } + let adapter = args.value("adapter")?; + let mut relation = render_source_read_relation(&adapter, &entry)?; + let columns = args + .value("source_warehouse_columns")? + .call_method1("get", (name,))?; + let expression = !entry.getattr("expression")?.is_none(); + if columns.is_truthy()? + && (!expression + || entry + .getattr("contract")? + .extract::>()? + .as_deref() + == Some("enforced")) + { + let label = if expression { + format!("source expression '{name}'") + } else { + relation.clone() + }; + check_columns(&entry, &columns, &adapter, &label)?; + } + if entry.getattr("type_enforcement")?.is_truthy()? { + relation = cast_source(&entry, &columns, &adapter, relation)?; + } + cursor_wrap(args, relation, name, false) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn resolve_source_references( + py: Python<'_>, + kwargs: Option>, +) -> PyResult { + let args = KeywordArguments::read( + py, + "resolve_source_references", + kwargs, + ( + &[ + "query_sql", + "source_map", + "source_warehouse_columns", + "star_exclude_keyword", + "cursor_bounds", + "cursor_filter_inputs", + "adapter", + "cursor_type", + "lower_bound_inclusive", + ], + &[], + ), + )?; + let sql: String = args.value("query_sql")?.extract()?; + let aliases = args + .value("adapter")? + .call_method0("requires_derived_table_aliases")? + .is_truthy()?; + let mut output = String::new(); + let mut last = 0; + for span in reference_spans(&sql, "source") { + let mut relation = resolve_one(&args, &span.name, &sql[span.start..span.end])?; + output.push_str(sql.get(last..span.start).unwrap_or_default()); + last = span.end; + if aliases && relation.trim_start().starts_with('(') { + let (end, alias) = source_alias(&sql, span.end, &span.name); + relation.push_str(&format!(" AS {alias}")); + last = end; + } + output.push_str(&relation); + } + output.push_str(&sql[last..]); + Ok(output) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_source_references, module)?)?; + module.add_function(wrap_pyfunction!(render_source_read_relation, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs new file mode 100644 index 0000000000..47d885cc62 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs @@ -0,0 +1,74 @@ +//! Boundary calls shared by reference and source resolution. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_sqltext::resolution::main::cursor_subquery::cursor_subquery; +use sqlbuild_sqltext::resolution::models::CursorFilter; + +pub(super) const NATIVE_MODULE: &str = "sqlbuild._native"; + +pub(super) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { + PyString::new(py, text).into_any() +} + +pub(super) fn method<'py>( + object: &Bound<'py, PyAny>, + name: &str, + values: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with(&object.getattr(name)?, values) +} + +pub(super) fn cursor_wrap( + arguments: &KeywordArguments<'_>, + relation: String, + name: &str, + inject_alias: bool, +) -> PyResult { + let bounds = arguments.value("cursor_bounds")?; + if bounds.is_none() { + return Ok(relation); + } + let column: Option = arguments + .value("cursor_filter_inputs")? + .call_method1("get", (name,))? + .extract()?; + let Some(column) = column else { + return Ok(relation); + }; + let adapter = arguments.value("adapter")?; + let py = adapter.py(); + let cursor_type = arguments.value("cursor_type")?; + let literal = |bound: &str| -> PyResult { + let token = call_with( + &imported(py, NATIVE_MODULE, "cursor_sentinel_to_token")?, + &[("sentinel", bounds.getattr(bound)?)], + )?; + method( + &adapter, + "render_cursor_bound_literal", + &[("value", token), ("cursor_type", cursor_type.clone())], + )? + .extract() + }; + let start = literal("start")?; + let end = literal("end")?; + let alias = inject_alias + && adapter + .call_method0("requires_derived_table_aliases")? + .is_truthy()?; + Ok(cursor_subquery( + &relation, + &CursorFilter { + column: &column, + start: &start, + end: &end, + lower_inclusive: arguments.value("lower_bound_inclusive")?.is_truthy()?, + typed: !cursor_type.is_none(), + }, + alias, + )) +} diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 23cb732f73..87a62e7eed 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -52,6 +52,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { planner_selection::registration::register(module)?; planner_scope::registration::register(module)?; planner_strategy::registration::register(module)?; + crate::bindings::_helpers::planner_resolution::registration::register(module)?; references::registration::register(module)?; authored_values::registration::register(module)?; auditing::registration::register(module)?; diff --git a/crates/sqlbuild-sqltext/src/lib.rs b/crates/sqlbuild-sqltext/src/lib.rs index 8e5ea36cbe..59a15f977b 100644 --- a/crates/sqlbuild-sqltext/src/lib.rs +++ b/crates/sqlbuild-sqltext/src/lib.rs @@ -5,5 +5,6 @@ pub mod code_scanner; pub mod compiler; mod constants; +pub mod resolution; pub mod sql_references; pub mod sql_scan; diff --git a/crates/sqlbuild-sqltext/src/resolution/_helpers/alias_tokens.rs b/crates/sqlbuild-sqltext/src/resolution/_helpers/alias_tokens.rs new file mode 100644 index 0000000000..27ee6fb965 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/_helpers/alias_tokens.rs @@ -0,0 +1,24 @@ +//! Alias token reading shared by source normalization. + +pub(crate) fn skip_space(sql: &str, start: usize) -> usize { + start + sql.get(start..).unwrap_or_default().len() + - sql.get(start..).unwrap_or_default().trim_start().len() +} + +pub(crate) fn alias_token(sql: &str, start: usize) -> Option<(usize, &str)> { + let suffix = sql.get(start..)?; + let first = suffix.chars().next()?; + if matches!(first, '[' | '"' | '`') { + let close = if first == '[' { ']' } else { first }; + let end = start + 1 + suffix[1..].find(close)? + 1; + return Some((end, &sql[start..end])); + } + if !(first.is_ascii_alphabetic() || first == '_') { + return None; + } + let length = suffix + .bytes() + .take_while(|c| c.is_ascii_alphanumeric() || matches!(c, b'_' | b'$')) + .count(); + Some((start + length, &suffix[..length])) +} diff --git a/crates/sqlbuild-sqltext/src/resolution/_helpers/mod.rs b/crates/sqlbuild-sqltext/src/resolution/_helpers/mod.rs new file mode 100644 index 0000000000..d3fd88c7ee --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/_helpers/mod.rs @@ -0,0 +1,3 @@ +//! Internal resolution token readers. + +pub(crate) mod alias_tokens; diff --git a/crates/sqlbuild-sqltext/src/resolution/constants.rs b/crates/sqlbuild-sqltext/src/resolution/constants.rs new file mode 100644 index 0000000000..7f086a7682 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/constants.rs @@ -0,0 +1,28 @@ +//! Alias boundaries retained from the planner resolution contract. + +pub(super) const EXTERNAL_REFERENCE: &str = "dbt_ref"; + +pub(super) const REF_CLAUSES: &[&str] = &[ + "WHERE", "JOIN", "INNER", "LEFT", "RIGHT", "FULL", "CROSS", "ON", "GROUP", "ORDER", "HAVING", + "LIMIT", "UNION", "QUALIFY", +]; +pub(super) const SOURCE_CLAUSES: &[&str] = &[ + "APPLY", + "CROSS", + "EXCEPT", + "FULL", + "GROUP", + "HAVING", + "INNER", + "INTERSECT", + "JOIN", + "LEFT", + "LIMIT", + "OFFSET", + "ON", + "ORDER", + "OUTER", + "RIGHT", + "UNION", + "WHERE", +]; diff --git a/crates/sqlbuild-sqltext/src/resolution/main/cursor_subquery.rs b/crates/sqlbuild-sqltext/src/resolution/main/cursor_subquery.rs new file mode 100644 index 0000000000..0de883a911 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/main/cursor_subquery.rs @@ -0,0 +1,14 @@ +//! Shared cursor-filtered relation construction. + +use crate::resolution::models::CursorFilter; + +/// Wrap an adapter-rendered relation and bounds in a cursor filter. +pub fn cursor_subquery(relation: &str, filter: &CursorFilter<'_>, alias: bool) -> String { + let lower = if filter.lower_inclusive { ">=" } else { ">" }; + let upper = if filter.typed { "<" } else { "<=" }; + let suffix = if alias { " AS __cursor_ref" } else { "" }; + format!( + "(SELECT * FROM {relation} WHERE {} {lower} {} AND {} {upper} {}){suffix}", + filter.column, filter.start, filter.column, filter.end + ) +} diff --git a/crates/sqlbuild-sqltext/src/resolution/main/mod.rs b/crates/sqlbuild-sqltext/src/resolution/main/mod.rs new file mode 100644 index 0000000000..8b223a84d6 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/main/mod.rs @@ -0,0 +1,7 @@ +//! Public SQL resolution decisions. + +pub mod cursor_subquery; +pub mod ref_alias; +pub mod reference_spans; +pub mod source_alias; +pub mod source_cast_plan; diff --git a/crates/sqlbuild-sqltext/src/resolution/main/ref_alias.rs b/crates/sqlbuild-sqltext/src/resolution/main/ref_alias.rs new file mode 100644 index 0000000000..8544127cb1 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/main/ref_alias.rs @@ -0,0 +1,44 @@ +//! Detect the aliases that suppress an injected cursor-reference alias. + +use crate::resolution::_helpers::alias_tokens::skip_space; +use crate::resolution::constants::REF_CLAUSES; + +/// Whether the original SQL supplies an alias after this reference. +pub fn ref_alias(sql: &str, start: usize) -> bool { + let index = skip_space(sql, start); + let Some(suffix) = sql.get(index..) else { + return false; + }; + let leading = |ch: char| ch.is_alphabetic() || matches!(ch, '_' | '['); + if suffix + .get(..2) + .is_some_and(|s| s.eq_ignore_ascii_case("AS")) + { + return sql + .get(skip_space(sql, index + 2)..) + .and_then(|s| s.chars().next()) + .is_some_and(leading); + } + if !suffix.chars().next().is_some_and(leading) { + return false; + } + let token = if let Some(rest) = suffix.strip_prefix('[') { + let Some(end) = rest.find(']') else { + return false; + }; + if end == 0 { + return false; + } + &rest[..end] + } else { + let length = suffix + .bytes() + .take_while(|ch| ch.is_ascii_alphanumeric() || *ch == b'_') + .count(); + if length == 0 { + return false; + } + &suffix[..length] + }; + !REF_CLAUSES.contains(&token.to_uppercase().as_str()) +} diff --git a/crates/sqlbuild-sqltext/src/resolution/main/reference_spans.rs b/crates/sqlbuild-sqltext/src/resolution/main/reference_spans.rs new file mode 100644 index 0000000000..7f1d881d41 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/main/reference_spans.rs @@ -0,0 +1,60 @@ +//! Locate planner reference markers with their historical lexical policy. + +use crate::resolution::constants::EXTERNAL_REFERENCE; +use crate::resolution::models::ReferenceSpan; + +/// Locate exact reference markers, retaining the external-reference whitespace grammar. +pub fn reference_spans(sql: &str, kind: &str) -> Vec { + let prefix = format!("__{kind}("); + let external = kind == EXTERNAL_REFERENCE; + let mut spans: Vec = Vec::new(); + let mut consumed = 0; + for (start, _) in sql.char_indices() { + if start < consumed { + continue; + } + let Some(candidate) = sql.get(start..start + prefix.len()) else { + continue; + }; + if candidate != prefix && !(external && candidate.eq_ignore_ascii_case(&prefix)) { + continue; + } + let mut tail = &sql[start + prefix.len()..]; + if external { + tail = tail.trim_start(); + } + let Some((first, rest)) = quoted_name(tail) else { + continue; + }; + tail = if external { rest.trim_start() } else { rest }; + let mut second = None; + if external && tail.starts_with(',') { + let Some((name, rest)) = quoted_name(tail[1..].trim_start()) else { + continue; + }; + second = Some(name); + tail = rest.trim_start(); + } + if !tail.starts_with(')') { + continue; + } + let end = sql.len() - tail.len() + 1; + spans.push(ReferenceSpan { + start, + end, + name: second.unwrap_or(first).to_owned(), + package: second.map(|_| first.to_owned()), + }); + consumed = end; + } + spans +} + +fn quoted_name(text: &str) -> Option<(&str, &str)> { + let text = text.strip_prefix('"')?; + let end = text.find('"')?; + if end == 0 { + return None; + } + Some((&text[..end], &text[end + 1..])) +} diff --git a/crates/sqlbuild-sqltext/src/resolution/main/source_alias.rs b/crates/sqlbuild-sqltext/src/resolution/main/source_alias.rs new file mode 100644 index 0000000000..058b5fd9a0 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/main/source_alias.rs @@ -0,0 +1,36 @@ +//! Preserve source aliases when normalizing derived table factors. + +use crate::resolution::_helpers::alias_tokens::{alias_token, skip_space}; +use crate::resolution::constants::SOURCE_CLAUSES; + +/// Return the consumed alias end and the explicit or generated source alias. +pub fn source_alias(sql: &str, start: usize, source_name: &str) -> (usize, String) { + let token = alias_token(sql, skip_space(sql, start)); + let token = match token { + Some((end, token)) if token.eq_ignore_ascii_case("AS") => { + alias_token(sql, skip_space(sql, end)) + } + other => other, + }; + if let Some((end, token)) = token + && !SOURCE_CLAUSES.contains(&token.to_uppercase().as_str()) + { + return (end, token.to_owned()); + } + let mut suffix = String::new(); + for ch in source_name.chars() { + if ch.is_ascii_alphanumeric() || ch == '_' { + suffix.push(ch.to_ascii_lowercase()); + } else if !suffix.ends_with('_') { + suffix.push('_'); + } + } + let suffix = suffix.trim_matches('_'); + ( + start, + format!( + "__sqb_source_{}", + if suffix.is_empty() { "source" } else { suffix } + ), + ) +} diff --git a/crates/sqlbuild-sqltext/src/resolution/main/source_cast_plan.rs b/crates/sqlbuild-sqltext/src/resolution/main/source_cast_plan.rs new file mode 100644 index 0000000000..b9ad75fcd2 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/main/source_cast_plan.rs @@ -0,0 +1,67 @@ +//! Plan source casts without making adapter rendering decisions. + +use crate::resolution::models::{SourceCastPlan, SourceProjection}; +use crate::resolution::types::SourceColumns; +use std::collections::{HashMap, HashSet}; + +/// Match declared casts to the actual source columns with the source-kind case policy. +pub fn source_cast_plan( + declared: &SourceColumns, + available: &SourceColumns, + expression: bool, +) -> SourceCastPlan { + let mut enforced: Vec<(&str, &str)> = Vec::new(); + for (name, kind) in declared { + let Some(kind) = kind else { + continue; + }; + if let Some((_, previous)) = enforced.iter_mut().find(|(key, _)| *key == name) { + *previous = kind; + } else { + enforced.push((name, kind)); + } + } + if expression { + let names: HashMap = available + .iter() + .map(|(name, _)| (name.to_lowercase(), name.as_str())) + .collect(); + let mut casts: HashMap<&str, (&str, &str)> = HashMap::new(); + for (declared, kind) in enforced { + if let Some(actual) = names.get(&declared.to_lowercase()) { + casts.insert(*actual, (declared, kind)); + } + } + let projections: Vec = available + .iter() + .map(|(name, _)| expression_projection(name, &casts)) + .collect(); + return SourceCastPlan { + projections, + all_columns_cast: false, + }; + } + let names: HashSet<&str> = available.iter().map(|(name, _)| name.as_str()).collect(); + let projections: Vec = enforced + .into_iter() + .filter(|(name, _)| names.contains(name)) + .map(|(name, kind)| SourceProjection { + name: name.to_owned(), + cast: Some((name.to_owned(), kind.to_owned())), + }) + .collect(); + let all_columns_cast = projections.len() == names.len(); + SourceCastPlan { + projections, + all_columns_cast, + } +} + +fn expression_projection(name: &str, casts: &HashMap<&str, (&str, &str)>) -> SourceProjection { + SourceProjection { + name: name.to_owned(), + cast: casts + .get(name) + .map(|(declared, kind)| ((*declared).to_owned(), (*kind).to_owned())), + } +} diff --git a/crates/sqlbuild-sqltext/src/resolution/mod.rs b/crates/sqlbuild-sqltext/src/resolution/mod.rs new file mode 100644 index 0000000000..cf29545fd5 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/mod.rs @@ -0,0 +1,7 @@ +//! SQL resolution decisions independent of adapter rendering. + +mod _helpers; +mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-sqltext/src/resolution/models.rs b/crates/sqlbuild-sqltext/src/resolution/models.rs new file mode 100644 index 0000000000..16110564a2 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/models.rs @@ -0,0 +1,34 @@ +//! Resolution spans and rendered cursor inputs. + +/// A reference marker at UTF-8 byte offsets. +#[derive(Debug)] +pub struct ReferenceSpan { + pub start: usize, + pub end: usize, + pub name: String, + pub package: Option, +} + +/// The adapter-rendered bounds and comparison policy of a cursor filter. +#[derive(Debug)] +pub struct CursorFilter<'a> { + pub column: &'a str, + pub start: &'a str, + pub end: &'a str, + pub lower_inclusive: bool, + pub typed: bool, +} + +/// One source projection whose identifiers and type are rendered by the adapter. +#[derive(Debug)] +pub struct SourceProjection { + pub name: String, + pub cast: Option<(String, String)>, +} + +/// A source cast plan in stable declaration or warehouse order. +#[derive(Debug)] +pub struct SourceCastPlan { + pub projections: Vec, + pub all_columns_cast: bool, +} diff --git a/crates/sqlbuild-sqltext/src/resolution/types.rs b/crates/sqlbuild-sqltext/src/resolution/types.rs new file mode 100644 index 0000000000..cdcc959dc5 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/types.rs @@ -0,0 +1,4 @@ +//! Source boundary column collection types. + +/// Column names and optional declared types at the source boundary. +pub type SourceColumns = Vec<(String, Option)>; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index e7d0cc1095..3be466b32f 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3242,6 +3242,54 @@ def runtime_project_relations( *, relations: Mapping[SqlResourceRef, str], dialect: str | None ) -> ProjectRelationIndex: ... def render_source_relation(*, entry: SourceEntry, adapter: StrictAdapter | None = None) -> str: ... +def render_source_read_relation(*, adapter: BaseAdapter, source_entry: SourceEntry) -> str: ... +def resolve_ref_references( + *, + query_sql: str, + model_locations: dict[str, CompiledRelationLocation], + seed_locations: dict[str, CompiledRelationLocation], + cursor_bounds: CursorBounds | None, + cursor_filter_inputs: dict[str, str], + adapter: BaseAdapter, + cursor_type: str | None, + lower_bound_inclusive: bool, +) -> str: ... +def resolve_source_references( + *, + query_sql: str, + source_map: dict[str, SourceEntry], + source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]], + star_exclude_keyword: str, + cursor_bounds: CursorBounds | None, + cursor_filter_inputs: dict[str, str], + adapter: BaseAdapter, + cursor_type: str | None, + lower_bound_inclusive: bool, +) -> str: ... +def resolve_dbt_ref_references( + *, + query_sql: str, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, +) -> str: ... +def resolve_udf_references( + *, + query_sql: str, + function_locations: dict[str, CompiledRelationLocation], + adapter: BaseAdapter, +) -> str: ... +def resolve_table_function_references( + *, + query_sql: str, + function_locations: dict[str, CompiledRelationLocation], + adapter: BaseAdapter, +) -> str: ... +def apply_deferred_locations( + *, + model_locations: dict[str, CompiledRelationLocation], + seed_locations: dict[str, CompiledRelationLocation], + deferred_locations: dict[str, CompiledRelationLocation], + selected_keys: frozenset[CompiledObjectKey], +) -> tuple[dict[str, CompiledRelationLocation], dict[str, CompiledRelationLocation]]: ... def substitute_placeholder_defaults(sql: str, placeholders: list[tuple[str, str]], /) -> str: ... def fixture_projection_facts( sql: str, dialect: str | None, / diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py index ea4cf8a5be..085697c523 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py @@ -1,4 +1,4 @@ -"""Ref and dbt_ref resolution with optional cursor-filtered subquery wrapping.""" +"""Native reference resolution and generic executable-SQL callback support.""" from __future__ import annotations @@ -6,248 +6,20 @@ from collections.abc import Callable, Iterator import sqlbuild._native as _native -from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( - resolve_relation_location_qualified_name, -) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) +from sqlbuild._native import apply_deferred_locations as apply_deferred_locations +from sqlbuild._native import resolve_dbt_ref_references as resolve_dbt_ref_references +from sqlbuild._native import resolve_ref_references as resolve_ref_references +from sqlbuild._native import resolve_table_function_references as resolve_table_function_references +from sqlbuild._native import resolve_udf_references as resolve_udf_references +from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.compile.types import ( CompiledFunctionView, CompiledModelView, CompiledSeedView, ) -from sqlbuild.compiler.planner.constants import ( - SQL_ALIAS_BOUNDARY_CHARACTERS, - SQL_ALIAS_KEYWORD, - SQL_FUNCTION_CALL_OPEN_PAREN, - SQL_IDENTIFIER_LEADING_CHARACTERS, -) -from sqlbuild.compiler.planner.models import CursorBounds -from sqlbuild.compiler.references.constants import ( - DBT_REF_CALL_PREFIX, - REF_PATTERN, - SEED_PATTERN, - TABLE_FUNCTION_PATTERN, - UDF_PATTERN, -) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -_REF_PATTERN: re.Pattern[str] = REF_PATTERN -_SEED_PATTERN: re.Pattern[str] = SEED_PATTERN -_DBT_REF_PATTERN: re.Pattern[str] = re.compile( - rf'{DBT_REF_CALL_PREFIX}\s*"([^"]+)"\s*' - r'(?:,\s*"([^"]+)"\s*)?\)', - re.IGNORECASE, -) -_UDF_PATTERN: re.Pattern[str] = UDF_PATTERN -_TABLE_FUNCTION_PATTERN: re.Pattern[str] = TABLE_FUNCTION_PATTERN _REFERENCE_SCAN_CONTEXT: str = "SQL reference resolution" -_CLAUSE_KEYWORDS: frozenset[str] = frozenset( - { - "WHERE", - "JOIN", - "INNER", - "LEFT", - "RIGHT", - "FULL", - "CROSS", - "ON", - "GROUP", - "ORDER", - "HAVING", - "LIMIT", - "UNION", - "QUALIFY", - } -) - - -def resolve_ref_references( - *, - query_sql: str, - model_locations: dict[str, CompiledRelationLocation], - seed_locations: dict[str, CompiledRelationLocation], - cursor_bounds: CursorBounds | None, - cursor_filter_inputs: dict[str, str], - adapter: BaseAdapter, - cursor_type: str | None, - lower_bound_inclusive: bool, -) -> str: - """Replace all __ref() and __seed() calls with qualified names or cursor subqueries.""" - - def _replace_ref(match: re.Match[str]) -> str: - ref_name: str = match.group(1) - target: CompiledRelationLocation | None = model_locations.get(ref_name) - if target is None: - return match.group(0) - qualified_name: str = resolve_relation_location_qualified_name( - adapter=adapter, location=target - ) - if cursor_bounds is None: - return qualified_name - cursor_column: str | None = cursor_filter_inputs.get(ref_name) - if cursor_column is None: - return qualified_name - has_user_alias: bool = _has_following_alias(sql=query_sql, start=match.end()) - return _build_cursor_subquery( - qualified_name=qualified_name, - cursor_column=cursor_column, - bounds=cursor_bounds, - adapter=adapter, - cursor_type=cursor_type, - lower_bound_inclusive=lower_bound_inclusive, - inject_alias=not has_user_alias, - ) - - def _replace_seed(match: re.Match[str]) -> str: - seed_name: str = match.group(1) - target: CompiledRelationLocation | None = seed_locations.get(seed_name) - if target is None: - return match.group(0) - qualified_name: str = resolve_relation_location_qualified_name( - adapter=adapter, location=target - ) - if cursor_bounds is None: - return qualified_name - cursor_column: str | None = cursor_filter_inputs.get(seed_name) - if cursor_column is None: - return qualified_name - has_user_alias: bool = _has_following_alias(sql=query_sql, start=match.end()) - return _build_cursor_subquery( - qualified_name=qualified_name, - cursor_column=cursor_column, - bounds=cursor_bounds, - adapter=adapter, - cursor_type=cursor_type, - lower_bound_inclusive=lower_bound_inclusive, - inject_alias=not has_user_alias, - ) - - return _SEED_PATTERN.sub(_replace_seed, _REF_PATTERN.sub(_replace_ref, query_sql)) - - -def resolve_dbt_ref_references( - *, query_sql: str, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None -) -> str: - """Replace all __dbt_ref() calls with external relation names.""" - - def _replace_dbt_ref(match: re.Match[str]) -> str: - if external_sql_reference_resolver is None: - return match.group(0) - first_arg: str = match.group(1) - second_arg: str | None = match.group(2) - relation_name: str | None = external_sql_reference_resolver.resolve_reference( - ref_kind="dbt_ref", - ref_package=first_arg if second_arg is not None else None, - ref_name=second_arg if second_arg is not None else first_arg, - ) - return match.group(0) if relation_name is None else relation_name - - return _DBT_REF_PATTERN.sub(_replace_dbt_ref, query_sql) - - -def resolve_udf_references( - *, - query_sql: str, - function_locations: dict[str, CompiledRelationLocation], - adapter: BaseAdapter, -) -> str: - """Replace __udf() calls with adapter-specific scalar UDF calls.""" - - parts: list[str] = [] - last_index: int = 0 - match: re.Match[str] - for match in _UDF_PATTERN.finditer(query_sql): - parts.append(query_sql[last_index : match.start()]) - function_name: str = match.group(1) - target: CompiledRelationLocation | None = function_locations.get(function_name) - if target is None or target.qualified_name is None: - parts.append(match.group(0)) - last_index = match.end() - continue - - call_suffix_start: int = _skip_whitespace(sql=query_sql, start=match.end()) - if ( - call_suffix_start >= len(query_sql) - or query_sql[call_suffix_start] != SQL_FUNCTION_CALL_OPEN_PAREN - ): - parts.append(match.group(0)) - last_index = match.end() - continue - - call_suffix_end: int = _native.SqlCodeScanner( - query_sql, None, "SQL UDF call" - ).matching_paren(call_suffix_start) - call_suffix_sql: str = query_sql[call_suffix_start : call_suffix_end + 1] - parts.append( - adapter.render_udf_call( - target=target.qualified_name, - call_suffix_sql=call_suffix_sql, - ) - ) - last_index = call_suffix_end + 1 - - parts.append(query_sql[last_index:]) - return "".join(parts) - - -def _skip_whitespace(*, sql: str, start: int) -> int: - index: int = start - while index < len(sql) and sql[index].isspace(): - index += 1 - return index - - -def resolve_table_function_references( - *, - query_sql: str, - function_locations: dict[str, CompiledRelationLocation], - adapter: BaseAdapter, -) -> str: - """Replace __table_fn() calls with adapter-specific table function calls.""" - - if _TABLE_FUNCTION_PATTERN.search(query_sql) is None: - return query_sql - parts: list[str] = [] - last_index: int = 0 - match: re.Match[str] - for match in _iter_executable_matches(sql=query_sql, pattern=_TABLE_FUNCTION_PATTERN): - parts.append(query_sql[last_index : match.start()]) - function_name: str = match.group(1) - target: CompiledRelationLocation | None = function_locations.get(function_name) - if target is None or target.qualified_name is None: - parts.append(match.group(0)) - last_index = match.end() - continue - - call_suffix_start: int = _skip_whitespace(sql=query_sql, start=match.end()) - if ( - call_suffix_start >= len(query_sql) - or query_sql[call_suffix_start] != SQL_FUNCTION_CALL_OPEN_PAREN - ): - parts.append(match.group(0)) - last_index = match.end() - continue - - call_suffix_end: int = _native.SqlCodeScanner( - query_sql, None, "SQL table function call" - ).matching_paren(call_suffix_start) - call_suffix_sql: str = query_sql[call_suffix_start : call_suffix_end + 1] - parts.append( - adapter.render_table_function_call( - target=target.qualified_name, - call_suffix_sql=call_suffix_sql, - ) - ) - last_index = call_suffix_end + 1 - - parts.append(query_sql[last_index:]) - return "".join(parts) def replace_executable_matches( @@ -261,8 +33,10 @@ def replace_executable_matches( parts: list[str] = [] last_index: int = 0 - for match in _iter_dialect_executable_matches( - sql=sql, pattern=pattern, lexical_syntax=lexical_syntax + for match in _iter_code_matches( + sql=sql, + pattern=pattern, + scanner=_native.SqlCodeScanner(sql, lexical_syntax.native_mapping, _REFERENCE_SCAN_CONTEXT), ): parts.append(sql[last_index : match.start()]) parts.append(replace(match)) @@ -271,24 +45,6 @@ def replace_executable_matches( return "".join(parts) -def _iter_executable_matches(*, sql: str, pattern: re.Pattern[str]) -> Iterator[re.Match[str]]: - return _iter_code_matches( - sql=sql, - pattern=pattern, - scanner=_native.SqlCodeScanner(sql, None, _REFERENCE_SCAN_CONTEXT), - ) - - -def _iter_dialect_executable_matches( - *, sql: str, pattern: re.Pattern[str], lexical_syntax: SqlLexicalSyntax -) -> Iterator[re.Match[str]]: - return _iter_code_matches( - sql=sql, - pattern=pattern, - scanner=_native.SqlCodeScanner(sql, lexical_syntax.native_mapping, _REFERENCE_SCAN_CONTEXT), - ) - - def _iter_code_matches( *, sql: str, pattern: re.Pattern[str], scanner: _native.SqlCodeScanner ) -> Iterator[re.Match[str]]: @@ -306,54 +62,6 @@ def _iter_code_matches( index = match.end() -def _build_cursor_subquery( - *, - qualified_name: str, - cursor_column: str, - bounds: CursorBounds, - adapter: BaseAdapter, - cursor_type: str | None, - lower_bound_inclusive: bool, - inject_alias: bool, -) -> str: - """Wrap a qualified name in a cursor-filtered subquery.""" - - lower_operator: str = ">=" if lower_bound_inclusive else ">" - upper_operator: str = "<=" if cursor_type is None else "<" - start_literal: str = adapter.render_cursor_bound_literal( - value=sentinel_to_token(sentinel=bounds.start), cursor_type=cursor_type - ) - end_literal: str = adapter.render_cursor_bound_literal( - value=sentinel_to_token(sentinel=bounds.end), cursor_type=cursor_type - ) - derived_alias: str = ( - " AS __cursor_ref" if inject_alias and adapter.requires_derived_table_aliases() else "" - ) - return ( - f"(SELECT * FROM {qualified_name}" - f" WHERE {cursor_column} {lower_operator} {start_literal}" - f" AND {cursor_column} {upper_operator} {end_literal}){derived_alias}" - ) - - -def _has_following_alias(*, sql: str, start: int) -> bool: - index: int = _skip_whitespace(sql=sql, start=start) - if index >= len(sql) or sql[index] in SQL_ALIAS_BOUNDARY_CHARACTERS: - return False - if sql[index : index + 2].upper() == SQL_ALIAS_KEYWORD: - after_as: int = _skip_whitespace(sql=sql, start=index + 2) - return after_as < len(sql) and ( - sql[after_as].isalpha() or sql[after_as] in SQL_IDENTIFIER_LEADING_CHARACTERS - ) - if not (sql[index].isalpha() or sql[index] in SQL_IDENTIFIER_LEADING_CHARACTERS): - return False - match: re.Match[str] | None = re.match(r"[A-Za-z_][A-Za-z0-9_]*|\[[^\]]+\]", sql[index:]) - if match is None: - return False - token: str = match.group(0).strip("[]").upper() - return token not in _CLAUSE_KEYWORDS - - def build_model_locations( models: tuple[CompiledModelView, ...], ) -> dict[str, CompiledRelationLocation]: @@ -376,27 +84,3 @@ def build_function_locations( """Build a lookup of function name to compiled relation location.""" return {function.name: function.destination for function in functions} - - -def apply_deferred_locations( - *, - model_locations: dict[str, CompiledRelationLocation], - seed_locations: dict[str, CompiledRelationLocation], - deferred_locations: dict[str, CompiledRelationLocation], - selected_keys: frozenset[CompiledObjectKey], -) -> tuple[dict[str, CompiledRelationLocation], dict[str, CompiledRelationLocation]]: - """Return model/seed locations with non-selected entries pointed at deferred targets.""" - - selected_names: frozenset[str] = frozenset(k.name for k in selected_keys) - updated_model_locations: dict[str, CompiledRelationLocation] = dict(model_locations) - updated_seed_locations: dict[str, CompiledRelationLocation] = dict(seed_locations) - name: str - deferred_location: CompiledRelationLocation - for name, deferred_location in deferred_locations.items(): - if name in selected_names: - continue - if name in updated_model_locations: - updated_model_locations[name] = deferred_location - if name in updated_seed_locations: - updated_seed_locations[name] = deferred_location - return updated_model_locations, updated_seed_locations diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py b/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py index 729848a6f7..b285a5aa70 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py @@ -1,438 +1,4 @@ -"""Source reference resolution with optional type enforcement CAST wrapping.""" +"""Native source resolution with adapter rendering callbacks.""" -from __future__ import annotations - -import re - -from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild._native import render_source_relation -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.planner.constants import ( - SOURCE_ALIAS_BOUNDARY_CHARACTERS, - SQL_ALIAS_KEYWORD, - SQL_BRACKETED_IDENTIFIER_START, - SQL_QUOTED_IDENTIFIER_DELIMITERS, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import CursorBounds -from sqlbuild.compiler.planner.types import ContractPolicy -from sqlbuild.compiler.references.constants import SOURCE_PATTERN -from sqlbuild.spec.contracts.models import SourceColumnEntry, SourceEntry - -_SOURCE_PATTERN: re.Pattern[str] = SOURCE_PATTERN -_DERIVED_TABLE_ALIAS_KEYWORDS: frozenset[str] = frozenset( - { - "APPLY", - "CROSS", - "EXCEPT", - "FULL", - "GROUP", - "HAVING", - "INNER", - "INTERSECT", - "JOIN", - "LEFT", - "LIMIT", - "OFFSET", - "ON", - "ORDER", - "OUTER", - "RIGHT", - "UNION", - "WHERE", - } -) - - -def resolve_source_references( - *, - query_sql: str, - source_map: dict[str, SourceEntry], - source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]], - star_exclude_keyword: str, - cursor_bounds: CursorBounds | None, - cursor_filter_inputs: dict[str, str], - adapter: BaseAdapter, - cursor_type: str | None, - lower_bound_inclusive: bool, -) -> str: - """Replace all __source() calls in query SQL with resolved names or CAST subqueries.""" - - def _replacement(match: re.Match[str]) -> str: - source_name: str = match.group(1) - return _resolve_source_reference( - source_name=source_name, - source_map=source_map, - source_warehouse_columns=source_warehouse_columns, - cursor_bounds=cursor_bounds, - cursor_filter_inputs=cursor_filter_inputs, - adapter=adapter, - cursor_type=cursor_type, - lower_bound_inclusive=lower_bound_inclusive, - unknown_source_sql=match.group(0), - ) - - if not adapter.requires_derived_table_aliases(): - return _SOURCE_PATTERN.sub(_replacement, query_sql) - - resolved_chunks: list[str] = [] - cursor: int = 0 - for match in _SOURCE_PATTERN.finditer(query_sql): - source_name: str = match.group(1) - resolved_source: str = _resolve_source_reference( - source_name=source_name, - source_map=source_map, - source_warehouse_columns=source_warehouse_columns, - cursor_bounds=cursor_bounds, - cursor_filter_inputs=cursor_filter_inputs, - adapter=adapter, - cursor_type=cursor_type, - lower_bound_inclusive=lower_bound_inclusive, - unknown_source_sql=match.group(0), - ) - alias_span: tuple[int, str | None] = (match.end(), None) - if _is_derived_table_factor(resolved_source): - alias_span = _consume_source_alias(query_sql=query_sql, start=match.end()) - alias: str = alias_span[1] or _internal_source_alias(source_name) - resolved_source = f"{resolved_source} AS {alias}" - resolved_chunks.append(query_sql[cursor : match.start()]) - resolved_chunks.append(resolved_source) - cursor = alias_span[0] - resolved_chunks.append(query_sql[cursor:]) - return "".join(resolved_chunks) - - -def _resolve_source_reference( - *, - source_name: str, - source_map: dict[str, SourceEntry], - source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]], - cursor_bounds: CursorBounds | None, - cursor_filter_inputs: dict[str, str], - adapter: BaseAdapter, - cursor_type: str | None, - lower_bound_inclusive: bool, - unknown_source_sql: str, -) -> str: - source_entry: SourceEntry | None = source_map.get(source_name) - if source_entry is None: - return unknown_source_sql - resolved_source: str = render_source_read_relation(adapter=adapter, source_entry=source_entry) - warehouse_cols: tuple[ColumnInfo, ...] | None = source_warehouse_columns.get(source_name) - if source_entry.expression is None and warehouse_cols: - _validate_declared_columns( - qualified_name=resolved_source, - declared_columns=source_entry.columns, - available_columns=warehouse_cols, - contract_enforced=source_entry.contract == ContractPolicy.ENFORCED, - dialect=adapter.sql_analysis_dialect_name, - ) - if ( - source_entry.expression is not None - and warehouse_cols - and source_entry.contract == ContractPolicy.ENFORCED - ): - _validate_declared_columns( - qualified_name=f"source expression '{source_name}'", - declared_columns=source_entry.columns, - available_columns=warehouse_cols, - contract_enforced=source_entry.contract == ContractPolicy.ENFORCED, - dialect=adapter.sql_analysis_dialect_name, - ) - if source_entry.type_enforcement: - if source_entry.expression is not None: - resolved_source = _build_expression_cast_subquery( - source_name=source_entry.name, - source_relation=resolved_source, - declared_columns=source_entry.columns, - expression_columns=warehouse_cols, - adapter=adapter, - ) - elif warehouse_cols is not None and warehouse_cols: - resolved_source = _build_relation_cast_subquery( - qualified_name=resolved_source, - declared_columns=source_entry.columns, - warehouse_columns=warehouse_cols, - adapter=adapter, - ) - if cursor_bounds is None: - return resolved_source - cursor_column: str | None = cursor_filter_inputs.get(source_name) - if cursor_column is None: - return resolved_source - return _build_cursor_subquery( - resolved_source=resolved_source, - cursor_column=cursor_column, - bounds=cursor_bounds, - adapter=adapter, - cursor_type=cursor_type, - lower_bound_inclusive=lower_bound_inclusive, - ) - - -def _is_derived_table_factor(source_sql: str) -> bool: - return source_sql.lstrip().startswith("(") - - -def _consume_source_alias(*, query_sql: str, start: int) -> tuple[int, str | None]: - index: int = _skip_whitespace(query_sql=query_sql, start=start) - token_end: int - token: str | None - token_end, token = _read_alias_token(query_sql=query_sql, start=index) - if token is None: - return start, None - if token.upper() == SQL_ALIAS_KEYWORD: - alias_start: int = _skip_whitespace(query_sql=query_sql, start=token_end) - alias_end: int - alias: str | None - alias_end, alias = _read_alias_token(query_sql=query_sql, start=alias_start) - if alias is None or _is_alias_keyword(alias): - return start, None - return alias_end, alias - if _is_alias_keyword(token): - return start, None - return token_end, token - - -def _skip_whitespace(*, query_sql: str, start: int) -> int: - index: int = start - while index < len(query_sql) and query_sql[index].isspace(): - index += 1 - return index - - -def _read_alias_token(*, query_sql: str, start: int) -> tuple[int, str | None]: - if start >= len(query_sql): - return start, None - first_char: str = query_sql[start] - if first_char in SOURCE_ALIAS_BOUNDARY_CHARACTERS: - return start, None - if first_char == SQL_BRACKETED_IDENTIFIER_START: - end_bracket: int = query_sql.find("]", start + 1) - if end_bracket == -1: - return start, None - return end_bracket + 1, query_sql[start : end_bracket + 1] - if first_char in SQL_QUOTED_IDENTIFIER_DELIMITERS: - end_quote: int = query_sql.find(first_char, start + 1) - if end_quote == -1: - return start, None - return end_quote + 1, query_sql[start : end_quote + 1] - match: re.Match[str] | None = re.match(r"[A-Za-z_][A-Za-z0-9_$]*", query_sql[start:]) - if match is None: - return start, None - return start + match.end(), match.group(0) - - -def _is_alias_keyword(token: str) -> bool: - return token.upper() in _DERIVED_TABLE_ALIAS_KEYWORDS - - -def _internal_source_alias(source_name: str) -> str: - alias_suffix: str = re.sub(r"[^A-Za-z0-9_]+", "_", source_name).strip("_").lower() - if not alias_suffix: - alias_suffix = "source" - return f"__sqb_source_{alias_suffix}" - - -def render_source_read_relation(*, adapter: BaseAdapter, source_entry: SourceEntry) -> str: - """Render the relation name that planned model SQL reads for one source.""" - - if source_entry.expression is not None: - return render_source_relation(entry=source_entry, adapter=adapter) - table_name: str = source_entry.table if source_entry.table is not None else source_entry.name - rendered: str | None = adapter.render_qualified_name( - database=source_entry.database, - schema=source_entry.schema, - name=table_name, - ) - if rendered is not None: - return rendered - return render_source_relation(entry=source_entry) - - -def _build_relation_cast_subquery( - *, - qualified_name: str, - declared_columns: tuple[SourceColumnEntry, ...], - warehouse_columns: tuple[ColumnInfo, ...], - adapter: BaseAdapter, -) -> str: - """Build a CAST subquery for type-enforced sources.""" - - enforced_map: dict[str, str] = { - col.name: col.type for col in declared_columns if col.type is not None - } - if not enforced_map: - return qualified_name - - warehouse_names: set[str] = {col.name for col in warehouse_columns} - cast_names: list[str] = [name for name in enforced_map if name in warehouse_names] - if not cast_names: - return qualified_name - - cast_expressions: tuple[str, ...] = tuple( - adapter.render_source_expression_cast( - expression=adapter.render_identifier(name), - target_type=enforced_map[name], - alias=adapter.render_identifier(name), - ) - for name in cast_names - ) - all_enforced: bool = len(cast_names) == len(warehouse_names) - return adapter._render_source_relation_cast_subquery_with_columns( - source_relation=qualified_name, - cast_projections=cast_expressions, - cast_column_names=tuple(cast_names), - warehouse_column_names=tuple(col.name for col in warehouse_columns), - all_columns_cast=all_enforced, - ) - - -def _validate_declared_columns( - *, - qualified_name: str, - declared_columns: tuple[SourceColumnEntry, ...], - available_columns: tuple[ColumnInfo, ...], - contract_enforced: bool = False, - dialect: str | None = None, -) -> None: - """Ensure declared source columns exist and enforced contracts match metadata.""" - - available_by_name: dict[str, ColumnInfo] = {col.name.lower(): col for col in available_columns} - missing_names: tuple[str, ...] = tuple( - col.name for col in declared_columns if col.name.lower() not in available_by_name - ) - if missing_names: - missing_columns: str = ", ".join(missing_names) - raise PlannerInputError( - f"source {qualified_name} declares columns not found in warehouse: {missing_columns}", - code="S401", - ) - - if not contract_enforced: - return - - declared_column: SourceColumnEntry - for declared_column in declared_columns: - if declared_column.type is None: - continue - available_column: ColumnInfo = available_by_name[declared_column.name.lower()] - if not available_column.type: - continue - if types_equal(left=available_column.type, right=declared_column.type, dialect=dialect): - continue - raise PlannerInputError( - f"source {qualified_name} column '{declared_column.name}' has type " - f"{available_column.type} but contract declares {declared_column.type}", - code="S404", - ) - - -def _build_expression_cast_subquery( - *, - source_name: str, - source_relation: str, - declared_columns: tuple[SourceColumnEntry, ...], - expression_columns: tuple[ColumnInfo, ...] | None, - adapter: BaseAdapter, -) -> str: - """Build a CAST projection for expression sources using probed column names.""" - - enforced_map: dict[str, str] = { - col.name: col.type for col in declared_columns if col.type is not None - } - if not enforced_map: - return source_relation - if expression_columns is None: - raise PlannerInputError( - f"source expression '{source_name}' type enforcement requires query output " - "column metadata", - code="S402", - ) - - expression_names: tuple[str, ...] = tuple(col.name for col in expression_columns) - expression_name_map: dict[str, str] = {name.lower(): name for name in expression_names} - missing_names: tuple[str, ...] = tuple( - col.name for col in declared_columns if col.name.lower() not in expression_name_map - ) - if missing_names: - missing_columns: str = ", ".join(missing_names) - available_columns: str = ", ".join(expression_names) if expression_names else "" - raise PlannerInputError( - f"source expression '{source_name}' declares columns not found in query output: " - f"{missing_columns}. Available query output columns: {available_columns}", - code="S403", - ) - - projections: list[str] = _build_expression_source_projections( - expression_names=expression_names, - declared_columns=declared_columns, - expression_name_map=expression_name_map, - adapter=adapter, - ) - return adapter.render_source_expression_cast_subquery( - source_relation=source_relation, - projections=tuple(projections), - ) - - -def _build_expression_source_projections( - *, - expression_names: tuple[str, ...], - declared_columns: tuple[SourceColumnEntry, ...], - expression_name_map: dict[str, str], - adapter: BaseAdapter, -) -> list[str]: - enforced_map: dict[str, str] = { - col.name: col.type for col in declared_columns if col.type is not None - } - enforced_by_expression_name: dict[str, tuple[str, str]] = { - expression_name_map[name.lower()]: (name, column_type) - for name, column_type in enforced_map.items() - if name.lower() in expression_name_map - } - projections: list[str] = [] - for name in expression_names: - enforced_entry: tuple[str, str] | None = enforced_by_expression_name.get(name) - if enforced_entry is None: - projections.append(adapter.render_identifier(name)) - continue - declared_name: str - column_type: str - declared_name, column_type = enforced_entry - projections.append( - adapter.render_source_expression_cast( - expression=adapter.render_identifier(name), - target_type=column_type, - alias=adapter.render_identifier(declared_name), - ) - ) - return projections - - -def _build_cursor_subquery( - *, - resolved_source: str, - cursor_column: str, - bounds: CursorBounds, - adapter: BaseAdapter, - cursor_type: str | None, - lower_bound_inclusive: bool, -) -> str: - """Wrap a resolved source relation in a cursor-filtered subquery.""" - - lower_operator: str = ">=" if lower_bound_inclusive else ">" - upper_operator: str = "<=" if cursor_type is None else "<" - start_literal: str = adapter.render_cursor_bound_literal( - value=sentinel_to_token(sentinel=bounds.start), cursor_type=cursor_type - ) - end_literal: str = adapter.render_cursor_bound_literal( - value=sentinel_to_token(sentinel=bounds.end), cursor_type=cursor_type - ) - return ( - f"(SELECT * FROM {resolved_source}" - f" WHERE {cursor_column} {lower_operator} {start_literal}" - f" AND {cursor_column} {upper_operator} {end_literal})" - ) +from sqlbuild._native import render_source_read_relation as render_source_read_relation +from sqlbuild._native import resolve_source_references as resolve_source_references diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py index 2da79324d3..e6be6b7bbd 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py @@ -8,6 +8,15 @@ from sqlbuild.spec.contracts.models import SourceEntry +@dataclass(frozen=True) +class AdapterOverrideTestCase: + """Expected SQL and rows from adapter-owned source rendering.""" + + description: str + expected_sql: str + expected_rows: list[tuple[int]] + + @dataclass(frozen=True) class ResolveAndExecuteTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index f530674685..dbd5824bc3 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -29,6 +29,26 @@ from sqlbuild.spec.contracts.models import SourceEntry +class ResolutionOverrideAdapter(DuckDbAdapter): + """Render an expression source using adapter-owned SQL at every boundary.""" + + def render_source_expression_relation(self, *, expression: str) -> str: + return '(SELECT 7 AS "OrderId")' + + def render_identifier(self, name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + def render_source_expression_cast( + self, *, expression: str, target_type: str, alias: str + ) -> str: + return f"CAST({expression} AS {target_type}) + 1 AS {alias}" + + def render_source_expression_cast_subquery( + self, *, source_relation: str, projections: tuple[str, ...] + ) -> str: + return f"(SELECT {', '.join(projections)} FROM {source_relation})" + + def build_model( *, name: str, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py new file mode 100644 index 0000000000..13cd341c29 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py @@ -0,0 +1,58 @@ +"""Execute SQL resolved through custom adapter hooks.""" + +from typing import Any + +import pytest + +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.compiler.planner._helpers.resolve.sources import resolve_source_references +from sqlbuild.spec.contracts.models import SourceColumnEntry, SourceEntry +from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( + AdapterOverrideTestCase, +) +from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve.helpers import ( + ResolutionOverrideAdapter, +) + + +@pytest.mark.parametrize( + "test_case", + [ + AdapterOverrideTestCase( + description="custom source rendering preserves case and changes data", + expected_sql=( + 'SELECT orderid FROM (SELECT CAST("OrderId" AS BIGINT) + 1 AS "orderid" ' + 'FROM (SELECT 7 AS "OrderId"))' + ), + expected_rows=[(8,)], + ) + ], + ids=lambda case: case.description, +) +def test_given_custom_adapter_when_resolving_source_then_executes_overridden_rendering( + test_case: AdapterOverrideTestCase, + connection: Any, +) -> None: + source: SourceEntry = SourceEntry( + name="orders", + expression='SELECT 1 AS "OrderId"', + type_enforcement=True, + columns=(SourceColumnEntry(name="orderid", type="BIGINT"),), + ) + sql: str = resolve_source_references( + query_sql='SELECT orderid FROM __source("orders")', + source_map={"orders": source}, + source_warehouse_columns={"orders": (ColumnInfo(name="OrderId", type="INTEGER"),)}, + star_exclude_keyword="EXCLUDE", + cursor_bounds=None, + cursor_filter_inputs={}, + adapter=ResolutionOverrideAdapter(), + cursor_type=None, + lower_bound_inclusive=True, + ) + assert sql == test_case.expected_sql + assert connection.execute(sql).fetchall() == test_case.expected_rows + + +if __name__ == "__main__": + pytest.main([__file__]) From 95cc0f8ea64e377edb7646cbcef9721a7fe94996 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:58:09 +0100 Subject: [PATCH 04/29] chore: update Python boundary after reference resolution port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 78fddcfcae..550e3d06e5 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 39006 +remaining_lines = 38256 [[stays]] path = "executor" From ac7c350fc180d4963d3b07f187c1da93039a5deb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 14:13:59 +0100 Subject: [PATCH 05/29] refactor: simplify fixture null projections natively --- .../_helpers/rules/fixture_formatting.rs | 29 +++ .../src/bindings/_helpers/rules/mod.rs | 1 + .../bindings/_helpers/rules/registration.rs | 1 + .../lint_text/_helpers/fixture_formatting.rs | 194 ++++++++++++++++++ .../src/lint_text/_helpers/mod.rs | 1 + .../sqlbuild-rules/src/lint_text/main/mod.rs | 1 + .../lint_text/main/simplify_fixture_nulls.rs | 9 + crates/sqlbuild-rules/src/lint_text/tests.rs | 2 + .../tests/test_fixture_formatting.rs | 74 +++++++ .../src/lint_text/tests/test_types.rs | 6 + src/sqlbuild/_native.pyi | 2 + .../_helpers/sql_tests/fixture_formatting.py | 185 +---------------- 12 files changed, 321 insertions(+), 184 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_formatting.rs create mode 100644 crates/sqlbuild-rules/src/lint_text/_helpers/fixture_formatting.rs create mode 100644 crates/sqlbuild-rules/src/lint_text/main/simplify_fixture_nulls.rs create mode 100644 crates/sqlbuild-rules/src/lint_text/tests/test_fixture_formatting.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_formatting.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_formatting.rs new file mode 100644 index 0000000000..66da2880a4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_formatting.rs @@ -0,0 +1,29 @@ +//! Fixture schema conversion at the Python model boundary. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_rules::lint_text::main::simplify_fixture_nulls::simplify_fixture_nulls; + +use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; + +#[pyfunction] +#[pyo3(signature = (*, sql, relation))] +fn simplify_fixture_null_projections(sql: &str, relation: &Bound<'_, PyAny>) -> PyResult { + let columns: Vec<(String, Option, Option)> = relation + .getattr("columns")? + .try_iter()? + .map(|column| { + let column = column?; + Ok(( + column.getattr("name")?.extract()?, + column.getattr("type")?.extract()?, + column.getattr("nullable")?.extract()?, + )) + }) + .collect::>()?; + compiler_guard(|| simplify_fixture_nulls(sql, &columns).map_err(value_error)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(simplify_fixture_null_projections, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index d2234ad0e6..6c5c446e0f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -9,6 +9,7 @@ pub(crate) mod findings; pub(crate) mod fix_facts; pub(crate) mod fix_session; pub(crate) mod fix_verdict; +pub(crate) mod fixture_formatting; pub(crate) mod guidance; pub(crate) mod lint; pub(crate) mod lint_expansion; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index 657cdb351b..5abc574c40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -44,6 +44,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { evaluation::register(module)?; lint::register(module)?; lint_text::register(module)?; + crate::bindings::_helpers::rules::fixture_formatting::register(module)?; lint_expansion::register(module)?; lint_sql_bodies::register(module)?; dsl_headers::register(module)?; diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/fixture_formatting.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/fixture_formatting.rs new file mode 100644 index 0000000000..ab80e592f1 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/fixture_formatting.rs @@ -0,0 +1,194 @@ +//! Schema-aware simplification of authored fixture projections. + +use regex::Regex; +use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlbuild_core::text::main::python_splitlines::python_splitlines; +use std::collections::HashMap; +use std::sync::LazyLock; + +static PATTERNS: LazyLock> = LazyLock::new(|| { + Ok([ + Regex::new(concat!( + r#"(?i)^[ \t]*(?:CAST\s*\(\s*NULL\s+AS\s+(?:"#, + r#"[A-Za-z_][A-Za-z0-9_]*(?:\s*\([^()]*\))?"#, + r#"(?:\s+[A-Za-z_][A-Za-z0-9_]*(?:\s*\([^()]*\))?)*"#, + r#")\)|NULL\s*::\s*(?:[A-Za-z_][A-Za-z0-9_]*(?:\([^)]*\))?))"#, + r#"\s+AS\s+(?P"[^"]+"|`[^`]+`|[A-Za-z_][A-Za-z0-9_]*)(?:\s*,)?[ \t]*$"#, + ))?, + Regex::new(r"(?i)\b(?:UNION|INTERSECT|EXCEPT)\b")?, + Regex::new(r"(?is)^\s*SELECT\s+(?P.+?)\s+WHERE\s+FALSE\s*$")?, + Regex::new(r"(?i)^\s*SELECT\b(?P.*)$")?, + Regex::new( + r"(?i)^\s*(?:FROM|WHERE|GROUP|HAVING|QUALIFY|ORDER|LIMIT|UNION|INTERSECT|EXCEPT)\b", + )?, + Regex::new(r#"(?i)\bAS\s+(?P"[^"]+"|`[^`]+`|[A-Za-z_][A-Za-z0-9_]*)\s*,?\s*$"#)?, + Regex::new(r",\s*$")?, + Regex::new(r"(?i)^\s*SELECT\s+WHERE\s+FALSE\s*$")?, + ]) +}); + +/// Remove only typed-null projections that completion can reconstruct in schema order. +pub(crate) fn simplify_fixture_nulls( + sql: &str, + columns: &[(String, Option, Option)], +) -> Result { + let [typed, set, empty, _, _, _, _, empty_remaining] = + PATTERNS.as_ref().map_err(Clone::clone)?; + if set.is_match(sql) { + return Ok(sql.to_string()); + } + let metadata_by_name: HashMap, Option)> = columns + .iter() + .map(|column| (python_casefold(&column.0), column)) + .collect(); + let known_column = |projection: &str| { + typed.captures(projection).and_then(|captures| { + let alias = python_casefold(captures["alias"].trim_matches(['"', '`'])); + metadata_by_name + .get(&alias) + .copied() + .filter(|column| column.1.is_some()) + }) + }; + if let Some(query) = empty.captures(sql) { + let projections = split_expressions(&query["projection"]); + if !projections.is_empty() + && projections + .iter() + .all(|projection| known_column(projection.trim()).is_some()) + { + return Ok("SELECT * FROM __empty_fixture()".to_string()); + } + } + let lines: Vec = python_splitlines(sql) + .into_iter() + .map(str::to_string) + .collect(); + let mut empty_retained: Vec = Vec::new(); + let mut retained: Vec = Vec::new(); + let mut removed = false; + for line in &lines { + if let Some(column) = known_column(line) { + if column.2 == Some(false) { + retained.push(line.clone()); + } else { + removed = true; + } + } else { + empty_retained.push(line.clone()); + retained.push(line.clone()); + } + } + trim_comma(&mut empty_retained)?; + if empty_remaining.is_match(&empty_retained.join("\n")) { + return Ok("SELECT * FROM __empty_fixture()".to_string()); + } + if !removed || !schema_prefix(&lines, columns)? { + return Ok(sql.to_string()); + } + trim_comma(&mut retained)?; + if aliases(&retained)?.is_none_or(|names| names.is_empty()) + || !schema_prefix(&retained, columns)? + { + return Ok(sql.to_string()); + } + Ok(retained.join("\n")) +} + +fn split_expressions(value: &str) -> Vec<&str> { + let mut expressions: Vec<&str> = Vec::new(); + let mut start = 0; + let mut depth: usize = 0; + let mut quote = None; + let mut characters = value.char_indices().peekable(); + while let Some((index, character)) = characters.next() { + if let Some(delimiter) = quote { + if character == delimiter { + if characters + .peek() + .is_some_and(|(_, next)| *next == delimiter) + { + characters.next(); + } else { + quote = None; + } + } + continue; + } + match character { + '\'' | '"' | '`' => quote = Some(character), + '(' => depth += 1, + ')' => depth = depth.saturating_sub(1), + ',' if depth == 0 => { + expressions.push(&value[start..index]); + start = index + 1; + } + _ => {} + } + } + expressions.push(&value[start..]); + expressions + .into_iter() + .filter(|expression| !expression.trim().is_empty()) + .collect() +} + +fn aliases(lines: &[String]) -> Result>, regex::Error> { + let [_, _, _, select, clause, alias, _, _] = PATTERNS.as_ref().map_err(Clone::clone)?; + let mut saw_select = false; + let mut names: Vec = Vec::new(); + for line in lines { + let mut candidate = line.as_str(); + if !saw_select { + if let Some(captures) = select.captures(line) { + saw_select = true; + candidate = captures.name("body").map_or("", |body| body.as_str()); + } else if line.trim().is_empty() { + continue; + } else { + return Ok(None); + } + } + if clause.is_match(candidate) { + break; + } + if candidate.trim().is_empty() { + continue; + } + let Some(captures) = alias.captures(candidate) else { + return Ok(None); + }; + names.push(python_casefold(captures["alias"].trim_matches(['"', '`']))); + } + Ok(saw_select.then_some(names)) +} + +fn schema_prefix( + lines: &[String], + columns: &[(String, Option, Option)], +) -> Result { + Ok(aliases(lines)?.is_some_and(|names| { + names.iter().eq(columns + .iter() + .take(names.len()) + .map(|column| python_casefold(&column.0)) + .collect::>() + .iter()) + })) +} + +fn trim_comma(lines: &mut [String]) -> Result<(), regex::Error> { + let [_, _, _, _, clause, _, trailing, _] = PATTERNS.as_ref().map_err(Clone::clone)?; + let end = lines + .iter() + .position(|line| clause.is_match(line)) + .unwrap_or(lines.len()); + if let Some(line) = lines[..end] + .iter_mut() + .rev() + .find(|line| !line.trim().is_empty()) + { + *line = trailing.replace(line, "").into_owned(); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs index 5726c1c9b0..a49c95e613 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod fixture_formatting; pub(crate) mod fixtures; pub(crate) mod format_bodies; pub(crate) mod lines; diff --git a/crates/sqlbuild-rules/src/lint_text/main/mod.rs b/crates/sqlbuild-rules/src/lint_text/main/mod.rs index b599ea6761..1fb780ee99 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/mod.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/mod.rs @@ -11,3 +11,4 @@ pub mod neutralize_interpolation; pub mod resolve_format_settings; pub mod restore_interpolation; pub mod scan_fixture_null_candidates; +pub mod simplify_fixture_nulls; diff --git a/crates/sqlbuild-rules/src/lint_text/main/simplify_fixture_nulls.rs b/crates/sqlbuild-rules/src/lint_text/main/simplify_fixture_nulls.rs new file mode 100644 index 0000000000..baaa7c5478 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/simplify_fixture_nulls.rs @@ -0,0 +1,9 @@ +//! Contract-aware fixture projection formatting. + +/// Remove typed nulls only when fixture completion restores the authored schema order. +pub fn simplify_fixture_nulls( + sql: &str, + columns: &[(String, Option, Option)], +) -> Result { + crate::lint_text::_helpers::fixture_formatting::simplify_fixture_nulls(sql, columns) +} diff --git a/crates/sqlbuild-rules/src/lint_text/tests.rs b/crates/sqlbuild-rules/src/lint_text/tests.rs index a080a424d5..57c38b8979 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_fixture_formatting.rs"] +mod fixture_formatting; #[path = "tests/test_format_settings.rs"] mod format_settings; #[path = "tests/test_lint_text.rs"] diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_fixture_formatting.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_fixture_formatting.rs new file mode 100644 index 0000000000..e7bb9515da --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_fixture_formatting.rs @@ -0,0 +1,74 @@ +use crate::lint_text::main::simplify_fixture_nulls::simplify_fixture_nulls; +use crate::lint_text::tests::test_types::FixtureFormattingTestCase; + +#[test] +fn given_authoritative_columns_when_simplifying_then_preserves_schema_order() +-> Result<(), regex::Error> { + let columns: Vec<(String, Option, Option)> = vec![ + ("order_id".into(), Some("INTEGER".into()), Some(false)), + ("amount".into(), Some("DECIMAL(12, 3)".into()), Some(true)), + ]; + let test_cases = [ + FixtureFormattingTestCase { + description: "nullable trailing column", + sql: "SELECT\n 1 AS order_id,\n CAST(NULL AS DECIMAL(12, 3)) AS amount\nWHERE TRUE", + columns: columns.clone(), + expected_sql: "SELECT\n 1 AS order_id\nWHERE TRUE", + }, + FixtureFormattingTestCase { + description: "out-of-order projection retained", + sql: "SELECT\n CAST(NULL AS DECIMAL(12, 3)) AS amount,\n 1 AS order_id", + columns: columns.clone(), + expected_sql: "SELECT\n CAST(NULL AS DECIMAL(12, 3)) AS amount,\n 1 AS order_id", + }, + FixtureFormattingTestCase { + description: "empty fixture with decimal precision", + sql: "SELECT CAST(NULL AS INTEGER) AS order_id, CAST(NULL AS DECIMAL(12, 3)) AS amount WHERE FALSE", + columns: columns.clone(), + expected_sql: "SELECT * FROM __empty_fixture()", + }, + FixtureFormattingTestCase { + description: "non-nullable projection retained", + sql: "SELECT\n CAST(NULL AS INTEGER) AS order_id,\n 2 AS amount", + columns: columns.clone(), + expected_sql: "SELECT\n CAST(NULL AS INTEGER) AS order_id,\n 2 AS amount", + }, + FixtureFormattingTestCase { + description: "unsupported array spelling retained", + sql: "SELECT\n 1 AS order_id,\n CAST(NULL AS INTEGER[]) AS amount", + columns: columns.clone(), + expected_sql: "SELECT\n 1 AS order_id,\n CAST(NULL AS INTEGER[]) AS amount", + }, + FixtureFormattingTestCase { + description: "set operation retained", + sql: "SELECT\n 1 AS order_id,\n CAST(NULL AS INTEGER) AS amount\nUNION ALL SELECT 2, 3", + columns: columns.clone(), + expected_sql: "SELECT\n 1 AS order_id,\n CAST(NULL AS INTEGER) AS amount\nUNION ALL SELECT 2, 3", + }, + FixtureFormattingTestCase { + description: "unicode line boundaries", + sql: "SELECT\u{2028} 1 AS order_id,\u{2028} CAST(NULL AS DECIMAL(12, 3)) AS amount", + columns, + expected_sql: "SELECT\n 1 AS order_id", + }, + FixtureFormattingTestCase { + description: "last duplicate column has unknown type", + sql: "SELECT\n 1 AS order_id,\n CAST(NULL AS INTEGER) AS amount", + columns: vec![ + ("order_id".into(), Some("INTEGER".into()), None), + ("amount".into(), Some("INTEGER".into()), None), + ("AMOUNT".into(), None, None), + ], + expected_sql: "SELECT\n 1 AS order_id,\n CAST(NULL AS INTEGER) AS amount", + }, + ]; + for test_case in test_cases { + assert_eq!( + simplify_fixture_nulls(test_case.sql, &test_case.columns)?, + test_case.expected_sql, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs index 8dde7c9547..0411bbca6e 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs @@ -73,3 +73,9 @@ pub(super) struct RoleKindTestCase { pub(super) relative_parts: &'static [&'static str], pub(super) expected_kind: Option<&'static str>, } +pub(crate) struct FixtureFormattingTestCase { + pub description: &'static str, + pub sql: &'static str, + pub columns: Vec<(String, Option, Option)>, + pub expected_sql: &'static str, +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 3be466b32f..890013040c 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -207,6 +207,7 @@ from sqlbuild.compiler.planner.models import ( CursorBounds, DeferralInputs, DirectModelVersionIdentities, + FixtureRelationMetadata, FutureCursorSafetyEvidence, MaximumStartSafetyEvidence, MigrationCompatibilityResult, @@ -2314,6 +2315,7 @@ def lint_interpolation_text_at(body: str, start: int, dialect: str) -> str | Non def scan_fixture_typed_null_candidates( project_dir: Path, selected_paths: frozenset[Path] | None = None ) -> tuple[frozenset[Path], frozenset[str]]: ... +def simplify_fixture_null_projections(*, sql: str, relation: FixtureRelationMetadata) -> str: ... def format_lint_sql_bodies( files: list[tuple[Path, str]], project_dir: Path, config: LintConfig ) -> tuple[dict[Path, str], list[LintViolation]]: ... diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py index bd6a1bb835..c026c7bd89 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py @@ -6,6 +6,7 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import simplify_fixture_null_projections as _remove_redundant_typed_null_lines from sqlbuild.compiler.compile.classes.sql_test_cte_extractor import SqlTestCteExtractor from sqlbuild.compiler.compile.constants import ( REF_TEST_CTE_PREFIX, @@ -30,41 +31,10 @@ from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import SchemaColumn, SchemaSeedEntry, SourceEntry -_TYPED_NULL_LINE_PATTERN: re.Pattern[str] = re.compile( - r"^(?P[ \t]*)(?:" - r"CAST\s*\(\s*NULL\s+AS\s+(?P" - r"[A-Za-z_][A-Za-z0-9_]*(?:\s*\([^()]*\))?" - r"(?:\s+[A-Za-z_][A-Za-z0-9_]*(?:\s*\([^()]*\))?)*" - r")\)" - r"|NULL\s*::\s*(?P[A-Za-z_][A-Za-z0-9_]*(?:\([^)]*\))?)" - r")\s+AS\s+(?P\"[^\"]+\"|`[^`]+`|[A-Za-z_][A-Za-z0-9_]*)" - r"(?P\s*,)?[ \t]*$", - re.IGNORECASE, -) -_CLAUSE_LINE_PATTERN: re.Pattern[str] = re.compile( - r"^\s*(?:FROM|WHERE|GROUP|HAVING|QUALIFY|ORDER|LIMIT|UNION|INTERSECT|EXCEPT)\b", - re.IGNORECASE, -) -_SET_OPERATION_PATTERN: re.Pattern[str] = re.compile( - r"\b(?:UNION|INTERSECT|EXCEPT)\b", re.IGNORECASE -) _MACRO_INVOCATION_MARKER: str = "@" _TYPED_NULL_CANDIDATE_PATTERN: re.Pattern[str] = re.compile( r"(?:CAST\s*\(\s*NULL\s+AS\b|NULL\s*::)", re.IGNORECASE ) -_PROJECTION_ALIAS_PATTERN: re.Pattern[str] = re.compile( - r"\bAS\s+(?P\"[^\"]+\"|`[^`]+`|[A-Za-z_][A-Za-z0-9_]*)\s*,?\s*$", - re.IGNORECASE, -) -_SELECT_LINE_PATTERN: re.Pattern[str] = re.compile(r"^\s*SELECT\b(?P.*)$", re.IGNORECASE) -_EMPTY_TYPED_NULL_QUERY_PATTERN: re.Pattern[str] = re.compile( - r"^\s*SELECT\s+(?P.+?)\s+WHERE\s+FALSE\s*$", - re.IGNORECASE | re.DOTALL, -) -_SQL_QUOTE_CHARACTERS: frozenset[str] = frozenset({"'", '"', "`"}) -_OPEN_PARENTHESIS_CHARACTER: str = "(" -_CLOSE_PARENTHESIS_CHARACTER: str = ")" -_COMMA_CHARACTER: str = "," def format_redundant_fixture_nulls( @@ -228,159 +198,6 @@ def _relation_metadata(*, columns: tuple[SchemaColumn, ...]) -> FixtureRelationM ) -def _remove_redundant_typed_null_lines(*, sql: str, relation: FixtureRelationMetadata) -> str: - if _SET_OPERATION_PATTERN.search(sql) is not None: - return sql - metadata_by_name: dict[str, FixtureColumnMetadata] = { - column.name.casefold(): column for column in relation.columns - } - if _is_empty_typed_null_query(sql=sql, metadata_by_name=metadata_by_name): - return "SELECT * FROM __empty_fixture()" - lines: list[str] = sql.splitlines() - typed_null_indexes: set[int] = set() - remove_indexes: set[int] = set() - for index, line in enumerate(lines): - match: re.Match[str] | None = _TYPED_NULL_LINE_PATTERN.fullmatch(line) - if match is None: - continue - alias: str = match.group("alias").strip('"`') - column: FixtureColumnMetadata | None = metadata_by_name.get(alias.casefold()) - if column is None or column.type is None: - continue - typed_null_indexes.add(index) - if column.nullable is False: - continue - remove_indexes.add(index) - empty_retained: list[str] = [ - line for index, line in enumerate(lines) if index not in typed_null_indexes - ] - empty_retained = _without_trailing_projection_comma(lines=empty_retained) - empty_candidate: str = "\n".join(empty_retained) - if re.fullmatch(r"\s*SELECT\s+WHERE\s+FALSE\s*", empty_candidate, re.IGNORECASE): - return "SELECT * FROM __empty_fixture()" - if not remove_indexes: - return sql - authored_names: tuple[str, ...] | None = _projection_aliases(lines=lines) - if authored_names is None or not _names_form_schema_prefix( - names=authored_names, - relation=relation, - ): - return sql - retained: list[str] = [line for index, line in enumerate(lines) if index not in remove_indexes] - retained = _without_trailing_projection_comma(lines=retained) - candidate: str = "\n".join(retained) - retained_names: tuple[str, ...] | None = _projection_aliases(lines=retained) - if retained_names is None or not retained_names: - return sql - if not _names_form_schema_prefix(names=retained_names, relation=relation): - return sql - return candidate - - -def _is_empty_typed_null_query( - *, sql: str, metadata_by_name: dict[str, FixtureColumnMetadata] -) -> bool: - """Return whether an empty fixture projects only known typed-null columns.""" - - query_match: re.Match[str] | None = _EMPTY_TYPED_NULL_QUERY_PATTERN.fullmatch(sql) - if query_match is None: - return False - projections: tuple[str, ...] = _split_top_level_expressions( - value=query_match.group("projection") - ) - if not projections: - return False - for projection in projections: - typed_null_match: re.Match[str] | None = _TYPED_NULL_LINE_PATTERN.fullmatch( - projection.strip() - ) - if typed_null_match is None: - return False - alias: str = typed_null_match.group("alias").strip('"`') - column: FixtureColumnMetadata | None = metadata_by_name.get(alias.casefold()) - if column is None or column.type is None: - return False - return True - - -def _split_top_level_expressions(*, value: str) -> tuple[str, ...]: - """Split a projection list without splitting commas inside types or quoted values.""" - - expressions: list[str] = [] - start: int = 0 - depth: int = 0 - quote: str | None = None - index: int = 0 - while index < len(value): - character: str = value[index] - if quote is not None: - if character == quote: - if index + 1 < len(value) and value[index + 1] == quote: - index += 2 - continue - quote = None - index += 1 - continue - if character in _SQL_QUOTE_CHARACTERS: - quote = character - elif character == _OPEN_PARENTHESIS_CHARACTER: - depth += 1 - elif character == _CLOSE_PARENTHESIS_CHARACTER: - depth = max(0, depth - 1) - elif character == _COMMA_CHARACTER and depth == 0: - expressions.append(value[start:index]) - start = index + 1 - index += 1 - expressions.append(value[start:]) - return tuple(expression for expression in expressions if expression.strip()) - - -def _projection_aliases(*, lines: list[str]) -> tuple[str, ...] | None: - aliases: list[str] = [] - saw_select: bool = False - for line in lines: - candidate: str = line - if not saw_select: - select_match: re.Match[str] | None = _SELECT_LINE_PATTERN.match(line) - if select_match is None: - if line.strip(): - return None - continue - saw_select = True - candidate = select_match.group("body") - if _CLAUSE_LINE_PATTERN.match(candidate): - break - if not candidate.strip(): - continue - alias_match: re.Match[str] | None = _PROJECTION_ALIAS_PATTERN.search(candidate) - if alias_match is None: - return None - aliases.append(alias_match.group("alias").strip('"`').casefold()) - return tuple(aliases) if saw_select else None - - -def _names_form_schema_prefix(*, names: tuple[str, ...], relation: FixtureRelationMetadata) -> bool: - schema_prefix: tuple[str, ...] = tuple( - column.name.casefold() for column in relation.columns[: len(names)] - ) - return names == schema_prefix - - -def _without_trailing_projection_comma(*, lines: list[str]) -> list[str]: - updated: list[str] = list(lines) - clause_index: int | None = next( - (index for index, line in enumerate(updated) if _CLAUSE_LINE_PATTERN.match(line)), - None, - ) - search_end: int = len(updated) if clause_index is None else clause_index - for index in range(search_end - 1, -1, -1): - if not updated[index].strip(): - continue - updated[index] = re.sub(r",\s*$", "", updated[index]) - break - return updated - - def _fixture_key(name: str) -> tuple[CompiledResourceType, str] | None: for prefix, resource_type in ( (REF_TEST_CTE_PREFIX, CompiledResourceType.MODEL), From 7dc5c95d4a758dbc66f3b2193387e4d632c3ecaa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 14:17:27 +0100 Subject: [PATCH 06/29] chore: update Python boundary after fixture simplification port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 550e3d06e5..02f3f1c4a4 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 38256 +remaining_lines = 38073 [[stays]] path = "executor" From 8451d05479e2dbf171ed6c9ef61f0e516aa4d1ae Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 14:19:19 +0100 Subject: [PATCH 07/29] refactor: host SQL resolution orchestration in native bindings --- .../_helpers/planner_resolution/locations.rs | 35 ++ .../_helpers/planner_resolution/mod.rs | 2 + .../planner_resolution/orchestration.rs | 347 ++++++++++++++++++ .../planner_resolution/registration.rs | 2 + src/sqlbuild/_native.pyi | 43 +++ .../compiler/planner/_helpers/resolve/refs.py | 33 +- .../planner/_helpers/resolve/resolve.py | 279 +------------- .../planner/_helpers/resolve/_test_types.py | 10 + .../planner/_helpers/resolve/helpers.py | 65 +++- .../_helpers/resolve/test_orchestration.py | 109 ++++++ 10 files changed, 622 insertions(+), 303 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs create mode 100644 tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs new file mode 100644 index 0000000000..e2b3f6c066 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs @@ -0,0 +1,35 @@ +//! Location maps retaining the original compiled Python destination objects. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; + +fn locations<'py>(items: &Bound<'py, PyAny>) -> PyResult> { + let result = PyDict::new(items.py()); + for item in items.try_iter()? { + let item = item?; + result.set_item(item.getattr("name")?, item.getattr("destination")?)?; + } + Ok(result) +} + +#[pyfunction] +fn build_model_locations<'py>(models: &Bound<'py, PyAny>) -> PyResult> { + locations(models) +} + +#[pyfunction] +fn build_seed_locations<'py>(seeds: &Bound<'py, PyAny>) -> PyResult> { + locations(seeds) +} + +#[pyfunction] +fn build_function_locations<'py>(functions: &Bound<'py, PyAny>) -> PyResult> { + locations(functions) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_model_locations, module)?)?; + module.add_function(wrap_pyfunction!(build_seed_locations, module)?)?; + module.add_function(wrap_pyfunction!(build_function_locations, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs index bae6e5ba20..c7be48b95d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs @@ -1,5 +1,7 @@ //! Planner reference resolution with Python adapter rendering callbacks. +mod locations; +mod orchestration; mod refs; pub(crate) mod registration; mod sources; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs new file mode 100644 index 0000000000..88b5321fe9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -0,0 +1,347 @@ +//! Model, function and SQL-hook orchestration through adapter-owned rendering. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_resolution::values::{NATIVE_MODULE, string}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; + +const RESOLVE_MODULE: &str = "sqlbuild.compiler.planner._helpers.resolve.resolve"; +const CONFIG_MODULE: &str = "sqlbuild.compiler.planner._helpers.resolve.config"; +const INTRINSICS_MODULE: &str = "sqlbuild.compiler.planner._helpers.resolve.cursor_intrinsics"; +const MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const HOOK_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const ERROR_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; + +struct Resolution<'py> { + adapter: Bound<'py, PyAny>, + context: Bound<'py, PyAny>, + external: Bound<'py, PyAny>, +} + +impl<'py> Resolution<'py> { + fn resolve( + &self, + sql: Bound<'py, PyAny>, + cursor: &[(&str, Bound<'py, PyAny>)], + ) -> PyResult> { + let py = self.adapter.py(); + let mut source = vec![("query_sql", sql), ("adapter", self.adapter.clone())]; + for key in [ + "source_map", + "source_warehouse_columns", + "star_exclude_keyword", + ] { + source.push((key, self.context.getattr(key)?)); + } + source.extend_from_slice(cursor); + let mut resolved = call_with( + &imported(py, NATIVE_MODULE, "resolve_source_references")?, + &source, + )?; + let mut refs = vec![("query_sql", resolved), ("adapter", self.adapter.clone())]; + for key in ["model_locations", "seed_locations"] { + refs.push((key, self.context.getattr(key)?)); + } + refs.extend_from_slice(cursor); + resolved = call_with( + &imported(py, NATIVE_MODULE, "resolve_ref_references")?, + &refs, + )?; + resolved = call_with( + &imported(py, NATIVE_MODULE, "resolve_dbt_ref_references")?, + &[ + ("query_sql", resolved), + ("external_sql_reference_resolver", self.external.clone()), + ], + )?; + let locations = self.context.getattr("function_locations")?; + let locations = if locations.is_truthy()? { + locations + } else { + PyDict::new(py).into_any() + }; + for name in [ + "resolve_udf_references", + "resolve_table_function_references", + ] { + resolved = call_with( + &imported(py, NATIVE_MODULE, name)?, + &[ + ("query_sql", resolved), + ("function_locations", locations.clone()), + ("adapter", self.adapter.clone()), + ], + )?; + } + Ok(resolved) + } + + fn assert_resolved(&self, sql: &Bound<'py, PyAny>, label: &str) -> PyResult<()> { + call_with( + &imported( + self.adapter.py(), + NATIVE_MODULE, + "assert_no_unresolved_sql_markers", + )?, + &[ + ("sql", sql.clone()), + ( + "context", + string(self.adapter.py(), &format!("{label} planned SQL")), + ), + ], + )?; + Ok(()) + } + + fn unbounded(&self, sql: Bound<'py, PyAny>, label: &str) -> PyResult> { + let py = self.adapter.py(); + let resolved = self.resolve( + sql, + &[ + ("cursor_bounds", py.None().into_bound(py)), + ("cursor_filter_inputs", PyDict::new(py).into_any()), + ("cursor_type", py.None().into_bound(py)), + ( + "lower_bound_inclusive", + true.into_pyobject(py)?.to_owned().into_any(), + ), + ], + )?; + self.assert_resolved(&resolved, label)?; + Ok(resolved) + } + + fn hook(&self, value: Bound<'py, PyAny>, label: &str) -> PyResult> { + let py = value.py(); + if value.is_instance(&imported(py, HOOK_MODULE, "SqlHookEntry")?)? { + let kwargs = PyDict::new(py); + kwargs.set_item( + "statement", + self.unbounded(value.getattr("statement")?, label)?, + )?; + return imported(py, "dataclasses", "replace")?.call((value,), Some(&kwargs)); + } + let is_list = value.is_instance_of::(); + if is_list || value.is_instance_of::() { + let mut resolved: Vec> = Vec::new(); + for (index, item) in value.try_iter()?.enumerate() { + resolved.push(self.hook(item?, &format!("{label}[{index}]"))?); + } + return if is_list { + Ok(PyList::new(py, resolved)?.into_any()) + } else { + Ok(PyTuple::new(py, resolved)?.into_any()) + }; + } + Ok(value) + } +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn resolve_model_sql<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "resolve_model_sql", + kwargs, + ( + &[ + "adapter", + "model", + "snapshot", + "context", + "backfill", + "full_refresh", + "cursor_overrides", + ], + &[ + "external_sql_reference_resolver", + "replaces_relation", + "full_rebuild_cause", + ], + ), + )?; + let model = args.value("model")?; + let resolution = Resolution { + adapter: args.value("adapter")?, + context: args.value("context")?, + external: args.value("external_sql_reference_resolver")?, + }; + let cursor_type = call_with( + &imported(py, NATIVE_MODULE, "get_config_str")?, + &[ + ("values", model.getattr("config")?.getattr("values")?), + ("key", string(py, "cursor_type")), + ], + )?; + let overrides = args.value("cursor_overrides")?; + let bounds = call_with( + &imported(py, RESOLVE_MODULE, "_compute_model_cursor_bounds")?, + &[ + ("model", model.clone()), + ("snapshot", args.value("snapshot")?), + ("backfill", args.value("backfill")?), + ("full_refresh", args.value("full_refresh")?), + ( + "start_cursor_override", + overrides.getattr("start_cursor_override")?, + ), + ( + "end_cursor_override", + overrides.getattr("end_cursor_override")?, + ), + ( + "runtime_cursor_producer_names", + resolution + .context + .getattr("runtime_cursor_producer_names")?, + ), + ("context", resolution.context.clone()), + ( + "replaces_relation", + args.value_or_false("replaces_relation")?, + ), + ], + )?; + let roles = call_with( + &imported(py, NATIVE_MODULE, "resolve_cursor_input_roles")?, + &[("model", model.clone())], + )?; + let inclusive = + imported(py, CONFIG_MODULE, "get_config_append_cursor_inclusive")?.call1((&model,))?; + let mut sql = resolution.resolve( + model.getattr("query_sql")?, + &[ + ("cursor_bounds", bounds.clone()), + ("cursor_filter_inputs", roles.getattr("filter_inputs")?), + ("cursor_type", cursor_type.clone()), + ("lower_bound_inclusive", inclusive), + ], + )?; + let intrinsics = call_with( + &imported(py, NATIVE_MODULE, "resolve_cursor_intrinsics")?, + &[("sql", sql.clone())], + )?; + let name: String = model.getattr("name")?.extract()?; + if intrinsics.get_item(1)?.is_truthy()? { + if bounds.is_none() { + let cause = args.value("full_rebuild_cause")?; + let cause = if cause.is_truthy()? { + Some(cause.extract::()?) + } else if args.flag("full_refresh")? { + Some("--full-refresh".to_owned()) + } else { + None + }; + return Err(unresolved_bounds_error(py, &name, cause.as_deref())?); + } + sql = call_with( + &imported(py, INTRINSICS_MODULE, "render_cursor_intrinsic_bounds")?, + &[ + ("sql", sql), + ("bounds", bounds), + ("cursor_type", cursor_type), + ("adapter", resolution.adapter.clone()), + ], + )?; + } + resolution.assert_resolved(&sql, &format!("Model '{name}'"))?; + Ok(sql) +} + +fn unresolved_bounds_error(py: Python<'_>, name: &str, cause: Option<&str>) -> PyResult { + let error = imported(py, ERROR_MODULE, "PlannerInputError")?; + let kwargs = PyDict::new(py); + let message = if let Some(cause) = cause { + kwargs.set_item("help", "rebuild it with an explicit cursor interval (--start-cursor-ts and --end-cursor-ts, or the -int forms), or make it a microbatch model so a full rebuild can run batch by batch")?; + format!( + "Model '{name}' was given a FULL backfill ({cause}), but it uses cursor intrinsics and a non-microbatch full rebuild has no cursor interval" + ) + } else { + format!("Model '{name}' uses cursor intrinsics, but cursor bounds could not be resolved") + }; + Ok(PyErr::from_value(error.call((message,), Some(&kwargs))?)) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn resolve_function_sql<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "resolve_function_sql", + kwargs, + ( + &[ + "adapter", + "function", + "model_locations", + "seed_locations", + "function_locations", + "source_map", + "source_warehouse_columns", + "star_exclude_keyword", + ], + &[], + ), + )?; + let mut context = vec![("models_by_name", PyDict::new(py).into_any())]; + for key in [ + "model_locations", + "seed_locations", + "function_locations", + "source_map", + "source_warehouse_columns", + "star_exclude_keyword", + ] { + context.push((key, args.value(key)?)); + } + let resolution = Resolution { + adapter: args.value("adapter")?, + context: call_with(&imported(py, MODELS_MODULE, "ModelPlanContext")?, &context)?, + external: py.None().into_bound(py), + }; + let function = args.value("function")?; + let name: String = function.getattr("name")?.extract()?; + resolution.unbounded(function.getattr("body_sql")?, &format!("Function '{name}'")) +} + +#[pyfunction] +#[pyo3(signature = (*, model, adapter, context, external_sql_reference_resolver))] +fn resolve_model_hook_entries<'py>( + model: &Bound<'py, PyAny>, + adapter: Bound<'py, PyAny>, + context: Bound<'py, PyAny>, + external_sql_reference_resolver: Bound<'py, PyAny>, +) -> PyResult> { + let resolution = Resolution { + adapter, + context, + external: external_sql_reference_resolver, + }; + let values = model.getattr("config")?.getattr("values")?; + let name: String = model.getattr("name")?.extract()?; + let mut hooks: Vec> = Vec::new(); + for key in ["pre_hooks", "post_hooks"] { + hooks.push(resolution.hook( + values.call_method1("get", (key,))?, + &format!("Model '{name}' {key}"), + )?); + } + PyTuple::new(model.py(), hooks) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_model_sql, module)?)?; + module.add_function(wrap_pyfunction!(resolve_function_sql, module)?)?; + module.add_function(wrap_pyfunction!(resolve_model_hook_entries, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs index b58c9098f8..0d1ba24f3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs @@ -4,5 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::refs::register(module)?; + super::locations::register(module)?; + super::orchestration::register(module)?; super::sources::register(module) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 890013040c..0afefd460d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -202,9 +202,11 @@ from sqlbuild.compiler.planner.classes.migration_state_inspection import ( from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( AuditPlanEntry, + BackfillResult, ChangeDetectionResult, ColumnRenameHint, CursorBounds, + CursorOverridePair, DeferralInputs, DirectModelVersionIdentities, FixtureRelationMetadata, @@ -213,6 +215,7 @@ from sqlbuild.compiler.planner.models import ( MigrationCompatibilityResult, MissingUpstream, ModelMigrationRequest, + ModelPlanContext, ModelPlanEntry, OldNameView, ParsedSelector, @@ -1063,6 +1066,46 @@ class NativeCompiledProject: def compiled_model_record(model: CompiledModelView) -> CompiledModel: ... def compiled_audit_record(audit: CompiledAuditView) -> CompiledAudit: ... +def resolve_model_sql( + *, + adapter: BaseAdapter, + model: CompiledModelView, + snapshot: WarehouseSnapshot, + context: ModelPlanContext, + backfill: BackfillResult, + full_refresh: bool, + cursor_overrides: CursorOverridePair, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, + replaces_relation: bool = False, + full_rebuild_cause: str | None = None, +) -> str: ... +def resolve_function_sql( + *, + adapter: BaseAdapter, + function: CompiledFunctionView, + model_locations: dict[str, CompiledRelationLocation], + seed_locations: dict[str, CompiledRelationLocation], + function_locations: dict[str, CompiledRelationLocation], + source_map: dict[str, SourceEntry], + source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]], + star_exclude_keyword: str, +) -> str: ... +def resolve_model_hook_entries( + *, + model: CompiledModelView, + adapter: BaseAdapter, + context: ModelPlanContext, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None, +) -> tuple[object, object]: ... +def build_model_locations( + models: tuple[CompiledModelView, ...], +) -> dict[str, CompiledRelationLocation]: ... +def build_seed_locations( + seeds: tuple[CompiledSeedView, ...], +) -> dict[str, CompiledRelationLocation]: ... +def build_function_locations( + functions: tuple[CompiledFunctionView, ...], +) -> dict[str, CompiledRelationLocation]: ... def compiled_function_record(function: CompiledFunctionView) -> CompiledFunction: ... def compiled_source_record(source: CompiledSourceView) -> CompiledSource: ... def compiled_seed_record(seed: CompiledSeedView) -> CompiledSeed: ... diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py index 085697c523..8ad3e3c0e5 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py @@ -7,16 +7,13 @@ import sqlbuild._native as _native from sqlbuild._native import apply_deferred_locations as apply_deferred_locations +from sqlbuild._native import build_function_locations as build_function_locations +from sqlbuild._native import build_model_locations as build_model_locations +from sqlbuild._native import build_seed_locations as build_seed_locations from sqlbuild._native import resolve_dbt_ref_references as resolve_dbt_ref_references from sqlbuild._native import resolve_ref_references as resolve_ref_references from sqlbuild._native import resolve_table_function_references as resolve_table_function_references from sqlbuild._native import resolve_udf_references as resolve_udf_references -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.compile.types import ( - CompiledFunctionView, - CompiledModelView, - CompiledSeedView, -) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax _REFERENCE_SCAN_CONTEXT: str = "SQL reference resolution" @@ -60,27 +57,3 @@ def _iter_code_matches( continue yield match index = match.end() - - -def build_model_locations( - models: tuple[CompiledModelView, ...], -) -> dict[str, CompiledRelationLocation]: - """Build a lookup of model name to compiled relation location.""" - - return {model.name: model.destination for model in models} - - -def build_seed_locations( - seeds: tuple[CompiledSeedView, ...], -) -> dict[str, CompiledRelationLocation]: - """Build a lookup of seed name to compiled relation location.""" - - return {seed.name: seed.destination for seed in seeds} - - -def build_function_locations( - functions: tuple[CompiledFunctionView, ...], -) -> dict[str, CompiledRelationLocation]: - """Build a lookup of function name to compiled relation location.""" - - return {function.name: function.destination for function in functions} diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py index 2a1d38d872..b2d83cadbf 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py @@ -1,29 +1,20 @@ -"""Per-model SQL resolution orchestration.""" +"""Cursor-bound computation and native per-model SQL resolution.""" from __future__ import annotations -from dataclasses import replace from datetime import datetime from sqlbuild._native import ( - assert_no_unresolved_sql_markers, get_config_cursor_bound, get_config_str, resolve_bounded_cursor_override, resolve_cursor_input_roles, - resolve_cursor_intrinsics, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationLookup -from sqlbuild.compiler.compile.models import ( - CompiledRelationLocation, - CursorInputRoles, -) -from sqlbuild.compiler.compile.types import CompiledFunctionView, CompiledModelView -from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.planner._helpers.resolve.config import ( - get_config_append_cursor_inclusive, ) +from sqlbuild._native import resolve_function_sql as resolve_function_sql +from sqlbuild._native import resolve_model_hook_entries as resolve_model_hook_entries +from sqlbuild._native import resolve_model_sql as resolve_model_sql +from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner._helpers.resolve.cursor import ( compute_cursor_bounds, cursor_inputs_without_rows, @@ -32,181 +23,23 @@ unreadable_cursor_inputs_error, without_destination_cursor, ) -from sqlbuild.compiler.planner._helpers.resolve.cursor_intrinsics import ( - render_cursor_intrinsic_bounds, -) from sqlbuild.compiler.planner._helpers.resolve.cursor_policies import ( resolve_future_cursor_config, resolve_start_cursor_config, ) -from sqlbuild.compiler.planner._helpers.resolve.refs import ( - resolve_dbt_ref_references, - resolve_ref_references, - resolve_table_function_references, - resolve_udf_references, -) -from sqlbuild.compiler.planner._helpers.resolve.sources import ( - resolve_source_references, -) -from sqlbuild.compiler.planner.constants import ( - MODEL_POST_HOOKS_CONFIG_KEY, - MODEL_PRE_HOOKS_CONFIG_KEY, -) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety from sqlbuild.compiler.planner.models import ( BackfillResult, CursorBounds, - CursorOverridePair, MaximumStartPolicyInputs, ModelCursorSnapshot, ModelPlanContext, WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import BackfillAction, IncrementalMode, MaterializationType -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.cursor_algebra.types import BoundSentinel -from sqlbuild.spec.contracts.models import FutureCursorsConfig, SourceEntry, StartCursorsConfig - - -def resolve_model_sql( - *, - adapter: BaseAdapter, - model: CompiledModelView, - snapshot: WarehouseSnapshot, - context: ModelPlanContext, - backfill: BackfillResult, - full_refresh: bool, - cursor_overrides: CursorOverridePair, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, - replaces_relation: bool = False, - full_rebuild_cause: str | None = None, -) -> str: - """Resolve all references in a model's query SQL to produce executable SQL.""" - - model_locations: dict[str, CompiledRelationLocation] = context.model_locations - seed_locations: dict[str, CompiledRelationLocation] = context.seed_locations - function_locations: dict[str, CompiledRelationLocation] = context.function_locations - source_map: dict[str, SourceEntry] = context.source_map - source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]] = context.source_warehouse_columns - star_exclude_keyword: str = context.star_exclude_keyword - query_sql: str = model.query_sql - cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") - - cursor_bounds: CursorBounds | None = _compute_model_cursor_bounds( - model=model, - snapshot=snapshot, - backfill=backfill, - full_refresh=full_refresh, - start_cursor_override=cursor_overrides.start_cursor_override, - end_cursor_override=cursor_overrides.end_cursor_override, - runtime_cursor_producer_names=context.runtime_cursor_producer_names, - context=context, - replaces_relation=replaces_relation, - ) - - cursor_roles: CursorInputRoles = resolve_cursor_input_roles(model=model) - lower_bound_inclusive: bool = get_config_append_cursor_inclusive(model) - - query_sql = resolve_source_references( - query_sql=query_sql, - source_map=source_map, - source_warehouse_columns=source_warehouse_columns, - star_exclude_keyword=star_exclude_keyword, - cursor_bounds=cursor_bounds, - cursor_filter_inputs=cursor_roles.filter_inputs, - adapter=adapter, - cursor_type=cursor_type, - lower_bound_inclusive=lower_bound_inclusive, - ) - - query_sql = resolve_ref_references( - query_sql=query_sql, - model_locations=model_locations, - seed_locations=seed_locations, - cursor_bounds=cursor_bounds, - cursor_filter_inputs=cursor_roles.filter_inputs, - adapter=adapter, - cursor_type=cursor_type, - lower_bound_inclusive=lower_bound_inclusive, - ) - - query_sql = resolve_dbt_ref_references( - query_sql=query_sql, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - query_sql = resolve_udf_references( - query_sql=query_sql, - function_locations=function_locations or {}, - adapter=adapter, - ) - query_sql = resolve_table_function_references( - query_sql=query_sql, - function_locations=function_locations or {}, - adapter=adapter, - ) - _, has_intrinsics = resolve_cursor_intrinsics(sql=query_sql) - if has_intrinsics: - if cursor_bounds is None: - raise _unresolved_cursor_bounds_error( - model_name=model.name, - full_rebuild_cause=full_rebuild_cause - or ("--full-refresh" if full_refresh else None), - ) - query_sql = render_cursor_intrinsic_bounds( - sql=query_sql, bounds=cursor_bounds, cursor_type=cursor_type, adapter=adapter - ) - - assert_no_unresolved_sql_markers(sql=query_sql, context=f"Model '{model.name}' planned SQL") - return query_sql - - -def _unresolved_cursor_bounds_error( - *, model_name: str, full_rebuild_cause: str | None -) -> PlannerInputError: - if full_rebuild_cause is None: - return PlannerInputError( - f"Model '{model_name}' uses cursor intrinsics, but cursor bounds could not be resolved" - ) - return PlannerInputError( - f"Model '{model_name}' was given a FULL backfill ({full_rebuild_cause}), but it uses " - "cursor intrinsics and a non-microbatch full rebuild has no cursor interval", - help=( - "rebuild it with an explicit cursor interval (--start-cursor-ts and " - "--end-cursor-ts, or the -int forms), or make it a microbatch model so a full " - "rebuild can run batch by batch" - ), - ) - - -def resolve_function_sql( - *, - adapter: BaseAdapter, - function: CompiledFunctionView, - model_locations: dict[str, CompiledRelationLocation], - seed_locations: dict[str, CompiledRelationLocation], - function_locations: dict[str, CompiledRelationLocation], - source_map: dict[str, SourceEntry], - source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]], - star_exclude_keyword: str, -) -> str: - """Resolve relation and function references in a SQL function body.""" - - return _resolve_unbounded_sql( - sql=function.body_sql, - context=ModelPlanContext( - model_locations=model_locations, - models_by_name={}, - seed_locations=seed_locations, - function_locations=function_locations, - source_map=source_map, - source_warehouse_columns=source_warehouse_columns, - star_exclude_keyword=star_exclude_keyword, - ), - adapter=adapter, - external_sql_reference_resolver=None, - label=f"Function '{function.name}'", - ) +from sqlbuild.spec.contracts.models import FutureCursorsConfig, StartCursorsConfig def _compute_model_cursor_bounds( @@ -362,101 +195,3 @@ def _history_redirected(*, model: CompiledModelView, cursor_snapshot: ModelCurso cursor_snapshot.target_relation is not None and cursor_snapshot.target_relation != model.destination.qualified_name ) - - -def resolve_model_hook_entries( - *, - model: CompiledModelView, - adapter: BaseAdapter, - context: ModelPlanContext, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None, -) -> tuple[object, object]: - """Return the model's pre and post hooks with SQL hook references resolved like model SQL.""" - - pre_hooks, post_hooks = ( - _resolve_hook_value( - value=model.config.values.get(hook_key), - label=f"Model '{model.name}' {hook_key}", - adapter=adapter, - context=context, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - for hook_key in (MODEL_PRE_HOOKS_CONFIG_KEY, MODEL_POST_HOOKS_CONFIG_KEY) - ) - return pre_hooks, post_hooks - - -def _resolve_hook_value( - *, - value: object, - label: str, - adapter: BaseAdapter, - context: ModelPlanContext, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None, -) -> object: - if isinstance(value, SqlHookEntry): - return replace( - value, - statement=_resolve_unbounded_sql( - sql=value.statement, - label=label, - adapter=adapter, - context=context, - external_sql_reference_resolver=external_sql_reference_resolver, - ), - ) - if isinstance(value, list | tuple): - resolved: list[object] = [ - _resolve_hook_value( - value=item, - label=f"{label}[{index}]", - adapter=adapter, - context=context, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - for index, item in enumerate(value) - ] - return resolved if isinstance(value, list) else tuple(resolved) - return value - - -def _resolve_unbounded_sql( - *, - sql: str, - label: str, - adapter: BaseAdapter, - context: ModelPlanContext, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None, -) -> str: - resolved: str = resolve_source_references( - query_sql=sql, - source_map=context.source_map, - source_warehouse_columns=context.source_warehouse_columns, - star_exclude_keyword=context.star_exclude_keyword, - cursor_bounds=None, - cursor_filter_inputs={}, - adapter=adapter, - cursor_type=None, - lower_bound_inclusive=True, - ) - resolved = resolve_ref_references( - query_sql=resolved, - model_locations=context.model_locations, - seed_locations=context.seed_locations, - cursor_bounds=None, - cursor_filter_inputs={}, - adapter=adapter, - cursor_type=None, - lower_bound_inclusive=True, - ) - resolved = resolve_dbt_ref_references( - query_sql=resolved, external_sql_reference_resolver=external_sql_reference_resolver - ) - resolved = resolve_udf_references( - query_sql=resolved, function_locations=context.function_locations or {}, adapter=adapter - ) - resolved = resolve_table_function_references( - query_sql=resolved, function_locations=context.function_locations or {}, adapter=adapter - ) - assert_no_unresolved_sql_markers(sql=resolved, context=f"{label} planned SQL") - return resolved diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py index e6be6b7bbd..a8db051a8f 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py @@ -8,6 +8,16 @@ from sqlbuild.spec.contracts.models import SourceEntry +@dataclass(frozen=True) +class OrchestrationTestCase: + """Expected rows and ordered adapter calls across the resolution entry points.""" + + description: str + expected_rows: list[tuple[int]] + expected_model_calls: list[str] + expected_relation_calls: list[str] + + @dataclass(frozen=True) class AdapterOverrideTestCase: """Expected SQL and rows from adapter-owned source rendering.""" diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index dbd5824bc3..f51c1f12e5 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -6,16 +6,22 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( + CompileAdapterContext, CompiledModel, CompiledObjectKey, CompiledRelationLocation, CompileModelConfig, CompileSqlReference, ) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_model_sql from sqlbuild.compiler.planner.models import ( BackfillResult, @@ -27,6 +33,63 @@ from sqlbuild.compiler.planner.types import BackfillAction from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import SourceEntry +from sqlbuild.sql_values.types import CollectionRendering + + +class OrchestrationOverrideAdapter(DuckDbAdapter): + """Record orchestration callbacks and render executable custom relations and functions.""" + + def __init__(self) -> None: + self.calls: list[str] = [] + + def render_qualified_name(self, *, database: str | None, schema: str | None, name: str) -> str: + self.calls.append(f"relation:{schema}.{name}") + return "staging.actual_orders" + + def render_table_function_call(self, *, target: str, call_suffix_sql: str) -> str: + self.calls.append(f"function:{target}{call_suffix_sql}") + return "(SELECT 9 AS order_id)" + + +def compile_resolution_project(project_dir: Path) -> CompiledProjectView: + """Compile a real model, dependent table function and SQL hook project.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "resolution"\nadapter = "duckdb"\n' + '[connection]\ndatabase = ":memory:"\n[defaults]\nschema = "staging"\n' + ), + "models/orders.sql": ( + 'MODEL (description "Orders", alias "orders_alias"); SELECT 9 AS order_id' + ), + "functions/sql/order_lookup.sql": ( + 'FUNCTION (description "Order lookup", arguments (id INTEGER), ' + "returns table (order_id INTEGER)); " + 'SELECT order_id FROM __ref("orders") WHERE order_id = id' + ), + "models/report.sql": ( + 'MODEL (description "Report", pre_hooks [inline_sql(\'SELECT order_id ' + 'FROM __ref("orders")\')]); SELECT o.order_id FROM __ref("orders") o ' + 'JOIN __table_fn("order_lookup")(9) f ON o.order_id = f.order_id' + ), + } + for path, text in files.items(): + destination: Path = project_dir / path + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(text) + return assemble_project( + inputs=build_compile_inputs( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter_context=CompileAdapterContext( + value_renderer=DuckDbAdapter(), + collection_rendering=CollectionRendering.VALUE_LIST, + python_functions_inherit_default_namespace=True, + sql_lexical_syntax=DuckDbAdapter.sql_lexical_syntax, + ), + run_id="resolution", + ), + skip_column_inference=True, + ) class ResolutionOverrideAdapter(DuckDbAdapter): diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py new file mode 100644 index 0000000000..1042f9d7aa --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py @@ -0,0 +1,109 @@ +"""Execute compiled model, function and hook SQL with custom adapter overrides.""" + +from dataclasses import replace +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild._native import ( + build_function_locations, + build_model_locations, + build_seed_locations, + resolve_function_sql, + resolve_model_hook_entries, + resolve_model_sql, +) +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView +from sqlbuild.compiler.discovery.models import SqlHookEntry +from sqlbuild.compiler.planner.models import ( + BackfillResult, + CursorOverridePair, + ModelPlanContext, + WarehouseSnapshot, +) +from sqlbuild.compiler.planner.types import BackfillAction +from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( + OrchestrationTestCase, +) +from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve.helpers import ( + OrchestrationOverrideAdapter, + compile_resolution_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + OrchestrationTestCase( + description="compiled aliases and function dependencies honor adapter rendering", + expected_rows=[(9,)], + expected_model_calls=[ + "relation:staging.orders_alias", + "function:staging.order_lookup(9)", + ], + expected_relation_calls=["relation:staging.orders_alias"], + ) + ], + ids=lambda case: case.description, +) +def test_given_compiled_dependencies_when_resolving_then_adapter_overrides_execute_in_order( + test_case: OrchestrationTestCase, tmp_path: Path, connection: Any +) -> None: + project: CompiledProjectView = compile_resolution_project(tmp_path) + model: CompiledModelView = {model.name: model for model in project.models}["report"] + adapter: OrchestrationOverrideAdapter = OrchestrationOverrideAdapter() + context: ModelPlanContext = ModelPlanContext( + model_locations=build_model_locations(project.models), + models_by_name={model.name: model for model in project.models}, + seed_locations=build_seed_locations(project.seeds), + function_locations=build_function_locations(project.functions), + source_map={}, + source_warehouse_columns={}, + star_exclude_keyword="EXCLUDE", + ) + assert not project.diagnostics + assert context.model_locations["orders"].name == "orders_alias" + context.function_locations["order_lookup"] = replace( + context.function_locations["order_lookup"], qualified_name="staging.order_lookup" + ) + connection.execute("CREATE TABLE staging.actual_orders AS SELECT 9 AS order_id") + sql: str = resolve_model_sql( + adapter=adapter, + model=model, + snapshot=WarehouseSnapshot(existing_relations={}), + context=context, + backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + full_refresh=False, + cursor_overrides=CursorOverridePair(), + ) + assert connection.execute(sql).fetchall() == test_case.expected_rows + assert adapter.calls == test_case.expected_model_calls + adapter.calls.clear() + function_sql: str = resolve_function_sql( + adapter=adapter, + function=project.functions[0], + model_locations=context.model_locations, + seed_locations=context.seed_locations, + function_locations=context.function_locations, + source_map={}, + source_warehouse_columns={}, + star_exclude_keyword="EXCLUDE", + ) + connection.execute(f"CREATE MACRO order_lookup(id) AS TABLE {function_sql}") + assert connection.execute("SELECT * FROM order_lookup(9)").fetchall() == test_case.expected_rows + assert adapter.calls == test_case.expected_relation_calls + adapter.calls.clear() + pre_hooks, post_hooks = resolve_model_hook_entries( + model=model, adapter=adapter, context=context, external_sql_reference_resolver=None + ) + assert isinstance(pre_hooks, tuple | list) + assert len(pre_hooks) == 1 + assert isinstance(pre_hooks[0], SqlHookEntry) + assert connection.execute(pre_hooks[0].statement).fetchall() == test_case.expected_rows + assert adapter.calls == test_case.expected_relation_calls + assert not post_hooks + + +if __name__ == "__main__": + pytest.main([__file__]) From 0b5fd0202cb3043fd391b3a29357e45fcf9135a3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 14:22:43 +0100 Subject: [PATCH 08/29] chore: update Python boundary after resolution orchestration port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 02f3f1c4a4..017159143e 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 38073 +remaining_lines = 37781 [[stays]] path = "executor" From 7996ca56975b2633532097435ebfd51b2003d938 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 14:47:33 +0100 Subject: [PATCH 09/29] refactor: host fixture formatting orchestration in Rust --- Cargo.lock | 1 + .../src/bindings/_helpers/rules/constants.rs | 3 + .../_helpers/rules/fixture_formatting.rs | 89 ++++++- .../_helpers/rules/fixture_schemas.rs | 121 ++++++++++ .../src/bindings/_helpers/rules/mod.rs | 2 + crates/sqlbuild-rules/Cargo.toml | 1 + .../src/lint_text/_helpers/fixture_blocks.rs | 57 +++++ .../src/lint_text/_helpers/mod.rs | 1 + .../lint_text/main/format_fixture_blocks.rs | 16 ++ .../sqlbuild-rules/src/lint_text/main/mod.rs | 1 + crates/sqlbuild-rules/src/lint_text/mod.rs | 1 + crates/sqlbuild-rules/src/lint_text/models.rs | 15 ++ .../tests/test_fixture_formatting.rs | 67 ++++++ .../src/lint_text/tests/test_types.rs | 7 + crates/sqlbuild-rules/src/lint_text/types.rs | 4 + python-boundary.toml | 4 - src/sqlbuild/_native.pyi | 7 + .../_helpers/sql_tests/fixture_formatting.py | 224 +----------------- .../lint/test_adapter_lexical_syntax.py | 11 +- .../unit/src/sqlbuild/lint/test_cli_format.py | 4 - 20 files changed, 386 insertions(+), 250 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs create mode 100644 crates/sqlbuild-rules/src/lint_text/_helpers/fixture_blocks.rs create mode 100644 crates/sqlbuild-rules/src/lint_text/main/format_fixture_blocks.rs create mode 100644 crates/sqlbuild-rules/src/lint_text/types.rs diff --git a/Cargo.lock b/Cargo.lock index a4d63b81c4..8736887afc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1319,6 +1319,7 @@ dependencies = [ "sqlbuild-analysis", "sqlbuild-attachments", "sqlbuild-core", + "sqlbuild-scopes", "sqlbuild-sqltext", "sqlparser", "stacker", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs new file mode 100644 index 0000000000..3c352fc657 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs @@ -0,0 +1,3 @@ +//! Authored SQL-test mode identifiers. + +pub(crate) const MODEL_TEST_MODE: &str = "model"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_formatting.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_formatting.rs index 66da2880a4..8bb4a852ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_formatting.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_formatting.rs @@ -1,29 +1,92 @@ //! Fixture schema conversion at the Python model boundary. +use pyo3::exceptions::{PyOSError, PySyntaxError, PyUnicodeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyStringMethods}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_rules::lint_text::main::format_fixture_blocks::format_fixture_blocks; use sqlbuild_rules::lint_text::main::simplify_fixture_nulls::simplify_fixture_nulls; +use sqlbuild_rules::lint_text::models::FixtureFormatBlock; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; +use crate::bindings::_helpers::discovery::sql_test_records::parse_sql_test_file; +use crate::bindings::_helpers::rules::constants::MODEL_TEST_MODE; +use crate::bindings::_helpers::rules::fixture_schemas; +use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; #[pyfunction] #[pyo3(signature = (*, sql, relation))] fn simplify_fixture_null_projections(sql: &str, relation: &Bound<'_, PyAny>) -> PyResult { - let columns: Vec<(String, Option, Option)> = relation - .getattr("columns")? - .try_iter()? - .map(|column| { - let column = column?; - Ok(( - column.getattr("name")?.extract()?, - column.getattr("type")?.extract()?, - column.getattr("nullable")?.extract()?, - )) - }) - .collect::>()?; + let columns = fixture_schemas::columns(&relation.getattr("columns")?)?; compiler_guard(|| simplify_fixture_nulls(sql, &columns).map_err(value_error)) } +#[pyfunction] +#[pyo3(signature = (*, files, project_dir, discovered_inputs, sql_lexical_syntax))] +fn format_redundant_fixture_nulls<'py>( + files: &Bound<'py, PyDict>, + project_dir: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, + sql_lexical_syntax: &Bound<'py, PyAny>, +) -> PyResult> { + compiler_guard(|| { + let py = files.py(); + let relations = fixture_schemas::relations(discovered_inputs)?; + let syntax: LexicalSyntax = sql_lexical_syntax + .getattr("native_mapping")? + .extract::()? + .into(); + let updated = PyDict::new(py); + for file in discovered_inputs.getattr("test_files")?.try_iter()? { + let file = file?; + let relative = file.getattr("relative_path")?; + let path = project_dir + .call_method1("__truediv__", (&relative,))? + .call_method0("resolve")?; + let Some(original) = files.get_item(&path)? else { + continue; + }; + let contents: String = updated.get_item(&path)?.unwrap_or(original).extract()?; + let blocks = match parse_sql_test_file(contents.clone(), &path) { + Ok(blocks) => blocks, + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) + || error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + continue; + } + Err(error) => return Err(error), + }; + let blocks = blocks + .try_iter()? + .map(|block| { + let block = block?; + Ok(FixtureFormatBlock { + sql_body: block.getattr("sql_body")?.extract()?, + model_mode: block + .getattr("mode")? + .getattr("value")? + .extract::()? + == MODEL_TEST_MODE, + }) + }) + .collect::>>()?; + let label = relative.str()?.to_str()?.to_owned(); + if let Some(result) = + format_fixture_blocks((&contents, &label), &blocks, &relations, &syntax) + .map_err(value_error)? + { + updated.set_item(path, result)?; + } + } + Ok(updated) + }) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(simplify_fixture_null_projections, module)?) + module.add_function(wrap_pyfunction!(simplify_fixture_null_projections, module)?)?; + module.add_function(wrap_pyfunction!(format_redundant_fixture_nulls, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs new file mode 100644 index 0000000000..75c88a8b82 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs @@ -0,0 +1,121 @@ +//! Discovered contract objects converted to authoritative fixture schemas. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyStringMethods}; +use sqlbuild_model_config::path_defaults::main::select_path_default::select_path_default; +use sqlbuild_model_config::path_defaults::models::PathDefaultChoice; +use sqlbuild_rules::lint_text::models::FixtureFormatRelation; +use sqlbuild_rules::lint_text::types::FixtureFormatColumn; +use std::collections::HashMap; + +use crate::bindings::_helpers::model_config::header_columns::schema_columns; +use crate::bindings::_helpers::model_config::model_config_builder::raised_config_error; + +pub(crate) fn columns(columns: &Bound<'_, PyAny>) -> PyResult> { + columns + .try_iter()? + .map(|column| { + let column = column?; + Ok(( + column.getattr("name")?.extract()?, + column.getattr("type")?.extract()?, + column.getattr("nullable")?.extract()?, + )) + }) + .collect() +} + +pub(crate) fn relations( + inputs: &Bound<'_, PyAny>, +) -> PyResult> { + let py = inputs.py(); + let config = inputs.getattr("project_config")?; + let default_contract: Option = + config.getattr("defaults")?.getattr("contract")?.extract()?; + let defaults = config.getattr("path_defaults")?.cast_into::()?; + let keys: Vec = defaults + .keys() + .iter() + .map(|key| key.extract()) + .collect::>()?; + let mut result: HashMap = HashMap::new(); + for model in inputs.getattr("model_files")?.try_iter()? { + let model = model?; + let path = model.getattr("relative_path")?.str()?.to_str()?.to_owned(); + let mut contract = default_contract.clone(); + match select_path_default(&path, &keys) { + PathDefaultChoice::Selected(Some(key)) => { + let values = defaults + .get_item(key)? + .ok_or_else(|| { + pyo3::exceptions::PyKeyError::new_err("selected path default is absent") + })? + .cast_into::()?; + if let Some(value) = values.get_item("contract")? { + contract = value.extract()?; + } + } + PathDefaultChoice::Selected(None) => {} + PathDefaultChoice::Conflict(error) => return Err(raised_config_error(py, error)), + } + let header = model.getattr("header_values")?.cast_into::()?; + if let Some(value) = header.get_item("contract")? { + contract = value.extract()?; + } + if contract.as_deref() != Some("enforced") { + continue; + } + let file = model.getattr("file_path")?; + let file_name = file.str()?.to_str()?.to_owned(); + let raw_columns = header + .get_item("columns")? + .unwrap_or_else(|| py.None().into_bound(py)); + let locations = model + .getattr("header_column_locations")? + .cast_into::()?; + let parsed = schema_columns( + &raw_columns, + (&file_name, "model"), + (false, true, false), + &locations, + )?; + let columns = columns(parsed.as_any())?; + if !columns.is_empty() { + result.insert( + format!("__ref__{}", file.getattr("stem")?.extract::()?), + FixtureFormatRelation { columns }, + ); + } + } + for source_file in inputs.getattr("source_files")?.try_iter()? { + for source in source_file?.getattr("source_entries")?.try_iter()? { + let source = source?; + let contract: Option = source.getattr("contract")?.extract()?; + let contract = contract + .filter(|value| !value.is_empty()) + .or_else(|| default_contract.clone()); + if contract.as_deref() != Some("enforced") { + continue; + } + result.insert( + format!("__source__{}", source.getattr("name")?.extract::()?), + FixtureFormatRelation { + columns: columns(&source.getattr("columns")?)?, + }, + ); + } + } + for schema_file in inputs.getattr("schema_files")?.try_iter()? { + for seed in schema_file?.getattr("seed_entries")?.try_iter()? { + let seed = seed?; + let columns = columns(&seed.getattr("columns")?)?; + if !columns.is_empty() { + result.insert( + format!("__seed__{}", seed.getattr("name")?.extract::()?), + FixtureFormatRelation { columns }, + ); + } + } + } + Ok(result) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 6c5c446e0f..f13e0bc557 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod builtin_catalogue; pub(crate) mod catalogue; +pub(crate) mod constants; pub(crate) mod custom_rule_cache; pub(crate) mod custom_rule_evidence; pub(crate) mod dsl_headers; @@ -10,6 +11,7 @@ pub(crate) mod fix_facts; pub(crate) mod fix_session; pub(crate) mod fix_verdict; pub(crate) mod fixture_formatting; +pub(crate) mod fixture_schemas; pub(crate) mod guidance; pub(crate) mod lint; pub(crate) mod lint_expansion; diff --git a/crates/sqlbuild-rules/Cargo.toml b/crates/sqlbuild-rules/Cargo.toml index 51945d206b..16312b86c4 100644 --- a/crates/sqlbuild-rules/Cargo.toml +++ b/crates/sqlbuild-rules/Cargo.toml @@ -19,6 +19,7 @@ sha2.workspace = true sqlbuild-analysis.workspace = true sqlbuild-attachments.workspace = true sqlbuild-core.workspace = true +sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true stacker.workspace = true sqlparser.workspace = true diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/fixture_blocks.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/fixture_blocks.rs new file mode 100644 index 0000000000..080c7e9bab --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/fixture_blocks.rs @@ -0,0 +1,57 @@ +//! Fixture block eligibility, extraction, and ordered text replacement. + +use regex::Regex; +use sqlbuild_scopes::relationship_names::main::top_level_ctes::scan_top_level_ctes; +use sqlbuild_scopes::relationship_names::models::{RelationshipSource, TopLevelCtes}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; +use std::collections::HashMap; +use std::sync::LazyLock; + +use crate::lint_text::_helpers::fixture_formatting::simplify_fixture_nulls; +use crate::lint_text::models::{FixtureFormatBlock, FixtureFormatRelation}; + +static CANDIDATE: LazyLock> = + LazyLock::new(|| Regex::new(r"(?i)(?:CAST\s*\(\s*NULL\s+AS\b|NULL\s*::)")); + +pub(crate) fn format( + (contents, label): (&str, &str), + blocks: &[FixtureFormatBlock], + relations: &HashMap, + syntax: &LexicalSyntax, +) -> Result, regex::Error> { + let candidate = CANDIDATE.as_ref().map_err(Clone::clone)?; + let mut result = contents.to_owned(); + let mut changed = false; + for block in blocks { + if !block.model_mode || block.sql_body.contains('@') || !candidate.is_match(&block.sql_body) + { + continue; + } + let TopLevelCtes::Scanned(ctes) = + scan_top_level_ctes(&block.sql_body, label, RelationshipSource::Test, syntax) + else { + continue; + }; + let mut replacements: Vec<(String, String)> = Vec::new(); + for (name, body) in ctes { + let Some(relation) = relations.get(&name) else { + continue; + }; + let replacement = simplify_fixture_nulls(&body, &relation.columns)?; + if replacement != body { + replacements.push((body, replacement)); + } + } + let mut fixed = block.sql_body.clone(); + for (original, replacement) in replacements { + if fixed.matches(&original).count() == 1 { + fixed = fixed.replacen(&original, &replacement, 1); + } + } + if fixed != block.sql_body { + result = result.replacen(&block.sql_body, &fixed, 1); + changed = true; + } + } + Ok(changed.then_some(result)) +} diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs index a49c95e613..a666bb0e05 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod fixture_blocks; pub(crate) mod fixture_formatting; pub(crate) mod fixtures; pub(crate) mod format_bodies; diff --git a/crates/sqlbuild-rules/src/lint_text/main/format_fixture_blocks.rs b/crates/sqlbuild-rules/src/lint_text/main/format_fixture_blocks.rs new file mode 100644 index 0000000000..81e907d624 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/format_fixture_blocks.rs @@ -0,0 +1,16 @@ +//! Contract-aware formatting of SQL-test fixture bodies. + +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; +use std::collections::HashMap; + +use crate::lint_text::models::{FixtureFormatBlock, FixtureFormatRelation}; + +/// Format parsed blocks, retaining ambiguous CTE replacements and invalid SQL verbatim. +pub fn format_fixture_blocks( + source: (&str, &str), + blocks: &[FixtureFormatBlock], + relations: &HashMap, + syntax: &LexicalSyntax, +) -> Result, regex::Error> { + crate::lint_text::_helpers::fixture_blocks::format(source, blocks, relations, syntax) +} diff --git a/crates/sqlbuild-rules/src/lint_text/main/mod.rs b/crates/sqlbuild-rules/src/lint_text/main/mod.rs index 1fb780ee99..d09920580a 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/mod.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/mod.rs @@ -1,6 +1,7 @@ pub mod apply_suppressions; pub mod dependency_sites; pub mod first_header_only; +pub mod format_fixture_blocks; pub mod format_sql_bodies; pub mod interpolation_text_at; pub mod lint_sql_bodies; diff --git a/crates/sqlbuild-rules/src/lint_text/mod.rs b/crates/sqlbuild-rules/src/lint_text/mod.rs index e0a0e363a4..bb458f2892 100644 --- a/crates/sqlbuild-rules/src/lint_text/mod.rs +++ b/crates/sqlbuild-rules/src/lint_text/mod.rs @@ -5,6 +5,7 @@ pub mod constants; pub mod errors; pub mod main; pub mod models; +pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-rules/src/lint_text/models.rs b/crates/sqlbuild-rules/src/lint_text/models.rs index bef8d138d3..fc0b21277b 100644 --- a/crates/sqlbuild-rules/src/lint_text/models.rs +++ b/crates/sqlbuild-rules/src/lint_text/models.rs @@ -1,5 +1,20 @@ //! Interpolation sites, suppression outcomes and fixture-null candidates. +use crate::lint_text::types::FixtureFormatColumn; + +/// A schema whose names are authoritative for an authored fixture CTE. +#[derive(Clone, Debug)] +pub struct FixtureFormatRelation { + pub columns: Vec, +} + +/// An authored SQL-test block eligible for model fixture formatting. +#[derive(Clone, Debug)] +pub struct FixtureFormatBlock { + pub sql_body: String, + pub model_mode: bool, +} + /// Dependency sentinels and keyed relations derived from authored interpolation sites. #[derive(Debug, Default)] pub struct DependencySites { diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_fixture_formatting.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_fixture_formatting.rs index e7bb9515da..045836b2ba 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_fixture_formatting.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_fixture_formatting.rs @@ -1,5 +1,72 @@ +use crate::lint_text::main::format_fixture_blocks::format_fixture_blocks; use crate::lint_text::main::simplify_fixture_nulls::simplify_fixture_nulls; +use crate::lint_text::models::{FixtureFormatBlock, FixtureFormatRelation}; +use crate::lint_text::tests::test_types::FixtureBlockTestCase; use crate::lint_text::tests::test_types::FixtureFormattingTestCase; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; +use std::collections::HashMap; + +#[test] +fn given_authored_blocks_when_formatting_then_only_unique_model_fixtures_change() +-> Result<(), regex::Error> { + let relations = HashMap::from([( + "__ref__orders".to_owned(), + FixtureFormatRelation { + columns: vec![("id".into(), Some("INTEGER".into()), Some(true))], + }, + )]); + let test_cases = [ + FixtureBlockTestCase { + description: "unique fixture body changes", + sql: "WITH __ref__orders AS (SELECT CAST(NULL AS INTEGER) AS id WHERE FALSE) SELECT 1", + model_mode: true, + expected_sql: Some("WITH __ref__orders AS (SELECT * FROM __empty_fixture()) SELECT 1"), + }, + FixtureBlockTestCase { + description: "ambiguous duplicate bodies are retained", + sql: "WITH __ref__orders AS (SELECT CAST(NULL AS INTEGER) AS id WHERE FALSE), other AS (SELECT CAST(NULL AS INTEGER) AS id WHERE FALSE) SELECT 1", + model_mode: true, + expected_sql: None, + }, + FixtureBlockTestCase { + description: "expected output is not an input fixture", + sql: "WITH __expected__orders AS (SELECT CAST(NULL AS INTEGER) AS id WHERE FALSE) SELECT 1", + model_mode: true, + expected_sql: None, + }, + FixtureBlockTestCase { + description: "macro-bearing blocks are retained", + sql: "WITH __ref__orders AS (SELECT CAST(NULL AS INTEGER) AS id WHERE FALSE) SELECT @value()", + model_mode: true, + expected_sql: None, + }, + FixtureBlockTestCase { + description: "non-model blocks are retained", + sql: "WITH __ref__orders AS (SELECT CAST(NULL AS INTEGER) AS id WHERE FALSE) SELECT 1", + model_mode: false, + expected_sql: None, + }, + ]; + for test_case in test_cases { + let blocks = [FixtureFormatBlock { + sql_body: test_case.sql.into(), + model_mode: test_case.model_mode, + }]; + assert_eq!( + format_fixture_blocks( + (test_case.sql, "test_orders.sql"), + &blocks, + &relations, + &LexicalSyntax::default() + )? + .as_deref(), + test_case.expected_sql, + "{}", + test_case.description + ); + } + Ok(()) +} #[test] fn given_authoritative_columns_when_simplifying_then_preserves_schema_order() diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs index 0411bbca6e..264314566b 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs @@ -79,3 +79,10 @@ pub(crate) struct FixtureFormattingTestCase { pub columns: Vec<(String, Option, Option)>, pub expected_sql: &'static str, } +/// Authored block formatting across fixture roles and ambiguous replacements. +pub(super) struct FixtureBlockTestCase { + pub description: &'static str, + pub sql: &'static str, + pub model_mode: bool, + pub expected_sql: Option<&'static str>, +} diff --git a/crates/sqlbuild-rules/src/lint_text/types.rs b/crates/sqlbuild-rules/src/lint_text/types.rs new file mode 100644 index 0000000000..4e21467032 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/types.rs @@ -0,0 +1,4 @@ +//! Schema values shared by fixture formatting and its object bridge. + +/// A fixture column name, optional type, and optional nullability. +pub type FixtureFormatColumn = (String, Option, Option); diff --git a/python-boundary.toml b/python-boundary.toml index 017159143e..c45e0bc152 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -617,10 +617,6 @@ note = "A bound that is not integer text is parsed as a timestamp instead; Curso location = "compiler/planner/_helpers/warehouse/snapshot.py::_parse_warehouse_cursor" note = "A warehouse cursor value that is not an integer is parsed as a timestamp instead; CursorAlgebraError is an input error, and native compiler failures are not caught." -[[handled_native_errors]] -location = "compiler/planner/_helpers/sql_tests/fixture_formatting.py::format_redundant_fixture_nulls" -note = "A test file that no longer parses is left unformatted, as before; no Python implementation runs." - [[handled_native_errors]] location = "cli/commands/_helpers/refactor/compile_facts.py::compile_for_refactor" note = "Reports the Python stage errors the native compile driver propagates as refactor diagnostics; a native compiler failure is a RuntimeError and is not caught." diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 0afefd460d..3073708da5 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2359,6 +2359,13 @@ def scan_fixture_typed_null_candidates( project_dir: Path, selected_paths: frozenset[Path] | None = None ) -> tuple[frozenset[Path], frozenset[str]]: ... def simplify_fixture_null_projections(*, sql: str, relation: FixtureRelationMetadata) -> str: ... +def format_redundant_fixture_nulls( + *, + files: dict[Path, str], + project_dir: Path, + discovered_inputs: DiscoveredProjectInputs, + sql_lexical_syntax: SqlLexicalSyntax, +) -> dict[Path, str]: ... def format_lint_sql_bodies( files: list[tuple[Path, str]], project_dir: Path, config: LintConfig ) -> tuple[dict[Path, str], list[LintViolation]]: ... diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py index c026c7bd89..df9d703e7e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py @@ -1,225 +1,3 @@ """Contract-aware formatting of SQL test fixture projections.""" -from __future__ import annotations - -import re -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild._native import simplify_fixture_null_projections as _remove_redundant_typed_null_lines -from sqlbuild.compiler.compile.classes.sql_test_cte_extractor import SqlTestCteExtractor -from sqlbuild.compiler.compile.constants import ( - REF_TEST_CTE_PREFIX, - SEED_TEST_CTE_PREFIX, - SOURCE_TEST_CTE_PREFIX, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileSqlTestCte -from sqlbuild.compiler.compile.types import CompiledResourceType, SqlTestMode -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, - DiscoveredSqlModelFile, - DiscoveredSqlTestBlock, - DiscoveredSqlTestFile, -) -from sqlbuild.compiler.path_defaults.main._select import select_path_default -from sqlbuild.compiler.planner.models import ( - FixtureColumnMetadata, - FixtureRelationMetadata, -) -from sqlbuild.compiler.planner.types import ContractPolicy, FixtureKey -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import SchemaColumn, SchemaSeedEntry, SourceEntry - -_MACRO_INVOCATION_MARKER: str = "@" -_TYPED_NULL_CANDIDATE_PATTERN: re.Pattern[str] = re.compile( - r"(?:CAST\s*\(\s*NULL\s+AS\b|NULL\s*::)", re.IGNORECASE -) - - -def format_redundant_fixture_nulls( - *, - files: dict[Path, str], - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - sql_lexical_syntax: SqlLexicalSyntax, -) -> dict[Path, str]: - """Remove typed-null fixture projections that planner completion can reproduce.""" - - relations: dict[FixtureKey, FixtureRelationMetadata] = _fixture_relations( - discovered_inputs=discovered_inputs - ) - updated: dict[Path, str] = {} - test_file: DiscoveredSqlTestFile - for test_file in discovered_inputs.test_files: - file_path: Path = (project_dir / test_file.relative_path).resolve() - if file_path not in files: - continue - contents: str | None = updated.get(file_path, files.get(file_path)) - if contents is None: - continue - try: - current_blocks: tuple[DiscoveredSqlTestBlock, ...] = _native.parse_sql_test_file( - contents=contents, - file_path=file_path, - ) - except (OSError, UnicodeError, ValueError, SyntaxError): - continue - block: DiscoveredSqlTestBlock - for block in current_blocks: - fixed_block: str = _format_test_block( - block=block, - file_label=str(test_file.relative_path), - relations=relations, - syntax=sql_lexical_syntax, - ) - if fixed_block == block.sql_body: - continue - contents = _replace_test_block( - contents=contents, - original=block.sql_body, - replacement=fixed_block, - ) - updated[file_path] = contents - return updated - - -def _format_test_block( - *, - block: DiscoveredSqlTestBlock, - file_label: str, - relations: dict[FixtureKey, FixtureRelationMetadata], - syntax: SqlLexicalSyntax, -) -> str: - if ( - block.mode is not SqlTestMode.MODEL - or _MACRO_INVOCATION_MARKER in block.sql_body - or _TYPED_NULL_CANDIDATE_PATTERN.search(block.sql_body) is None - ): - return block.sql_body - try: - authored_ctes: tuple[CompileSqlTestCte, ...] = SqlTestCteExtractor.extract( - sql=block.sql_body, - file_label=file_label, - syntax=syntax, - ) - except CompileInputError: - return block.sql_body - changed_bodies: list[tuple[str, str]] = [] - for cte in authored_ctes: - key: FixtureKey | None = _fixture_key(cte.name) - if key is None: - continue - relation: FixtureRelationMetadata | None = relations.get(key) - if relation is None or not relation.authoritative_names: - continue - candidate: str = _remove_redundant_typed_null_lines( - sql=cte.sql_body, - relation=relation, - ) - if candidate == cte.sql_body: - continue - changed_bodies.append((cte.sql_body, candidate)) - if not changed_bodies: - return block.sql_body - return _replace_cte_bodies( - sql=block.sql_body, - replacements=tuple(changed_bodies), - ) - - -def _fixture_relations( - *, discovered_inputs: DiscoveredProjectInputs -) -> dict[FixtureKey, FixtureRelationMetadata]: - relations: dict[FixtureKey, FixtureRelationMetadata] = {} - default_contract: str | None = discovered_inputs.project_config.defaults.contract - path_defaults: dict[str, dict[str, object]] = discovered_inputs.project_config.path_defaults - model_file: DiscoveredSqlModelFile - for model_file in discovered_inputs.model_files: - selected_default: str | None = select_path_default( - model_path=str(model_file.relative_path), - path_keys=tuple(path_defaults), - ).selected_key - contract: object = default_contract - if selected_default is not None: - contract = path_defaults[selected_default].get("contract", contract) - contract = model_file.header_values.get("contract", contract) - if contract != ContractPolicy.ENFORCED.value: - continue - columns: tuple[SchemaColumn, ...] = _native.parse_schema_columns( - model_file.header_values.get("columns"), - (str(model_file.file_path), "model"), - (False, True, False), - model_file.header_column_locations, - ) - if not columns: - continue - relations[(CompiledResourceType.MODEL, model_file.file_path.stem)] = _relation_metadata( - columns=columns - ) - for source_file in discovered_inputs.source_files: - source: SourceEntry - for source in source_file.source_entries: - source_contract: str | None = source.contract or default_contract - if source_contract != ContractPolicy.ENFORCED.value: - continue - relations[(CompiledResourceType.SOURCE, source.name)] = FixtureRelationMetadata( - columns=tuple( - FixtureColumnMetadata( - name=column.name, - type=column.type, - nullable=column.nullable, - ) - for column in source.columns - ), - authoritative_names=bool(source.columns), - ) - for schema_file in discovered_inputs.schema_files: - seed: SchemaSeedEntry - for seed in schema_file.seed_entries: - if seed.columns: - relations[(CompiledResourceType.SEED, seed.name)] = _relation_metadata( - columns=seed.columns - ) - return relations - - -def _relation_metadata(*, columns: tuple[SchemaColumn, ...]) -> FixtureRelationMetadata: - return FixtureRelationMetadata( - columns=tuple( - FixtureColumnMetadata( - name=column.name, - type=column.type, - nullable=column.nullable, - ) - for column in columns - ), - authoritative_names=True, - ) - - -def _fixture_key(name: str) -> tuple[CompiledResourceType, str] | None: - for prefix, resource_type in ( - (REF_TEST_CTE_PREFIX, CompiledResourceType.MODEL), - (SOURCE_TEST_CTE_PREFIX, CompiledResourceType.SOURCE), - (SEED_TEST_CTE_PREFIX, CompiledResourceType.SEED), - ): - if name.startswith(prefix): - return resource_type, name.removeprefix(prefix) - return None - - -def _replace_cte_bodies(*, sql: str, replacements: tuple[tuple[str, str], ...]) -> str: - result: str = sql - for original, replacement in replacements: - if result.count(original) != 1: - continue - result = result.replace(original, replacement, 1) - return result - - -def _replace_test_block(*, contents: str, original: str, replacement: str) -> str: - start: int = contents.find(original) - if start < 0: - return contents - return f"{contents[:start]}{replacement}{contents[start + len(original) :]}" +from sqlbuild._native import format_redundant_fixture_nulls as format_redundant_fixture_nulls diff --git a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py index 8dafbf4b74..c20bee4cfb 100644 --- a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py +++ b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py @@ -34,23 +34,22 @@ class _CustomSyntaxDuckDbAdapter(DuckDbAdapter): def test_given_passed_syntax_when_formatting_fixture_nulls_then_scanner_uses_it( test_case: PassedAdapterSyntaxTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: test_file: Path = write_fixture_format_project( - tmp_path=tmp_path, fixture_sql="SELECT 1 AS id, CAST(NULL AS BOOLEAN) AS flag" + tmp_path=tmp_path, + fixture_sql="SELECT\n 1 AS id,\n CAST(NULL AS BOOLEAN) AS flag\nWHERE TRUE\n # ) custom comment\n", ) inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - extract: Mock = Mock(wraps=fixture_formatting_module.SqlTestCteExtractor.extract) - monkeypatch.setattr(fixture_formatting_module.SqlTestCteExtractor, "extract", extract) - _ = fixture_formatting_module.format_redundant_fixture_nulls( + updated: dict[Path, str] = fixture_formatting_module.format_redundant_fixture_nulls( files={test_file.resolve(): test_file.read_text(encoding="utf-8")}, project_dir=tmp_path, discovered_inputs=inputs, sql_lexical_syntax=test_case.expected_syntax, ) - assert extract.call_args.kwargs["syntax"] is test_case.expected_syntax + assert "CAST(NULL AS BOOLEAN)" not in updated[test_file.resolve()] + assert "# ) custom comment" in updated[test_file.resolve()] @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/lint/test_cli_format.py b/tests/unit/src/sqlbuild/lint/test_cli_format.py index 0d452ce596..3ef0cd87cf 100644 --- a/tests/unit/src/sqlbuild/lint/test_cli_format.py +++ b/tests/unit/src/sqlbuild/lint/test_cli_format.py @@ -9,7 +9,6 @@ from _pytest.capture import CaptureResult import sqlbuild._native as native_module -import sqlbuild.compiler.planner._helpers.sql_tests.fixture_formatting as fixture_formatting_module from sqlbuild._native import run_cli as main from tests.unit.src.sqlbuild.lint._test_types import ( FormatCliTestCase, @@ -549,11 +548,9 @@ def test_given_test_path_selector_when_formatting_then_work_is_bounded_to_select unselected_file: Path = tmp_path / "tests" / "unit" / "customers" / "test_other.sql" unselected_file.parent.mkdir() unselected_file.write_text(fixture_sql, encoding="utf-8") - fixture_parse: Mock = Mock(wraps=fixture_formatting_module.SqlTestCteExtractor.extract) native_format: Mock = Mock(wraps=native_module.format_lint_sql_bodies) full_discovery: Mock = Mock(side_effect=AssertionError("full discovery must not run")) project_graph: Mock = Mock(side_effect=AssertionError("project graph must not build")) - monkeypatch.setattr(fixture_formatting_module.SqlTestCteExtractor, "extract", fixture_parse) monkeypatch.setattr(native_module, "format_lint_sql_bodies", native_format) monkeypatch.setattr(native_module, "discover_project_inputs", full_discovery) monkeypatch.setattr(native_module, "build_project_graph", project_graph) @@ -571,7 +568,6 @@ def test_given_test_path_selector_when_formatting_then_work_is_bounded_to_select captured: CaptureResult[str] = capsys.readouterr() assert exit_code == 0 assert f"{test_case.selected_file_count} files checked" in captured.err - assert fixture_parse.call_count == test_case.expected_fixture_parse_count assert native_format.call_count == test_case.expected_native_batch_count assert len(native_format.call_args.args[0]) == test_case.expected_native_file_count assert full_discovery.call_count == 0 From 2885af6db144db5f9079ddafb741bd9732bb6ca6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 14:54:57 +0100 Subject: [PATCH 10/29] chore: update Python boundary after fixture formatting port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index c45e0bc152..e32b053154 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 37781 +remaining_lines = 37559 [[stays]] path = "executor" From a990e98bfd74c539af326d0cfd064191ef9b83aa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:02:55 +0100 Subject: [PATCH 11/29] refactor: host column migration planning in Rust --- .../main/column_decision.rs | 19 + .../planner_migrations/main/column_warning.rs | 52 ++ .../src/planner_migrations/main/mod.rs | 2 + .../src/planner_migrations/models.rs | 12 + .../src/planner_migrations/tests.rs | 2 + .../tests/test_column_decision.rs | 67 +++ .../planner_migrations/tests/test_types.rs | 8 + .../planner_migrations/column_candidates.rs | 113 ++++ .../planner_migrations/column_entries.rs | 245 ++++++++ .../planner_migrations/column_objects.rs | 67 +++ .../planner_migrations/column_overlay.rs | 182 ++++++ .../planner_migrations/column_planning.rs | 131 ++++ .../planner_migrations/column_warnings.rs | 113 ++++ .../_helpers/planner_migrations/constants.rs | 5 + .../_helpers/planner_migrations/mod.rs | 8 + .../_helpers/planner_migrations/models.rs | 17 + .../planner_migrations/registration.rs | 1 + src/sqlbuild/_native.pyi | 13 + .../planner/_helpers/migrations/columns.py | 564 ------------------ .../_helpers/planning/warehouse_state.py | 2 +- .../test_column_migration_candidates.py | 2 +- 21 files changed, 1059 insertions(+), 566 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/main/column_decision.rs create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/main/column_warning.rs create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/tests/test_column_decision.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/migrations/columns.py diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/column_decision.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/column_decision.rs new file mode 100644 index 0000000000..2b9a0cf71a --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/column_decision.rs @@ -0,0 +1,19 @@ +//! Live column evidence takes precedence over recorded rename history. + +/// Decide a column rename from recorded history and live output evidence. +pub fn column_decision(recorded: bool, live: (bool, bool), still_produced: bool) -> &'static str { + let (origin, destination) = live; + if recorded && (destination || !origin) { + "done" + } else if still_produced { + "still_produced" + } else if origin && !destination { + "rename" + } else if !origin && destination { + "record" + } else if origin { + "conflict" + } else { + "source_missing" + } +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/column_warning.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/column_warning.rs new file mode 100644 index 0000000000..8016247290 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/column_warning.rs @@ -0,0 +1,52 @@ +//! Column migration diagnostic codes and explanations. + +use crate::planner_migrations::models::ColumnWarning; + +/// Format a column migration diagnostic from its decision and display facts. +pub fn column_warning(decision: &str, facts: &ColumnWarning<'_>) -> Option<(&'static str, String)> { + let ColumnWarning { + model_name: name, + relation, + origin, + destination: column, + completed, + target, + missing_outcome: outcome, + message, + } = facts; + let declaration = format!("column '{column}' migrate_from {origin}"); + Some(match decision { + "done" => ( + "M108", + format!( + "column rename {origin} -> {column} on {relation} completed at {completed} on target '{target}'; migrate_from can be removed from column '{column}' of '{name}'" + ), + ), + "source_missing" => ( + "M109", + format!( + "model '{name}': {declaration}: column {origin} does not exist in {relation} and no recorded rename into {column} was found; {outcome}" + ), + ), + "conflict" => ( + "M110", + format!( + "model '{name}': {declaration}: {relation} already has both {origin} and {column}, so the column cannot be renamed in place; drop one of them, or remove migrate_from and let on_schema_change handle the columns" + ), + ), + "still_produced" => ( + "M111", + format!( + "model '{name}': {declaration}: the model still produces {origin}; remove it from the query, or remove migrate_from" + ), + ), + "unsupported" => ("M112", format!("model '{name}': {declaration}: {message}")), + "automatic_unavailable" => ( + "M112", + format!( + "column {column} looks renamed from {origin}, but it cannot be renamed in place: {message}; the change follows on_schema_change instead" + ), + ), + _ => return None, + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs index 5bc0d79def..10663c6718 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs @@ -1,3 +1,5 @@ +pub mod column_decision; +pub mod column_warning; pub mod compatibility_findings; pub mod declared_column_aliases; pub mod identical_renames; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/models.rs b/crates/sqlbuild-analysis/src/planner_migrations/models.rs index ad4e625547..3badb760fb 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/models.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/models.rs @@ -70,3 +70,15 @@ pub struct QueryShape { pub local_input_columns: BTreeSet, pub clause_references: BTreeSet, } +/// Display facts for a column migration warning. +#[derive(Debug)] +pub struct ColumnWarning<'a> { + pub model_name: &'a str, + pub relation: &'a str, + pub origin: &'a str, + pub destination: &'a str, + pub completed: &'a str, + pub target: &'a str, + pub missing_outcome: &'a str, + pub message: &'a str, +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/tests.rs b/crates/sqlbuild-analysis/src/planner_migrations/tests.rs index 2f5cf327e6..9a86b7352e 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/tests.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_column_decision.rs"] +mod test_column_decision; #[path = "tests/test_column_renames.rs"] mod test_column_renames; #[path = "tests/test_decisions.rs"] diff --git a/crates/sqlbuild-analysis/src/planner_migrations/tests/test_column_decision.rs b/crates/sqlbuild-analysis/src/planner_migrations/tests/test_column_decision.rs new file mode 100644 index 0000000000..55239e7571 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/tests/test_column_decision.rs @@ -0,0 +1,67 @@ +//! Decision precedence preserves live evidence across repeated and reversed renames. + +use crate::planner_migrations::main::column_decision::column_decision; +use crate::planner_migrations::tests::test_types::ColumnDecisionTestCase; + +#[test] +fn given_live_and_recorded_evidence_when_deciding_then_preserves_precedence() { + let test_cases = [ + ColumnDecisionTestCase { + description: "recorded destination wins over output", + recorded: true, + live: (true, true), + still_produced: true, + expected_decision: "done", + }, + ColumnDecisionTestCase { + description: "recorded missing origin is done", + recorded: true, + live: (false, false), + still_produced: false, + expected_decision: "done", + }, + ColumnDecisionTestCase { + description: "recreated origin must rename again", + recorded: true, + live: (true, false), + still_produced: false, + expected_decision: "rename", + }, + ColumnDecisionTestCase { + description: "still produced blocks rename", + recorded: false, + live: (true, false), + still_produced: true, + expected_decision: "still_produced", + }, + ColumnDecisionTestCase { + description: "missing original records existing destination", + recorded: false, + live: (false, true), + still_produced: false, + expected_decision: "record", + }, + ColumnDecisionTestCase { + description: "two live columns conflict", + recorded: false, + live: (true, true), + still_produced: false, + expected_decision: "conflict", + }, + ColumnDecisionTestCase { + description: "neither column is source missing", + recorded: false, + live: (false, false), + still_produced: false, + expected_decision: "source_missing", + }, + ]; + for test_case in test_cases { + assert_eq!( + column_decision(test_case.recorded, test_case.live, test_case.still_produced), + test_case.expected_decision, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_migrations/tests/test_types.rs index 147726e883..89e925ace4 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/tests/test_types.rs @@ -23,3 +23,11 @@ pub(super) struct MigrationMatchTestCase { pub expected_matches: &'static [(usize, usize)], pub expected_ambiguous: usize, } +#[derive(Debug)] +pub(super) struct ColumnDecisionTestCase { + pub description: &'static str, + pub recorded: bool, + pub live: (bool, bool), + pub still_produced: bool, + pub expected_decision: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs new file mode 100644 index 0000000000..27a15d694e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs @@ -0,0 +1,113 @@ +//! Gather live candidate evidence without changing adapter inspection ownership. + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_migrations::column_objects::{ + config, definition_hash, lower, native, +}; +use crate::bindings::_helpers::planner_migrations::models::ColumnCandidate; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; + +pub(super) fn candidates<'py>( + args: &KeywordArguments<'py>, + dialect: &Bound<'py, PyAny>, +) -> PyResult>> { + let scope = args.value("scope")?; + let snapshot = args.value("snapshot")?; + let py = scope.py(); + let mut result: Vec> = Vec::new(); + for key in scope.getattr("execution_order")?.try_iter()? { + let key = key?; + if !scope.getattr("selected_keys")?.contains(&key)? + || !key.getattr("resource_type")?.eq("model")? + { + continue; + } + let model = scope + .getattr("models_by_name")? + .call_method1("get", (key.getattr("name")?,))?; + if model.is_none() { + continue; + } + let materialized = config(&model, "materialized")?; + if !materialized.eq("incremental")? && !materialized.eq("snapshot")? { + continue; + } + let name = model.getattr("name")?; + if args.value("full_refresh_model_names")?.contains(&name)? + || !snapshot.getattr("existing_relations")?.contains(&name)? + { + continue; + } + let warehouse = snapshot + .getattr("existing_columns")? + .call_method1("get", (&name, PyTuple::empty(py)))?; + let declared = PyDict::new(py); + let schema = model.getattr("schema_entry")?; + if !schema.is_none() { + for column in schema.getattr("columns")?.try_iter()? { + let column = column?; + let origin = column.getattr("migrate_from")?; + if !origin.is_none() { + declared.set_item(column.getattr("name")?, origin.call_method0("strip")?)?; + } + } + } + let fingerprint = snapshot + .getattr("fingerprints")? + .getattr("models")? + .call_method1("get", (&name,))?; + let changed = !fingerprint.is_none() + && !fingerprint + .getattr("definition_hash")? + .eq(definition_hash(&model, &snapshot)?)?; + if !warehouse.is_truthy()? || (declared.is_empty() && !changed) { + continue; + } + let by_name = PyDict::new(py); + for column in warehouse.try_iter()? { + let column = column?; + by_name.set_item(lower(&column.getattr("name")?)?, column)?; + } + let previous = if changed { + native( + py, + "parse_query_shape", + &[ + ("query_sql", fingerprint.getattr("definition")?), + ("dialect", dialect.clone()), + ], + )? + } else { + py.None().into_bound(py) + }; + let current = native( + py, + "parse_query_shape", + &[ + ("query_sql", model.getattr("query_sql")?), + ("dialect", dialect.clone()), + ], + )?; + result.push(ColumnCandidate { + model, + warehouse: by_name, + declared, + fingerprint, + previous, + current, + }); + } + Ok(result) +} + +pub(super) fn declared_names<'py>( + candidate: &ColumnCandidate<'py>, +) -> PyResult> { + let mut names: Vec = Vec::new(); + for (destination, origin) in candidate.declared.iter() { + names.push(lower(&destination)?); + names.push(lower(&origin)?); + } + PyFrozenSet::new(candidate.model.py(), names) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs new file mode 100644 index 0000000000..64584d334c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs @@ -0,0 +1,245 @@ +//! Materialize column decisions from native proofs and live evidence. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_migrations::column_candidates::declared_names; +use crate::bindings::_helpers::planner_migrations::column_objects::{ + lower, model, native, replace, +}; +use crate::bindings::_helpers::planner_migrations::constants::{AUTOMATIC, DONE, MIGRATION_TYPES}; +use crate::bindings::_helpers::planner_migrations::models::{CheckedColumns, ColumnCandidate}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; +use sqlbuild_analysis::planner_migrations::main::column_decision::column_decision; + +pub(super) fn decide<'py>( + runtime: &Bound<'py, PyAny>, + candidate: &ColumnCandidate<'py>, + events: &Bound<'py, PyAny>, + inputs: &Bound<'py, PyAny>, +) -> PyResult>> { + let py = runtime.py(); + let mut pairs: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>, &str)> = Vec::new(); + let declared = PyDict::new(py); + for (destination, origin) in candidate.declared.iter() { + declared.set_item(&origin, &destination)?; + pairs.push((origin, destination, "manual")); + } + if !candidate.previous.is_none() && !candidate.current.is_none() { + let detected = native( + py, + "identical_renames", + &[ + ("previous", candidate.previous.clone()), + ("current", candidate.current.clone()), + ("excluded", declared_names(candidate)?.into_any()), + ("declared", declared.into_any()), + ("input_columns", inputs.clone()), + ], + )?; + for pair in detected.try_iter()? { + let pair = pair?; + pairs.push((pair.get_item(0)?, pair.get_item(1)?, "automatic")); + } + } + let mut outputs: Vec = Vec::new(); + if !candidate.current.is_none() { + for projection in candidate.current.getattr("projections")?.try_iter()? { + outputs.push(projection?.getattr("key")?.extract::()?); + } + } else { + let inferred = candidate.model.getattr("inferred_columns")?; + if !inferred.is_none() { + for column in inferred.try_iter()? { + outputs.push(lower(&column?.getattr("name")?)?); + } + } + } + let relation = imported( + py, + "sqlbuild.compiler.migrations.main.relation_for_location", + "migration_relation_for_location", + )? + .call1((candidate.model.getattr("destination")?,))?; + let newest_event = imported( + py, + "sqlbuild.compiler.migrations.main._newest_column_event", + "newest_column_migration_event_mentioning", + )?; + let mut entries: Vec> = Vec::new(); + for (origin_name, destination, discovery) in pairs { + let origin_key = lower(&origin_name)?; + let origin = candidate.warehouse.get_item(&origin_key)?; + let has_destination = candidate.warehouse.contains(lower(&destination)?)?; + let newest = call_with( + &newest_event, + &[ + ("events", events.clone()), + ("relation", relation.clone()), + ("origin_column", origin_name.clone()), + ("destination_column", destination.clone()), + ], + )?; + let recorded = !newest.is_none() + && call_with( + &newest.getattr("renamed")?, + &[ + ("origin_column", origin_name.clone()), + ("destination_column", destination.clone()), + ], + )? + .is_truthy()?; + let decision_name = column_decision( + recorded, + (origin.is_some(), has_destination), + outputs.contains(&origin_key), + ); + let decision = + imported(py, MIGRATION_TYPES, "ColumnMigrationDecision")?.call1((decision_name,))?; + if discovery == AUTOMATIC && decision.getattr("blocks_build")?.is_truthy()? { + continue; + } + let done = decision_name == DONE && !newest.is_none(); + entries.push(model( + py, + "ColumnMigrationPlanEntry", + &[ + ("model_name", candidate.model.getattr("name")?), + ("destination", candidate.model.getattr("destination")?), + ( + "origin_column", + if let Some(origin) = origin { + origin.getattr("name")? + } else { + origin_name + }, + ), + ("destination_column", destination), + ( + "discovery", + imported(py, MIGRATION_TYPES, "MigrationDiscovery")?.call1((discovery,))?, + ), + ("decision", decision), + ( + "target_name", + if done { + newest.getattr("target_name")? + } else { + runtime + .getattr("project")? + .getattr("effective_target_name")? + }, + ), + ( + "completed_at", + if done { + newest.getattr("created_at")? + } else { + py.None().into_bound(py) + }, + ), + ], + )?); + } + Ok(entries) +} + +pub(super) fn checked<'py>( + runtime: &Bound<'py, PyAny>, + candidate: &ColumnCandidate<'py>, + entries: Vec>, + physical: &Bound<'py, PyAny>, +) -> PyResult> { + let py = runtime.py(); + let mut pending = false; + for entry in &entries { + pending |= entry.getattr("decision")?.getattr("renames")?.is_truthy()?; + } + if !pending { + return Ok(CheckedColumns { + entries, + warnings: Vec::new(), + }); + } + let destination = candidate.model.getattr("destination")?; + let qualified = destination.getattr("qualified_name")?; + let relation = if physical.is_truthy()? { + physical.clone() + } else if qualified.is_truthy()? { + qualified + } else { + destination.getattr("name")? + }; + let reason = call_with( + &runtime + .getattr("adapter")? + .getattr("column_rename_unavailable_reason")?, + &[ + ("connection", runtime.getattr("connection")?), + ("destination", relation), + ], + )?; + if reason.is_none() { + return Ok(CheckedColumns { + entries, + warnings: Vec::new(), + }); + } + let mut checked: Vec> = Vec::new(); + let mut warnings: Vec> = Vec::new(); + for entry in entries { + if !entry.getattr("decision")?.getattr("renames")?.is_truthy()? { + checked.push(entry); + } else if entry.getattr("discovery")?.eq("manual")? { + checked.push(replace( + &entry, + &[ + ( + "decision", + imported(py, MIGRATION_TYPES, "ColumnMigrationDecision")? + .getattr("UNSUPPORTED")?, + ), + ("message", reason.clone()), + ], + )?); + } else { + warnings.push(super::column_warnings::unavailable( + &entry, + &reason.extract::()?, + )?); + } + } + Ok(CheckedColumns { + entries: checked, + warnings, + }) +} + +pub(super) fn hints<'py>( + candidate: &ColumnCandidate<'py>, + entries: &[Bound<'py, PyAny>], +) -> PyResult> { + let py = candidate.model.py(); + if candidate.previous.is_none() || candidate.current.is_none() { + return Ok(PyTuple::empty(py).into_any()); + } + let excluded = declared_names(candidate)?.call_method0("copy")?; + let claimed = PyDict::new(py); + for entry in entries { + claimed.set_item(lower(&entry.getattr("origin_column")?)?, py.None())?; + claimed.set_item(lower(&entry.getattr("destination_column")?)?, py.None())?; + } + native( + py, + "rename_hints", + &[ + ("model_name", candidate.model.getattr("name")?), + ("previous", candidate.previous.clone()), + ("current", candidate.current.clone()), + ("excluded", excluded.call_method1("union", (claimed,))?), + ( + "live_columns", + PyFrozenSet::new(py, candidate.warehouse.keys().iter())?.into_any(), + ), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs new file mode 100644 index 0000000000..b70bde8ce7 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs @@ -0,0 +1,67 @@ +//! Boundary conversions for the column migration planner. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_migrations::constants::PLANNER_MODELS; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; + +pub(super) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +pub(super) fn lower(value: &Bound<'_, PyAny>) -> PyResult { + value.call_method0("lower")?.extract() +} + +pub(super) fn replace<'py>( + value: &Bound<'py, PyAny>, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let kwargs = PyDict::new(value.py()); + for (name, field) in fields { + kwargs.set_item(name, field)?; + } + imported(value.py(), "dataclasses", "replace")?.call((value,), Some(&kwargs)) +} + +pub(super) fn model<'py>( + py: Python<'py>, + name: &str, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with(&imported(py, PLANNER_MODELS, name)?, fields) +} + +pub(super) fn native<'py>( + py: Python<'py>, + name: &str, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with(&imported(py, "sqlbuild._native", name)?, fields) +} + +pub(super) fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { + native( + model.py(), + "get_config_str", + &[ + ("values", model.getattr("config")?.getattr("values")?), + ("key", text(model.py(), key)), + ], + ) +} + +pub(super) fn definition_hash<'py>( + model: &Bound<'py, PyAny>, + snapshot: &Bound<'py, PyAny>, +) -> PyResult> { + native( + model.py(), + "model_definition_hash", + &[ + ("model_name", model.getattr("name")?), + ("query_sql", model.getattr("query_sql")?), + ("dialect", snapshot.getattr("column_dialect")?), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs new file mode 100644 index 0000000000..9ba85bcbb2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs @@ -0,0 +1,182 @@ +//! Project column renames onto the existing snapshot and redirect cursor reads. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_migrations::column_objects::{ + config, definition_hash, lower, model, native, replace, +}; +use crate::bindings::_helpers::planner_migrations::constants::PLANNING; +use crate::bindings::_helpers::planner_migrations::models::ColumnCandidate; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +pub(super) fn overlay<'py>( + snapshot: &Bound<'py, PyAny>, + candidates: &[ColumnCandidate<'py>], + entries: &[Bound<'py, PyAny>], + inputs: &Bound<'py, PyAny>, +) -> PyResult> { + if entries.is_empty() { + return Ok(snapshot.clone()); + } + let py = snapshot.py(); + let columns = + imported(py, "builtins", "dict")?.call1((snapshot.getattr("existing_columns")?,))?; + let stored = snapshot.getattr("fingerprints")?; + let fingerprints = imported(py, "builtins", "dict")?.call1((stored.getattr("models")?,))?; + for candidate in candidates { + let name = candidate.model.getattr("name")?; + let renames = PyDict::new(py); + let pending = PyDict::new(py); + let mut blocked = false; + for entry in entries { + if !entry.getattr("model_name")?.eq(&name)? { + continue; + } + blocked |= entry.getattr("blocks_build")?.is_truthy()? + || entry.getattr("origin_missing")?.is_truthy()?; + renames.set_item( + entry.getattr("origin_column")?, + entry.getattr("destination_column")?, + )?; + if entry.getattr("decision")?.getattr("renames")?.is_truthy()? { + pending.set_item( + lower(&entry.getattr("origin_column")?)?, + entry.getattr("destination_column")?, + )?; + } + } + if renames.is_empty() || blocked { + continue; + } + let mut renamed: Vec> = Vec::new(); + for column in snapshot + .getattr("existing_columns")? + .call_method1("get", (&name, PyTuple::empty(py)))? + .try_iter()? + { + let column = column?; + let old = column.getattr("name")?; + let new = pending.get_item(lower(&old)?)?.unwrap_or(old); + renamed.push(replace(&column, &[("name", new)])?); + } + columns.set_item(&name, PyTuple::new(py, renamed)?)?; + if !candidate.fingerprint.is_none() + && !candidate.previous.is_none() + && !candidate.current.is_none() + && native( + py, + "renames_explain_change", + &[ + ("previous", candidate.previous.clone()), + ("current", candidate.current.clone()), + ("renames", renames.into_any()), + ("input_columns", inputs.clone()), + ], + )? + .is_truthy()? + { + fingerprints.set_item( + &name, + replace( + &candidate.fingerprint, + &[ + ("definition", candidate.model.getattr("query_sql")?), + ( + "definition_hash", + definition_hash(&candidate.model, snapshot)?, + ), + ], + )?, + )?; + } + } + replace( + snapshot, + &[ + ("existing_columns", columns), + ( + "fingerprints", + model( + py, + "WarehouseFingerprints", + &[ + ("models", fingerprints), + ("functions", stored.getattr("functions")?), + ("seeds", stored.getattr("seeds")?), + ("python_nodes", stored.getattr("python_nodes")?), + ], + )?, + ), + ], + ) +} + +pub(super) fn cursors<'py>( + args: &KeywordArguments<'py>, + snapshot: Bound<'py, PyAny>, + entries: &[Bound<'py, PyAny>], +) -> PyResult> { + let py = snapshot.py(); + let scope = args.value("scope")?; + let models = scope.getattr("models_by_name")?; + let origins = PyDict::new(py); + for entry in entries { + let name = entry.getattr("model_name")?; + let model = models.call_method1("get", (&name,))?; + let cursor = if model.is_truthy()? { + config(&model, "cursor")? + } else { + py.None().into_bound(py) + }; + if entry.getattr("decision")?.getattr("renames")?.is_truthy()? + && !cursor.is_none() + && lower(&cursor)? == lower(&entry.getattr("destination_column")?)? + { + origins.set_item(name, entry.getattr("origin_column")?)?; + } + } + if origins.is_empty() { + return Ok(snapshot); + } + let targets = PyDict::new(py); + for (name, _) in origins.iter() { + let physical = args + .value("physical_relations")? + .call_method1("get", (&name,))?; + let target = if physical.is_truthy()? { + physical + } else { + let qualified = models + .get_item(&name)? + .getattr("destination")? + .getattr("qualified_name")?; + if qualified.is_truthy()? { + qualified + } else { + name.clone() + } + }; + targets.set_item(name, target)?; + } + let redirected = call_with( + &imported(py, PLANNING, "redirected_cursor_snapshots")?, + &[ + ("runtime", args.value("runtime")?), + ("scope", scope), + ( + "existing_relations", + snapshot.getattr("existing_relations")?, + ), + ("target_relations", targets.into_any()), + ("overrides", args.value("overrides")?), + ("deferral", args.value("deferral")?), + ("origin_cursor_columns", origins.into_any()), + ], + )?; + let cursors = + imported(py, "builtins", "dict")?.call1((snapshot.getattr("cursor_snapshots")?,))?; + cursors.call_method1("update", (redirected,))?; + replace(&snapshot, &[("cursor_snapshots", cursors)]) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs new file mode 100644 index 0000000000..9fcb001fd2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -0,0 +1,131 @@ +//! Host column migration planning while Python owns warehouse inspection. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_migrations::column_objects::{model, replace}; +use crate::bindings::_helpers::planner_migrations::constants::PLANNING; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PySet, PySetMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +/// Decide column renames and return the existing planning result model. +#[pyfunction] +#[pyo3(signature = (**options))] +fn plan_column_migrations<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "plan_column_migrations", + options, + ( + &[ + "runtime", + "scope", + "snapshot", + "full_refresh_model_names", + "physical_relations", + "overrides", + "deferral", + "source_columns", + ], + &[], + ), + )?; + let runtime = args.value("runtime")?; + let snapshot = args.value("snapshot")?; + let adapter = runtime.getattr("adapter")?; + let candidates = super::column_candidates::candidates( + &args, + &adapter.call_method0("sql_analysis_dialect")?, + )?; + if candidates.is_empty() { + return model(py, "ColumnMigrationPlanning", &[("snapshot", snapshot)]); + } + let database = call_with( + &imported(py, PLANNING, "planning_database")?, + &[("runtime", runtime.clone())], + )?; + let state = call_with( + &imported( + py, + "sqlbuild.compiler.planner.classes.migration_state_inspection", + "MigrationStateInspection", + )?, + &[ + ("adapter", adapter), + ("connection", runtime.getattr("connection")?), + ("database", database), + ], + )?; + let schemas = PySet::empty(py)?; + for candidate in &candidates { + let schema = candidate.model.getattr("destination")?.getattr("schema")?; + if !schema.is_none() { + schemas.add(schema)?; + } + } + call_with( + &state.getattr("inspect_column_events")?, + &[("schemas", schemas.into_any())], + )?; + let inputs = call_with( + &imported( + py, + "sqlbuild.compiler.planner.classes.known_input_columns", + "KnownInputColumns", + )?, + &[ + ("model_columns", snapshot.getattr("existing_columns")?), + ("source_columns", args.value("source_columns")?), + ], + )?; + let mut entries: Vec> = Vec::new(); + let mut warnings: Vec> = Vec::new(); + let mut hints: Vec> = Vec::new(); + for candidate in &candidates { + let decided = super::column_entries::decide( + &runtime, + candidate, + &state.getattr("column_events")?, + &inputs, + )?; + let physical = args + .value("physical_relations")? + .call_method1("get", (candidate.model.getattr("name")?,))?; + let checked = super::column_entries::checked(&runtime, candidate, decided, &physical)?; + for hint in super::column_entries::hints(candidate, &checked.entries)?.try_iter()? { + hints.push(hint?); + } + entries.extend(checked.entries); + warnings.extend(checked.warnings); + } + let policy = call_with( + &imported(py, PLANNING, "missing_origin_policy")?, + &[("runtime", runtime)], + )?; + for entry in &mut entries { + *entry = replace(entry, &[("missing_origin_policy", policy.clone())])?; + if let Some(warning) = super::column_warnings::entry_warning(entry)? { + warnings.push(warning); + } + } + let overlaid = super::column_overlay::overlay(&snapshot, &candidates, &entries, &inputs)?; + let snapshot = super::column_overlay::cursors(&args, overlaid, &entries)?; + model( + py, + "ColumnMigrationPlanning", + &[ + ("snapshot", snapshot), + ("entries", PyTuple::new(py, entries)?.into_any()), + ("hints", PyTuple::new(py, hints)?.into_any()), + ("warnings", PyTuple::new(py, warnings)?.into_any()), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(plan_column_migrations, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs new file mode 100644 index 0000000000..9c8d7c5db8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs @@ -0,0 +1,113 @@ +//! Preserve column-migration diagnostic codes and public messages. + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::planner_migrations::column_objects::{model, text}; +use crate::bindings::_helpers::planner_migrations::constants::DONE; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::planner_migrations::main::column_warning::column_warning; +use sqlbuild_analysis::planner_migrations::models::ColumnWarning; + +pub(super) fn warning<'py>( + entry: &Bound<'py, PyAny>, + code: &str, + message: &str, + error: bool, +) -> PyResult> { + let py = entry.py(); + model( + py, + "PlanWarning", + &[ + ("model_name", entry.getattr("model_name")?), + ( + "severity", + imported(py, "sqlbuild.compiler.planner.types", "WarningSeverity")? + .getattr(if error { "ERROR" } else { "WARNING" })?, + ), + ("message", text(py, message)), + ("code", text(py, code)), + ], + ) +} + +pub(super) fn entry_warning<'py>(entry: &Bound<'py, PyAny>) -> PyResult>> { + let py = entry.py(); + let destination = entry.getattr("destination")?; + let qualified = destination.getattr("qualified_name")?; + let relation: String = if qualified.is_truthy()? { + qualified + } else { + destination.getattr("name")? + } + .extract()?; + let origin: String = entry.getattr("origin_column")?.extract()?; + let column: String = entry.getattr("destination_column")?.extract()?; + let name: String = entry.getattr("model_name")?.extract()?; + let decision: String = entry.getattr("decision")?.extract()?; + if decision == DONE && !entry.getattr("discovery")?.eq("manual")? { + return Ok(None); + } + let completed = entry.getattr("completed_at")?; + let completed: String = if completed.is_truthy()? { + completed.call_method0("isoformat")?.extract()? + } else { + "unknown time".into() + }; + let target = entry.getattr("target_name")?; + let target: String = if target.is_truthy()? { + target.extract()? + } else { + "default".into() + }; + let outcome: String = imported( + py, + "sqlbuild.compiler.planner.constants", + "MISSING_ORIGIN_OUTCOMES", + )? + .get_item(entry.getattr("missing_origin_policy")?)? + .extract()?; + let message: String = entry.getattr("message")?.str()?.extract()?; + let Some((code, message)) = column_warning( + &decision, + &ColumnWarning { + model_name: &name, + relation: &relation, + origin: &origin, + destination: &column, + completed: &completed, + target: &target, + missing_outcome: &outcome, + message: &message, + }, + ) else { + return Ok(None); + }; + Ok(Some(warning( + entry, + code, + &message, + decision != DONE && entry.getattr("blocks_build")?.is_truthy()?, + )?)) +} + +pub(super) fn unavailable<'py>( + entry: &Bound<'py, PyAny>, + reason: &str, +) -> PyResult> { + let origin: String = entry.getattr("origin_column")?.extract()?; + let destination: String = entry.getattr("destination_column")?.extract()?; + let facts = ColumnWarning { + model_name: "", + relation: "", + origin: &origin, + destination: &destination, + completed: "", + target: "", + missing_outcome: "", + message: reason, + }; + let (code, message) = column_warning("automatic_unavailable", &facts).ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err("missing column migration diagnostic") + })?; + warning(entry, code, &message, false) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs new file mode 100644 index 0000000000..e7ce1e1bdd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs @@ -0,0 +1,5 @@ +pub(super) const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +pub(super) const MIGRATION_TYPES: &str = "sqlbuild.compiler.migrations.types"; +pub(super) const PLANNING: &str = "sqlbuild.compiler.planner._helpers.migrations.planning"; +pub(super) const AUTOMATIC: &str = "automatic"; +pub(super) const DONE: &str = "done"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index cb1301eb3f..ab9df51a2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -1,8 +1,16 @@ +mod column_candidates; +mod column_entries; +mod column_objects; +mod column_overlay; +mod column_planning; +mod column_warnings; pub(crate) mod columns; mod compatibility; +mod constants; mod errors; mod fingerprint; mod matching; +mod models; mod old_names; mod projections; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs new file mode 100644 index 0000000000..a0bf4ad647 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs @@ -0,0 +1,17 @@ +use pyo3::prelude::{Bound, PyAny}; +use pyo3::types::PyDict; + +#[derive(Debug)] +pub(crate) struct ColumnCandidate<'py> { + pub model: Bound<'py, PyAny>, + pub warehouse: Bound<'py, PyDict>, + pub declared: Bound<'py, PyDict>, + pub fingerprint: Bound<'py, PyAny>, + pub previous: Bound<'py, PyAny>, + pub current: Bound<'py, PyAny>, +} +#[derive(Debug)] +pub(crate) struct CheckedColumns<'py> { + pub entries: Vec>, + pub warnings: Vec>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs index 464cc9cd0c..00b53a7bab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs @@ -7,6 +7,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::matching::register(module)?; super::old_names::register(module)?; super::columns::register(module)?; + super::column_planning::register(module)?; super::projections::register(module)?; super::fingerprint::register(module) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 3073708da5..cb1d787f8a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -204,6 +204,7 @@ from sqlbuild.compiler.planner.models import ( AuditPlanEntry, BackfillResult, ChangeDetectionResult, + ColumnMigrationPlanning, ColumnRenameHint, CursorBounds, CursorOverridePair, @@ -221,6 +222,7 @@ from sqlbuild.compiler.planner.models import ( ParsedSelector, PathSelector, PlannerChangeResults, + PlannerOverrides, PlannerResolvedActions, PlannerRuntime, PlannerScope, @@ -4446,6 +4448,17 @@ def migration_destination_fingerprints( ) -> dict[str, tuple[str, ...]]: ... def stored_migration_fingerprint(fingerprint: Fingerprint) -> str | None: ... def declared_column_aliases(model: CompiledModelView) -> tuple[tuple[str, str], ...]: ... +def plan_column_migrations( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + full_refresh_model_names: frozenset[str], + physical_relations: Mapping[str, str], + overrides: PlannerOverrides, + deferral: DeferralInputs, + source_columns: Mapping[str, tuple[ColumnInfo, ...]], +) -> ColumnMigrationPlanning: ... def old_name_schemas_to_read(*, snapshot: WarehouseSnapshot, needed: set[str]) -> set[str]: ... def old_name_claim_message(*, model_name: str, view: OldNameView, now: datetime) -> str: ... def old_name_claim_errors( diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py deleted file mode 100644 index 2c5d8723e4..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py +++ /dev/null @@ -1,564 +0,0 @@ -"""Per-run decisions for in-place column renames of incremental and snapshot models.""" - -from __future__ import annotations - -from collections.abc import Mapping -from dataclasses import dataclass, replace - -from sqlbuild._native import ( - get_config_str, - identical_renames, - model_definition_hash, - parse_query_shape, - rename_hints, - renames_explain_change, -) -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.migrations.main._newest_column_event import ( - newest_column_migration_event_mentioning, -) -from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent -from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDiscovery -from sqlbuild.compiler.planner._helpers.migrations.planning import ( - missing_origin_policy, - planning_database, - redirected_cursor_snapshots, -) -from sqlbuild.compiler.planner.classes.known_input_columns import KnownInputColumns -from sqlbuild.compiler.planner.classes.migration_state_inspection import ( - MigrationStateInspection, -) -from sqlbuild.compiler.planner.constants import MISSING_ORIGIN_OUTCOMES -from sqlbuild.compiler.planner.models import ( - ColumnMigrationPlanEntry, - ColumnMigrationPlanning, - ColumnRenameHint, - DeferralInputs, - PlannerOverrides, - PlannerRuntime, - PlannerScope, - PlanWarning, - QueryShape, - WarehouseFingerprints, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import InputColumns, MaterializationType, WarningSeverity -from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy - -_HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( - {MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT} -) - - -@dataclass(frozen=True) -class _ModelColumns: - """Inputs gathered for one model that may rename columns this run.""" - - model: CompiledModelView - warehouse: dict[str, ColumnInfo] - declared: dict[str, str] - fingerprint: Fingerprint | None - previous: QueryShape | None - current: QueryShape | None - - -@dataclass(frozen=True) -class _Pair: - """One old-to-new column rename and how it was requested.""" - - origin: str - destination: str - discovery: MigrationDiscovery - - -def plan_column_migrations( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - full_refresh_model_names: frozenset[str], - physical_relations: Mapping[str, str], - overrides: PlannerOverrides, - deferral: DeferralInputs, - source_columns: Mapping[str, tuple[ColumnInfo, ...]], -) -> ColumnMigrationPlanning: - """Decide declared and detected column renames and project the renamed snapshot.""" - - dialect: str | None = runtime.adapter.sql_analysis_dialect() - candidates: tuple[_ModelColumns, ...] = tuple( - candidate - for key in scope.execution_order - if ( - candidate := _candidate( - key=key, - scope=scope, - snapshot=snapshot, - full_refresh_model_names=full_refresh_model_names, - dialect=dialect, - ) - ) - is not None - ) - if not candidates: - return ColumnMigrationPlanning(snapshot=snapshot) - state: MigrationStateInspection = MigrationStateInspection( - adapter=runtime.adapter, - connection=runtime.connection, - database=planning_database(runtime=runtime), - ) - state.inspect_column_events( - schemas={ - candidate.model.destination.schema - for candidate in candidates - if candidate.model.destination.schema is not None - } - ) - input_columns: InputColumns = KnownInputColumns( - model_columns=snapshot.existing_columns, - source_columns=source_columns, - ) - entries: list[ColumnMigrationPlanEntry] = [] - hints: list[ColumnRenameHint] = [] - warnings: list[PlanWarning] = [] - candidate: _ModelColumns - for candidate in candidates: - model_entries: tuple[ColumnMigrationPlanEntry, ...] - model_warnings: tuple[PlanWarning, ...] - model_entries, model_warnings = _checked_entries( - runtime=runtime, - candidate=candidate, - entries=_decide( - runtime=runtime, - candidate=candidate, - events=state.column_events, - input_columns=input_columns, - ), - physical_relation=physical_relations.get(candidate.model.name), - ) - entries.extend(model_entries) - warnings.extend(model_warnings) - hints.extend(_hints(candidate=candidate, entries=model_entries)) - origin_policy: MissingMigrationOriginPolicy = missing_origin_policy(runtime=runtime) - entries = [replace(entry, missing_origin_policy=origin_policy) for entry in entries] - warnings.extend(warning for entry in entries if (warning := _entry_warning(entry))) - overlaid: WarehouseSnapshot = _overlay_snapshot( - snapshot=snapshot, - candidates=candidates, - entries=tuple(entries), - input_columns=input_columns, - ) - return ColumnMigrationPlanning( - snapshot=_with_renamed_cursors( - runtime=runtime, - scope=scope, - snapshot=overlaid, - entries=tuple(entries), - physical_relations=physical_relations, - overrides=overrides, - deferral=deferral, - ), - entries=tuple(entries), - hints=tuple(hints), - warnings=tuple(warnings), - ) - - -def _candidate( - *, - key: CompiledObjectKey, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - full_refresh_model_names: frozenset[str], - dialect: str | None, -) -> _ModelColumns | None: - if key not in scope.selected_keys or key.resource_type != CompiledResourceType.MODEL: - return None - model: CompiledModelView | None = scope.models_by_name.get(key.name) - if ( - model is None - or get_config_str(values=model.config.values, key="materialized") - not in _HISTORY_MATERIALIZATIONS - or model.name in full_refresh_model_names - or model.name not in snapshot.existing_relations - ): - return None - warehouse: tuple[ColumnInfo, ...] = snapshot.existing_columns.get(model.name, ()) - declared: dict[str, str] = { - column.name: column.migrate_from.strip() - for column in (model.schema_entry.columns if model.schema_entry is not None else ()) - if column.migrate_from is not None - } - fingerprint: Fingerprint | None = snapshot.fingerprints.models.get(model.name) - changed: bool = ( - fingerprint is not None - and fingerprint.definition_hash - != model_definition_hash( - model_name=model.name, query_sql=model.query_sql, dialect=snapshot.column_dialect - ) - ) - if not warehouse or (not declared and not changed): - return None - return _ModelColumns( - model=model, - warehouse={column.name.lower(): column for column in warehouse}, - declared=declared, - fingerprint=fingerprint, - previous=( - parse_query_shape(query_sql=fingerprint.definition, dialect=dialect) - if changed and fingerprint is not None - else None - ), - current=parse_query_shape(query_sql=model.query_sql, dialect=dialect), - ) - - -def _decide( - *, - runtime: PlannerRuntime, - candidate: _ModelColumns, - events: tuple[ColumnMigrationEvent, ...], - input_columns: InputColumns, -) -> tuple[ColumnMigrationPlanEntry, ...]: - entries: list[ColumnMigrationPlanEntry] = [] - pair: _Pair - for pair in ( - *_declared_pairs(candidate), - *_detected_pairs(candidate=candidate, input_columns=input_columns), - ): - entry: ColumnMigrationPlanEntry | None = _decide_pair( - runtime=runtime, candidate=candidate, pair=pair, events=events - ) - if entry is not None: - entries.append(entry) - return tuple(entries) - - -def _declared_pairs(candidate: _ModelColumns) -> tuple[_Pair, ...]: - return tuple( - _Pair(origin=origin, destination=destination, discovery=MigrationDiscovery.MANUAL) - for destination, origin in candidate.declared.items() - ) - - -def _detected_pairs(*, candidate: _ModelColumns, input_columns: InputColumns) -> tuple[_Pair, ...]: - if candidate.previous is None or candidate.current is None: - return () - declared: dict[str, str] = { - origin: destination for destination, origin in candidate.declared.items() - } - return tuple( - _Pair(origin=origin, destination=destination, discovery=MigrationDiscovery.AUTOMATIC) - for origin, destination in identical_renames( - previous=candidate.previous, - current=candidate.current, - excluded=_declared_names(candidate), - declared=declared, - input_columns=input_columns, - ) - ) - - -def _declared_names(candidate: _ModelColumns) -> frozenset[str]: - return frozenset( - {name.lower() for name in candidate.declared} - | {origin.lower() for origin in candidate.declared.values()} - ) - - -def _decide_pair( - *, - runtime: PlannerRuntime, - candidate: _ModelColumns, - pair: _Pair, - events: tuple[ColumnMigrationEvent, ...], -) -> ColumnMigrationPlanEntry | None: - """Decide one rename from live columns first, then from recorded events.""" - - origin: ColumnInfo | None = candidate.warehouse.get(pair.origin.lower()) - has_destination: bool = pair.destination.lower() in candidate.warehouse - newest: ColumnMigrationEvent | None = newest_column_migration_event_mentioning( - events=events, - relation=migration_relation_for_location(candidate.model.destination), - origin_column=pair.origin, - destination_column=pair.destination, - ) - recorded: bool = newest is not None and newest.renamed( - origin_column=pair.origin, destination_column=pair.destination - ) - decision: ColumnMigrationDecision - if recorded and (has_destination or origin is None): - decision = ColumnMigrationDecision.DONE - elif pair.origin.lower() in _output_names(candidate): - decision = ColumnMigrationDecision.STILL_PRODUCED - elif origin is not None and not has_destination: - decision = ColumnMigrationDecision.RENAME - elif origin is None and has_destination: - decision = ColumnMigrationDecision.RECORD - elif origin is not None: - decision = ColumnMigrationDecision.CONFLICT - else: - decision = ColumnMigrationDecision.SOURCE_MISSING - if pair.discovery == MigrationDiscovery.AUTOMATIC and decision.blocks_build: - return None - return ColumnMigrationPlanEntry( - model_name=candidate.model.name, - destination=candidate.model.destination, - origin_column=origin.name if origin is not None else pair.origin, - destination_column=pair.destination, - discovery=pair.discovery, - decision=decision, - target_name=( - newest.target_name - if decision == ColumnMigrationDecision.DONE and newest is not None - else runtime.project.effective_target_name - ), - completed_at=( - newest.created_at - if decision == ColumnMigrationDecision.DONE and newest is not None - else None - ), - ) - - -def _output_names(candidate: _ModelColumns) -> frozenset[str]: - if candidate.current is not None: - return frozenset(projection.key for projection in candidate.current.projections) - return frozenset(column.name.lower() for column in candidate.model.inferred_columns or ()) - - -def _checked_entries( - *, - runtime: PlannerRuntime, - candidate: _ModelColumns, - entries: tuple[ColumnMigrationPlanEntry, ...], - physical_relation: str | None, -) -> tuple[tuple[ColumnMigrationPlanEntry, ...], tuple[PlanWarning, ...]]: - """Ask the adapter once per model whether pending renames can run in place.""" - - if not any(entry.decision.renames for entry in entries): - return entries, () - relation: str = ( - physical_relation - or candidate.model.destination.qualified_name - or candidate.model.destination.name - ) - reason: str | None = runtime.adapter.column_rename_unavailable_reason( - connection=runtime.connection, destination=relation - ) - if reason is None: - return entries, () - checked: list[ColumnMigrationPlanEntry] = [] - warnings: list[PlanWarning] = [] - entry: ColumnMigrationPlanEntry - for entry in entries: - if not entry.decision.renames: - checked.append(entry) - elif entry.discovery == MigrationDiscovery.MANUAL: - checked.append( - replace(entry, decision=ColumnMigrationDecision.UNSUPPORTED, message=reason) - ) - else: - warnings.append( - PlanWarning( - model_name=entry.model_name, - severity=WarningSeverity.WARNING, - message=( - f"column {entry.destination_column} looks renamed from " - f"{entry.origin_column}, but it cannot be renamed in place: {reason}; " - "the change follows on_schema_change instead" - ), - code="M112", - ) - ) - return tuple(checked), tuple(warnings) - - -def _hints( - *, candidate: _ModelColumns, entries: tuple[ColumnMigrationPlanEntry, ...] -) -> tuple[ColumnRenameHint, ...]: - if candidate.previous is None or candidate.current is None: - return () - claimed: frozenset[str] = frozenset( - {entry.origin_column.lower() for entry in entries} - | {entry.destination_column.lower() for entry in entries} - ) - return rename_hints( - model_name=candidate.model.name, - previous=candidate.previous, - current=candidate.current, - excluded=_declared_names(candidate) | claimed, - live_columns=frozenset(candidate.warehouse), - ) - - -def _overlay_snapshot( - *, - snapshot: WarehouseSnapshot, - candidates: tuple[_ModelColumns, ...], - entries: tuple[ColumnMigrationPlanEntry, ...], - input_columns: InputColumns, -) -> WarehouseSnapshot: - """Plan each model as if its columns were already renamed and its query unchanged.""" - - if not entries: - return snapshot - blocked: frozenset[str] = frozenset( - entry.model_name for entry in entries if entry.blocks_build or entry.origin_missing - ) - columns: dict[str, tuple[ColumnInfo, ...]] = dict(snapshot.existing_columns) - fingerprints: dict[str, Fingerprint] = dict(snapshot.fingerprints.models) - candidate: _ModelColumns - for candidate in candidates: - renames: dict[str, str] = { - entry.origin_column: entry.destination_column - for entry in entries - if entry.model_name == candidate.model.name - } - if not renames or candidate.model.name in blocked: - continue - pending: dict[str, str] = { - entry.origin_column.lower(): entry.destination_column - for entry in entries - if entry.model_name == candidate.model.name and entry.decision.renames - } - columns[candidate.model.name] = tuple( - replace(column, name=pending.get(column.name.lower(), column.name)) - for column in snapshot.existing_columns.get(candidate.model.name, ()) - ) - if ( - candidate.fingerprint is not None - and candidate.previous is not None - and candidate.current is not None - and renames_explain_change( - previous=candidate.previous, - current=candidate.current, - renames=renames, - input_columns=input_columns, - ) - ): - fingerprints[candidate.model.name] = replace( - candidate.fingerprint, - definition=candidate.model.query_sql, - definition_hash=model_definition_hash( - model_name=candidate.model.name, - query_sql=candidate.model.query_sql, - dialect=snapshot.column_dialect, - ), - ) - return replace( - snapshot, - existing_columns=columns, - fingerprints=WarehouseFingerprints( - models=fingerprints, - functions=snapshot.fingerprints.functions, - seeds=snapshot.fingerprints.seeds, - python_nodes=snapshot.fingerprints.python_nodes, - ), - ) - - -def _with_renamed_cursors( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - entries: tuple[ColumnMigrationPlanEntry, ...], - physical_relations: Mapping[str, str], - overrides: PlannerOverrides, - deferral: DeferralInputs, -) -> WarehouseSnapshot: - """Read the target watermark from the old column when the cursor column is renamed.""" - - old_cursor_columns: dict[str, str] = {} - entry: ColumnMigrationPlanEntry - for entry in entries: - model: CompiledModelView | None = scope.models_by_name.get(entry.model_name) - cursor: str | None = ( - get_config_str(values=model.config.values, key="cursor") if model else None - ) - if ( - entry.decision.renames - and cursor is not None - and cursor.lower() == entry.destination_column.lower() - ): - old_cursor_columns[entry.model_name] = entry.origin_column - if not old_cursor_columns: - return snapshot - return replace( - snapshot, - cursor_snapshots={ - **snapshot.cursor_snapshots, - **redirected_cursor_snapshots( - runtime=runtime, - scope=scope, - existing_relations=snapshot.existing_relations, - target_relations={ - name: physical_relations.get(name) - or scope.models_by_name[name].destination.qualified_name - or name - for name in old_cursor_columns - }, - overrides=overrides, - deferral=deferral, - origin_cursor_columns=old_cursor_columns, - ), - }, - ) - - -def _entry_warning(entry: ColumnMigrationPlanEntry) -> PlanWarning | None: - relation: str = entry.destination.qualified_name or entry.destination.name - declaration: str = f"column '{entry.destination_column}' migrate_from {entry.origin_column}" - if entry.decision == ColumnMigrationDecision.DONE: - if entry.discovery != MigrationDiscovery.MANUAL: - return None - completed: str = entry.completed_at.isoformat() if entry.completed_at else "unknown time" - return PlanWarning( - model_name=entry.model_name, - severity=WarningSeverity.WARNING, - message=( - f"column rename {entry.origin_column} -> {entry.destination_column} on " - f"{relation} completed at {completed} on target " - f"'{entry.target_name or 'default'}'; " - f"migrate_from can be removed from column '{entry.destination_column}' of " - f"'{entry.model_name}'" - ), - code="M108", - ) - messages: dict[ColumnMigrationDecision, tuple[str, str]] = { - ColumnMigrationDecision.SOURCE_MISSING: ( - "M109", - f"model '{entry.model_name}': {declaration}: column {entry.origin_column} does not " - f"exist in {relation} and no recorded rename into {entry.destination_column} was " - f"found; {MISSING_ORIGIN_OUTCOMES[entry.missing_origin_policy]}", - ), - ColumnMigrationDecision.CONFLICT: ( - "M110", - f"model '{entry.model_name}': {declaration}: {relation} already has both " - f"{entry.origin_column} and {entry.destination_column}, so the column cannot be " - "renamed in place; drop one of them, or remove migrate_from and let " - "on_schema_change handle the columns", - ), - ColumnMigrationDecision.STILL_PRODUCED: ( - "M111", - f"model '{entry.model_name}': {declaration}: the model still produces " - f"{entry.origin_column}; remove it from the query, or remove migrate_from", - ), - ColumnMigrationDecision.UNSUPPORTED: ( - "M112", - f"model '{entry.model_name}': {declaration}: {entry.message}", - ), - } - coded: tuple[str, str] | None = messages.get(entry.decision) - if coded is None: - return None - return PlanWarning( - model_name=entry.model_name, - severity=WarningSeverity.ERROR if entry.blocks_build else WarningSeverity.WARNING, - message=coded[1], - code=coded[0], - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py index d8da979bcd..c01d8ea5fc 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py @@ -4,7 +4,7 @@ import time -from sqlbuild.compiler.planner._helpers.migrations.columns import plan_column_migrations +from sqlbuild._native import plan_column_migrations from sqlbuild.compiler.planner._helpers.migrations.old_names import plan_old_name_views from sqlbuild.compiler.planner._helpers.migrations.planning import plan_model_migrations from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py index 9092c3b15c..f302136b45 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py @@ -8,6 +8,7 @@ import pytest +from sqlbuild._native import plan_column_migrations from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -18,7 +19,6 @@ ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.migrations.columns import plan_column_migrations from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( DeferralInputs, From 54156c691866239fb9ed27ce2a07d65f55ec4f09 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:06:46 +0100 Subject: [PATCH 12/29] chore: update Python boundary after column migration port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index e32b053154..6dcf361f6e 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 37559 +remaining_lines = 36995 [[stays]] path = "executor" From 80ddb1f0055b69eebfad337868578825bb970d18 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:04:32 +0100 Subject: [PATCH 13/29] refactor: resolve planner cursor windows natively --- crates/sqlbuild-cursor/src/lib.rs | 1 + .../src/planner/main/empty_inputs.rs | 10 + .../sqlbuild-cursor/src/planner/main/mod.rs | 5 + .../src/planner/main/raw_bounds.rs | 99 ++++ .../src/planner/main/replay_start.rs | 104 ++++ crates/sqlbuild-cursor/src/planner/mod.rs | 4 + crates/sqlbuild-cursor/src/planner/models.rs | 33 ++ .../_helpers/planner_resolution/constants.rs | 3 + .../planner_resolution/cursor_bounds.rs | 305 +++++++++++ .../planner_resolution/cursor_model.rs | 294 ++++++++++ .../planner_resolution/cursor_windows.rs | 284 ++++++++++ .../_helpers/planner_resolution/mod.rs | 4 + .../planner_resolution/orchestration.rs | 3 +- .../planner_resolution/registration.rs | 3 + src/sqlbuild/_native.pyi | 76 +++ .../planner/_helpers/resolve/cursor.py | 508 ++---------------- .../planner/_helpers/resolve/resolve.py | 194 +------ .../planner/_helpers/resolve/_test_types.py | 11 + .../planner/_helpers/resolve/helpers.py | 12 + .../_helpers/resolve/test_cursor_planning.py | 56 ++ .../planner/_helpers/resolve/test_cursor.py | 45 +- 21 files changed, 1389 insertions(+), 665 deletions(-) create mode 100644 crates/sqlbuild-cursor/src/planner/main/empty_inputs.rs create mode 100644 crates/sqlbuild-cursor/src/planner/main/mod.rs create mode 100644 crates/sqlbuild-cursor/src/planner/main/raw_bounds.rs create mode 100644 crates/sqlbuild-cursor/src/planner/main/replay_start.rs create mode 100644 crates/sqlbuild-cursor/src/planner/mod.rs create mode 100644 crates/sqlbuild-cursor/src/planner/models.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs create mode 100644 tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_planning.py diff --git a/crates/sqlbuild-cursor/src/lib.rs b/crates/sqlbuild-cursor/src/lib.rs index d5963ee3a5..13b88eed8f 100644 --- a/crates/sqlbuild-cursor/src/lib.rs +++ b/crates/sqlbuild-cursor/src/lib.rs @@ -3,3 +3,4 @@ #![forbid(unsafe_code)] pub mod algebra; +pub mod planner; diff --git a/crates/sqlbuild-cursor/src/planner/main/empty_inputs.rs b/crates/sqlbuild-cursor/src/planner/main/empty_inputs.rs new file mode 100644 index 0000000000..b987f7fd1d --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/main/empty_inputs.rs @@ -0,0 +1,10 @@ +//! Whether known empty inputs leave a usable cursor window. + +/// Retain empty input names unless an any-input window has both endpoints. +pub fn empty_inputs( + waits_for_every_input: bool, + inputs_with_rows: bool, + has_window_start: bool, +) -> bool { + waits_for_every_input || !inputs_with_rows || !has_window_start +} diff --git a/crates/sqlbuild-cursor/src/planner/main/mod.rs b/crates/sqlbuild-cursor/src/planner/main/mod.rs new file mode 100644 index 0000000000..7b69754ff8 --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/main/mod.rs @@ -0,0 +1,5 @@ +//! Planner cursor algebra entry points. + +pub mod empty_inputs; +pub mod raw_bounds; +pub mod replay_start; diff --git a/crates/sqlbuild-cursor/src/planner/main/raw_bounds.rs b/crates/sqlbuild-cursor/src/planner/main/raw_bounds.rs new file mode 100644 index 0000000000..5273149f58 --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/main/raw_bounds.rs @@ -0,0 +1,99 @@ +//! Discover the effective half-open cursor window from snapshot facts. + +use crate::algebra::errors::CursorError; +use crate::algebra::main::cursor_sort_key::cursor_sort_key; +use crate::algebra::main::extreme_position::extreme_position; +use crate::algebra::main::floor_to_grain::floor_to_grain; +use crate::algebra::main::observed_partition::observed_partition; +use crate::algebra::models::{Extreme, Grain, Scalar, ScalarValue}; +use crate::algebra::types::ZoneRules; +use crate::planner::models::{CursorKind, CursorSnapshot}; + +fn extreme( + values: &[Scalar], + mode: Extreme, + zones: &impl ZoneRules, +) -> Result { + let keys = values + .iter() + .map(|value| cursor_sort_key(value, zones)) + .collect::, _>>()?; + Ok(values[extreme_position(&keys, mode)?].clone()) +} + +fn discovered_end(value: &Scalar, kind: &CursorKind<'_>) -> Result { + let grain = match kind.cursor_type { + Some("integer") => None, + Some("timestamp") => Some(kind.grain.unwrap_or({ + if matches!(value.value, ScalarValue::Timestamp(_)) { + Grain::Second + } else { + Grain::Day + } + })), + _ => return Ok(value.clone()), + }; + Ok(observed_partition(value, grain)?.end) +} + +/// Determine raw start and exclusive end without replay or override policy. +pub fn raw_bounds( + snapshot: &CursorSnapshot, + kind: &CursorKind<'_>, + zones: &impl ZoneRules, +) -> Result, CursorError> { + let mut ends: Vec = Vec::new(); + if !snapshot.availability_ends.is_empty() { + for value in &snapshot.availability_ends { + ends.push(if kind.cursor_type == Some("timestamp") { + if let Some(grain) = kind.grain { + floor_to_grain(value, Some(grain)) + } else { + value.clone() + } + } else { + value.clone() + }); + } + } else if !snapshot.end_inputs.is_empty() { + for (physical, terminal) in &snapshot.end_inputs { + if let Some(terminal) = terminal { + ends.push(terminal.clone()); + } else if let Some(physical) = physical { + ends.push(discovered_end(physical, kind)?); + } + } + } else { + for value in &snapshot.upstream_maxes { + ends.push(discovered_end(value, kind)?); + } + ends.extend(snapshot.terminal_ends.iter().cloned()); + } + if ends.is_empty() { + return Ok(None); + } + let end = extreme( + &ends, + if snapshot.any_input { + Extreme::Max + } else { + Extreme::Min + }, + zones, + )?; + let start = if let Some(value) = &snapshot.target_max { + value.clone() + } else { + let starts: Vec = snapshot + .upstream_mins + .iter() + .chain(&snapshot.terminal_starts) + .cloned() + .collect(); + if starts.is_empty() { + return Ok(None); + } + extreme(&starts, Extreme::Min, zones)? + }; + Ok(Some((start, end))) +} diff --git a/crates/sqlbuild-cursor/src/planner/main/replay_start.rs b/crates/sqlbuild-cursor/src/planner/main/replay_start.rs new file mode 100644 index 0000000000..94207c6e17 --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/main/replay_start.rs @@ -0,0 +1,104 @@ +//! Replay windows and configured cursor floors. + +use std::cmp::Ordering; + +use crate::algebra::errors::CursorError; +use crate::algebra::main::compare::compare; +use crate::algebra::main::shift_by_duration::shift_by_duration; +use crate::algebra::models::{ + DurationSpan, LocalDateTime, Moment, Scalar, ScalarValue, Timestamp, ZoneOffset, +}; +use crate::algebra::types::ZoneRules; +use crate::planner::models::{CursorKind, ReplayPolicy}; + +fn timestamp(value: &Scalar) -> Option { + match value.value { + ScalarValue::Timestamp(_) => Some(value.clone()), + ScalarValue::Date(date) => Some(Scalar { + value: ScalarValue::Timestamp(Timestamp { + local: LocalDateTime { + date, + hour: 0, + minute: 0, + second: 0, + microsecond: 0, + }, + fold: false, + offset: ZoneOffset::Naive, + template: None, + source_text: None, + }), + handle: None, + }), + ScalarValue::Integer(_) => None, + } +} + +fn subtract(value: &Scalar, span: &DurationSpan) -> Result { + if let ScalarValue::Integer(integer) = value.value { + let seconds = i128::from(span.days) * 86_400 + + i128::from(span.hours) * 3_600 + + i128::from(span.minutes) * 60 + + i128::from(span.seconds); + return Ok(Scalar { + value: ScalarValue::Integer(integer - seconds), + handle: None, + }); + } + let Some(mut result) = timestamp(value) else { + return Ok(value.clone()); + }; + if let ScalarValue::Timestamp(ref mut value) = result.value { + let shifted = shift_by_duration( + Moment::Timestamp { + local: value.local, + fold: value.fold, + }, + span, + -1, + )?; + if let Moment::Timestamp { local, fold } = shifted { + value.local = local; + value.fold = fold; + value.source_text = None; + } + } + result.handle = None; + Ok(result) +} + +/// Apply replay and lower-bound policy to a typed cursor start. +pub fn replay_start( + bounds: (&Scalar, &Scalar), + policy: &ReplayPolicy, + kind: &CursorKind<'_>, + zones: &impl ZoneRules, +) -> Result { + let (start, end) = bounds; + let mut effective = start.clone(); + if !policy.has_start_override { + if policy.has_backfill { + if let Some(span) = &policy.backfill { + effective = subtract(end, span)?; + } + } else if let Some(span) = &policy.lookback { + effective = subtract(&effective, span)?; + } + } + let Some(floor) = &policy.floor else { + return Ok(effective); + }; + let pair = match kind.cursor_type { + Some("timestamp") => timestamp(&effective).zip(timestamp(floor)), + Some("integer") => Some((effective.clone(), floor.clone())), + _ => None, + }; + if let Some((current, floor)) = pair { + return Ok(if compare(&floor, ¤t, zones)? == Ordering::Greater { + floor + } else { + current + }); + } + Ok(effective) +} diff --git a/crates/sqlbuild-cursor/src/planner/mod.rs b/crates/sqlbuild-cursor/src/planner/mod.rs new file mode 100644 index 0000000000..a10f585e28 --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/mod.rs @@ -0,0 +1,4 @@ +//! Cursor windows projected from authoritative planner snapshots. + +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-cursor/src/planner/models.rs b/crates/sqlbuild-cursor/src/planner/models.rs new file mode 100644 index 0000000000..0c51018bcc --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/models.rs @@ -0,0 +1,33 @@ +//! Inputs to planner cursor-window algebra. + +use crate::algebra::models::{DurationSpan, Grain, Scalar}; + +/// Discovered values used to determine a half-open cursor window. +#[derive(Debug)] +pub struct CursorSnapshot { + pub target_max: Option, + pub upstream_mins: Vec, + pub upstream_maxes: Vec, + pub terminal_starts: Vec, + pub terminal_ends: Vec, + pub end_inputs: Vec<(Option, Option)>, + pub availability_ends: Vec, + pub any_input: bool, +} + +/// Cursor representation and discovered partition grain. +#[derive(Debug)] +pub struct CursorKind<'a> { + pub cursor_type: Option<&'a str>, + pub grain: Option, +} + +/// Parsed replay durations and configured lower bound. +#[derive(Debug)] +pub struct ReplayPolicy { + pub floor: Option, + pub lookback: Option, + pub backfill: Option, + pub has_backfill: bool, + pub has_start_override: bool, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/constants.rs new file mode 100644 index 0000000000..b443c6acb1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/constants.rs @@ -0,0 +1,3 @@ +//! Planner cursor configuration keys. + +pub(super) const CURSOR_START: &str = "cursor_start"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs new file mode 100644 index 0000000000..7846d5bbf2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs @@ -0,0 +1,305 @@ +//! Snapshot conversion and cursor window policy boundary. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::cursor::conversion::{CursorSession, algebra_error}; +use crate::bindings::_helpers::cursor::scalars::try_parse; +use crate::bindings::_helpers::planner_resolution::values::string; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_cursor::algebra::main::inclusive_to_exclusive::inclusive_to_exclusive; +use sqlbuild_cursor::algebra::models::{DurationSpan, Grain, Scalar}; +use sqlbuild_cursor::planner::main::raw_bounds::raw_bounds; +use sqlbuild_cursor::planner::main::replay_start::replay_start; +use sqlbuild_cursor::planner::models::{CursorKind, CursorSnapshot, ReplayPolicy}; + +const MODELS: &str = "sqlbuild.compiler.planner.models"; +const MAXIMUM_START: &str = "sqlbuild.compiler.planner._helpers.resolve.maximum_start"; + +fn optional_scalar<'py>( + session: &CursorSession<'py>, + value: &Bound<'py, PyAny>, +) -> PyResult> { + if value.is_none() { + Ok(None) + } else { + session.scalar(value).map(Some) + } +} + +fn scalars<'py>( + session: &CursorSession<'py>, + object: &Bound<'py, PyAny>, + field: &str, +) -> PyResult> { + object + .getattr(field)? + .try_iter()? + .map(|value| session.scalar(&value?)) + .collect() +} + +fn read_snapshot<'py>( + session: &CursorSession<'py>, + object: &Bound<'py, PyAny>, +) -> PyResult { + let mut end_inputs: Vec<(Option, Option)> = Vec::new(); + for pair in object.getattr("upstream_end_inputs")?.try_iter()? { + let pair = pair?; + end_inputs.push(( + optional_scalar(session, &pair.get_item(0)?)?, + optional_scalar(session, &pair.get_item(1)?)?, + )); + } + Ok(CursorSnapshot { + target_max: optional_scalar(session, &object.getattr("target_max")?)?, + upstream_mins: scalars(session, object, "upstream_mins")?, + upstream_maxes: scalars(session, object, "upstream_maxes")?, + terminal_starts: scalars(session, object, "upstream_terminal_starts")?, + terminal_ends: scalars(session, object, "upstream_terminal_ends")?, + availability_ends: scalars(session, object, "upstream_availability_ends")?, + any_input: object.getattr("cursor_watermark_mode")?.eq("any")?, + end_inputs, + }) +} + +fn duration(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + let class = imported(value.py(), MODELS, "Duration")?; + let parsed = if value.is_instance(&class)? { + value.clone() + } else { + class.call_method1("parse", (value,))? + }; + if parsed.is_none() { + return Ok(None); + } + let field = |name: &str| -> PyResult { parsed.getattr(name)?.extract() }; + Ok(Some(DurationSpan { + months: field("years")? * 12 + field("months")?, + days: field("days")?, + hours: field("hours")?, + minutes: field("minutes")?, + seconds: field("seconds")?, + })) +} + +fn read_policy<'py>( + session: &CursorSession<'py>, + args: &KeywordArguments<'py>, +) -> PyResult { + let floor = args.value("cursor_start")?; + let cursor_type = args.value("cursor_type")?; + let floor = + if floor.is_none() || !(cursor_type.eq("timestamp")? || cursor_type.eq("integer")?) { + None + } else if session.scalar_typed(&floor)? { + Some(session.scalar(&floor)?) + } else { + try_parse(&floor, &cursor_type)? + .map(|value| session.scalar(&value)) + .transpose()? + }; + Ok(ReplayPolicy { + floor, + lookback: duration(&args.value("lookback")?)?, + backfill: duration(&args.value("backfill_duration")?)?, + has_backfill: !args.value("backfill_duration")?.is_none(), + has_start_override: args.flag("has_start_override")?, + }) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn apply_typed_cursor_replay_policy<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "apply_typed_cursor_replay_policy", + kwargs, + ( + &[ + "start", + "end", + "cursor_start", + "cursor_type", + "lookback", + "backfill_duration", + "has_start_override", + ], + &[], + ), + )?; + let session = CursorSession::new(py)?; + let cursor_type: Option = args.value("cursor_type")?.extract()?; + let start = session.scalar(&args.value("start")?)?; + let end = session.scalar(&args.value("end")?)?; + let policy = read_policy(&session, &args)?; + let result = session.checked(replay_start( + (&start, &end), + &policy, + &CursorKind { + cursor_type: cursor_type.as_deref(), + grain: None, + }, + &session, + ))?; + session.python_scalar(&result) +} + +fn required_scalar<'py>( + session: &CursorSession<'py>, + raw: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult { + let Some(value) = try_parse(raw, cursor_type)? else { + return Err(algebra_error( + raw.py(), + &format!("invalid cursor value: {}", raw.str()?), + )); + }; + session.scalar(&value) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn compute_cursor_bounds<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "compute_cursor_bounds", + kwargs, + ( + &[ + "cursor_snapshot", + "cursor_type", + "cursor_start", + "lookback", + "backfill_duration", + "start_cursor_override", + "end_cursor_override", + "is_microbatch", + ], + &["cursor_grain", "maximum_start_policy"], + ), + )?; + let bounds_class = imported(py, MODELS, "CursorBounds")?; + if args.flag("is_microbatch")? { + let sentinel = imported(py, "sqlbuild.cursor_algebra.types", "BoundSentinel")?; + return call_with( + &bounds_class, + &[ + ("start", sentinel.getattr("START")?), + ("end", sentinel.getattr("END")?), + ], + ); + } + let snapshot = args.value("cursor_snapshot")?; + if !snapshot.getattr("watermarks_available")?.is_truthy()? { + return Ok(py.None().into_bound(py)); + } + let session = CursorSession::new(py)?; + let cursor_type: Option = args.value("cursor_type")?.extract()?; + let grain = session.grain(&args.value("cursor_grain")?)?; + let kind = CursorKind { + cursor_type: cursor_type.as_deref(), + grain, + }; + let Some((mut start, mut end)) = session.checked(raw_bounds( + &read_snapshot(&session, &snapshot)?, + &kind, + &session, + ))? + else { + return Ok(py.None().into_bound(py)); + }; + let parse_type = string(py, cursor_type.as_deref().unwrap_or("timestamp")); + let start_override = args.value("start_cursor_override")?; + let end_override = args.value("end_cursor_override")?; + if !start_override.is_none() { + start = required_scalar(&session, &start_override, &parse_type)?; + } + if !end_override.is_none() { + let value = required_scalar(&session, &end_override, &parse_type)?; + end = session.checked(inclusive_to_exclusive( + &value, + if cursor_type.as_deref() == Some("integer") { + None + } else { + Some(grain.unwrap_or(Grain::Second)) + }, + ))?; + } + let mut replay_args: Vec<(&str, Bound<'py, PyAny>)> = Vec::new(); + for key in [ + "cursor_start", + "cursor_type", + "lookback", + "backfill_duration", + ] { + replay_args.push((key, args.value(key)?)); + } + replay_args.extend([ + ("start", session.python_scalar(&start)?), + ("end", session.python_scalar(&end)?), + ( + "has_start_override", + (!start_override.is_none()) + .into_pyobject(py)? + .to_owned() + .into_any(), + ), + ]); + let replayed = call_with( + &imported(py, "sqlbuild._native", "apply_typed_cursor_replay_policy")?, + &replay_args, + )?; + let bounds = call_with( + &bounds_class, + &[("start", replayed), ("end", session.python_scalar(&end)?)], + )?; + let policy = args.value("maximum_start_policy")?; + let policy = if policy.is_truthy()? { + policy + } else { + imported(py, MODELS, "MaximumStartPolicyInputs")?.call0()? + }; + let mut policy_args = vec![ + ("bounds", bounds), + ("snapshot", snapshot), + ("policy", policy), + ( + "has_start_override", + (!start_override.is_none()) + .into_pyobject(py)? + .to_owned() + .into_any(), + ), + ]; + for key in [ + "cursor_type", + "cursor_grain", + "cursor_start", + "lookback", + "backfill_duration", + ] { + policy_args.push((key, args.value(key)?)); + } + call_with( + &imported(py, MAXIMUM_START, "apply_maximum_start_policy")?, + &policy_args, + ) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(compute_cursor_bounds, module)?)?; + module.add_function(wrap_pyfunction!(apply_typed_cursor_replay_policy, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs new file mode 100644 index 0000000000..212257f4dd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -0,0 +1,294 @@ +//! Model cursor orchestration over live warehouse snapshot facts. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_resolution::constants::CURSOR_START; +use crate::bindings::_helpers::planner_resolution::values::{NATIVE_MODULE, string}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; + +const MODELS: &str = "sqlbuild.compiler.planner.models"; +const CURSOR: &str = "sqlbuild._native"; +const POLICIES: &str = "sqlbuild.compiler.planner._helpers.resolve.cursor_policies"; +const FUTURE: &str = "sqlbuild.compiler.planner.main.execution.future_cursor_safety"; + +fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { + call_with( + &imported( + model.py(), + NATIVE_MODULE, + if key == CURSOR_START { + "get_config_cursor_bound" + } else { + "get_config_str" + }, + )?, + &[ + ("values", model.getattr("config")?.getattr("values")?), + ("key", string(model.py(), key)), + ], + ) +} + +fn unbounded(py: Python<'_>) -> PyResult> { + let sentinel = imported(py, "sqlbuild.cursor_algebra.types", "BoundSentinel")?; + call_with( + &imported(py, MODELS, "CursorBounds")?, + &[ + ("start", sentinel.getattr("START")?), + ("end", sentinel.getattr("END")?), + ], + ) +} + +fn destination_exists(model: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>) -> PyResult { + let relation = snapshot + .getattr("existing_relations")? + .call_method1("get", (model.getattr("name")?,))?; + if relation.is_none() { + return Ok(false); + } + let destination = model.getattr("destination")?; + let key = call_with( + &imported( + model.py(), + "sqlbuild.adapter.contract.models", + "RelationLookup", + )? + .getattr("key")?, + &[ + ("database", destination.getattr("database")?), + ("schema", destination.getattr("schema")?), + ("name", destination.getattr("name")?), + ], + )?; + relation.getattr("identity")?.eq(key) +} + +fn history_redirected(model: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>) -> PyResult { + let target = snapshot.getattr("target_relation")?; + Ok(!target.is_none() + && target.ne(model.getattr("destination")?.getattr("qualified_name")?)?) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn compute_model_cursor_bounds<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "compute_model_cursor_bounds", + kwargs, + ( + &[ + "model", + "snapshot", + "backfill", + "full_refresh", + "start_cursor_override", + "end_cursor_override", + "runtime_cursor_producer_names", + "context", + ], + &["replaces_relation"], + ), + )?; + let model = args.value("model")?; + if !config(&model, "materialized")?.eq("incremental")? || config(&model, "cursor")?.is_none() { + return Ok(py.None().into_bound(py)); + } + let context = args.value("context")?; + let incremental_mode = config(&model, "incremental_mode")?; + let future_config = call_with( + &imported(py, POLICIES, "resolve_future_cursor_config")?, + &[ + ("model", model.clone()), + ("project_config", context.getattr("future_cursor_config")?), + ], + )?; + let start_config = call_with( + &imported(py, POLICIES, "resolve_start_cursor_config")?, + &[ + ("model", model.clone()), + ("project_config", context.getattr("start_cursor_config")?), + ], + )?; + if incremental_mode.eq("microbatch")? { + return unbounded(py); + } + let cursor_type = config(&model, "cursor_type")?; + let cursor_grain = config(&model, "cursor_grain")?; + let start_override = args.value("start_cursor_override")?; + let end_override = args.value("end_cursor_override")?; + let bounded = call_with( + &imported(py, NATIVE_MODULE, "resolve_bounded_cursor_override")?, + &[ + ("start_cursor_override", start_override.clone()), + ("end_cursor_override", end_override.clone()), + ("cursor_type", cursor_type.clone()), + ("cursor_grain", cursor_grain.clone()), + ], + )?; + if !bounded.is_none() { + return Ok(bounded); + } + let snapshot = args.value("snapshot")?; + let mut cursor_snapshot = snapshot + .getattr("cursor_snapshots")? + .call_method1("get", (model.getattr("name")?,))?; + let roles = call_with( + &imported(py, NATIVE_MODULE, "resolve_cursor_input_roles")?, + &[("model", model.clone())], + )?; + let producers = args.value("runtime_cursor_producer_names")?; + let mut runtime_owned = false; + for name in roles.getattr("watermark_inputs")?.try_iter()? { + if producers.contains(name?)? { + runtime_owned = true; + break; + } + } + let full_refresh = args.flag("full_refresh")?; + let replaces = args.flag("replaces_relation")?; + if !full_refresh && !runtime_owned && !cursor_snapshot.is_none() { + let empty = call_with( + &imported(py, CURSOR, "cursor_inputs_without_rows")?, + &[("cursor_snapshot", cursor_snapshot.clone())], + )? + .getattr("input_names")?; + if empty.is_truthy()? + && replaces + && (destination_exists(&model, &snapshot)? + || history_redirected(&model, &cursor_snapshot)?) + { + let refusal = call_with( + &imported(py, CURSOR, "empty_input_rebuild_refusal")?, + &[ + ("model_name", model.getattr("name")?), + ("input_names", empty.clone()), + ("cursor_type", cursor_type.clone()), + ], + )?; + return Err(PyErr::from_value(call_with( + &imported( + py, + "sqlbuild.compiler.planner.exceptions", + "PlannerInputError", + )?, + &[ + ("message", refusal.get_item(0)?), + ("code", string(py, "S302")), + ("help", refusal.get_item(1)?), + ], + )?)); + } + if empty.is_truthy()? { + return call_with( + &imported(py, CURSOR, "empty_cursor_window")?, + &[ + ("cursor_type", cursor_type), + ("cursor_start", config(&model, "cursor_start")?), + ], + ); + } + if !cursor_snapshot + .getattr("watermarks_available")? + .is_truthy()? + { + let unreadable = cursor_snapshot.getattr("unreadable_input_names")?; + let names = if unreadable.is_truthy()? { + unreadable + } else { + cursor_snapshot.getattr("unavailable_watermark_tags")? + }; + return Err(PyErr::from_value(call_with( + &imported(py, CURSOR, "unreadable_cursor_inputs_error")?, + &[ + ("model_name", model.getattr("name")?), + ("cursor_type", cursor_type), + ("input_names", names), + ], + )?)); + } + } + if full_refresh { + return Ok(py.None().into_bound(py)); + } + if runtime_owned { + return unbounded(py); + } + if cursor_snapshot.is_none() { + return Ok(py.None().into_bound(py)); + } + if replaces { + cursor_snapshot = call_with( + &imported(py, CURSOR, "without_destination_cursor")?, + &[("cursor_snapshot", cursor_snapshot)], + )?; + } + let backfill = args.value("backfill")?; + let duration = if backfill.getattr("action")?.eq("bounded")? { + backfill.getattr("duration")? + } else { + py.None().into_bound(py) + }; + let policy = call_with( + &imported(py, MODELS, "MaximumStartPolicyInputs")?, + &[ + ("config", start_config), + ("invocation_time", context.getattr("invocation_time")?), + ( + "incremental_strategy", + config(&model, "incremental_strategy")?, + ), + ("incremental_mode", incremental_mode), + ], + )?; + let bounds = call_with( + &imported(py, NATIVE_MODULE, "compute_cursor_bounds")?, + &[ + ("cursor_snapshot", cursor_snapshot.clone()), + ("cursor_type", cursor_type.clone()), + ("cursor_start", config(&model, "cursor_start")?), + ("lookback", config(&model, "lookback")?), + ("backfill_duration", duration), + ("start_cursor_override", start_override.clone()), + ("end_cursor_override", end_override.clone()), + ( + "is_microbatch", + false.into_pyobject(py)?.to_owned().into_any(), + ), + ("cursor_grain", cursor_grain.clone()), + ("maximum_start_policy", policy), + ], + )?; + if bounds.is_none() { + return Ok(bounds); + } + call_with( + &imported(py, FUTURE, "apply_future_cursor_safety")?, + &[ + ("bounds", bounds), + ("cursor_type", cursor_type), + ("cursor_grain", cursor_grain), + ("config", future_config), + ("invocation_time", context.getattr("invocation_time")?), + ( + "has_complete_override", + (!start_override.is_none() && !end_override.is_none()) + .into_pyobject(py)? + .to_owned() + .into_any(), + ), + ("input_evidence", cursor_snapshot.getattr("input_evidence")?), + ], + ) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(compute_model_cursor_bounds, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs new file mode 100644 index 0000000000..4c268b6fb0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs @@ -0,0 +1,284 @@ +//! Empty-window decisions and snapshot grain conversion. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cursor::conversion::CursorSession; +use crate::bindings::_helpers::cursor::scalars::parse_value; +use crate::bindings::_helpers::planner_resolution::values::string; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_cursor::algebra::main::floor_to_grain::floor_to_grain; +use sqlbuild_cursor::planner::main::empty_inputs::empty_inputs; + +const MODELS: &str = "sqlbuild.compiler.planner.models"; + +#[pyfunction] +#[pyo3(signature = (*, empty_input_names, waits_for_every_input, inputs_with_rows, has_window_start))] +fn decide_empty_cursor_inputs<'py>( + empty_input_names: &Bound<'py, PyAny>, + waits_for_every_input: bool, + inputs_with_rows: bool, + has_window_start: bool, +) -> PyResult> { + let py = empty_input_names.py(); + let class = imported(py, MODELS, "EmptyCursorInputDecision")?; + if !empty_input_names.is_truthy()? + || !empty_inputs(waits_for_every_input, inputs_with_rows, has_window_start) + { + return class.call0(); + } + call_with( + &class, + &[ + ("input_names", empty_input_names.clone()), + ( + "waiting_on_empty_inputs", + inputs_with_rows.into_pyobject(py)?.to_owned().into_any(), + ), + ], + ) +} + +#[pyfunction] +#[pyo3(signature = (*, cursor_snapshot))] +fn cursor_inputs_without_rows<'py>( + cursor_snapshot: &Bound<'py, PyAny>, +) -> PyResult> { + if cursor_snapshot + .getattr("unreadable_input_names")? + .is_truthy()? + { + return imported(cursor_snapshot.py(), MODELS, "EmptyCursorInputDecision")?.call0(); + } + let rows = cursor_snapshot.getattr("upstream_maxes")?.is_truthy()? + || cursor_snapshot + .getattr("upstream_terminal_ends")? + .is_truthy()? + || cursor_snapshot + .getattr("upstream_availability_ends")? + .is_truthy()?; + let start = !cursor_snapshot.getattr("target_max")?.is_none() + || cursor_snapshot.getattr("upstream_mins")?.is_truthy()? + || cursor_snapshot + .getattr("upstream_terminal_starts")? + .is_truthy()?; + decide_empty_cursor_inputs( + &cursor_snapshot.getattr("empty_input_names")?, + !cursor_snapshot + .getattr("cursor_watermark_mode")? + .eq("any")?, + rows, + start, + ) +} + +#[pyfunction] +#[pyo3(signature = (*, cursor_type, cursor_start))] +fn empty_cursor_window<'py>( + cursor_type: &Bound<'py, PyAny>, + cursor_start: &Bound<'py, PyAny>, +) -> PyResult> { + let py = cursor_type.py(); + let kind = if cursor_type.is_truthy()? { + cursor_type.clone() + } else { + string(py, "timestamp") + }; + let raw = if !cursor_start.is_none() { + cursor_start.clone() + } else if kind.eq("integer")? { + string(py, "0") + } else { + string(py, "1970-01-01T00:00:00") + }; + let mut bound = parse_value(&raw, &kind)?; + if cursor_start.is_none() && !kind.eq("integer")? { + bound = call_with( + &imported(py, "sqlbuild.cursor_algebra.models", "TimestampValue")?, + &[("value", bound.getattr("value")?)], + )?; + } + call_with( + &imported(py, MODELS, "CursorBounds")?, + &[("start", bound.clone()), ("end", bound)], + ) +} + +#[pyfunction] +#[pyo3(signature = (*, cursor_type))] +fn cursor_window_override_flags(cursor_type: Option<&str>) -> String { + let suffix = if cursor_type == Some("integer") { + "int" + } else { + "ts" + }; + format!("--start-cursor-{suffix} and --end-cursor-{suffix}") +} + +#[pyfunction] +#[pyo3(signature = (*, model_name, input_names, cursor_type))] +fn empty_input_rebuild_refusal( + model_name: &str, + input_names: Vec, + cursor_type: Option<&str>, +) -> (String, String) { + ( + format!( + "model '{model_name}' was not rebuilt: its cursor inputs have no rows ({}), so rebuilding it now would leave it empty", + input_names.join(", ") + ), + format!( + "load the inputs first, or set the window explicitly with {}", + cursor_window_override_flags(cursor_type) + ), + ) +} + +#[pyfunction] +#[pyo3(signature = (*, model_name, cursor_type, input_names))] +fn unreadable_cursor_inputs_error<'py>( + py: Python<'py>, + model_name: &str, + cursor_type: Option<&str>, + input_names: Vec, +) -> PyResult> { + call_with( + &imported( + py, + "sqlbuild.compiler.planner.exceptions", + "PlannerInputError", + )?, + &[ + ( + "message", + string( + py, + &format!( + "model '{model_name}' cannot work out its cursor window: cursor bounds could not be read from {}", + input_names.join(", ") + ), + ), + ), + ("code", string(py, "S302")), + ( + "help", + string( + py, + &format!( + "check that each input exists and has its cursor column (`sqb debug` checks the connection), or set the window explicitly with {}", + cursor_window_override_flags(cursor_type) + ), + ), + ), + ], + ) +} + +#[pyfunction] +#[pyo3(signature = (*, cursor_snapshot))] +fn without_destination_cursor<'py>( + cursor_snapshot: &Bound<'py, PyAny>, +) -> PyResult> { + let py = cursor_snapshot.py(); + let kwargs = PyDict::new(py); + for name in ["target_max", "physical_target_max", "target_eligible_max"] { + kwargs.set_item(name, py.None())?; + } + imported(py, "dataclasses", "replace")?.call((cursor_snapshot,), Some(&kwargs)) +} + +#[pyfunction] +#[pyo3(signature = (*, cursor_type, downstream_grain, cursor_input_grains, microbatch_strategy=None))] +fn resolve_effective_timestamp_grain<'py>( + cursor_type: &Bound<'py, PyAny>, + downstream_grain: &Bound<'py, PyAny>, + cursor_input_grains: &Bound<'py, PyAny>, + microbatch_strategy: Option<&str>, +) -> PyResult> { + let py = cursor_type.py(); + if !cursor_type.eq("timestamp")? { + return Ok(py.None().into_bound(py)); + } + let mut effective = if downstream_grain.is_truthy()? { + downstream_grain.clone() + } else { + string(py, "second") + }; + if microbatch_strategy == Some("watermark") { + return Ok(effective); + } + let session = CursorSession::new(py)?; + for value in cursor_input_grains.try_iter()? { + let value = value?; + let candidate = if value.is_truthy()? { + value + } else { + string(py, "second") + }; + if session.grain(&candidate)? > session.grain(&effective)? { + effective = candidate; + } + } + Ok(effective) +} + +#[pyfunction] +#[pyo3(signature = (*, cursor_snapshot, cursor_type, effective_grain))] +fn normalize_cursor_snapshot_grain<'py>( + cursor_snapshot: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, + effective_grain: &Bound<'py, PyAny>, +) -> PyResult> { + if !cursor_type.eq("timestamp")? || effective_grain.is_none() { + return Ok(cursor_snapshot.clone()); + } + let py = cursor_snapshot.py(); + let session = CursorSession::new(py)?; + let grain = session.grain(effective_grain)?; + let normalize = |value: Bound<'py, PyAny>| -> PyResult> { + if value.is_none() { + return Ok(value); + } + session.python_scalar(&floor_to_grain(&session.scalar(&value)?, grain)) + }; + let kwargs = PyDict::new(py); + for key in ["target_max", "target_eligible_max"] { + kwargs.set_item(key, normalize(cursor_snapshot.getattr(key)?)?)?; + } + for key in ["upstream_mins", "upstream_maxes"] { + let values = cursor_snapshot + .getattr(key)? + .try_iter()? + .map(|value| normalize(value?)) + .collect::>>()?; + kwargs.set_item(key, PyTuple::new(py, values)?)?; + } + let physical = cursor_snapshot.getattr("physical_target_max")?; + kwargs.set_item( + "physical_target_max", + if physical.is_truthy()? { + physical + } else { + cursor_snapshot.getattr("target_max")? + }, + )?; + let mut ends: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); + for pair in cursor_snapshot.getattr("upstream_end_inputs")?.try_iter()? { + let pair = pair?; + ends.push((normalize(pair.get_item(0)?)?, pair.get_item(1)?)); + } + kwargs.set_item("upstream_end_inputs", PyTuple::new(py, ends)?)?; + imported(py, "dataclasses", "replace")?.call((cursor_snapshot,), Some(&kwargs)) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(decide_empty_cursor_inputs, module)?)?; + module.add_function(wrap_pyfunction!(cursor_inputs_without_rows, module)?)?; + module.add_function(wrap_pyfunction!(empty_cursor_window, module)?)?; + module.add_function(wrap_pyfunction!(cursor_window_override_flags, module)?)?; + module.add_function(wrap_pyfunction!(empty_input_rebuild_refusal, module)?)?; + module.add_function(wrap_pyfunction!(unreadable_cursor_inputs_error, module)?)?; + module.add_function(wrap_pyfunction!(without_destination_cursor, module)?)?; + module.add_function(wrap_pyfunction!(resolve_effective_timestamp_grain, module)?)?; + module.add_function(wrap_pyfunction!(normalize_cursor_snapshot_grain, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs index c7be48b95d..a49f426198 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs @@ -1,5 +1,9 @@ //! Planner reference resolution with Python adapter rendering callbacks. +mod constants; +mod cursor_bounds; +mod cursor_model; +mod cursor_windows; mod locations; mod orchestration; mod refs; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index 88b5321fe9..fe8a260dbd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -8,7 +8,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; -const RESOLVE_MODULE: &str = "sqlbuild.compiler.planner._helpers.resolve.resolve"; const CONFIG_MODULE: &str = "sqlbuild.compiler.planner._helpers.resolve.config"; const INTRINSICS_MODULE: &str = "sqlbuild.compiler.planner._helpers.resolve.cursor_intrinsics"; const MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; @@ -183,7 +182,7 @@ fn resolve_model_sql<'py>( )?; let overrides = args.value("cursor_overrides")?; let bounds = call_with( - &imported(py, RESOLVE_MODULE, "_compute_model_cursor_bounds")?, + &imported(py, NATIVE_MODULE, "compute_model_cursor_bounds")?, &[ ("model", model.clone()), ("snapshot", args.value("snapshot")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs index 0d1ba24f3a..534450995d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs @@ -3,6 +3,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + super::cursor_bounds::register(module)?; + super::cursor_model::register(module)?; + super::cursor_windows::register(module)?; super::refs::register(module)?; super::locations::register(module)?; super::orchestration::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index cb1d787f8a..a83e0277c1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -210,11 +210,14 @@ from sqlbuild.compiler.planner.models import ( CursorOverridePair, DeferralInputs, DirectModelVersionIdentities, + EmptyCursorInputDecision, FixtureRelationMetadata, FutureCursorSafetyEvidence, + MaximumStartPolicyInputs, MaximumStartSafetyEvidence, MigrationCompatibilityResult, MissingUpstream, + ModelCursorSnapshot, ModelMigrationRequest, ModelPlanContext, ModelPlanEntry, @@ -1068,6 +1071,79 @@ class NativeCompiledProject: def compiled_model_record(model: CompiledModelView) -> CompiledModel: ... def compiled_audit_record(audit: CompiledAuditView) -> CompiledAudit: ... +def compute_model_cursor_bounds( + *, + model: CompiledModelView, + snapshot: WarehouseSnapshot, + backfill: BackfillResult, + full_refresh: bool, + start_cursor_override: str | None, + end_cursor_override: str | None, + runtime_cursor_producer_names: frozenset[str], + context: ModelPlanContext, + replaces_relation: bool = False, +) -> CursorBounds | None: ... +def compute_cursor_bounds( + *, + cursor_snapshot: ModelCursorSnapshot, + cursor_type: str | None, + cursor_start: str | None, + lookback: str | None, + backfill_duration: str | None, + start_cursor_override: str | None, + end_cursor_override: str | None, + is_microbatch: bool, + cursor_grain: str | None = None, + maximum_start_policy: MaximumStartPolicyInputs | None = None, +) -> CursorBounds | None: ... +def apply_typed_cursor_replay_policy( + *, + start: CursorScalar, + end: CursorScalar, + cursor_start: CursorScalar | str | None, + cursor_type: str | None, + lookback: Duration | str | None, + backfill_duration: Duration | str | None, + has_start_override: bool, +) -> CursorScalar: ... +def cursor_inputs_without_rows( + *, cursor_snapshot: ModelCursorSnapshot +) -> EmptyCursorInputDecision: ... +def decide_empty_cursor_inputs( + *, + empty_input_names: tuple[str, ...], + waits_for_every_input: bool, + inputs_with_rows: bool, + has_window_start: bool, +) -> EmptyCursorInputDecision: ... +def empty_cursor_window(*, cursor_type: str | None, cursor_start: str | None) -> CursorBounds: ... +def cursor_window_override_flags(*, cursor_type: str | None) -> str: ... +def empty_input_rebuild_refusal( + *, + model_name: str, + input_names: tuple[str, ...], + cursor_type: str | None, +) -> tuple[str, str]: ... +def unreadable_cursor_inputs_error( + *, + model_name: str, + cursor_type: str | None, + input_names: tuple[str, ...], +) -> PlannerInputError: ... +def without_destination_cursor(*, cursor_snapshot: ModelCursorSnapshot) -> ModelCursorSnapshot: ... +def resolve_effective_timestamp_grain( + *, + cursor_type: str | None, + downstream_grain: str | None, + cursor_input_grains: tuple[str | None, ...], + microbatch_strategy: str | None = None, +) -> str | None: ... +def normalize_cursor_snapshot_grain( + *, + cursor_snapshot: ModelCursorSnapshot, + cursor_type: str | None, + effective_grain: str | None, +) -> ModelCursorSnapshot: ... def resolve_model_sql( *, adapter: BaseAdapter, diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py index c0a19606e0..ae8602015f 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py @@ -1,483 +1,53 @@ -"""Cursor bounds computation from warehouse snapshot and model config.""" +"""Cursor override typing and native cursor-window exports.""" from __future__ import annotations import re from collections.abc import Iterable -from dataclasses import replace -from datetime import datetime -from sqlbuild._native import cursor_floor_to_grain as floor_to_grain -from sqlbuild._native import cursor_inclusive_to_exclusive as inclusive_to_exclusive -from sqlbuild._native import cursor_max_bound as max_bound -from sqlbuild._native import cursor_min_bound as min_bound -from sqlbuild._native import cursor_observed_partition as observed_partition -from sqlbuild._native import cursor_parse as parse -from sqlbuild._native import cursor_try_parse as try_parse -from sqlbuild._native import get_config_str -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - CursorBounds, - CursorOverrides, - Duration, - EmptyCursorInputDecision, - MaximumStartPolicyInputs, - ModelCursorSnapshot, +from sqlbuild._native import ( + apply_typed_cursor_replay_policy as apply_typed_cursor_replay_policy, +) +from sqlbuild._native import ( + compute_cursor_bounds as compute_cursor_bounds, +) +from sqlbuild._native import ( + cursor_inputs_without_rows as cursor_inputs_without_rows, +) +from sqlbuild._native import ( + cursor_window_override_flags as cursor_window_override_flags, +) +from sqlbuild._native import ( + decide_empty_cursor_inputs as decide_empty_cursor_inputs, +) +from sqlbuild._native import ( + empty_cursor_window as empty_cursor_window, +) +from sqlbuild._native import ( + empty_input_rebuild_refusal as empty_input_rebuild_refusal, ) -from sqlbuild.compiler.planner.types import ( - CursorGrain, - CursorType, - CursorWatermarkMode, - MaterializationType, - MicrobatchStrategy, +from sqlbuild._native import ( + get_config_str, ) -from sqlbuild.cursor_algebra.constants import GRAIN_ORDER -from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue -from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar +from sqlbuild._native import ( + normalize_cursor_snapshot_grain as normalize_cursor_snapshot_grain, +) +from sqlbuild._native import ( + resolve_effective_timestamp_grain as resolve_effective_timestamp_grain, +) +from sqlbuild._native import ( + unreadable_cursor_inputs_error as unreadable_cursor_inputs_error, +) +from sqlbuild._native import ( + without_destination_cursor as without_destination_cursor, +) +from sqlbuild.compiler.compile.types import CompiledModelView +from sqlbuild.compiler.planner.exceptions import PlannerInputError +from sqlbuild.compiler.planner.models import CursorOverrides +from sqlbuild.compiler.planner.types import CursorType, MaterializationType _INTEGER_LITERAL: re.Pattern[str] = re.compile(r"[+-]?\d+") _LISTED_MODEL_LIMIT: int = 3 -_EMPTY_INTEGER_CURSOR_BOUND: int = 0 -_EMPTY_TIMESTAMP_CURSOR_BOUND: datetime = datetime(1970, 1, 1) - - -def compute_cursor_bounds( - *, - cursor_snapshot: ModelCursorSnapshot, - cursor_type: str | None, - cursor_start: str | None, - lookback: str | None, - backfill_duration: str | None, - start_cursor_override: str | None, - end_cursor_override: str | None, - is_microbatch: bool, - cursor_grain: str | None = None, - maximum_start_policy: MaximumStartPolicyInputs | None = None, -) -> CursorBounds | None: - """Compute effective cursor bounds for one incremental model.""" - - if is_microbatch: - return CursorBounds(start=BoundSentinel.START, end=BoundSentinel.END) - if not cursor_snapshot.watermarks_available: - return None - - raw_end: CursorScalar | None = _compute_raw_end( - snapshot=cursor_snapshot, cursor_type=cursor_type, cursor_grain=cursor_grain - ) - if raw_end is None: - return None - - raw_start: CursorScalar | None = _compute_raw_start( - snapshot=cursor_snapshot, cursor_type=cursor_type - ) - if raw_start is None: - return None - - if start_cursor_override is not None: - raw_start = _parse_required(value=start_cursor_override, cursor_type=cursor_type) - if end_cursor_override is not None: - raw_end = inclusive_to_exclusive( - value=_parse_required(value=end_cursor_override, cursor_type=cursor_type), - grain=( - None - if cursor_type == CursorType.INTEGER - else CursorGrain(cursor_grain or CursorGrain.SECOND) - ), - ) - - raw_start = apply_typed_cursor_replay_policy( - start=raw_start, - end=raw_end, - cursor_start=cursor_start, - cursor_type=cursor_type, - lookback=lookback, - backfill_duration=backfill_duration, - has_start_override=start_cursor_override is not None, - ) - - bounds: CursorBounds = CursorBounds(start=raw_start, end=raw_end) - from sqlbuild.compiler.planner._helpers.resolve.maximum_start import ( - apply_maximum_start_policy, - ) - - return apply_maximum_start_policy( - bounds=bounds, - snapshot=cursor_snapshot, - cursor_type=cursor_type, - cursor_grain=cursor_grain, - cursor_start=cursor_start, - lookback=lookback, - backfill_duration=backfill_duration, - policy=maximum_start_policy or MaximumStartPolicyInputs(), - has_start_override=start_cursor_override is not None, - ) - - -def cursor_inputs_without_rows(*, cursor_snapshot: ModelCursorSnapshot) -> EmptyCursorInputDecision: - """Return empty cursor inputs that leave no window; any unreadable input is a failure.""" - - if cursor_snapshot.unreadable_input_names: - return EmptyCursorInputDecision() - return decide_empty_cursor_inputs( - empty_input_names=cursor_snapshot.empty_input_names, - waits_for_every_input=cursor_snapshot.cursor_watermark_mode != CursorWatermarkMode.ANY, - inputs_with_rows=bool( - cursor_snapshot.upstream_maxes - or cursor_snapshot.upstream_terminal_ends - or cursor_snapshot.upstream_availability_ends - ), - has_window_start=bool( - cursor_snapshot.target_max is not None - or cursor_snapshot.upstream_mins - or cursor_snapshot.upstream_terminal_starts - ), - ) - - -def decide_empty_cursor_inputs( - *, - empty_input_names: tuple[str, ...], - waits_for_every_input: bool, - inputs_with_rows: bool, - has_window_start: bool, -) -> EmptyCursorInputDecision: - """Decide whether empty cursor inputs leave no window, shared by planning and runtime.""" - - if not empty_input_names: - return EmptyCursorInputDecision() - if not waits_for_every_input and inputs_with_rows and has_window_start: - return EmptyCursorInputDecision() - return EmptyCursorInputDecision( - input_names=empty_input_names, waiting_on_empty_inputs=inputs_with_rows - ) - - -def empty_cursor_window(*, cursor_type: str | None, cursor_start: str | None) -> CursorBounds: - """Return a zero-width cursor window, which selects no rows from any filtered input.""" - - bound: CursorScalar - if cursor_start is not None: - bound = parse(raw=cursor_start, cursor_type=cursor_type or CursorType.TIMESTAMP) - elif cursor_type == CursorType.INTEGER: - bound = IntegerValue(value=_EMPTY_INTEGER_CURSOR_BOUND) - else: - bound = TimestampValue(value=_EMPTY_TIMESTAMP_CURSOR_BOUND) - return CursorBounds(start=bound, end=bound) - - -def cursor_window_override_flags(*, cursor_type: str | None) -> str: - """Name the CLI flags that set a cursor window explicitly for this cursor type.""" - - suffix: str = "int" if cursor_type == CursorType.INTEGER else "ts" - return f"--start-cursor-{suffix} and --end-cursor-{suffix}" - - -def empty_input_rebuild_refusal( - *, model_name: str, input_names: tuple[str, ...], cursor_type: str | None -) -> tuple[str, str]: - """Return the message and help for refusing to rebuild an existing table from empty inputs.""" - - return ( - f"model '{model_name}' was not rebuilt: its cursor inputs have no rows " - f"({', '.join(input_names)}), so rebuilding it now would leave it empty", - "load the inputs first, or set the window explicitly with " - + cursor_window_override_flags(cursor_type=cursor_type), - ) - - -def unreadable_cursor_inputs_error( - *, model_name: str, cursor_type: str | None, input_names: tuple[str, ...] -) -> PlannerInputError: - """Report cursor inputs whose bounds could not be read, in terms of the model's inputs.""" - - return PlannerInputError( - f"model '{model_name}' cannot work out its cursor window: cursor bounds could not be " - f"read from {', '.join(input_names)}", - code="S302", - help=( - "check that each input exists and has its cursor column (`sqb debug` checks the " - "connection), or set the window explicitly with " - f"{cursor_window_override_flags(cursor_type=cursor_type)}" - ), - ) - - -def without_destination_cursor(*, cursor_snapshot: ModelCursorSnapshot) -> ModelCursorSnapshot: - """Return the snapshot as a first run sees it, ignoring the target's existing cursor.""" - - return replace( - cursor_snapshot, - target_max=None, - physical_target_max=None, - target_eligible_max=None, - ) - - -def resolve_effective_timestamp_grain( - *, - cursor_type: str | None, - downstream_grain: str | None, - cursor_input_grains: tuple[str | None, ...], - microbatch_strategy: str | None = None, -) -> str | None: - """Return the coarsest timestamp grain participating in cursor replay.""" - - if cursor_type != CursorType.TIMESTAMP: - return None - if microbatch_strategy == MicrobatchStrategy.WATERMARK: - return downstream_grain or CursorGrain.SECOND - effective: str = downstream_grain or CursorGrain.SECOND - input_grain: str | None - for input_grain in cursor_input_grains: - candidate: str = input_grain or CursorGrain.SECOND - if GRAIN_ORDER[CursorGrain(candidate)] > GRAIN_ORDER[CursorGrain(effective)]: - effective = candidate - return effective - - -def normalize_cursor_snapshot_grain( - *, - cursor_snapshot: ModelCursorSnapshot, - cursor_type: str | None, - effective_grain: str | None, -) -> ModelCursorSnapshot: - """Floor timestamp snapshot values to the effective replay grain.""" - - if cursor_type != CursorType.TIMESTAMP or effective_grain is None: - return cursor_snapshot - return ModelCursorSnapshot( - target_max=( - _normalize_timestamp_grain(value=cursor_snapshot.target_max, grain=effective_grain) - if cursor_snapshot.target_max is not None - else None - ), - upstream_mins=tuple( - _normalize_timestamp_grain(value=value, grain=effective_grain) - for value in cursor_snapshot.upstream_mins - ), - upstream_maxes=tuple( - _normalize_timestamp_grain(value=value, grain=effective_grain) - for value in cursor_snapshot.upstream_maxes - ), - physical_target_max=cursor_snapshot.physical_target_max or cursor_snapshot.target_max, - target_eligible_max=( - _normalize_timestamp_grain( - value=cursor_snapshot.target_eligible_max, grain=effective_grain - ) - if cursor_snapshot.target_eligible_max is not None - else None - ), - target_relation=cursor_snapshot.target_relation, - destination_cursor_column=cursor_snapshot.destination_cursor_column, - input_evidence=cursor_snapshot.input_evidence, - expected_watermark_count=cursor_snapshot.expected_watermark_count, - unavailable_watermark_tags=cursor_snapshot.unavailable_watermark_tags, - cursor_watermark_mode=cursor_snapshot.cursor_watermark_mode, - upstream_terminal_starts=cursor_snapshot.upstream_terminal_starts, - upstream_terminal_ends=cursor_snapshot.upstream_terminal_ends, - upstream_end_inputs=tuple( - ( - _normalize_timestamp_grain(value=physical, grain=effective_grain) - if physical is not None - else None, - terminal, - ) - for physical, terminal in cursor_snapshot.upstream_end_inputs - ), - upstream_availability_ends=cursor_snapshot.upstream_availability_ends, - empty_input_names=cursor_snapshot.empty_input_names, - unreadable_input_names=cursor_snapshot.unreadable_input_names, - ) - - -def apply_typed_cursor_replay_policy( - *, - start: CursorScalar, - end: CursorScalar, - cursor_start: CursorScalar | str | None, - cursor_type: str | None, - lookback: Duration | str | None, - backfill_duration: Duration | str | None, - has_start_override: bool, -) -> CursorScalar: - """Apply replay, lookback, and configured lower-bound policy to a typed cursor start.""" - - effective_start: CursorScalar = start - if backfill_duration is not None and not has_start_override: - adjusted_start: CursorScalar | None = _subtract_duration( - value=end, duration=backfill_duration - ) - if adjusted_start is not None: - effective_start = adjusted_start - if lookback is not None and not has_start_override and backfill_duration is None: - adjusted_lookback: CursorScalar | None = _subtract_duration( - value=effective_start, duration=lookback - ) - if adjusted_lookback is not None: - effective_start = adjusted_lookback - return _apply_cursor_start_floor( - current_start=effective_start, - cursor_start=cursor_start, - cursor_type=cursor_type, - ) - - -def _compute_raw_start( - *, snapshot: ModelCursorSnapshot, cursor_type: str | None -) -> CursorScalar | None: - """Derive the raw start bound from snapshot data.""" - - if snapshot.target_max is not None: - return snapshot.target_max - starts: tuple[CursorScalar, ...] = ( - *snapshot.upstream_mins, - *snapshot.upstream_terminal_starts, - ) - if not starts: - return None - return min_bound(values=starts, cursor_type=cursor_type or CursorType.TIMESTAMP) - - -def _compute_raw_end( - *, snapshot: ModelCursorSnapshot, cursor_type: str | None, cursor_grain: str | None -) -> CursorScalar | None: - """Derive the raw end bound from snapshot data.""" - - if snapshot.upstream_availability_ends: - exclusive_ends: tuple[CursorScalar, ...] = tuple( - _normalize_timestamp_grain(value=value, grain=cursor_grain) - if cursor_type == CursorType.TIMESTAMP and cursor_grain is not None - else value - for value in snapshot.upstream_availability_ends - ) - elif snapshot.upstream_end_inputs: - exclusive_ends = tuple( - terminal - if terminal is not None - else _advance_typed_discovered_cursor_end( - value=physical, - cursor_type=cursor_type, - cursor_grain=cursor_grain, - ) - for physical, terminal in snapshot.upstream_end_inputs - if terminal is not None or physical is not None - ) - else: - exclusive_ends = ( - tuple( - _advance_typed_discovered_cursor_end( - value=value, cursor_type=cursor_type, cursor_grain=cursor_grain - ) - for value in snapshot.upstream_maxes - ) - + snapshot.upstream_terminal_ends - ) - if not exclusive_ends: - return None - if snapshot.cursor_watermark_mode == CursorWatermarkMode.ANY: - return max_bound(values=exclusive_ends, cursor_type=cursor_type or CursorType.TIMESTAMP) - return min_bound(values=exclusive_ends, cursor_type=cursor_type or CursorType.TIMESTAMP) - - -def _subtract_duration(*, value: CursorScalar, duration: Duration | str) -> CursorScalar | None: - """Subtract a duration string from a cursor value.""" - - parsed: Duration | None = ( - duration if isinstance(duration, Duration) else Duration.parse(duration) - ) - if parsed is None: - return None - - if isinstance(value, TimestampValue): - return TimestampValue(value=parsed.subtract_from(value.value)) - if isinstance(value, DateValue): - timestamp: datetime = datetime.combine(value.value, datetime.min.time()) - return TimestampValue(value=parsed.subtract_from(timestamp)) - if isinstance(value, IntegerValue): - return IntegerValue(value=value.value - parsed.fixed_seconds) - - return None - - -def _normalize_timestamp_grain(*, value: CursorScalar, grain: str) -> CursorScalar: - return floor_to_grain(value=value, grain=CursorGrain(grain)) - - -def _apply_cursor_start_floor( - *, - current_start: CursorScalar, - cursor_start: CursorScalar | str | None, - cursor_type: str | None, -) -> CursorScalar: - if cursor_start is None: - return current_start - if cursor_type == CursorType.TIMESTAMP: - current_scalar: CursorScalar = current_start - floor_scalar: CursorScalar | None = ( - cursor_start - if isinstance(cursor_start, TimestampValue | DateValue | IntegerValue) - else try_parse(raw=cursor_start, cursor_type=CursorType.TIMESTAMP) - ) - current_timestamp: TimestampValue | None = ( - current_scalar - if isinstance(current_scalar, TimestampValue) - else TimestampValue(value=datetime.combine(current_scalar.value, datetime.min.time())) - if isinstance(current_scalar, DateValue) - else None - ) - floor_timestamp: TimestampValue | None = ( - floor_scalar - if isinstance(floor_scalar, TimestampValue) - else TimestampValue(value=datetime.combine(floor_scalar.value, datetime.min.time())) - if isinstance(floor_scalar, DateValue) - else None - ) - if current_timestamp is not None and floor_timestamp is not None: - return max_bound( - values=(current_timestamp, floor_timestamp), cursor_type=CursorType.TIMESTAMP - ) - return current_start - if cursor_type == CursorType.INTEGER: - current_integer: CursorScalar = current_start - floor_integer: CursorScalar | None = ( - cursor_start - if isinstance(cursor_start, TimestampValue | DateValue | IntegerValue) - else try_parse(raw=cursor_start, cursor_type=CursorType.INTEGER) - ) - if current_integer is not None and floor_integer is not None: - return max_bound( - values=(current_integer, floor_integer), cursor_type=CursorType.INTEGER - ) - return current_start - - -def _advance_typed_discovered_cursor_end( - *, value: CursorScalar | None, cursor_type: str | None, cursor_grain: str | None -) -> CursorScalar: - if value is None: - raise CursorAlgebraError("a discovered cursor value is required") - if cursor_type not in {CursorType.INTEGER, CursorType.TIMESTAMP}: - return value - grain: CursorGrain | None = ( - None - if cursor_type == CursorType.INTEGER - else CursorGrain( - cursor_grain - or (CursorGrain.SECOND if isinstance(value, TimestampValue) else CursorGrain.DAY) - ) - ) - return observed_partition(value=value, grain=grain).end - - -def _parse_required(*, value: str, cursor_type: str | None) -> CursorScalar: - parsed: CursorScalar | None = try_parse( - raw=value, cursor_type=cursor_type or CursorType.TIMESTAMP - ) - if parsed is None: - raise CursorAlgebraError(f"invalid cursor value: {value}") - return parsed def type_cursor_overrides( diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py index b2d83cadbf..2802c8ba56 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py @@ -1,197 +1,5 @@ -"""Cursor-bound computation and native per-model SQL resolution.""" +"""Native model, function and hook SQL resolution exports.""" -from __future__ import annotations - -from datetime import datetime - -from sqlbuild._native import ( - get_config_cursor_bound, - get_config_str, - resolve_bounded_cursor_override, - resolve_cursor_input_roles, -) from sqlbuild._native import resolve_function_sql as resolve_function_sql from sqlbuild._native import resolve_model_hook_entries as resolve_model_hook_entries from sqlbuild._native import resolve_model_sql as resolve_model_sql -from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - compute_cursor_bounds, - cursor_inputs_without_rows, - empty_cursor_window, - empty_input_rebuild_refusal, - unreadable_cursor_inputs_error, - without_destination_cursor, -) -from sqlbuild.compiler.planner._helpers.resolve.cursor_policies import ( - resolve_future_cursor_config, - resolve_start_cursor_config, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety -from sqlbuild.compiler.planner.models import ( - BackfillResult, - CursorBounds, - MaximumStartPolicyInputs, - ModelCursorSnapshot, - ModelPlanContext, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import BackfillAction, IncrementalMode, MaterializationType -from sqlbuild.cursor_algebra.types import BoundSentinel -from sqlbuild.spec.contracts.models import FutureCursorsConfig, StartCursorsConfig - - -def _compute_model_cursor_bounds( - *, - model: CompiledModelView, - snapshot: WarehouseSnapshot, - backfill: BackfillResult, - full_refresh: bool, - start_cursor_override: str | None, - end_cursor_override: str | None, - runtime_cursor_producer_names: frozenset[str], - context: ModelPlanContext, - replaces_relation: bool = False, -) -> CursorBounds | None: - """Compute cursor bounds for a model if it is incremental with a cursor.""" - - materialized: str | None = get_config_str(values=model.config.values, key="materialized") - cursor_column: str | None = get_config_str(values=model.config.values, key="cursor") - - if materialized != MaterializationType.INCREMENTAL or cursor_column is None: - return None - - incremental_mode: str | None = get_config_str( - values=model.config.values, key="incremental_mode" - ) - future_cursor_config: FutureCursorsConfig | None = resolve_future_cursor_config( - model=model, project_config=context.future_cursor_config - ) - start_cursor_config: StartCursorsConfig | None = resolve_start_cursor_config( - model=model, project_config=context.start_cursor_config - ) - invocation_time: datetime | None = context.invocation_time - is_microbatch: bool = incremental_mode == IncrementalMode.MICROBATCH - if is_microbatch: - return CursorBounds(start=BoundSentinel.START, end=BoundSentinel.END) - - bounded_override: CursorBounds | None = resolve_bounded_cursor_override( - start_cursor_override=start_cursor_override, - end_cursor_override=end_cursor_override, - cursor_type=get_config_str(values=model.config.values, key="cursor_type"), - cursor_grain=get_config_str(values=model.config.values, key="cursor_grain"), - ) - if bounded_override is not None: - return bounded_override - - cursor_snapshot: ModelCursorSnapshot | None = snapshot.cursor_snapshots.get(model.name) - runtime_owned: bool = bool( - set(resolve_cursor_input_roles(model=model).watermark_inputs) - & runtime_cursor_producer_names - ) - if not full_refresh and not runtime_owned and cursor_snapshot is not None: - empty_inputs: tuple[str, ...] = cursor_inputs_without_rows( - cursor_snapshot=cursor_snapshot - ).input_names - if ( - empty_inputs - and replaces_relation - and ( - _destination_exists(model=model, snapshot=snapshot) - or _history_redirected(model=model, cursor_snapshot=cursor_snapshot) - ) - ): - message, help_text = empty_input_rebuild_refusal( - model_name=model.name, - input_names=empty_inputs, - cursor_type=get_config_str(values=model.config.values, key="cursor_type"), - ) - raise PlannerInputError(message, code="S302", help=help_text) - if empty_inputs: - return empty_cursor_window( - cursor_type=get_config_str(values=model.config.values, key="cursor_type"), - cursor_start=get_config_cursor_bound( - values=model.config.values, key="cursor_start" - ), - ) - if not cursor_snapshot.watermarks_available: - raise unreadable_cursor_inputs_error( - model_name=model.name, - cursor_type=get_config_str(values=model.config.values, key="cursor_type"), - input_names=( - cursor_snapshot.unreadable_input_names - or cursor_snapshot.unavailable_watermark_tags - ), - ) - - if full_refresh: - return None - - if runtime_owned: - return CursorBounds(start=BoundSentinel.START, end=BoundSentinel.END) - if cursor_snapshot is None: - return None - if replaces_relation: - cursor_snapshot = without_destination_cursor(cursor_snapshot=cursor_snapshot) - - lookback: str | None = get_config_str(values=model.config.values, key="lookback") - cursor_start: str | None = get_config_cursor_bound( - values=model.config.values, key="cursor_start" - ) - backfill_duration: str | None = None - if backfill.action == BackfillAction.BOUNDED: - backfill_duration = backfill.duration - - bounds: CursorBounds | None = compute_cursor_bounds( - cursor_snapshot=cursor_snapshot, - cursor_type=get_config_str(values=model.config.values, key="cursor_type"), - cursor_start=cursor_start, - lookback=lookback, - backfill_duration=backfill_duration, - start_cursor_override=start_cursor_override, - end_cursor_override=end_cursor_override, - is_microbatch=is_microbatch, - cursor_grain=get_config_str(values=model.config.values, key="cursor_grain"), - maximum_start_policy=MaximumStartPolicyInputs( - config=start_cursor_config, - invocation_time=invocation_time, - incremental_strategy=get_config_str( - values=model.config.values, key="incremental_strategy" - ), - incremental_mode=incremental_mode, - ), - ) - if bounds is None: - return None - return apply_future_cursor_safety( - bounds=bounds, - cursor_type=get_config_str(values=model.config.values, key="cursor_type"), - cursor_grain=get_config_str(values=model.config.values, key="cursor_grain"), - config=future_cursor_config, - invocation_time=invocation_time, - has_complete_override=( - start_cursor_override is not None and end_cursor_override is not None - ), - input_evidence=cursor_snapshot.input_evidence, - ) - - -def _destination_exists(*, model: CompiledModelView, snapshot: WarehouseSnapshot) -> bool: - """Return whether the model's resolved destination, alias included, exists in full identity.""" - - relation: RelationInfo | None = snapshot.existing_relations.get(model.name) - return relation is not None and relation.identity == RelationLookup.key( - database=model.destination.database, - schema=model.destination.schema, - name=model.destination.name, - ) - - -def _history_redirected(*, model: CompiledModelView, cursor_snapshot: ModelCursorSnapshot) -> bool: - """Return whether the model's cursor history lives in a relation other than its destination.""" - - return ( - cursor_snapshot.target_relation is not None - and cursor_snapshot.target_relation != model.destination.qualified_name - ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py index a8db051a8f..62d9cf8e2d 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py @@ -3,11 +3,22 @@ from __future__ import annotations from dataclasses import dataclass +from datetime import datetime from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, field from sqlbuild.spec.contracts.models import SourceEntry +@dataclass(frozen=True) +class CursorPlannerTestCase: + """Expected bounds and rows through a custom cursor-rendering planner.""" + + description: str + expected_start: str + expected_end: str + expected_rows: list[tuple[datetime]] + + @dataclass(frozen=True) class OrchestrationTestCase: """Expected rows and ordered adapter calls across the resolution entry points.""" diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index f51c1f12e5..65e2308310 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -51,6 +51,18 @@ def render_table_function_call(self, *, target: str, call_suffix_sql: str) -> st return "(SELECT 9 AS order_id)" +class CursorLiteralOverrideAdapter(DuckDbAdapter): + """Keep custom cursor rendering observable through complete planner execution.""" + + def __init__(self) -> None: + self.cursor_literals: list[tuple[str, str | None]] = [] + + def render_cursor_bound_literal(self, *, value: str, cursor_type: str | None) -> str: + self.cursor_literals.append((value, cursor_type)) + literal: str = super().render_cursor_bound_literal(value=value, cursor_type=cursor_type) + return f"{literal} /* custom cursor literal */" + + def compile_resolution_project(project_dir: Path) -> CompiledProjectView: """Compile a real model, dependent table function and SQL hook project.""" diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_planning.py new file mode 100644 index 0000000000..4b3e677317 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_planning.py @@ -0,0 +1,56 @@ +"""Native cursor resolution through the planner and a custom warehouse adapter.""" + +from datetime import datetime +from typing import Any + +import pytest + +from sqlbuild.compiler.planner.models import CursorBounds, ModelPlanEntry, PlanOutput +from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( + CursorPlannerTestCase, +) +from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve.helpers import ( + CursorLiteralOverrideAdapter, +) +from tests.integration.src.sqlbuild.compiler.planner.main.helpers import ( + build_execution_plan_from_kwargs, + build_future_cursor_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + CursorPlannerTestCase( + description="leap-day source window preserves the observed start and advances the end", + expected_start="2024-02-28 12:00:00", + expected_end="2024-03-01T00:00:00", + expected_rows=[(datetime(2024, 2, 28, 12),), (datetime(2024, 2, 29, 23, 59, 59),)], + ) + ], + ids=lambda case: case.description, +) +def test_given_live_source_when_planning_then_custom_cursor_literals_execute_exact_window( + test_case: CursorPlannerTestCase, + connection: Any, +) -> None: + connection.execute("CREATE TABLE raw.events (occurred_at TIMESTAMP)") + connection.execute( + "INSERT INTO raw.events VALUES ('2024-02-28 12:00:00'), ('2024-02-29 23:59:59')" + ) + adapter: CursorLiteralOverrideAdapter = CursorLiteralOverrideAdapter() + plan: PlanOutput = build_execution_plan_from_kwargs( + project=build_future_cursor_project(), adapter=adapter, connection=connection + ) + entry: ModelPlanEntry = plan.model_entries[0] + assert entry.cursor_bounds == CursorBounds( + start=test_case.expected_start, end=test_case.expected_end + ) + assert "/* custom cursor literal */" in entry.resolved_sql + assert (test_case.expected_start, "timestamp") in adapter.cursor_literals + assert (test_case.expected_end, "timestamp") in adapter.cursor_literals + assert connection.execute(entry.resolved_sql).fetchall() == test_case.expected_rows + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py index 0b23b9b933..3a41b522c7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py @@ -3,6 +3,7 @@ from __future__ import annotations from datetime import UTC, date, datetime, timedelta, timezone +from zoneinfo import ZoneInfo import pytest @@ -23,7 +24,7 @@ CursorType, CursorWatermarkMode, ) -from sqlbuild.cursor_algebra.models import AlignedInterval, DateValue +from sqlbuild.cursor_algebra.models import AlignedInterval, DateValue, TimestampValue from sqlbuild.cursor_algebra.types import CursorScalar from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( CursorBoundsTestCase, @@ -141,6 +142,48 @@ def test_given_observed_timestamp_when_resolving_partition_then_boundaries_are_g @pytest.mark.parametrize( "test_case", [ + CursorBoundsTestCase( + description="negative integer override through zero has an exclusive end of one", + cursor_snapshot=ModelCursorSnapshot( + target_max="-2", + upstream_mins=("-9",), + upstream_maxes=("0",), + ), + cursor_type="integer", + lookback="1d", + backfill_duration="2d", + start_cursor_override="-3", + end_cursor_override="0", + is_microbatch=False, + expected_bounds=CursorBounds(start="-3", end="1"), + ), + CursorBoundsTestCase( + description="external timezone replay retains wall time across daylight saving", + cursor_snapshot=ModelCursorSnapshot( + target_max=TimestampValue( + value=datetime(2024, 3, 31, 12, tzinfo=ZoneInfo("Europe/Paris")) + ), + upstream_mins=( + TimestampValue(value=datetime(2024, 3, 1, tzinfo=ZoneInfo("Europe/Paris"))), + ), + upstream_maxes=( + TimestampValue(value=datetime(2024, 4, 1, 18, tzinfo=ZoneInfo("Europe/Paris"))), + ), + ), + cursor_type="timestamp", + cursor_grain="day", + lookback="1d", + backfill_duration=None, + start_cursor_override=None, + end_cursor_override=None, + is_microbatch=False, + expected_bounds=CursorBounds( + start=TimestampValue( + value=datetime(2024, 3, 30, 12, tzinfo=ZoneInfo("Europe/Paris")) + ), + end=TimestampValue(value=datetime(2024, 4, 2, tzinfo=ZoneInfo("Europe/Paris"))), + ), + ), CursorBoundsTestCase( description="normal incremental uses target max as start and min of upstream maxes as end", cursor_snapshot=ModelCursorSnapshot( From 65d5a8f3bf9510d0ea6a9d604fc4b70e30bcb73b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:07:14 +0100 Subject: [PATCH 14/29] chore: update Python boundary after cursor resolution port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 6dcf361f6e..d893897805 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 36995 +remaining_lines = 36373 [[stays]] path = "executor" From 777fcd658154aa7b1c3c708d7ec6cb8c1510d8bd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:41:46 +0100 Subject: [PATCH 15/29] refactor: move fixture alignment and rendering into native core --- crates/sqlbuild-analysis/src/lib.rs | 1 + .../src/planner_fixtures/_helpers/mod.rs | 1 + .../planner_fixtures/_helpers/schema_order.rs | 15 + .../src/planner_fixtures/constants.rs | 3 + .../planner_fixtures/main/align_columns.rs | 96 +++++ .../main/empty_fixture_error.rs | 25 ++ .../main/empty_fixture_marker.rs | 15 + .../planner_fixtures/main/merge_metadata.rs | 35 ++ .../src/planner_fixtures/main/mod.rs | 6 + .../planner_fixtures/main/render_completed.rs | 40 ++ .../src/planner_fixtures/main/render_empty.rs | 22 + .../src/planner_fixtures/mod.rs | 9 + .../src/planner_fixtures/models.rs | 52 +++ .../src/planner_fixtures/tests.rs | 6 + .../src/planner_fixtures/tests/helpers.rs | 10 + .../planner_fixtures/tests/test_alignment.rs | 101 +++++ .../src/planner_fixtures/tests/test_types.rs | 12 + .../planner_strategy/fixture_completion.rs | 211 +++++++++ .../planner_strategy/fixture_metadata.rs | 39 ++ .../planner_strategy/fixture_preparation.rs | 76 ++++ .../_helpers/planner_strategy/fixture_sql.rs | 136 ++++++ .../planner_strategy/fixture_values.rs | 83 ++++ .../bindings/_helpers/planner_strategy/mod.rs | 5 + .../_helpers/planner_strategy/registration.rs | 6 +- src/sqlbuild/_native.pyi | 22 + .../planner/_helpers/fixtures/completion.py | 403 +----------------- .../_helpers/fixtures/empty_fixture.py | 13 +- 27 files changed, 1046 insertions(+), 397 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/_helpers/schema_order.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/constants.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/main/align_columns.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/main/empty_fixture_error.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/main/empty_fixture_marker.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/main/merge_metadata.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/main/render_completed.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/main/render_empty.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/models.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/tests.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/tests/test_alignment.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index dac314a359..4cf17bc857 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -20,6 +20,7 @@ pub mod lineage_command; pub mod lineage_graph; pub mod manifest; pub mod planner_changes; +pub mod planner_fixtures; pub mod planner_identity; pub mod planner_migrations; pub mod planner_scope; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/mod.rs new file mode 100644 index 0000000000..163629ef76 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod schema_order; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/schema_order.rs b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/schema_order.rs new file mode 100644 index 0000000000..7e775adc77 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/schema_order.rs @@ -0,0 +1,15 @@ +//! Fixture alignment phases. + +use crate::planner_fixtures::models::AlignmentInput; + +pub(crate) fn supplied_in_schema_order(input: &AlignmentInput<'_>) -> bool { + input + .supplied + .iter() + .map(|column| &column.folded_name) + .eq(input + .metadata + .iter() + .take(input.supplied.len()) + .map(|column| &column.folded_name)) +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/constants.rs b/crates/sqlbuild-analysis/src/planner_fixtures/constants.rs new file mode 100644 index 0000000000..ec8f90170a --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/constants.rs @@ -0,0 +1,3 @@ +//! Empty fixture marker pattern. + +pub(crate) const EMPTY_PATTERN: &str = r"(?i)^[\s\x1c-\x1f]*SELECT[\s\x1c-\x1f]+\*[\s\x1c-\x1f]+FROM[\s\x1c-\x1f]+__empty_fixture[\s\x1c-\x1f]*\([\s\x1c-\x1f]*\)[\s\x1c-\x1f]*$"; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/align_columns.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/align_columns.rs new file mode 100644 index 0000000000..6d7a0ca9cd --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/align_columns.rs @@ -0,0 +1,96 @@ +//! Align supplied columns to their authoritative schema without rendering adapter SQL. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use crate::planner_fixtures::_helpers::schema_order::supplied_in_schema_order; +use crate::planner_fixtures::models::{Alignment, AlignmentInput, RequiredColumn}; + +/// Resolve missing columns, diagnostics and star-prefix ordering for one fixture. +pub fn align_columns(label: &str, input: AlignmentInput<'_>) -> Alignment { + let missing = missing_columns(&input); + if missing.is_empty() && !input.typed_nulls { + return Alignment::default(); + } + let metadata: HashMap<&str, usize> = input + .metadata + .iter() + .enumerate() + .map(|(index, column)| (column.folded_name.as_str(), index)) + .collect(); + let mut unknown: Vec<&str> = Vec::new(); + let mut non_nullable: Vec<&str> = Vec::new(); + let mut readers: BTreeSet<&str> = BTreeSet::new(); + let mut completed: Vec = Vec::new(); + for required in missing { + match metadata.get(required.folded_name.as_str()).copied() { + Some(index) if input.metadata[index].sql_type.is_some() => { + if input.metadata[index].nullable == Some(false) { + non_nullable.push(required.name.as_str()); + } else { + completed.push(index); + } + } + _ => { + unknown.push(required.name.as_str()); + readers.extend(required.readers.iter().map(String::as_str)); + } + } + } + let diagnostics = missing_diagnostics(label, &unknown, &non_nullable, readers); + if !diagnostics.is_empty() { + return Alignment { + diagnostics, + completed: None, + }; + } + if input.star && !supplied_in_schema_order(&input) { + return Alignment { + diagnostics: vec![format!( + "{label} supplies columns out of authoritative order for a SELECT * closure; provide the leading relation columns in schema order" + )], + completed: None, + }; + } + completed.sort_unstable(); + Alignment { + diagnostics, + completed: Some(completed), + } +} + +fn missing_columns<'a>(input: &AlignmentInput<'a>) -> Vec<&'a RequiredColumn> { + let supplied: HashSet<&str> = input + .supplied + .iter() + .map(|column| column.folded_name.as_str()) + .collect(); + let mut missing: Vec<_> = input + .required + .iter() + .filter(|column| !supplied.contains(column.folded_name.as_str())) + .collect(); + missing.sort_by(|left, right| left.name.cmp(&right.name)); + missing +} + +fn missing_diagnostics( + label: &str, + unknown: &[&str], + non_nullable: &[&str], + readers: BTreeSet<&str>, +) -> Vec { + let mut diagnostics: Vec = Vec::new(); + if !unknown.is_empty() { + diagnostics.push(format!( + "{label} is missing required columns: {}; provide explicit values because their types are not authoritative (read by: {})", + unknown.join(", "), readers.into_iter().collect::>().join(", ") + )); + } + if !non_nullable.is_empty() { + diagnostics.push(format!( + "{label} must provide required non-nullable columns: {}", + non_nullable.join(", ") + )); + } + diagnostics +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/empty_fixture_error.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/empty_fixture_error.rs new file mode 100644 index 0000000000..8026ea41b1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/empty_fixture_error.rs @@ -0,0 +1,25 @@ +//! Authoritative schema requirements for empty fixtures. + +use crate::planner_fixtures::models::FixtureColumn; + +/// Return the first schema failure before rendering any empty-fixture column. +pub fn empty_fixture_error(authoritative: bool, columns: &[FixtureColumn]) -> Option { + if !authoritative { + return Some("uses __empty_fixture() but its column schema is not authoritative".into()); + } + let unknown: Vec<_> = columns + .iter() + .filter(|column| column.sql_type.is_none()) + .map(|column| column.name.as_str()) + .collect(); + if !unknown.is_empty() { + return Some(format!( + "uses __empty_fixture() but columns have unknown types: {}", + unknown.join(", ") + )); + } + if columns.is_empty() { + return Some("uses __empty_fixture() but its schema has no columns".into()); + } + None +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/empty_fixture_marker.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/empty_fixture_marker.rs new file mode 100644 index 0000000000..172a028698 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/empty_fixture_marker.rs @@ -0,0 +1,15 @@ +//! Exact empty-fixture marker recognition. + +use crate::planner_fixtures::constants::EMPTY_PATTERN; +use regex::Regex; +use std::sync::OnceLock; + +/// Whether SQL consists solely of the empty-fixture marker. +pub fn empty_fixture_marker(sql: &str) -> Result { + static PATTERN: OnceLock> = OnceLock::new(); + PATTERN + .get_or_init(|| Regex::new(EMPTY_PATTERN)) + .as_ref() + .map(|pattern| pattern.is_match(sql)) + .map_err(Clone::clone) +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/merge_metadata.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/merge_metadata.rs new file mode 100644 index 0000000000..cf2d56e737 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/merge_metadata.rs @@ -0,0 +1,35 @@ +//! Merge inferred model columns with declared schema metadata. + +use std::collections::HashMap; + +use crate::planner_fixtures::models::FixtureColumn; + +/// Merge declarations over inferred columns while preserving inferred order. +pub fn merge_metadata( + inferred: Vec, + declared: Vec, +) -> Vec { + let mut order: Vec = Vec::new(); + let mut columns: HashMap = HashMap::new(); + for column in inferred { + order.push(column.folded_name.clone()); + columns.insert(column.folded_name.clone(), column); + } + for mut column in declared { + if let Some(inferred) = columns.get(&column.folded_name) { + if column.sql_type.as_ref().is_none_or(String::is_empty) { + column.sql_type.clone_from(&inferred.sql_type); + } + if column.nullable.is_none() { + column.nullable = inferred.nullable; + } + } else { + order.push(column.folded_name.clone()); + } + columns.insert(column.folded_name.clone(), column); + } + order + .into_iter() + .filter_map(|name| columns.get(&name).cloned()) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs new file mode 100644 index 0000000000..1bc35b463a --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs @@ -0,0 +1,6 @@ +pub mod align_columns; +pub mod empty_fixture_error; +pub mod empty_fixture_marker; +pub mod merge_metadata; +pub mod render_completed; +pub mod render_empty; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/render_completed.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/render_completed.rs new file mode 100644 index 0000000000..f095238175 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/render_completed.rs @@ -0,0 +1,40 @@ +//! Adapter-independent completion SQL with identifier callbacks. + +use crate::planner_fixtures::models::FixtureSqlInput; + +/// Render completion SQL while preserving identifier callback order. +pub fn render_completed( + input: FixtureSqlInput<'_>, + mut render: impl FnMut(&str, bool) -> Result, +) -> Result { + let alias = render("__sqlbuild_partial_fixture", false)?; + let mut projections: Vec = Vec::new(); + if input.typed_nulls { + for column in input.supplied { + let name = if column.quoted { + render(&column.name, true)? + } else { + column.name.clone() + }; + let source = format!("{alias}.{name}"); + projections.push(match &column.expected_type { + Some(sql_type) => format!("CAST({source} AS {sql_type}) AS {name}"), + None => source, + }); + } + } else { + projections.push(format!("{alias}.*")); + } + for column in input.completed { + let name = render(&column.name, false)?; + projections.push(format!( + "CAST(NULL AS {}) AS {name}", + column.sql_type.as_deref().unwrap_or("None") + )); + } + Ok(format!( + "SELECT\n {}\nFROM (\n{}\n) AS {alias}", + projections.join(",\n "), + input.sql + )) +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/render_empty.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/render_empty.rs new file mode 100644 index 0000000000..3778c43af3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/render_empty.rs @@ -0,0 +1,22 @@ +//! Adapter-independent empty-fixture projection SQL. + +use crate::planner_fixtures::models::FixtureColumn; + +/// Render the validated authoritative empty-fixture schema. +pub fn render_empty( + columns: &[FixtureColumn], + mut render: impl FnMut(&str) -> Result, +) -> Result { + let mut projections: Vec = Vec::new(); + for column in columns { + let name = render(&column.name)?; + projections.push(format!( + "CAST(NULL AS {}) AS {name}", + column.sql_type.as_deref().unwrap_or("None") + )); + } + Ok(format!( + "SELECT\n {}\nWHERE FALSE", + projections.join(",\n ") + )) +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs b/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs new file mode 100644 index 0000000000..b010d05a00 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs @@ -0,0 +1,9 @@ +//! Schema alignment and preparation of relation fixtures. + +mod _helpers; +mod constants; + +pub mod main; +pub mod models; +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/models.rs b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs new file mode 100644 index 0000000000..3e33082170 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs @@ -0,0 +1,52 @@ +//! Passive fixture schema and alignment facts. + +/// One authoritative or inferred fixture column. +#[derive(Clone, Debug)] +pub struct FixtureColumn { + pub name: String, + pub folded_name: String, + pub sql_type: Option, + pub nullable: Option, +} + +/// A required column and the models reading it. +#[derive(Debug)] +pub struct RequiredColumn { + pub name: String, + pub folded_name: String, + pub readers: Vec, +} + +/// Schema facts needed to align a partial fixture. +#[derive(Debug)] +pub struct AlignmentInput<'a> { + pub supplied: &'a [FixtureColumn], + pub metadata: &'a [FixtureColumn], + pub required: &'a [RequiredColumn], + pub star: bool, + pub typed_nulls: bool, +} + +/// Missing-column diagnostics and schema-ordered completion indices. +#[derive(Debug, Default)] +pub struct Alignment { + pub diagnostics: Vec, + pub completed: Option>, +} + +/// A supplied projection and its optional authoritative null type. +#[derive(Debug)] +pub struct SuppliedProjection { + pub name: String, + pub quoted: bool, + pub expected_type: Option, +} + +/// Completed fixture inputs independent of adapter rendering. +#[derive(Debug)] +pub struct FixtureSqlInput<'a> { + pub sql: &'a str, + pub supplied: &'a [SuppliedProjection], + pub typed_nulls: bool, + pub completed: &'a [FixtureColumn], +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs new file mode 100644 index 0000000000..bcfaf02203 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_alignment.rs"] +mod test_alignment; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/helpers.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/helpers.rs new file mode 100644 index 0000000000..51ca2f5bc9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/helpers.rs @@ -0,0 +1,10 @@ +use crate::planner_fixtures::models::FixtureColumn; + +pub(super) fn column(name: &str, sql_type: &str, nullable: bool) -> FixtureColumn { + FixtureColumn { + name: name.into(), + folded_name: name.into(), + sql_type: Some(sql_type.into()), + nullable: Some(nullable), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_alignment.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_alignment.rs new file mode 100644 index 0000000000..0274d14185 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_alignment.rs @@ -0,0 +1,101 @@ +use crate::planner_fixtures::main::align_columns::align_columns; +use crate::planner_fixtures::models::{AlignmentInput, RequiredColumn}; +use crate::planner_fixtures::tests::helpers::column; +use crate::planner_fixtures::tests::test_types::AlignmentTestCase; + +#[test] +fn given_partial_fixtures_when_aligning_then_preserves_schema_and_diagnostic_order() { + let test_cases = [ + AlignmentTestCase { + description: "nullable completion", + supplied: &["order_id"], + required: &["status", "details"], + star: false, + typed_nulls: false, + expected_completed: Some(vec![1, 2]), + expected_diagnostics: &[], + }, + AlignmentTestCase { + description: "unknown and required diagnostics", + supplied: &["details"], + required: &["unknown", "order_id"], + star: false, + typed_nulls: false, + expected_completed: None, + expected_diagnostics: &[ + "mock source 'orders' is missing required columns: unknown; provide explicit values because their types are not authoritative (read by: orders)", + "mock source 'orders' must provide required non-nullable columns: order_id", + ], + }, + AlignmentTestCase { + description: "schema prefix order", + supplied: &["status", "order_id"], + required: &["details"], + star: true, + typed_nulls: false, + expected_completed: None, + expected_diagnostics: &[ + "mock source 'orders' supplies columns out of authoritative order for a SELECT * closure; provide the leading relation columns in schema order", + ], + }, + AlignmentTestCase { + description: "complete fixture", + supplied: &["order_id", "status", "details"], + required: &["status"], + star: true, + typed_nulls: false, + expected_completed: None, + expected_diagnostics: &[], + }, + AlignmentTestCase { + description: "typed null fixture", + supplied: &["order_id", "status", "details"], + required: &["status"], + star: true, + typed_nulls: true, + expected_completed: Some(vec![]), + expected_diagnostics: &[], + }, + ]; + let metadata = vec![ + column("order_id", "INTEGER", false), + column("status", "VARCHAR", true), + column("details", "STRUCT(value INTEGER)", true), + ]; + for test_case in test_cases { + let supplied: Vec<_> = test_case + .supplied + .iter() + .map(|name| column(name, "UNKNOWN", true)) + .collect(); + let required: Vec<_> = test_case + .required + .iter() + .map(|name| RequiredColumn { + name: (*name).into(), + folded_name: (*name).into(), + readers: vec!["orders".into()], + }) + .collect(); + let actual = align_columns( + "mock source 'orders'", + AlignmentInput { + supplied: &supplied, + metadata: &metadata, + required: &required, + star: test_case.star, + typed_nulls: test_case.typed_nulls, + }, + ); + assert_eq!( + actual.completed, test_case.expected_completed, + "{}", + test_case.description + ); + assert_eq!( + actual.diagnostics, test_case.expected_diagnostics, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs new file mode 100644 index 0000000000..5aed7a8a9f --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs @@ -0,0 +1,12 @@ +//! Fixture alignment regression inputs. + +#[derive(Debug)] +pub(crate) struct AlignmentTestCase { + pub(crate) description: &'static str, + pub(crate) supplied: &'static [&'static str], + pub(crate) required: &'static [&'static str], + pub(crate) star: bool, + pub(crate) typed_nulls: bool, + pub(crate) expected_completed: Option>, + pub(crate) expected_diagnostics: &'static [&'static str], +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs new file mode 100644 index 0000000000..811ad0edea --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs @@ -0,0 +1,211 @@ +//! Native-hosted relation fixture completion over Python boundary models. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_fixtures::main::align_columns::align_columns; +use sqlbuild_analysis::planner_fixtures::models::{AlignmentInput, RequiredColumn}; + +use crate::bindings::_helpers::planner_strategy::fixture_preparation::{PreparedFixtures, prepare}; +use crate::bindings::_helpers::planner_strategy::fixture_sql::{CompletionSql, render_completed}; +use crate::bindings::_helpers::planner_strategy::fixture_values::{ + COMPLETION, columns, diagnostic, folded, label, record, strings, +}; + +fn helper<'py>( + py: Python<'py>, + name: &str, + kwargs: &Bound<'py, PyDict>, +) -> PyResult> { + py.import(COMPLETION)?.getattr(name)?.call((), Some(kwargs)) +} + +#[pyfunction] +#[pyo3(signature = (*, project, adapter, request, planning_context = None))] +fn build_relation_fixture_completion<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + request: (Bound<'py, PyAny>, Bound<'py, PyAny>), + planning_context: Option>, +) -> PyResult> { + let (ordered_model_names, fixture_groups) = request; + let py = project.py(); + let context = match planning_context { + Some(context) if context.is_truthy()? => context, + _ => { + let kwargs = PyDict::new(py); + kwargs.set_item("project", project)?; + helper(py, "build_relation_fixture_context", &kwargs)? + } + }; + let relations = context.getattr("relations")?.cast_into::()?; + let prepared = prepare( + &fixture_groups, + &relations, + adapter, + &context.getattr("fixture_inferences")?, + )?; + let kwargs = PyDict::new(py); + kwargs.set_item("inferred_by_fixture", &prepared.inferred)?; + kwargs.set_item( + "authoritative_columns", + context.getattr("authoritative_columns")?, + )?; + let fixture_keys = helper(py, "_fixture_keys_requiring_analysis", &kwargs)?; + let kwargs = PyDict::new(py); + kwargs.set_item("ordered_model_names", &ordered_model_names)?; + kwargs.set_item("model_map", context.getattr("models_by_name")?)?; + kwargs.set_item("fixture_keys", fixture_keys)?; + kwargs.set_item("relations", &relations)?; + kwargs.set_item( + "authoritative_columns", + context.getattr("authoritative_columns")?, + )?; + kwargs.set_item("adapter", adapter)?; + let required_and_stars = helper(py, "_required_fixture_columns", &kwargs)?; + let required = required_and_stars.get_item(0)?.cast_into::()?; + let stars = required_and_stars.get_item(1)?; + let kwargs = PyDict::new(py); + kwargs.set_item("base_expected_types", context.getattr("expected_types")?)?; + kwargs.set_item("ordered_model_names", &ordered_model_names)?; + kwargs.set_item("model_map", context.getattr("models_by_name")?)?; + let expected = helper(py, "_expected_fixture_types", &kwargs)?; + complete_prepared(prepared, (relations, required, stars), expected, adapter) +} + +fn complete_prepared<'py>( + mut prepared: PreparedFixtures<'py>, + plan: (Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyAny>), + expected: Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py = adapter.py(); + let (relations, required, stars) = plan; + let typed_nulls = PyDict::new(py); + for (key, names) in prepared.null_names.iter() { + let types = expected.call_method1("get", (&key, PyDict::new(py)))?; + let selected = PyDict::new(py); + for name in names.try_iter()? { + let name = name?; + let value = types.call_method1("get", (&name,))?; + if !value.is_none() { + selected.set_item(name, value)?; + } + } + if !selected.is_empty() { + typed_nulls.set_item(key, selected)?; + } + } + let kwargs = PyDict::new(py); + kwargs.set_item("inferred_by_fixture", &prepared.inferred)?; + kwargs.set_item("relations", &relations)?; + for value in helper(py, "_unknown_fixture_column_diagnostics", &kwargs)?.try_iter()? { + prepared.diagnostics.push(value?); + } + let completed_keys = PySet::empty(py)?; + for (key, required_columns) in required.iter() { + let Some(inferred) = prepared.inferred.get_item(&key)? else { + continue; + }; + let typed = typed_nulls + .get_item(&key)? + .map(|value| value.cast_into::()) + .transpose()? + .unwrap_or_else(|| PyDict::new(py)); + let metadata = match relations.get_item(&key)? { + Some(relation) => relation.getattr("columns")?, + None => PyTuple::empty(py).into_any(), + }; + let requirements = required_columns + .cast_into::()? + .iter() + .map(|(name, readers)| { + Ok(RequiredColumn { + name: name.extract()?, + folded_name: folded(&name)?, + readers: strings(&readers)?, + }) + }) + .collect::>>()?; + let alignment = align_columns( + &label(&key)?, + AlignmentInput { + supplied: &columns(&inferred, true)?, + metadata: &columns(&metadata, false)?, + required: &requirements, + star: stars.contains(&key)?, + typed_nulls: !typed.is_empty(), + }, + ); + for message in alignment.diagnostics { + prepared.diagnostics.push(diagnostic(&key, &message)?); + } + let Some(completed) = alignment.completed else { + continue; + }; + let completed = completed + .into_iter() + .map(|index| metadata.get_item(index)) + .collect::>>()?; + let sql: String = prepared + .sql + .get_item(&key)? + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err("fixture SQL"))? + .extract()?; + let quoted = prepared + .quoted_names + .get_item(&key)? + .unwrap_or(PyFrozenSet::empty(py)?.into_any()); + let sql = render_completed( + CompletionSql { + sql: &sql, + inferred, + typed_nulls: typed, + quoted_names: quoted, + completed: PyTuple::new(py, completed)?.into_any(), + }, + adapter, + )?; + prepared.sql.set_item(&key, sql)?; + completed_keys.add(key)?; + } + for (key, typed) in typed_nulls.iter() { + if completed_keys.contains(&key)? { + continue; + } + let inferred = prepared + .inferred + .get_item(&key)? + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err("fixture inference"))?; + let sql: String = prepared + .sql + .get_item(&key)? + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err("fixture SQL"))? + .extract()?; + let quoted = prepared + .quoted_names + .get_item(&key)? + .unwrap_or(PyFrozenSet::empty(py)?.into_any()); + let sql = render_completed( + CompletionSql { + sql: &sql, + inferred, + typed_nulls: typed.cast_into::()?, + quoted_names: quoted, + completed: PyTuple::empty(py).into_any(), + }, + adapter, + )?; + prepared.sql.set_item(key, sql)?; + } + let kwargs = PyDict::new(py); + kwargs.set_item("fixture_sql_by_key", prepared.sql)?; + kwargs.set_item("inferred_by_fixture", prepared.inferred)?; + kwargs.set_item("expected_types", expected)?; + kwargs.set_item("diagnostics", PyTuple::new(py, prepared.diagnostics)?)?; + record(py, "RelationFixtureCompletion", &kwargs) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_relation_fixture_completion, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs new file mode 100644 index 0000000000..f5801b126d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs @@ -0,0 +1,39 @@ +//! Model fixture schema materialization at the Python boundary. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::PyTuple; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_fixtures::main::merge_metadata::merge_metadata; + +use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, metadata}; + +#[pyfunction] +#[pyo3(signature = (*, model))] +fn model_fixture_metadata<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py = model.py(); + let inferred = model.getattr("inferred_columns")?; + let schema = model.getattr("schema_entry")?; + let declared = if schema.is_none() { + PyTuple::empty(py).into_any() + } else { + schema.getattr("columns")? + }; + let inferred_columns = if inferred.is_none() { + Vec::new() + } else { + columns(&inferred, true)? + }; + let merged = merge_metadata(inferred_columns, columns(&declared, false)?); + let authoritative = !model.getattr("fast_lineage_has_star")?.is_truthy()? + && (model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("contract",))? + .eq("enforced")? + || !inferred.is_none()); + metadata(py, &merged, authoritative) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(model_fixture_metadata, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs new file mode 100644 index 0000000000..b0b1446345 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs @@ -0,0 +1,76 @@ +//! Inference and schema preparation at the fixture Python boundary. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_analysis::planner_fixtures::main::empty_fixture_marker::empty_fixture_marker; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::planner_strategy::fixture_sql::complete_empty_fixture_sql; +use crate::bindings::_helpers::planner_strategy::fixture_values::{diagnostic, label}; + +pub(crate) struct PreparedFixtures<'py> { + pub(crate) sql: Bound<'py, PyDict>, + pub(crate) inferred: Bound<'py, PyDict>, + pub(crate) null_names: Bound<'py, PyDict>, + pub(crate) quoted_names: Bound<'py, PyDict>, + pub(crate) diagnostics: Vec>, +} + +pub(crate) fn prepare<'py>( + groups: &Bound<'py, PyAny>, + relations: &Bound<'py, PyDict>, + adapter: &Bound<'py, PyAny>, + inferences: &Bound<'py, PyAny>, +) -> PyResult> { + let py = groups.py(); + let mut prepared = PreparedFixtures { + sql: PyDict::new(py), + inferred: PyDict::new(py), + null_names: PyDict::new(py), + quoted_names: PyDict::new(py), + diagnostics: Vec::new(), + }; + for group in groups.try_iter()? { + let group = group?; + let kind = group.get_item(0)?; + let fixtures = group.get_item(1)?.cast_into::()?; + for (name, sql) in fixtures.iter() { + let key = PyTuple::new(py, [&kind, &name])?.into_any(); + let mut sql: String = sql.extract()?; + if empty_fixture_marker(&sql).map_err(value_error)? { + let relation = relations + .get_item(&key)? + .unwrap_or_else(|| py.None().into_bound(py)); + let (empty_sql, error) = complete_empty_fixture_sql(&sql, &relation, adapter)?; + if let Some(error) = error { + prepared + .diagnostics + .push(diagnostic(&key, &format!("{} {error}", label(&key)?))?); + prepared.sql.set_item(&key, &sql)?; + continue; + } + sql = empty_sql; + } + prepared.sql.set_item(&key, &sql)?; + let kwargs = PyDict::new(py); + kwargs.set_item("query_sql", &sql)?; + kwargs.set_item( + "inference_profile", + adapter.call_method0("expression_inference_profile")?, + )?; + let inference = inferences.call_method("infer", (), Some(&kwargs))?; + if !inference.is_none() { + prepared + .inferred + .set_item(&key, inference.getattr("columns")?)?; + prepared + .null_names + .set_item(&key, inference.getattr("null_literal_names")?)?; + prepared + .quoted_names + .set_item(&key, inference.getattr("quoted_names")?)?; + } + } + } + Ok(prepared) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs new file mode 100644 index 0000000000..e5f4eb6767 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs @@ -0,0 +1,136 @@ +//! Adapter-bound rendering of fixture completion SQL. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_fixtures::main::{ + empty_fixture_error::empty_fixture_error, empty_fixture_marker::empty_fixture_marker, + render_completed::render_completed as native_render_completed, render_empty::render_empty, +}; +use sqlbuild_analysis::planner_fixtures::models::{FixtureSqlInput, SuppliedProjection}; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::planner_strategy::fixture_values::columns; + +#[pyfunction(name = "is_empty_fixture_query")] +pub(crate) fn fixture_marker(sql: &str) -> PyResult { + empty_fixture_marker(sql).map_err(value_error) +} + +#[pyfunction] +#[pyo3(signature = (*, sql, relation, adapter))] +pub(crate) fn complete_empty_fixture_sql( + sql: &str, + relation: &Bound<'_, PyAny>, + adapter: &Bound<'_, PyAny>, +) -> PyResult<(String, Option)> { + if !fixture_marker(sql)? { + return Ok((sql.into(), None)); + } + let authoritative = + !relation.is_none() && relation.getattr("authoritative_names")?.is_truthy()?; + let metadata = if authoritative { + columns(&relation.getattr("columns")?, false)? + } else { + Vec::new() + }; + if let Some(error) = empty_fixture_error(authoritative, &metadata) { + return Ok((sql.into(), Some(error))); + } + Ok(( + render_empty(&metadata, |name| { + adapter + .call_method1("render_identifier", (name,))? + .extract::() + })?, + None, + )) +} + +pub(crate) struct CompletionSql<'py, 'a> { + pub(crate) sql: &'a str, + pub(crate) inferred: Bound<'py, PyAny>, + pub(crate) typed_nulls: Bound<'py, PyDict>, + pub(crate) quoted_names: Bound<'py, PyAny>, + pub(crate) completed: Bound<'py, PyAny>, +} + +pub(crate) fn render_completed( + input: CompletionSql<'_, '_>, + adapter: &Bound<'_, PyAny>, +) -> PyResult { + let mut supplied: Vec = Vec::new(); + if !input.typed_nulls.is_empty() { + for column in input.inferred.try_iter()? { + let column = column?; + let name = column.getattr("name")?; + let folded = name.call_method0("casefold")?; + supplied.push(SuppliedProjection { + name: name.extract()?, + quoted: input.quoted_names.contains(&folded)?, + expected_type: input + .typed_nulls + .get_item(folded)? + .map(|value| value.extract()) + .transpose()?, + }); + } + } + native_render_completed( + FixtureSqlInput { + sql: input.sql, + supplied: &supplied, + typed_nulls: !input.typed_nulls.is_empty(), + completed: &columns(&input.completed, false)?, + }, + |name, exact| { + adapter + .call_method1( + if exact { + "render_exact_identifier" + } else { + "render_identifier" + }, + (name,), + )? + .extract::() + }, + ) +} + +#[pyfunction] +#[pyo3(signature = (*, sql, inference, expected_types, adapter))] +pub(crate) fn complete_typed_null_columns( + sql: &str, + inference: &Bound<'_, PyAny>, + expected_types: &Bound<'_, PyDict>, + adapter: &Bound<'_, PyAny>, +) -> PyResult { + let py = inference.py(); + let typed_nulls = PyDict::new(py); + for name in inference.getattr("null_literal_names")?.try_iter()? { + let name = name?; + if let Some(expected) = expected_types.get_item(&name)? { + typed_nulls.set_item(name, expected)?; + } + } + if typed_nulls.is_empty() { + return Ok(sql.into()); + } + render_completed( + CompletionSql { + sql, + inferred: inference.getattr("columns")?, + typed_nulls, + quoted_names: inference.getattr("quoted_names")?, + completed: pyo3::types::PyTuple::empty(py).into_any(), + }, + adapter, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(fixture_marker, module)?)?; + module.add_function(wrap_pyfunction!(complete_empty_fixture_sql, module)?)?; + module.add_function(wrap_pyfunction!(complete_typed_null_columns, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs new file mode 100644 index 0000000000..0f35b03b71 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs @@ -0,0 +1,83 @@ +//! Fixture boundary records and adapter callbacks. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; + +pub(crate) const MODELS: &str = "sqlbuild.compiler.planner.models"; +pub(crate) const COMPLETION: &str = "sqlbuild.compiler.planner._helpers.fixtures.completion"; + +pub(crate) fn folded(value: &Bound<'_, PyAny>) -> PyResult { + value.call_method0("casefold")?.extract() +} + +pub(crate) fn strings(values: &Bound<'_, PyAny>) -> PyResult> { + values.try_iter()?.map(|value| value?.extract()).collect() +} + +pub(crate) fn columns(values: &Bound<'_, PyAny>, inferred: bool) -> PyResult> { + values + .try_iter()? + .map(|value| { + let value = value?; + let name = value.getattr("name")?; + Ok(FixtureColumn { + name: name.extract()?, + folded_name: folded(&name)?, + sql_type: value.getattr("type")?.extract()?, + nullable: if inferred { + None + } else { + value.getattr("nullable")?.extract()? + }, + }) + }) + .collect() +} + +pub(crate) fn record<'py>( + py: Python<'py>, + name: &str, + kwargs: &Bound<'py, PyDict>, +) -> PyResult> { + py.import(MODELS)?.getattr(name)?.call((), Some(kwargs)) +} + +pub(crate) fn metadata<'py>( + py: Python<'py>, + columns: &[FixtureColumn], + authoritative: bool, +) -> PyResult> { + let values = columns + .iter() + .map(|column| { + let kwargs = PyDict::new(py); + kwargs.set_item("name", &column.name)?; + kwargs.set_item("type", &column.sql_type)?; + kwargs.set_item("nullable", column.nullable)?; + record(py, "FixtureColumnMetadata", &kwargs) + }) + .collect::>>()?; + let kwargs = PyDict::new(py); + kwargs.set_item("columns", PyTuple::new(py, values)?)?; + kwargs.set_item("authoritative_names", authoritative)?; + record(py, "FixtureRelationMetadata", &kwargs) +} + +pub(crate) fn label(key: &Bound<'_, PyAny>) -> PyResult { + Ok(format!( + "mock {} '{}'", + key.get_item(0)?.getattr("value")?.extract::()?, + key.get_item(1)?.extract::()? + )) +} + +pub(crate) fn diagnostic<'py>( + key: &Bound<'py, PyAny>, + message: &str, +) -> PyResult> { + let kwargs = PyDict::new(key.py()); + kwargs.set_item("key", key)?; + kwargs.set_item("message", message)?; + record(key.py(), "RelationFixtureDiagnostic", &kwargs) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs index 0ec94542c6..0099559538 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs @@ -1,5 +1,10 @@ pub(crate) mod audits; pub(crate) mod cursor_types; +pub(crate) mod fixture_completion; +pub(crate) mod fixture_metadata; +pub(crate) mod fixture_preparation; +pub(crate) mod fixture_sql; +pub(crate) mod fixture_values; pub(crate) mod lineage; pub(crate) mod microbatches; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs index e4a631cc0f..d0332ef440 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -3,12 +3,16 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::planner_strategy::{ - audits, cursor_types, lineage, microbatches, strategy, + audits, cursor_types, fixture_completion, fixture_metadata, fixture_sql, lineage, microbatches, + strategy, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { audits::register(module)?; cursor_types::register(module)?; + fixture_completion::register(module)?; + fixture_metadata::register(module)?; + fixture_sql::register(module)?; lineage::register(module)?; microbatches::register(module)?; strategy::register(module) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index a83e0277c1..42194e0d05 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -233,6 +233,8 @@ from sqlbuild.compiler.planner.models import ( PlanWarning, QueryShape, ReferenceRename, + RelationFixtureCompletion, + RelationFixturePlanningContext, SchemaAction, SchemaFinding, SelectionStalenessGraph, @@ -247,6 +249,7 @@ from sqlbuild.compiler.planner.models import ( from sqlbuild.compiler.planner.types import ( CursorGrain, CursorType, + FixtureGroups, InputColumns, MaterializationType, OnSchemaChange, @@ -2437,6 +2440,25 @@ def scan_fixture_typed_null_candidates( project_dir: Path, selected_paths: frozenset[Path] | None = None ) -> tuple[frozenset[Path], frozenset[str]]: ... def simplify_fixture_null_projections(*, sql: str, relation: FixtureRelationMetadata) -> str: ... +def is_empty_fixture_query(sql: str) -> bool: ... +def complete_empty_fixture_sql( + *, sql: str, relation: FixtureRelationMetadata | None, adapter: BaseAdapter +) -> tuple[str, str | None]: ... +def complete_typed_null_columns( + *, + sql: str, + inference: FixtureColumnInference, + expected_types: dict[str, str], + adapter: BaseAdapter, +) -> str: ... +def model_fixture_metadata(*, model: CompiledModelView) -> FixtureRelationMetadata: ... +def build_relation_fixture_completion( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + request: tuple[tuple[str, ...], FixtureGroups], + planning_context: RelationFixturePlanningContext | None = None, +) -> RelationFixtureCompletion: ... def format_redundant_fixture_nulls( *, files: dict[Path, str], diff --git a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py b/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py index 1f1389a977..fd24ef88cc 100644 --- a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py +++ b/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py @@ -8,14 +8,27 @@ from functools import lru_cache from typing import Any -from sqlbuild._native import analyze_resolved_column_reads +from sqlbuild._native import ( + analyze_resolved_column_reads, +) +from sqlbuild._native import ( + build_relation_fixture_completion as _native_fixture_completion, +) +from sqlbuild._native import ( + complete_empty_fixture_sql as complete_empty_fixture_sql, +) +from sqlbuild._native import ( + complete_typed_null_columns as complete_typed_null_columns, +) +from sqlbuild._native import ( + model_fixture_metadata as _model_fixture_metadata, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, CompiledLineageSourceFact, CompileSqlReference, DynamicColumnContractProof, - FixtureColumnInference, InferredColumn, ) from sqlbuild.compiler.compile.types import ( @@ -27,8 +40,6 @@ ColumnLineageConfidence, ColumnTransformKind, ) -from sqlbuild.compiler.planner._helpers.fixtures.empty_fixture import is_empty_fixture_query -from sqlbuild.compiler.planner.classes.fixture_column_inferences import FixtureColumnInferences from sqlbuild.compiler.planner.models import ( FixtureColumnMetadata, FixtureRelationMetadata, @@ -41,7 +52,6 @@ from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql -_PARTIAL_FIXTURE_ALIAS: str = "__sqlbuild_partial_fixture" _DBT_REF_PATTERN: re.Pattern[str] = re.compile(r'__dbt_ref\("([^"]+)"\s*,\s*"([^"]+)"\)') @@ -55,294 +65,14 @@ def build_relation_fixture_completion( ) -> RelationFixtureCompletion: """Complete required omitted columns with typed nulls unless explicitly non-nullable.""" - context: RelationFixturePlanningContext = planning_context or build_relation_fixture_context( - project=project - ) - model_map: dict[str, CompiledModelView] = context.models_by_name - relations: dict[FixtureKey, FixtureRelationMetadata] = context.relations - fixture_sql_by_key: dict[FixtureKey, str] - inferred_by_fixture: dict[FixtureKey, tuple[InferredColumn, ...]] - null_literal_names_by_fixture: dict[FixtureKey, frozenset[str]] - quoted_names_by_fixture: dict[FixtureKey, frozenset[str]] - diagnostics: list[RelationFixtureDiagnostic] - ( - fixture_sql_by_key, - inferred_by_fixture, - null_literal_names_by_fixture, - quoted_names_by_fixture, - diagnostics, - ) = _prepare_fixture_inputs( - fixture_groups=fixture_groups, - relations=relations, - adapter=adapter, - inferences=context.fixture_inferences, - ) - - fixture_keys_requiring_analysis: frozenset[FixtureKey] = _fixture_keys_requiring_analysis( - inferred_by_fixture=inferred_by_fixture, - authoritative_columns=context.authoritative_columns, - ) - required: dict[FixtureKey, dict[str, set[str]]] - star_fixture_keys: frozenset[FixtureKey] - required, star_fixture_keys = _required_fixture_columns( - ordered_model_names=ordered_model_names, - model_map=model_map, - fixture_keys=fixture_keys_requiring_analysis, - relations=relations, - authoritative_columns=context.authoritative_columns, - adapter=adapter, - ) - expected_types: Mapping[FixtureKey, dict[str, str]] = _expected_fixture_types( - base_expected_types=context.expected_types, - ordered_model_names=ordered_model_names, - model_map=model_map, - ) - typed_nulls_by_fixture: dict[FixtureKey, dict[str, str]] = _typed_null_fixture_types( - null_literal_names_by_fixture=null_literal_names_by_fixture, - expected_types=expected_types, - ) - diagnostics.extend( - _unknown_fixture_column_diagnostics( - inferred_by_fixture=inferred_by_fixture, relations=relations - ) - ) - - completed_keys: set[FixtureKey] = set() - for key, required_columns in required.items(): - inferred: tuple[InferredColumn, ...] | None = inferred_by_fixture.get(key) - if inferred is None: - continue - available_names: set[str] = {column.name.casefold() for column in inferred} - missing: tuple[str, ...] = tuple( - sorted(name for name in required_columns if name.casefold() not in available_names) - ) - typed_nulls: dict[str, str] = typed_nulls_by_fixture.get(key, {}) - if not missing and not typed_nulls: - continue - relation: FixtureRelationMetadata | None = relations.get(key) - metadata: tuple[FixtureColumnMetadata, ...] = ( - relation.columns if relation is not None else () - ) - metadata_by_name: dict[str, FixtureColumnMetadata] = { - column.name.casefold(): column for column in metadata - } - nullable_columns: list[FixtureColumnMetadata] = [] - unknown: list[str] = [] - non_nullable: list[str] = [] - for name in missing: - column: FixtureColumnMetadata | None = metadata_by_name.get(name.casefold()) - if column is None or column.type is None: - unknown.append(name) - elif column.nullable is False: - non_nullable.append(name) - else: - nullable_columns.append(column) - if unknown: - read_by: tuple[str, ...] = _models_reading_columns( - column_names=tuple(unknown), - required_columns=required_columns, - ) - diagnostics.append( - RelationFixtureDiagnostic( - key=key, - message=( - f"mock {key[0].value} '{key[1]}' is missing required columns: " - f"{', '.join(unknown)}; provide explicit values because their types are " - f"not authoritative (read by: {', '.join(read_by)})" - ), - ) - ) - if non_nullable: - diagnostics.append( - RelationFixtureDiagnostic( - key=key, - message=( - f"mock {key[0].value} '{key[1]}' must provide required non-nullable " - f"columns: {', '.join(non_nullable)}" - ), - ) - ) - if unknown or non_nullable: - continue - if key in star_fixture_keys and not _supplied_columns_form_schema_prefix( - inferred=inferred, - metadata=metadata, - ): - diagnostics.append( - RelationFixtureDiagnostic( - key=key, - message=( - f"mock {key[0].value} '{key[1]}' supplies columns out of authoritative " - "order for a SELECT * closure; provide the leading relation columns in " - "schema order" - ), - ) - ) - continue - metadata_order: dict[str, int] = { - column.name.casefold(): index for index, column in enumerate(metadata) - } - nullable_columns.sort(key=lambda column: metadata_order[column.name.casefold()]) - fixture_sql_by_key[key] = _completed_fixture_sql( - sql=fixture_sql_by_key[key], - inferred=inferred, - typed_nulls=typed_nulls, - quoted_names=quoted_names_by_fixture.get(key, frozenset()), - completed=tuple(nullable_columns), - adapter=adapter, - ) - completed_keys.add(key) - - for key, typed_nulls in typed_nulls_by_fixture.items(): - if key in completed_keys: - continue - inferred = inferred_by_fixture[key] - fixture_sql_by_key[key] = _completed_fixture_sql( - sql=fixture_sql_by_key[key], - inferred=inferred, - typed_nulls=typed_nulls, - quoted_names=quoted_names_by_fixture.get(key, frozenset()), - completed=(), - adapter=adapter, - ) - - return RelationFixtureCompletion( - fixture_sql_by_key=fixture_sql_by_key, - inferred_by_fixture=inferred_by_fixture, - expected_types=expected_types, - diagnostics=tuple(diagnostics), - ) - - -def _prepare_fixture_inputs( - *, - fixture_groups: FixtureGroups, - relations: dict[FixtureKey, FixtureRelationMetadata], - adapter: BaseAdapter, - inferences: FixtureColumnInferences, -) -> tuple[ - dict[FixtureKey, str], - dict[FixtureKey, tuple[InferredColumn, ...]], - dict[FixtureKey, frozenset[str]], - dict[FixtureKey, frozenset[str]], - list[RelationFixtureDiagnostic], -]: - fixture_sql_by_key: dict[FixtureKey, str] = {} - inferred_by_fixture: dict[FixtureKey, tuple[InferredColumn, ...]] = {} - null_literal_names_by_fixture: dict[FixtureKey, frozenset[str]] = {} - quoted_names_by_fixture: dict[FixtureKey, frozenset[str]] = {} - diagnostics: list[RelationFixtureDiagnostic] = [] - for resource_type, fixtures in fixture_groups: - for name, sql in fixtures.items(): - key: FixtureKey = (resource_type, name) - fixture_sql: str = sql - if is_empty_fixture_query(fixture_sql): - empty_sql, empty_error = complete_empty_fixture_sql( - sql=fixture_sql, - relation=relations.get(key), - adapter=adapter, - ) - if empty_error is not None: - diagnostics.append( - RelationFixtureDiagnostic( - key=key, - message=f"mock {key[0].value} '{key[1]}' {empty_error}", - ) - ) - fixture_sql_by_key[key] = fixture_sql - continue - fixture_sql = empty_sql - fixture_sql_by_key[key] = fixture_sql - inference: FixtureColumnInference | None = inferences.infer( - query_sql=fixture_sql, - inference_profile=adapter.expression_inference_profile(), - ) - if inference is not None: - inferred_by_fixture[key] = inference.columns - null_literal_names_by_fixture[key] = inference.null_literal_names - quoted_names_by_fixture[key] = inference.quoted_names - return ( - fixture_sql_by_key, - inferred_by_fixture, - null_literal_names_by_fixture, - quoted_names_by_fixture, - diagnostics, - ) - - -def complete_typed_null_columns( - *, - sql: str, - inference: FixtureColumnInference, - expected_types: dict[str, str], - adapter: BaseAdapter, -) -> str: - """Apply authoritative types to direct bare-NULL projections.""" - - typed_nulls: dict[str, str] = { - name: expected_types[name] - for name in inference.null_literal_names - if name in expected_types - } - if not typed_nulls: - return sql - return _completed_fixture_sql( - sql=sql, - inferred=inference.columns, - typed_nulls=typed_nulls, - quoted_names=inference.quoted_names, - completed=(), + return _native_fixture_completion( + project=project, adapter=adapter, + request=(ordered_model_names, fixture_groups), + planning_context=planning_context, ) -def complete_empty_fixture_sql( - *, - sql: str, - relation: FixtureRelationMetadata | None, - adapter: BaseAdapter, -) -> tuple[str, str | None]: - """Expand an empty-fixture marker from authoritative relation metadata.""" - - if not is_empty_fixture_query(sql): - return sql, None - if relation is None or not relation.authoritative_names: - return sql, "uses __empty_fixture() but its column schema is not authoritative" - unknown_types: tuple[str, ...] = tuple( - column.name for column in relation.columns if column.type is None - ) - if unknown_types: - return ( - sql, - f"uses __empty_fixture() but columns have unknown types: {', '.join(unknown_types)}", - ) - if not relation.columns: - return sql, "uses __empty_fixture() but its schema has no columns" - projections: str = ",\n ".join( - f"CAST(NULL AS {column.type}) AS {adapter.render_identifier(column.name)}" - for column in relation.columns - ) - return f"SELECT\n {projections}\nWHERE FALSE", None - - -def _typed_null_fixture_types( - *, - null_literal_names_by_fixture: dict[FixtureKey, frozenset[str]], - expected_types: Mapping[FixtureKey, dict[str, str]], -) -> dict[FixtureKey, dict[str, str]]: - result: dict[FixtureKey, dict[str, str]] = {} - for key, null_literal_names in null_literal_names_by_fixture.items(): - fixture_types: dict[str, str] = {} - expected_fixture_types: dict[str, str] = expected_types.get(key, {}) - for name in null_literal_names: - expected_type: str | None = expected_fixture_types.get(name) - if expected_type is not None: - fixture_types[name] = expected_type - if fixture_types: - result[key] = fixture_types - return result - - def _fixture_keys_requiring_analysis( *, inferred_by_fixture: dict[FixtureKey, tuple[InferredColumn, ...]], @@ -624,43 +354,6 @@ def _fixture_relation_metadata( return relations -def _model_fixture_metadata(*, model: CompiledModelView) -> FixtureRelationMetadata: - columns_by_name: dict[str, FixtureColumnMetadata] = {} - order: list[str] = [] - for column in model.inferred_columns or (): - key: str = column.name.casefold() - order.append(key) - columns_by_name[key] = FixtureColumnMetadata( - name=column.name, - type=column.type, - nullable=None, - ) - for column in model.schema_entry.columns if model.schema_entry is not None else (): - key = column.name.casefold() - if key not in columns_by_name: - order.append(key) - inferred: FixtureColumnMetadata | None = columns_by_name.get(key) - columns_by_name[key] = FixtureColumnMetadata( - name=column.name, - type=column.type or (inferred.type if inferred is not None else None), - nullable=( - column.nullable - if column.nullable is not None - else (inferred.nullable if inferred is not None else None) - ), - ) - return FixtureRelationMetadata( - columns=tuple(columns_by_name[key] for key in order), - authoritative_names=( - not model.fast_lineage_has_star - and ( - model.config.values.get("contract") == ContractPolicy.ENFORCED - or model.inferred_columns is not None - ) - ), - ) - - def _expected_fixture_types( *, base_expected_types: dict[FixtureKey, dict[str, str]], @@ -732,61 +425,3 @@ def _unknown_fixture_column_diagnostics( ) ) return tuple(diagnostics) - - -def _models_reading_columns( - *, column_names: tuple[str, ...], required_columns: dict[str, set[str]] -) -> tuple[str, ...]: - model_names: set[str] = set() - for name in column_names: - model_names.update(required_columns[name]) - return tuple(sorted(model_names)) - - -def _supplied_columns_form_schema_prefix( - *, - inferred: tuple[InferredColumn, ...], - metadata: tuple[FixtureColumnMetadata, ...], -) -> bool: - supplied_names: tuple[str, ...] = tuple(column.name.casefold() for column in inferred) - schema_prefix: tuple[str, ...] = tuple( - column.name.casefold() for column in metadata[: len(supplied_names)] - ) - return supplied_names == schema_prefix - - -def _completed_fixture_sql( - *, - sql: str, - inferred: tuple[InferredColumn, ...], - typed_nulls: dict[str, str], - quoted_names: frozenset[str], - completed: tuple[FixtureColumnMetadata, ...], - adapter: BaseAdapter, -) -> str: - alias: str = adapter.render_identifier(_PARTIAL_FIXTURE_ALIAS) - projections: list[str] - if typed_nulls: - projections = [] - for supplied_column in inferred: - rendered_name: str = ( - adapter.render_exact_identifier(supplied_column.name) - if supplied_column.name.casefold() in quoted_names - else supplied_column.name - ) - source_expression: str = f"{alias}.{rendered_name}" - expected_type: str | None = typed_nulls.get(supplied_column.name.casefold()) - projections.append( - source_expression - if expected_type is None - else f"CAST({source_expression} AS {expected_type}) AS {rendered_name}" - ) - else: - projections = [f"{alias}.*"] - for completed_column in completed: - projections.append( - f"CAST(NULL AS {completed_column.type}) AS " - f"{adapter.render_identifier(completed_column.name)}" - ) - projection_sql: str = ",\n ".join(projections) - return f"SELECT\n {projection_sql}\nFROM (\n{sql}\n) AS {alias}" diff --git a/src/sqlbuild/compiler/planner/_helpers/fixtures/empty_fixture.py b/src/sqlbuild/compiler/planner/_helpers/fixtures/empty_fixture.py index 868fdbf5ad..e5f142cbe5 100644 --- a/src/sqlbuild/compiler/planner/_helpers/fixtures/empty_fixture.py +++ b/src/sqlbuild/compiler/planner/_helpers/fixtures/empty_fixture.py @@ -1,14 +1,3 @@ """Recognition for the planner's empty relation fixture marker.""" -import re - -_EMPTY_FIXTURE_PATTERN: re.Pattern[str] = re.compile( - r"^\s*SELECT\s+\*\s+FROM\s+__empty_fixture\s*\(\s*\)\s*$", - re.IGNORECASE, -) - - -def is_empty_fixture_query(sql: str) -> bool: - """Return whether SQL is exactly the empty relation fixture marker.""" - - return _EMPTY_FIXTURE_PATTERN.fullmatch(sql) is not None +from sqlbuild._native import is_empty_fixture_query as is_empty_fixture_query From e4342a83a6a26c987b38338ca36411a00fbc0ab1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:43:07 +0100 Subject: [PATCH 16/29] chore: update Python boundary after fixture alignment port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index d893897805..446a526d92 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 36373 +remaining_lines = 35997 [[stays]] path = "executor" From b0d0cf8d759db39f471ed936496e0d4a793d27c7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:50:11 +0100 Subject: [PATCH 17/29] refactor: move cursor safety policies into native planner --- .../src/planner/main/eligible_start.rs | 31 +++ .../sqlbuild-cursor/src/planner/main/mod.rs | 3 + .../src/planner/main/override_kind.rs | 42 +++ .../src/planner/main/safety_horizon.rs | 49 ++++ crates/sqlbuild-cursor/src/planner/mod.rs | 3 + crates/sqlbuild-cursor/src/planner/tests.rs | 8 + .../src/planner/tests/helpers.rs | 26 ++ .../src/planner/tests/test_safety_horizon.rs | 64 +++++ .../src/planner/tests/test_types.rs | 9 + .../planner_resolution/cursor_bounds.rs | 10 +- .../planner_resolution/cursor_overrides.rs | 77 ++++++ .../planner_resolution/cursor_safety.rs | 105 ++++++++ .../planner_resolution/future_safety.rs | 164 ++++++++++++ .../planner_resolution/maximum_safety.rs | 252 ++++++++++++++++++ .../_helpers/planner_resolution/mod.rs | 4 + .../planner_resolution/registration.rs | 3 + src/sqlbuild/_native.pyi | 31 +++ .../planner/_helpers/resolve/cursor.py | 83 +----- .../_helpers/resolve/future_cursor_safety.py | 158 +---------- .../planner/_helpers/resolve/maximum_start.py | 172 +----------- 20 files changed, 885 insertions(+), 409 deletions(-) create mode 100644 crates/sqlbuild-cursor/src/planner/main/eligible_start.rs create mode 100644 crates/sqlbuild-cursor/src/planner/main/override_kind.rs create mode 100644 crates/sqlbuild-cursor/src/planner/main/safety_horizon.rs create mode 100644 crates/sqlbuild-cursor/src/planner/tests.rs create mode 100644 crates/sqlbuild-cursor/src/planner/tests/helpers.rs create mode 100644 crates/sqlbuild-cursor/src/planner/tests/test_safety_horizon.rs create mode 100644 crates/sqlbuild-cursor/src/planner/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs diff --git a/crates/sqlbuild-cursor/src/planner/main/eligible_start.rs b/crates/sqlbuild-cursor/src/planner/main/eligible_start.rs new file mode 100644 index 0000000000..c072a5e875 --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/main/eligible_start.rs @@ -0,0 +1,31 @@ +//! Eligible automatic-start fallback selection. + +use crate::algebra::errors::CursorError; +use crate::algebra::main::compare::compare; +use crate::algebra::models::Scalar; +use crate::algebra::types::ZoneRules; +use std::cmp::Ordering; + +/// Prefer a discovered eligible target, then an eligible upstream minimum. +pub fn eligible_start( + target: Option<&Scalar>, + upstream: &[Scalar], + horizon: &Scalar, + zones: &impl ZoneRules, +) -> Result, CursorError> { + if let Some(target) = target { + return Ok(Some(target.clone())); + } + let mut minimum: Option<&Scalar> = None; + for value in upstream { + if minimum.is_none() || compare(value, minimum.unwrap_or(value), zones)? == Ordering::Less { + minimum = Some(value); + } + } + if let Some(minimum) = minimum + && compare(minimum, horizon, zones)? != Ordering::Greater + { + return Ok(Some(minimum.clone())); + } + Ok(None) +} diff --git a/crates/sqlbuild-cursor/src/planner/main/mod.rs b/crates/sqlbuild-cursor/src/planner/main/mod.rs index 7b69754ff8..9c989bded1 100644 --- a/crates/sqlbuild-cursor/src/planner/main/mod.rs +++ b/crates/sqlbuild-cursor/src/planner/main/mod.rs @@ -1,5 +1,8 @@ //! Planner cursor algebra entry points. +pub mod eligible_start; pub mod empty_inputs; +pub mod override_kind; pub mod raw_bounds; pub mod replay_start; +pub mod safety_horizon; diff --git a/crates/sqlbuild-cursor/src/planner/main/override_kind.rs b/crates/sqlbuild-cursor/src/planner/main/override_kind.rs new file mode 100644 index 0000000000..af451032cb --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/main/override_kind.rs @@ -0,0 +1,42 @@ +//! Untyped override selection and mixed-model diagnostics. + +use std::collections::BTreeMap; + +/// Resolve the selected type or return the mixed-selection diagnostic. +pub fn override_kind( + models: &[(String, String)], + integer_literals: bool, +) -> Result { + let mut grouped: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); + for (name, kind) in models { + grouped.entry(kind).or_default().push(name); + } + if grouped.len() <= 1 { + return Ok(grouped + .keys() + .next() + .copied() + .unwrap_or(if integer_literals { + "integer" + } else { + "timestamp" + }) + .into()); + } + let mut descriptions: Vec = Vec::new(); + for (kind, mut names) in grouped { + names.sort_unstable(); + let shown = names.iter().take(3).copied().collect::>().join(", "); + let hidden = names.len().saturating_sub(3); + let names = if hidden > 0 { + format!("{shown} and {hidden} more") + } else { + shown + }; + descriptions.push(format!("{kind} cursor models ({names})")); + } + Err(format!( + "--start-cursor and --end-cursor need one cursor type, but the selection mixes {}", + descriptions.join(" and ") + )) +} diff --git a/crates/sqlbuild-cursor/src/planner/main/safety_horizon.rs b/crates/sqlbuild-cursor/src/planner/main/safety_horizon.rs new file mode 100644 index 0000000000..44021c7228 --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/main/safety_horizon.rs @@ -0,0 +1,49 @@ +//! Invocation-anchored automatic cursor horizon shared by both safety policies. + +use crate::algebra::errors::CursorError; +use crate::algebra::main::floor_to_grain::floor_to_grain; +use crate::algebra::main::shift_by_duration::shift_by_duration; +use crate::algebra::models::{DurationSpan, Grain, Moment, Scalar, ScalarValue}; + +/// Shift a UTC invocation clock, retaining the discovered cursor's date representation. +pub fn safety_horizon( + clock: &Scalar, + duration: &DurationSpan, + grain: Grain, + date_only: bool, +) -> Result { + let ScalarValue::Timestamp(timestamp) = &clock.value else { + return Err(CursorError::Type( + "invocation clock must be a timestamp".into(), + )); + }; + let Moment::Timestamp { local, fold } = shift_by_duration( + Moment::Timestamp { + local: timestamp.local, + fold: timestamp.fold, + }, + duration, + 1, + )? + else { + return Err(CursorError::Type( + "invocation clock must be a timestamp".into(), + )); + }; + let value = if date_only { + ScalarValue::Date(local.date) + } else { + let mut maximum = timestamp.clone(); + maximum.local = local; + maximum.fold = fold; + maximum.source_text = None; + ScalarValue::Timestamp(maximum) + }; + Ok(floor_to_grain( + &Scalar { + value, + handle: None, + }, + Some(grain), + )) +} diff --git a/crates/sqlbuild-cursor/src/planner/mod.rs b/crates/sqlbuild-cursor/src/planner/mod.rs index a10f585e28..26a799970e 100644 --- a/crates/sqlbuild-cursor/src/planner/mod.rs +++ b/crates/sqlbuild-cursor/src/planner/mod.rs @@ -2,3 +2,6 @@ pub mod main; pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cursor/src/planner/tests.rs b/crates/sqlbuild-cursor/src/planner/tests.rs new file mode 100644 index 0000000000..d695d13453 --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/tests.rs @@ -0,0 +1,8 @@ +//! Focused planner policy regressions. + +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_safety_horizon.rs"] +mod test_safety_horizon; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cursor/src/planner/tests/helpers.rs b/crates/sqlbuild-cursor/src/planner/tests/helpers.rs new file mode 100644 index 0000000000..b9b6ef1bdd --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/tests/helpers.rs @@ -0,0 +1,26 @@ +use crate::algebra::models::{ + CivilDate, LocalDateTime, Scalar, ScalarValue, Timestamp, ZoneOffset, +}; + +pub(super) fn timestamp(month: u32, day: u32, hour: u32, minute: u32) -> Scalar { + Scalar { + handle: None, + value: ScalarValue::Timestamp(Timestamp { + local: LocalDateTime { + date: CivilDate { + year: 2026, + month, + day, + }, + hour, + minute, + second: 0, + microsecond: 0, + }, + fold: false, + offset: ZoneOffset::Fixed(0), + template: None, + source_text: None, + }), + } +} diff --git a/crates/sqlbuild-cursor/src/planner/tests/test_safety_horizon.rs b/crates/sqlbuild-cursor/src/planner/tests/test_safety_horizon.rs new file mode 100644 index 0000000000..4be53f2a4a --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/tests/test_safety_horizon.rs @@ -0,0 +1,64 @@ +use crate::algebra::models::{CivilDate, DurationSpan, Grain, Scalar, ScalarValue}; +use crate::planner::main::safety_horizon::safety_horizon; +use crate::planner::tests::helpers::timestamp; +use crate::planner::tests::test_types::HorizonTestCase; + +#[test] +fn given_invocation_clock_when_resolving_horizon_then_calendar_and_grain_are_preserved() { + let test_cases = [ + HorizonTestCase { + description: "calendar month clamps day", + months: 1, + grain: Grain::Hour, + date_only: false, + expected_horizon: timestamp(2, 28, 12, 0), + }, + HorizonTestCase { + description: "date retains final day", + months: 1, + grain: Grain::Day, + date_only: true, + expected_horizon: Scalar { + handle: None, + value: ScalarValue::Date(CivilDate { + year: 2026, + month: 2, + day: 28, + }), + }, + }, + HorizonTestCase { + description: "month grain floors shifted clock", + months: 1, + grain: Grain::Month, + date_only: false, + expected_horizon: timestamp(2, 1, 0, 0), + }, + HorizonTestCase { + description: "zero duration retains current partition", + months: 0, + grain: Grain::Day, + date_only: false, + expected_horizon: timestamp(1, 31, 0, 0), + }, + ]; + for test_case in test_cases { + assert_eq!( + safety_horizon( + ×tamp(1, 31, 12, 30), + &DurationSpan { + months: test_case.months, + days: 0, + hours: 0, + minutes: 0, + seconds: 0 + }, + test_case.grain, + test_case.date_only + ), + Ok(test_case.expected_horizon), + "{}", + test_case.description, + ); + } +} diff --git a/crates/sqlbuild-cursor/src/planner/tests/test_types.rs b/crates/sqlbuild-cursor/src/planner/tests/test_types.rs new file mode 100644 index 0000000000..f8d56d203a --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/tests/test_types.rs @@ -0,0 +1,9 @@ +use crate::algebra::models::{Grain, Scalar}; + +pub(super) struct HorizonTestCase { + pub(super) description: &'static str, + pub(super) months: i64, + pub(super) grain: Grain, + pub(super) date_only: bool, + pub(super) expected_horizon: Scalar, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs index 7846d5bbf2..f968c85d17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs @@ -16,9 +16,9 @@ use sqlbuild_cursor::planner::main::replay_start::replay_start; use sqlbuild_cursor::planner::models::{CursorKind, CursorSnapshot, ReplayPolicy}; const MODELS: &str = "sqlbuild.compiler.planner.models"; -const MAXIMUM_START: &str = "sqlbuild.compiler.planner._helpers.resolve.maximum_start"; +const MAXIMUM_START: &str = "sqlbuild._native"; -fn optional_scalar<'py>( +pub(super) fn optional_scalar<'py>( session: &CursorSession<'py>, value: &Bound<'py, PyAny>, ) -> PyResult> { @@ -29,7 +29,7 @@ fn optional_scalar<'py>( } } -fn scalars<'py>( +pub(super) fn scalars<'py>( session: &CursorSession<'py>, object: &Bound<'py, PyAny>, field: &str, @@ -65,7 +65,7 @@ fn read_snapshot<'py>( }) } -fn duration(value: &Bound<'_, PyAny>) -> PyResult> { +pub(super) fn duration(value: &Bound<'_, PyAny>) -> PyResult> { if value.is_none() { return Ok(None); } @@ -153,7 +153,7 @@ fn apply_typed_cursor_replay_policy<'py>( session.python_scalar(&result) } -fn required_scalar<'py>( +pub(super) fn required_scalar<'py>( session: &CursorSession<'py>, raw: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs new file mode 100644 index 0000000000..a7d34b0743 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs @@ -0,0 +1,77 @@ +//! Selected-model inspection and untyped override conversion. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_resolution::values::{method, string}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_cursor::planner::main::override_kind::override_kind; + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn type_cursor_overrides<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "type_cursor_overrides", + kwargs, + (&["cursor_overrides", "selected_models"], &[]), + )?; + let overrides = args.value("cursor_overrides")?; + if overrides.is_none() || !overrides.getattr("has_untyped")?.is_truthy()? { + return Ok(overrides); + } + let get_config = imported(py, "sqlbuild._native", "get_config_str")?; + let mut models: Vec<(String, String)> = Vec::new(); + for model in args.value("selected_models")?.try_iter()? { + let model = model?; + let values = model.getattr("config")?.getattr("values")?; + let config = |key: &str| { + call_with( + &get_config, + &[("values", values.clone()), ("key", string(py, key))], + ) + }; + if !config("materialized")?.eq("incremental")? || config("cursor")?.is_none() { + continue; + } + let kind = config("cursor_type")?; + if kind.eq("integer")? || kind.eq("timestamp")? { + models.push((model.getattr("name")?.extract()?, kind.extract()?)); + } + } + let mut integer_literals = true; + if models.is_empty() { + for field in ["start", "end"] { + let value = overrides.getattr(field)?; + if value.is_none() { + continue; + } + let stripped = value.call_method0("strip")?; + let text: String = stripped.extract()?; + let digits = text.strip_prefix(['+', '-']).unwrap_or(&text); + if !string(py, digits).call_method0("isdecimal")?.is_truthy()? { + integer_literals = false; + break; + } + } + } + let kind = override_kind(&models, integer_literals).map_err(|message| { + match imported(py, "sqlbuild.compiler.planner.exceptions", "PlannerInputError").and_then(|class| call_with(&class, &[ + ("message", string(py, &message)), ("code", string(py, "S303")), + ("help", string(py, "select models with one cursor type, or set each type explicitly with --start-cursor-ts/--end-cursor-ts and --start-cursor-int/--end-cursor-int")), + ])) { Ok(error) => PyErr::from_value(error), Err(error) => error } + })?; + let kind = imported(py, "sqlbuild.compiler.planner.types", "CursorType")?.call1((kind,))?; + method(&overrides, "typed_as", &[("cursor_type", kind)]) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(type_cursor_overrides, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs new file mode 100644 index 0000000000..e26c003d12 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs @@ -0,0 +1,105 @@ +//! Shared safety horizon conversion and Python evidence construction. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::cursor::conversion::CursorSession; +use crate::bindings::_helpers::planner_resolution::cursor_bounds::duration; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use sqlbuild_cursor::algebra::models::{DurationSpan, Grain, Scalar}; +use sqlbuild_cursor::planner::main::safety_horizon::safety_horizon; + +pub(super) const MODELS: &str = "sqlbuild.compiler.planner.models"; +pub(super) const EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; + +pub(super) fn safety_error(py: Python<'_>, class: &str, message: &str) -> PyErr { + match imported(py, EXCEPTIONS, class).and_then(|class| class.call1((message,))) { + Ok(value) => PyErr::from_value(value), + Err(error) => error, + } +} + +pub(super) fn safety_duration(value: &Bound<'_, PyAny>, future: bool) -> PyResult { + if let Some(parsed) = duration(value)? { + return Ok(parsed); + } + if value.eq("0d")? { + return Ok(DurationSpan { + months: 0, + days: 0, + hours: 0, + minutes: 0, + seconds: 0, + }); + } + let (class, field) = if future { + ("FutureCursorSafetyError", "future.max_distance") + } else { + ("MaximumAutomaticStartError", "start.max_ahead") + }; + Err(safety_error( + value.py(), + class, + &format!("invalid cursors.{field} '{}'", value.str()?), + )) +} + +pub(super) fn timestamp_bound(value: &Bound<'_, PyAny>) -> PyResult { + let py = value.py(); + Ok(value.is_instance(&imported( + py, + "sqlbuild.cursor_algebra.models", + "DateValue", + )?)? || value.is_instance(&imported( + py, + "sqlbuild.cursor_algebra.models", + "TimestampValue", + )?)?) +} + +pub(super) fn utc_clock<'py>(clock: &Bound<'py, PyAny>) -> PyResult> { + clock.call_method1("astimezone", (imported(clock.py(), "datetime", "UTC")?,)) +} + +pub(super) fn horizon<'py>( + session: &CursorSession<'py>, + discovered: &Bound<'py, PyAny>, + clock_and_grain: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + span: &DurationSpan, +) -> PyResult { + let py = session.py(); + let (clock, grain) = clock_and_grain; + let mut clock = utc_clock(clock)?; + let date_only = discovered.is_instance(&imported( + py, + "sqlbuild.cursor_algebra.models", + "DateValue", + )?)?; + if date_only || discovered.getattr("value")?.getattr("tzinfo")?.is_none() { + clock = call_with( + &clock.getattr("replace")?, + &[("tzinfo", py.None().into_bound(py))], + )?; + } + let clock = call_with( + &imported(py, "sqlbuild.cursor_algebra.models", "TimestampValue")?, + &[("value", clock)], + )?; + let grain = if grain.is_truthy()? { + session.grain(grain)?.unwrap_or(Grain::Second) + } else { + Grain::Second + }; + session.checked(safety_horizon( + &session.scalar(&clock)?, + span, + grain, + date_only, + )) +} + +pub(super) fn replace<'py>( + object: &Bound<'py, PyAny>, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + imported(object.py(), "dataclasses", "replace")? + .call((object,), Some(&keywords(object.py(), fields)?)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs new file mode 100644 index 0000000000..0163650f9e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs @@ -0,0 +1,164 @@ +//! Future cursor safety evidence and exception boundary. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::cursor::conversion::CursorSession; +use crate::bindings::_helpers::planner_resolution::cursor_safety::{ + MODELS, horizon, replace, safety_duration, safety_error, timestamp_bound, utc_clock, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_cursor::algebra::main::compare::compare; +use sqlbuild_cursor::algebra::main::next_boundary::next_boundary; +use sqlbuild_cursor::algebra::main::render::render; +use sqlbuild_cursor::algebra::models::{Grain, Scalar}; +use std::cmp::Ordering; + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn resolve_future_cursor_safety<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "resolve_future_cursor_safety", + kwargs, + ( + &[ + "bounds", + "cursor_type", + "cursor_grain", + "config", + "invocation_time", + "has_complete_override", + ], + &["input_evidence"], + ), + )?; + let bounds = args.value("bounds")?; + let config = args.value("config")?; + let clock = args.value("invocation_time")?; + if config.is_none() + || config.getattr("max_distance")?.is_none() + || !args.value("cursor_type")?.eq("timestamp")? + || clock.is_none() + || args.flag("has_complete_override")? + { + return Ok(bounds); + } + let distance = config.getattr("max_distance")?; + let duration = safety_duration(&distance, true)?; + let start = bounds.getattr("start")?; + let end = bounds.getattr("end")?; + if !timestamp_bound(&start)? || !timestamp_bound(&end)? { + return Ok(bounds); + } + let session = CursorSession::new(py)?; + let grain = args.value("cursor_grain")?; + let allowed_start = horizon(&session, &end, (&clock, &grain), &duration)?; + let grain = if grain.is_truthy()? { + session.grain(&grain)?.unwrap_or(Grain::Second) + } else { + Grain::Second + }; + let allowed_end = session.checked(next_boundary(&allowed_start, Some(grain)))?; + let typed_start = session.scalar(&start)?; + let typed_end = session.scalar(&end)?; + let future_start = + session.checked(compare(&typed_start, &allowed_start, &session))? == Ordering::Greater; + let future_end = + session.checked(compare(&typed_end, &allowed_end, &session))? == Ordering::Greater; + if !future_start && !future_end { + return Ok(bounds); + } + let evidence = args.value("input_evidence")?; + let evidence = if evidence.is_none() { + PyTuple::empty(py).into_any() + } else { + evidence + }; + let mut determining: Option<(Bound<'py, PyAny>, Scalar)> = None; + for item in evidence.try_iter()? { + let item = item?; + let maximum = session.scalar(&item.getattr("maximum")?)?; + let smaller = if let Some((_, previous)) = &determining { + session.checked(compare(&maximum, previous, &session))? == Ordering::Less + } else { + true + }; + if smaller { + determining = Some((item, maximum)); + } + } + if config.getattr("action")?.eq("error")? { + return Err(safety_error( + py, + "FutureCursorSafetyError", + &format!( + "future cursor safety limit exceeded: effective bounds ['{}', '{}') exceed maximum allowed bounds ['{}', '{}') (cursors.future.max_distance={})", + session.checked(render(&typed_start, &session))?, + session.checked(render(&typed_end, &session))?, + session.checked(render(&allowed_start, &session))?, + session.checked(render(&allowed_end, &session))?, + distance.str()? + ), + )); + } + let applied_end = if future_end { + session.python_scalar(&allowed_end)? + } else { + end.clone() + }; + let determining_field = |field: &str| -> PyResult> { + if let Some((item, _)) = &determining { + item.getattr(field) + } else { + Ok(py.None().into_bound(py)) + } + }; + let safety = call_with( + &imported(py, MODELS, "FutureCursorSafetyEvidence")?, + &[ + ("action", config.getattr("action")?), + ("max_distance", distance), + ( + "invocation_time", + call_with( + &imported(py, "sqlbuild.cursor_algebra.models", "TimestampValue")?, + &[("value", utc_clock(&clock)?)], + )?, + ), + ("discovered_start", start.clone()), + ("discovered_end", end), + ("applied_start", start), + ("applied_end", applied_end.clone()), + ( + "maximum_allowed_start", + session.python_scalar(&allowed_start)?, + ), + ("maximum_allowed_end", session.python_scalar(&allowed_end)?), + ( + "future_start_detected", + future_start.into_pyobject(py)?.to_owned().into_any(), + ), + ( + "future_end_detected", + future_end.into_pyobject(py)?.to_owned().into_any(), + ), + ("determining_relation", determining_field("relation")?), + ( + "determining_cursor_column", + determining_field("cursor_column")?, + ), + ("inputs", evidence), + ], + )?; + replace(&bounds, &[("end", applied_end), ("future_safety", safety)]) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_future_cursor_safety, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs new file mode 100644 index 0000000000..86631c049f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs @@ -0,0 +1,252 @@ +//! Maximum automatic-start recovery and evidence boundary. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::cursor::conversion::CursorSession; +use crate::bindings::_helpers::planner_resolution::cursor_bounds::{ + optional_scalar, required_scalar, scalars, +}; +use crate::bindings::_helpers::planner_resolution::cursor_safety::{ + EXCEPTIONS, MODELS, horizon, replace, safety_duration, safety_error, timestamp_bound, utc_clock, +}; +use crate::bindings::_helpers::planner_resolution::values::string; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyDict; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_cursor::algebra::main::compare::compare; +use sqlbuild_cursor::algebra::main::render::render; +use sqlbuild_cursor::planner::main::eligible_start::eligible_start; +use std::cmp::Ordering; + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn maximum_allowed_start<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult { + let args = KeywordArguments::read( + py, + "maximum_allowed_start", + kwargs, + ( + &[ + "discovered_value", + "cursor_grain", + "invocation_time", + "max_ahead", + ], + &[], + ), + )?; + let session = CursorSession::new(py)?; + let discovered = required_scalar( + &session, + &args.value("discovered_value")?, + &string(py, "timestamp"), + )?; + let duration = safety_duration(&args.value("max_ahead")?, false)?; + session.checked(render( + &horizon( + &session, + &session.python_scalar(&discovered)?, + ( + &args.value("invocation_time")?, + &args.value("cursor_grain")?, + ), + &duration, + )?, + &session, + )) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn apply_maximum_start_policy<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "apply_maximum_start_policy", + kwargs, + ( + &[ + "bounds", + "snapshot", + "cursor_type", + "cursor_grain", + "cursor_start", + "lookback", + "backfill_duration", + "policy", + "has_start_override", + ], + &[], + ), + )?; + let bounds = args.value("bounds")?; + let snapshot = args.value("snapshot")?; + let physical = snapshot.getattr("physical_target_max")?; + let physical = if physical.is_truthy()? { + physical + } else { + snapshot.getattr("target_max")? + }; + let policy = args.value("policy")?; + let config = policy.getattr("config")?; + let clock = policy.getattr("invocation_time")?; + if config.is_none() + || config.getattr("max_ahead")?.is_none() + || !args.value("cursor_type")?.eq("timestamp")? + || clock.is_none() + || args.flag("has_start_override")? + || physical.is_none() + { + return Ok(bounds); + } + if !timestamp_bound(&bounds.getattr("start")?)? || !timestamp_bound(&bounds.getattr("end")?)? { + return Ok(bounds); + } + let duration = safety_duration(&config.getattr("max_ahead")?, false)?; + if !timestamp_bound(&physical)? { + return Err(safety_error( + py, + "MaximumAutomaticStartError", + "automatic start requires a timestamp cursor", + )); + } + let session = CursorSession::new(py)?; + let horizon = horizon( + &session, + &physical, + (&clock, &args.value("cursor_grain")?), + &duration, + )?; + let physical_typed = session.scalar(&physical)?; + if session.checked(compare(&physical_typed, &horizon, &session))? != Ordering::Greater { + return Ok(bounds); + } + let target = optional_scalar(&session, &snapshot.getattr("target_eligible_max")?)?; + let upstream = if target.is_none() { + scalars(&session, &snapshot, "upstream_mins")? + } else { + Vec::new() + }; + let mut eligible = session.checked(eligible_start( + target.as_ref(), + &upstream, + &horizon, + &session, + ))?; + if eligible.is_none() { + let floor = args.value("cursor_start")?; + if !floor.is_none() { + let floor = if timestamp_bound(&floor)? { + session.scalar(&floor)? + } else { + required_scalar(&session, &floor, &string(py, "timestamp"))? + }; + if session.checked(compare(&floor, &horizon, &session))? != Ordering::Greater { + eligible = Some(floor); + } + } + } + let effective = if let Some(eligible) = &eligible { + let mut replay = vec![ + ("start", session.python_scalar(eligible)?), + ("end", bounds.getattr("end")?), + ( + "has_start_override", + false.into_pyobject(py)?.to_owned().into_any(), + ), + ]; + for key in [ + "cursor_start", + "cursor_type", + "lookback", + "backfill_duration", + ] { + replay.push((key, args.value(key)?)); + } + call_with( + &imported(py, "sqlbuild._native", "apply_typed_cursor_replay_policy")?, + &replay, + )? + } else { + bounds.getattr("start")? + }; + let relation = snapshot.getattr("target_relation")?; + let column = snapshot.getattr("destination_cursor_column")?; + let evidence = call_with( + &imported(py, MODELS, "MaximumStartSafetyEvidence")?, + &[ + ("action", config.getattr("action")?), + ("max_ahead", config.getattr("max_ahead")?), + ( + "invocation_time", + call_with( + &imported(py, "sqlbuild.cursor_algebra.models", "TimestampValue")?, + &[("value", utc_clock(&clock)?)], + )?, + ), + ("physical_target_max", physical), + ( + "highest_eligible_target_max", + snapshot.getattr("target_eligible_max")?, + ), + ("effective_start", effective.clone()), + ("maximum_allowed_start", session.python_scalar(&horizon)?), + ( + "target_relation", + if relation.is_truthy()? { + relation + } else { + string(py, "") + }, + ), + ( + "cursor_column", + if column.is_truthy()? { + column + } else { + string(py, "") + }, + ), + ], + )?; + let error = |non_idempotent: bool| -> PyResult> { + Err(PyErr::from_value(call_with( + &imported(py, EXCEPTIONS, "MaximumAutomaticStartError")?, + &[ + ("evidence", evidence.clone()), + ( + "non_idempotent", + non_idempotent.into_pyobject(py)?.to_owned().into_any(), + ), + ], + )?)) + }; + if config.getattr("action")?.eq("error")? + || eligible.is_none() + || session.checked(compare(&session.scalar(&effective)?, &horizon, &session))? + == Ordering::Greater + { + return error(false); + } + let strategy = policy.getattr("incremental_strategy")?; + if !strategy.eq("delete_insert")? && !strategy.eq("merge")? { + return error(true); + } + replace( + &bounds, + &[("start", effective), ("maximum_start_safety", evidence)], + ) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(maximum_allowed_start, module)?)?; + module.add_function(wrap_pyfunction!(apply_maximum_start_policy, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs index a49f426198..b90f4dc22a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs @@ -3,8 +3,12 @@ mod constants; mod cursor_bounds; mod cursor_model; +mod cursor_overrides; +mod cursor_safety; mod cursor_windows; +mod future_safety; mod locations; +mod maximum_safety; mod orchestration; mod refs; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs index 534450995d..6626407126 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs @@ -5,6 +5,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::cursor_bounds::register(module)?; super::cursor_model::register(module)?; + super::cursor_overrides::register(module)?; + super::future_safety::register(module)?; + super::maximum_safety::register(module)?; super::cursor_windows::register(module)?; super::refs::register(module)?; super::locations::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 42194e0d05..c12fd82743 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -207,7 +207,9 @@ from sqlbuild.compiler.planner.models import ( ColumnMigrationPlanning, ColumnRenameHint, CursorBounds, + CursorInputEvidence, CursorOverridePair, + CursorOverrides, DeferralInputs, DirectModelVersionIdentities, EmptyCursorInputDecision, @@ -402,6 +404,7 @@ from sqlbuild.spec.contracts.models import ( DbtConfig, DefaultsConfig, ExecutionLimitsConfig, + FutureCursorsConfig, LoaderDestinationParts, LocalConfig, LocalDbtConfig, @@ -4261,6 +4264,34 @@ def cursor_render(*, value: CursorScalar) -> str: ... def cursor_sentinel_to_token(*, sentinel: Bound) -> str: ... def cursor_sentinel_from_token(*, token: str) -> BoundSentinel | None: ... def cursor_compare(*, left: CursorScalar, right: CursorScalar) -> int: ... +def type_cursor_overrides( + *, cursor_overrides: CursorOverrides | None, selected_models: Iterable[CompiledModelView] +) -> CursorOverrides | None: ... +def maximum_allowed_start( + *, discovered_value: str, cursor_grain: str | None, invocation_time: datetime, max_ahead: str +) -> str: ... +def apply_maximum_start_policy( + *, + bounds: CursorBounds, + snapshot: ModelCursorSnapshot, + cursor_type: str | None, + cursor_grain: str | None, + cursor_start: CursorScalar | str | None, + lookback: Duration | str | None, + backfill_duration: Duration | str | None, + policy: MaximumStartPolicyInputs, + has_start_override: bool, +) -> CursorBounds: ... +def resolve_future_cursor_safety( + *, + bounds: CursorBounds, + cursor_type: str | None, + cursor_grain: str | None, + config: FutureCursorsConfig | None, + invocation_time: datetime | None, + has_complete_override: bool, + input_evidence: tuple[CursorInputEvidence, ...] = (), +) -> CursorBounds: ... def cursor_sort_key(*, raw: object, cursor_type: str) -> datetime | int: ... def cursor_max_bound[T](*, values: tuple[T, ...] | list[T], cursor_type: str) -> T: ... def cursor_min_bound[T](*, values: tuple[T, ...] | list[T], cursor_type: str) -> T: ... diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py index ae8602015f..cfa1fa0d30 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py @@ -1,9 +1,4 @@ -"""Cursor override typing and native cursor-window exports.""" - -from __future__ import annotations - -import re -from collections.abc import Iterable +"""Native cursor override typing and cursor-window exports.""" from sqlbuild._native import ( apply_typed_cursor_replay_policy as apply_typed_cursor_replay_policy, @@ -26,88 +21,18 @@ from sqlbuild._native import ( empty_input_rebuild_refusal as empty_input_rebuild_refusal, ) -from sqlbuild._native import ( - get_config_str, -) from sqlbuild._native import ( normalize_cursor_snapshot_grain as normalize_cursor_snapshot_grain, ) from sqlbuild._native import ( resolve_effective_timestamp_grain as resolve_effective_timestamp_grain, ) +from sqlbuild._native import ( + type_cursor_overrides as type_cursor_overrides, +) from sqlbuild._native import ( unreadable_cursor_inputs_error as unreadable_cursor_inputs_error, ) from sqlbuild._native import ( without_destination_cursor as without_destination_cursor, ) -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import CursorOverrides -from sqlbuild.compiler.planner.types import CursorType, MaterializationType - -_INTEGER_LITERAL: re.Pattern[str] = re.compile(r"[+-]?\d+") -_LISTED_MODEL_LIMIT: int = 3 - - -def type_cursor_overrides( - *, - cursor_overrides: CursorOverrides | None, - selected_models: Iterable[CompiledModelView], -) -> CursorOverrides | None: - """Type `--start-cursor`/`--end-cursor` from the selected cursor models, else the literal.""" - - if cursor_overrides is None or not cursor_overrides.has_untyped: - return cursor_overrides - model_names_by_type: dict[CursorType, list[str]] = {} - model: CompiledModelView - for model in selected_models: - cursor_type: CursorType | None = _model_cursor_type(model=model) - if cursor_type is not None: - model_names_by_type.setdefault(cursor_type, []).append(model.name) - if len(model_names_by_type) > 1: - raise PlannerInputError( - "--start-cursor and --end-cursor need one cursor type, but the selection mixes " - + " and ".join( - f"{cursor_type} cursor models ({_format_names(names)})" - for cursor_type, names in sorted(model_names_by_type.items()) - ), - code="S303", - help=( - "select models with one cursor type, or set each type explicitly with " - "--start-cursor-ts/--end-cursor-ts and --start-cursor-int/--end-cursor-int" - ), - ) - resolved_type: CursorType = ( - next(iter(model_names_by_type)) - if model_names_by_type - else _literal_cursor_type(cursor_overrides=cursor_overrides) - ) - return cursor_overrides.typed_as(cursor_type=resolved_type) - - -def _model_cursor_type(*, model: CompiledModelView) -> CursorType | None: - materialized: str | None = get_config_str(values=model.config.values, key="materialized") - cursor_column: str | None = get_config_str(values=model.config.values, key="cursor") - if materialized != MaterializationType.INCREMENTAL or cursor_column is None: - return None - cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") - if cursor_type not in {CursorType.TIMESTAMP, CursorType.INTEGER}: - return None - return CursorType(cursor_type) - - -def _literal_cursor_type(*, cursor_overrides: CursorOverrides) -> CursorType: - values: tuple[str, ...] = tuple( - value for value in (cursor_overrides.start, cursor_overrides.end) if value is not None - ) - if all(_INTEGER_LITERAL.fullmatch(value.strip()) for value in values): - return CursorType.INTEGER - return CursorType.TIMESTAMP - - -def _format_names(names: list[str]) -> str: - ordered: list[str] = sorted(names) - shown: str = ", ".join(ordered[:_LISTED_MODEL_LIMIT]) - hidden: int = len(ordered) - _LISTED_MODEL_LIMIT - return f"{shown} and {hidden} more" if hidden > 0 else shown diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/future_cursor_safety.py b/src/sqlbuild/compiler/planner/_helpers/resolve/future_cursor_safety.py index 12c3b57ab3..ac87304474 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/future_cursor_safety.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/future_cursor_safety.py @@ -1,157 +1,3 @@ -"""Future cursor safety calculation.""" +"""Native future cursor safety policy.""" -from __future__ import annotations - -from dataclasses import replace -from datetime import UTC, datetime - -from sqlbuild._native import cursor_compare as compare -from sqlbuild._native import cursor_floor_to_grain as floor_to_grain -from sqlbuild._native import cursor_next_boundary as next_boundary -from sqlbuild._native import cursor_render as render -from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.compiler.planner.exceptions import FutureCursorSafetyError -from sqlbuild.compiler.planner.models import ( - CursorBounds, - CursorInputEvidence, - Duration, - FutureCursorSafetyEvidence, -) -from sqlbuild.compiler.planner.types import CursorGrain, CursorType -from sqlbuild.cursor_algebra.models import DateValue, TimestampValue -from sqlbuild.cursor_algebra.types import CursorScalar -from sqlbuild.spec.contracts.constants import ZERO_DAY_CURSOR_DURATION -from sqlbuild.spec.contracts.models import FutureCursorsConfig -from sqlbuild.spec.contracts.types import FutureCursorAction - - -def resolve_future_cursor_safety( - *, - bounds: CursorBounds, - cursor_type: str | None, - cursor_grain: str | None, - config: FutureCursorsConfig | None, - invocation_time: datetime | None, - has_complete_override: bool, - input_evidence: tuple[CursorInputEvidence, ...] = (), -) -> CursorBounds: - """Apply future cursor policy to effective timestamp bounds.""" - - if ( - config is None - or config.max_distance is None - or cursor_type != CursorType.TIMESTAMP - or invocation_time is None - or has_complete_override - ): - return bounds - duration: Duration | None = ( - Duration() - if config.max_distance == ZERO_DAY_CURSOR_DURATION - else Duration.parse(config.max_distance) - ) - if duration is None: - raise FutureCursorSafetyError( - f"invalid cursors.future.max_distance '{config.max_distance}'" - ) - if not isinstance(bounds.start, DateValue | TimestampValue) or not isinstance( - bounds.end, DateValue | TimestampValue - ): - return bounds - allowed_start, allowed_end = _allowed_bounds( - discovered_end=bounds.end, - cursor_grain=cursor_grain, - invocation_time=invocation_time, - duration=duration, - ) - future_start_detected: bool = compare(left=bounds.start, right=allowed_start) > 0 - future_end_detected: bool = compare(left=bounds.end, right=allowed_end) > 0 - if not future_start_detected and not future_end_detected: - return bounds - determining: CursorInputEvidence | None = _determining_input(input_evidence) - if config.action == FutureCursorAction.ERROR: - raise FutureCursorSafetyError( - _error_message( - bounds=bounds, - allowed_start=allowed_start, - allowed_end=allowed_end, - max_distance=config.max_distance, - ) - ) - applied_end: CursorScalar = allowed_end if future_end_detected else bounds.end - return replace( - bounds, - end=applied_end, - future_safety=FutureCursorSafetyEvidence( - action=config.action, - max_distance=config.max_distance, - invocation_time=TimestampValue(value=invocation_time.astimezone(UTC)), - discovered_start=bounds.start, - discovered_end=bounds.end, - applied_start=bounds.start, - applied_end=applied_end, - maximum_allowed_start=allowed_start, - maximum_allowed_end=allowed_end, - future_start_detected=future_start_detected, - future_end_detected=future_end_detected, - determining_relation=determining.relation if determining is not None else None, - determining_cursor_column=( - determining.cursor_column if determining is not None else None - ), - inputs=input_evidence, - ), - ) - - -def _allowed_bounds( - *, - discovered_end: CursorScalar, - cursor_grain: str | None, - invocation_time: datetime, - duration: Duration, -) -> tuple[CursorScalar, CursorScalar]: - clock: datetime = invocation_time.astimezone(UTC).replace(tzinfo=None) - maximum: datetime = duration.add_to(clock) - grain: CursorGrain = CursorGrain(cursor_grain or CursorGrain.SECOND) - if isinstance(discovered_end, DateValue): - allowed_start: CursorScalar = floor_to_grain( - value=DateValue(value=maximum.date()), grain=grain - ) - elif isinstance(discovered_end, TimestampValue): - allowed_timestamp: datetime = maximum - if discovered_end.value.tzinfo is not None: - allowed_timestamp = allowed_timestamp.replace(tzinfo=UTC) - allowed_start = floor_to_grain(value=TimestampValue(value=allowed_timestamp), grain=grain) - else: - raise FutureCursorSafetyError("future cursor safety requires a timestamp cursor") - return allowed_start, next_boundary(value=allowed_start, grain=grain) - - -def _determining_input( - evidence: tuple[CursorInputEvidence, ...], -) -> CursorInputEvidence | None: - if not evidence: - return None - determining: CursorInputEvidence = evidence[0] - for item in evidence[1:]: - if compare(left=item.maximum, right=determining.maximum) < 0: - determining = item - return determining - - -def _error_message( - *, - bounds: CursorBounds, - allowed_start: CursorScalar, - allowed_end: CursorScalar, - max_distance: str, -) -> str: - start: str = sentinel_to_token(sentinel=bounds.start) - end: str = sentinel_to_token(sentinel=bounds.end) - return ( - "future cursor safety limit exceeded: effective bounds " - f"['{start}', '{end}') " - "exceed maximum allowed bounds " - f"['{render(value=allowed_start)}', '{render(value=allowed_end)}') " - f"(cursors.future.max_distance={max_distance})" - ) +from sqlbuild._native import resolve_future_cursor_safety as resolve_future_cursor_safety diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/maximum_start.py b/src/sqlbuild/compiler/planner/_helpers/resolve/maximum_start.py index 85d9df6458..267593051d 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/maximum_start.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/maximum_start.py @@ -1,170 +1,4 @@ -"""Maximum automatic cursor-start eligibility policy.""" +"""Native maximum automatic cursor-start eligibility policy.""" -from __future__ import annotations - -from dataclasses import replace -from datetime import UTC, datetime - -from sqlbuild._native import cursor_compare as compare -from sqlbuild._native import cursor_floor_to_grain as floor_to_grain -from sqlbuild._native import cursor_min_bound as min_bound -from sqlbuild._native import cursor_parse as parse -from sqlbuild._native import cursor_render as render -from sqlbuild.compiler.planner._helpers.resolve.cursor import apply_typed_cursor_replay_policy -from sqlbuild.compiler.planner.exceptions import MaximumAutomaticStartError -from sqlbuild.compiler.planner.models import ( - CursorBounds, - Duration, - MaximumStartPolicyInputs, - MaximumStartSafetyEvidence, - ModelCursorSnapshot, -) -from sqlbuild.compiler.planner.types import ( - CursorGrain, - CursorType, - IncrementalStrategy, -) -from sqlbuild.cursor_algebra.models import DateValue, TimestampValue -from sqlbuild.cursor_algebra.types import CursorScalar -from sqlbuild.spec.contracts.constants import ZERO_DAY_CURSOR_DURATION -from sqlbuild.spec.contracts.types import FutureCursorAction - - -def maximum_allowed_start( - *, discovered_value: str, cursor_grain: str | None, invocation_time: datetime, max_ahead: str -) -> str: - """Return the invocation-anchored maximum automatic start in cursor representation.""" - - discovered: CursorScalar = parse(raw=discovered_value, cursor_type=CursorType.TIMESTAMP) - return render( - value=_maximum_allowed_start( - discovered_value=discovered, - cursor_grain=cursor_grain, - invocation_time=invocation_time, - max_ahead=max_ahead, - ) - ) - - -def _maximum_allowed_start( - *, - discovered_value: CursorScalar, - cursor_grain: str | None, - invocation_time: datetime, - max_ahead: str, -) -> CursorScalar: - """Return the invocation-anchored maximum automatic typed start.""" - - duration: Duration = _parse_duration(max_ahead) - clock: datetime = invocation_time.astimezone(UTC).replace(tzinfo=None) - maximum: datetime = duration.add_to(clock) - grain: CursorGrain = CursorGrain(cursor_grain or CursorGrain.SECOND) - if isinstance(discovered_value, DateValue): - return floor_to_grain(value=DateValue(value=maximum.date()), grain=grain) - if not isinstance(discovered_value, TimestampValue): - raise MaximumAutomaticStartError(message="automatic start requires a timestamp cursor") - if discovered_value.value.tzinfo is not None: - maximum = maximum.replace(tzinfo=UTC) - return floor_to_grain(value=TimestampValue(value=maximum), grain=grain) - - -def apply_maximum_start_policy( - *, - bounds: CursorBounds, - snapshot: ModelCursorSnapshot, - cursor_type: str | None, - cursor_grain: str | None, - cursor_start: CursorScalar | str | None, - lookback: Duration | str | None, - backfill_duration: Duration | str | None, - policy: MaximumStartPolicyInputs, - has_start_override: bool, -) -> CursorBounds: - """Replace an ineligible physical start before applying normal replay policy.""" - - physical_max: CursorScalar | None = snapshot.physical_target_max or snapshot.target_max - if ( - policy.config is None - or policy.config.max_ahead is None - or cursor_type != CursorType.TIMESTAMP - or policy.invocation_time is None - or has_start_override - or physical_max is None - ): - return bounds - if not isinstance(bounds.start, DateValue | TimestampValue) or not isinstance( - bounds.end, DateValue | TimestampValue - ): - return bounds - horizon: CursorScalar = _maximum_allowed_start( - discovered_value=physical_max, - cursor_grain=cursor_grain, - invocation_time=policy.invocation_time, - max_ahead=policy.config.max_ahead, - ) - if compare(left=physical_max, right=horizon) <= 0: - return bounds - eligible_start: CursorScalar | None = snapshot.target_eligible_max - if eligible_start is None and snapshot.upstream_mins: - upstream_minimum: CursorScalar = min_bound( - values=snapshot.upstream_mins, cursor_type=CursorType.TIMESTAMP - ) - if compare(left=upstream_minimum, right=horizon) <= 0: - eligible_start = upstream_minimum - if eligible_start is None: - parsed_cursor_start: CursorScalar | None = ( - cursor_start - if isinstance(cursor_start, DateValue | TimestampValue) - else parse(raw=cursor_start, cursor_type=CursorType.TIMESTAMP) - if cursor_start is not None - else None - ) - if ( - parsed_cursor_start is not None - and compare(left=parsed_cursor_start, right=horizon) <= 0 - ): - eligible_start = parsed_cursor_start - effective_start: CursorScalar = ( - apply_typed_cursor_replay_policy( - start=eligible_start, - end=bounds.end, - cursor_start=cursor_start, - cursor_type=cursor_type, - lookback=lookback, - backfill_duration=backfill_duration, - has_start_override=False, - ) - if eligible_start is not None - else bounds.start - ) - evidence: MaximumStartSafetyEvidence = MaximumStartSafetyEvidence( - action=policy.config.action, - max_ahead=policy.config.max_ahead, - invocation_time=TimestampValue(value=policy.invocation_time.astimezone(UTC)), - physical_target_max=physical_max, - highest_eligible_target_max=snapshot.target_eligible_max, - effective_start=effective_start, - maximum_allowed_start=horizon, - target_relation=snapshot.target_relation or "", - cursor_column=snapshot.destination_cursor_column or "", - ) - if policy.config.action == FutureCursorAction.ERROR: - raise MaximumAutomaticStartError(evidence=evidence) - if eligible_start is None or compare(left=effective_start, right=horizon) > 0: - raise MaximumAutomaticStartError(evidence=evidence) - idempotent: bool = policy.incremental_strategy in { - IncrementalStrategy.DELETE_INSERT, - IncrementalStrategy.MERGE, - } - if not idempotent: - raise MaximumAutomaticStartError(evidence=evidence, non_idempotent=True) - return replace(bounds, start=effective_start, maximum_start_safety=evidence) - - -def _parse_duration(value: str) -> Duration: - parsed: Duration | None = Duration.parse(value) - if parsed is not None: - return parsed - if value == ZERO_DAY_CURSOR_DURATION: - return Duration() - raise MaximumAutomaticStartError(message=f"invalid cursors.start.max_ahead '{value}'") +from sqlbuild._native import apply_maximum_start_policy as apply_maximum_start_policy +from sqlbuild._native import maximum_allowed_start as maximum_allowed_start From 55c7df6a27c61380b55f9be70d2ca52ab95a3a1a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:51:36 +0100 Subject: [PATCH 18/29] chore: update Python boundary after cursor safety port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 446a526d92..df759ce888 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 35997 +remaining_lines = 35602 [[stays]] path = "executor" From da557843520c4530129004d030322f115f9b2a18 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:56:16 +0100 Subject: [PATCH 19/29] refactor: host model migration orchestration in native planner --- .../src/planner_migrations/constants.rs | 3 + .../src/planner_migrations/main/mod.rs | 2 + .../planner_migrations/main/model_decision.rs | 21 + .../planner_migrations/main/model_warning.rs | 61 ++ .../src/planner_migrations/models.rs | 18 + .../src/planner_migrations/tests.rs | 2 + .../tests/test_model_decision.rs | 64 ++ .../planner_migrations/tests/test_types.rs | 7 + .../_helpers/planner_migrations/mod.rs | 5 + .../planner_migrations/model_entries.rs | 293 ++++++++ .../planner_migrations/model_handover.rs | 115 ++++ .../planner_migrations/model_overlay.rs | 134 ++++ .../planner_migrations/model_planning.rs | 367 ++++++++++ .../planner_migrations/model_warnings.rs | 68 ++ .../planner_migrations/registration.rs | 1 + src/sqlbuild/_native.pyi | 22 + .../planner/_helpers/migrations/planning.py | 624 +----------------- 17 files changed, 1192 insertions(+), 615 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/main/model_decision.rs create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/main/model_warning.rs create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/tests/test_model_decision.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs diff --git a/crates/sqlbuild-analysis/src/planner_migrations/constants.rs b/crates/sqlbuild-analysis/src/planner_migrations/constants.rs index dd09a82c85..b760634f78 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/constants.rs @@ -46,3 +46,6 @@ pub(crate) const COLUMN_ADDED: &str = "column_added"; pub(crate) const COLUMN_TYPE_CHANGED: &str = "column_type_changed"; pub(crate) const POLICY_DENY: &str = "deny"; pub(crate) const POLICY_FAIL: &str = "fail"; +pub(crate) const MODEL_DONE: &str = "done"; +pub(crate) const MODEL_RENAMED: &str = "renamed"; +pub(crate) const MODEL_CONFLICT: &str = "conflict"; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs index 10663c6718..d984291613 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs @@ -6,6 +6,8 @@ pub mod identical_renames; pub mod match_migrations; pub mod migration_fingerprint; pub mod migration_ref_names; +pub mod model_decision; +pub mod model_warning; pub mod old_name_claim_message; pub mod old_name_schemas; pub mod old_name_skip_reason; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/model_decision.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/model_decision.rs new file mode 100644 index 0000000000..f128ddff8c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/model_decision.rs @@ -0,0 +1,21 @@ +//! Model migration precedence from live identity and append-only history. + +/// Decide a data migration after origin existence and compatibility inspection. +pub fn model_decision( + destination_exists: bool, + history: (bool, bool, bool), + force: bool, +) -> &'static str { + let (has_event, destination_was_origin, has_fingerprint) = history; + if !destination_exists { + "migrate" + } else if destination_was_origin { + "superseded_replace" + } else if !has_event && !has_fingerprint { + "redo" + } else if force { + "forced_replace" + } else { + "conflict" + } +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/model_warning.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/model_warning.rs new file mode 100644 index 0000000000..3c0eb5a546 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/model_warning.rs @@ -0,0 +1,61 @@ +//! Ordered model migration diagnostics from the public planning result. + +use crate::planner_migrations::constants::{MODEL_CONFLICT, MODEL_DONE, MODEL_RENAMED}; +use crate::planner_migrations::models::ModelWarning; + +/// Select and format the first applicable model migration diagnostic. +pub fn model_warning(facts: &ModelWarning<'_>) -> Option<(&'static str, String, bool)> { + let ModelWarning { + name, + origin, + destination, + decision, + manual, + completed, + target, + hidden, + missing, + deny, + blocked, + remedy, + outcome, + findings, + } = facts; + if *decision == MODEL_DONE || (*decision == MODEL_RENAMED && completed.is_some()) { + return manual.then(|| ("M101", format!("migration from {origin} completed at {} on target '{target}'; migrate_from can be removed from '{name}'", completed.unwrap_or("unknown time")), false)); + } + if *hidden { + return Some(( + "M102", + format!( + "model '{name}': migrate_from origin {origin} was built in this target but is not visible; {remedy}" + ), + true, + )); + } + if *missing { + return Some(( + "M102", + format!( + "model '{name}': migrate_from origin {origin} does not exist in this target and no recorded migration into it was found; {outcome}" + ), + *deny, + )); + } + if *decision == MODEL_CONFLICT { + return Some(( + "M103", + format!( + "migration conflict: {destination} already exists with its own build history and has no recorded migration from {origin}; set migrate_force true to replace it (the replaced table is kept under a _sqb_archive__ name until janitor expires it)" + ), + true, + )); + } + blocked.then(|| { + ( + "M104", + format!("migration from {origin} is incompatible with '{name}': {findings}"), + true, + ) + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/models.rs b/crates/sqlbuild-analysis/src/planner_migrations/models.rs index 3badb760fb..c368905c69 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/models.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/models.rs @@ -82,3 +82,21 @@ pub struct ColumnWarning<'a> { pub missing_outcome: &'a str, pub message: &'a str, } +/// Display facts and ordered gates for one model migration diagnostic. +#[derive(Debug)] +pub struct ModelWarning<'a> { + pub name: &'a str, + pub origin: &'a str, + pub destination: &'a str, + pub decision: &'a str, + pub manual: bool, + pub completed: Option<&'a str>, + pub target: &'a str, + pub hidden: bool, + pub missing: bool, + pub deny: bool, + pub blocked: bool, + pub remedy: &'a str, + pub outcome: &'a str, + pub findings: &'a str, +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/tests.rs b/crates/sqlbuild-analysis/src/planner_migrations/tests.rs index 9a86b7352e..5023a7ca72 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/tests.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/tests.rs @@ -6,5 +6,7 @@ mod test_column_decision; mod test_column_renames; #[path = "tests/test_decisions.rs"] mod test_decisions; +#[path = "tests/test_model_decision.rs"] +mod test_model_decision; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/tests/test_model_decision.rs b/crates/sqlbuild-analysis/src/planner_migrations/tests/test_model_decision.rs new file mode 100644 index 0000000000..186d5efb8b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/tests/test_model_decision.rs @@ -0,0 +1,64 @@ +//! Live destination evidence and history determine model migration precedence. + +use crate::planner_migrations::main::model_decision::model_decision; +use crate::planner_migrations::tests::test_types::ModelDecisionTestCase; + +#[test] +fn given_live_and_recorded_identity_when_deciding_then_preserves_replacement_precedence() { + let test_cases = [ + ModelDecisionTestCase { + description: "missing destination overrides stale history", + destination_exists: false, + history: (true, true, true), + force: true, + expected_decision: "migrate", + }, + ModelDecisionTestCase { + description: "reverse move supersedes even with force", + destination_exists: true, + history: (true, true, true), + force: true, + expected_decision: "superseded_replace", + }, + ModelDecisionTestCase { + description: "unrecorded destination redoes before force", + destination_exists: true, + history: (false, false, false), + force: true, + expected_decision: "redo", + }, + ModelDecisionTestCase { + description: "own history conflicts", + destination_exists: true, + history: (false, false, true), + force: false, + expected_decision: "conflict", + }, + ModelDecisionTestCase { + description: "force replaces own history", + destination_exists: true, + history: (false, false, true), + force: true, + expected_decision: "forced_replace", + }, + ModelDecisionTestCase { + description: "unrelated event prevents redo", + destination_exists: true, + history: (true, false, false), + force: false, + expected_decision: "conflict", + }, + ]; + for test_case in test_cases { + assert_eq!( + model_decision( + test_case.destination_exists, + test_case.history, + test_case.force + ), + test_case.expected_decision, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_migrations/tests/test_types.rs index 89e925ace4..9cc3909611 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/tests/test_types.rs @@ -31,3 +31,10 @@ pub(super) struct ColumnDecisionTestCase { pub still_produced: bool, pub expected_decision: &'static str, } +pub(super) struct ModelDecisionTestCase { + pub description: &'static str, + pub destination_exists: bool, + pub history: (bool, bool, bool), + pub force: bool, + pub expected_decision: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index ab9df51a2e..626c5b1a8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -10,6 +10,11 @@ mod constants; mod errors; mod fingerprint; mod matching; +mod model_entries; +mod model_handover; +mod model_overlay; +mod model_planning; +mod model_warnings; mod models; mod old_names; mod projections; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs new file mode 100644 index 0000000000..81178554ef --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs @@ -0,0 +1,293 @@ +//! Assemble model migration decisions from inspected warehouse facts. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_migrations::column_objects::{ + lower, model, native, replace, text, +}; +use crate::bindings::_helpers::planner_migrations::constants::{MIGRATION_TYPES, PLANNING}; +use crate::bindings::_helpers::planner_migrations::model_handover::{ + applies, empty_version, handover, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyBool, PyTuple}; +use sqlbuild_analysis::planner_migrations::main::model_decision::model_decision; + +pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { + imported( + location.py(), + "sqlbuild.compiler.migrations.main.relation_for_location", + "migration_relation_for_location", + )? + .call1((location,)) +} + +fn key(location: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + let schema = location.getattr("schema")?; + Ok(( + if schema.is_truthy()? { + lower(&schema)? + } else { + String::new() + }, + lower(&location.getattr("name")?)?, + )) +} + +pub(super) fn decision<'py>(entry: &Bound<'py, PyAny>, name: &str) -> PyResult> { + replace( + entry, + &[( + "decision", + imported(entry.py(), MIGRATION_TYPES, "MigrationDecision")?.call1((name,))?, + )], + ) +} + +fn result<'py>( + entry: Bound<'py, PyAny>, + fingerprint: Bound<'py, PyAny>, + applies: bool, +) -> PyResult> { + let py = entry.py(); + Ok(PyTuple::new( + py, + [ + entry, + fingerprint, + PyBool::new(py, applies).to_owned().into_any(), + ], + )? + .into_any()) +} + +pub(super) fn decide<'py>(args: &KeywordArguments<'py>) -> PyResult> { + let runtime = args.value("runtime")?; + let py = runtime.py(); + let request = args.value("request")?; + let item = request.getattr("model")?; + let destination = item.getattr("destination")?; + let origin = args.value("origin")?; + let snapshot = args.value("snapshot")?; + let state = args.value("state")?; + if key(&origin)? == key(&destination)? { + let name: String = item.getattr("name")?.extract()?; + return Err(pyo3::PyErr::from_value( + imported( + py, + "sqlbuild.compiler.planner.exceptions", + "PlannerInputError", + )? + .call1((format!( + "model '{name}': migrate_from names the model's own relation" + ),))?, + )); + } + let newest = call_with( + &imported( + py, + "sqlbuild.compiler.migrations.main._newest_event", + "newest_migration_event_mentioning", + )?, + &[ + ("events", state.getattr("events")?), + ("relation", relation(&destination)?), + ], + )?; + let origin_relation = state.call_method1("relation", (&origin,))?; + let fingerprint = call_with( + &state.getattr("model_fingerprint")?, + &[ + ("location", origin.clone()), + ("model_name", args.value("origin_model")?), + ], + )?; + let base = model( + py, + "ModelMigrationPlanEntry", + &[ + ("model_name", item.getattr("name")?), + ("discovery", request.getattr("discovery")?), + ( + "decision", + imported(py, MIGRATION_TYPES, "MigrationDecision")?.getattr("MIGRATE")?, + ), + ( + "compatibility", + imported(py, MIGRATION_TYPES, "MigrationCompatibility")?.getattr("NOT_CHECKED")?, + ), + ("origin_model", args.value("origin_model")?), + ("origin", origin.clone()), + ("destination", destination.clone()), + ( + "target_name", + runtime + .getattr("project")? + .getattr("effective_target_name")?, + ), + ( + "origin_version_hash", + if fingerprint.is_none() { + text(py, "") + } else { + fingerprint.getattr("version_hash")? + }, + ), + ( + "origin_is_transient", + PyBool::new( + py, + !origin_relation.is_none() + && origin_relation.getattr("is_transient")?.is_truthy()?, + ) + .to_owned() + .into_any(), + ), + ( + "origin_tracked", + PyBool::new(py, !fingerprint.is_none()) + .to_owned() + .into_any(), + ), + ], + )?; + let recorded = !newest.is_none() + && newest + .getattr("destination")? + .call_method1("matches", (relation(&destination)?,))? + .is_truthy()? + && newest + .getattr("origin")? + .call_method1("matches", (relation(&origin)?,))? + .is_truthy()?; + if request.getattr("identity_only")?.is_truthy()? { + let recorded = recorded && newest.getattr("decision")?.eq("renamed")?; + if request.getattr("discovery")?.eq("manual")? && !recorded && origin_relation.is_none() { + return result( + decision(&base, "origin_missing")?, + py.None().into_bound(py), + false, + ); + } + let inherited = if recorded && !applies(&item, &newest, &snapshot)? { + snapshot + .getattr("fingerprints")? + .getattr("models")? + .call_method1("get", (item.getattr("name")?,))? + } else { + empty_version(&item, &fingerprint)? + }; + return result( + replace( + &decision(&base, "renamed")?, + &[( + "completed_at", + if recorded { + newest.getattr("created_at")? + } else { + py.None().into_bound(py) + }, + )], + )?, + inherited, + true, + ); + } + if recorded { + let applies = applies(&item, &newest, &snapshot)?; + let inherited = if applies { + handover(&item, &fingerprint, &newest.getattr("origin_version_hash")?)? + } else { + snapshot + .getattr("fingerprints")? + .getattr("models")? + .call_method1("get", (item.getattr("name")?,))? + }; + return result( + replace( + &decision(&base, "done")?, + &[ + ("completed_at", newest.getattr("created_at")?), + ( + "origin_version_hash", + newest.getattr("origin_version_hash")?, + ), + ("target_name", newest.getattr("target_name")?), + ], + )?, + inherited, + applies, + ); + } + if origin_relation.is_none() { + return result( + decision(&base, "origin_missing")?, + py.None().into_bound(py), + false, + ); + } + let compatibility = native( + py, + "check_migration_compatibility", + &[ + ("model", item.clone()), + ("origin_relation", origin_relation), + ( + "origin_columns", + state.call_method1("relation_columns", (&origin,))?, + ), + ( + "sql_analysis_enabled", + runtime + .getattr("project")? + .getattr("settings")? + .getattr("sql_analysis")?, + ), + ("column_dialect", snapshot.getattr("column_dialect")?), + ("snapshots_config", args.value("snapshots_config")?), + ], + )?; + let name = model_decision( + snapshot + .getattr("existing_relations")? + .contains(item.getattr("name")?)?, + ( + !newest.is_none(), + !newest.is_none() + && newest + .getattr("origin")? + .call_method1("matches", (relation(&destination)?,))? + .is_truthy()?, + snapshot + .getattr("fingerprints")? + .getattr("models")? + .contains(item.getattr("name")?)?, + ), + request.getattr("force")?.is_truthy()?, + ); + let entry = replace( + &decision(&base, name)?, + &[ + ("compatibility", compatibility.getattr("status")?), + ("compatibility_findings", compatibility.getattr("findings")?), + ], + )?; + if !entry + .getattr("decision")? + .getattr("moves_data")? + .is_truthy()? + || entry.getattr("blocks_build")?.is_truthy()? + { + return result(entry, py.None().into_bound(py), false); + } + let inherited = handover(&item, &fingerprint, &entry.getattr("origin_version_hash")?)?; + result( + call_with( + &imported(py, PLANNING, "_with_execution")?, + &[("runtime", runtime), ("model", item), ("entry", entry)], + )?, + inherited, + true, + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs new file mode 100644 index 0000000000..e059dae71b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs @@ -0,0 +1,115 @@ +//! Adapt immutable migration fingerprints without changing their persisted representation. + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::planner_migrations::column_objects::{native, replace, text}; +use crate::bindings::_helpers::state_rows::timestamps::presumed_utc; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; + +pub(super) fn applies( + model: &Bound<'_, PyAny>, + event: &Bound<'_, PyAny>, + snapshot: &Bound<'_, PyAny>, +) -> PyResult { + let own = snapshot + .getattr("fingerprints")? + .getattr("models")? + .call_method1("get", (model.getattr("name")?,))?; + Ok(own.is_none() + || presumed_utc(&own.getattr("ts")?)? < presumed_utc(&event.getattr("created_at")?)?) +} + +pub(super) fn handover<'py>( + model: &Bound<'py, PyAny>, + fingerprint: &Bound<'py, PyAny>, + version: &Bound<'py, PyAny>, +) -> PyResult> { + if fingerprint.is_none() { + return Ok(fingerprint.clone()); + } + let py = model.py(); + let mut metadata = fingerprint.getattr("metadata_json")?; + let json = py.import("json")?; + let payload = match json.call_method1("loads", (&metadata,)) { + Ok(payload) => payload, + Err(error) if error.is_instance(py, &json.getattr("JSONDecodeError")?) => { + py.None().into_bound(py) + } + Err(error) => return Err(error), + }; + let key = imported( + py, + "sqlbuild.compiler.planner.constants", + "MIGRATION_MODEL_NAME_METADATA_KEY", + )?; + if let Ok(payload) = payload.cast::() + && payload.contains(&key)? + { + payload.set_item(key, model.getattr("name")?)?; + let kwargs = PyDict::new(py); + kwargs.set_item("sort_keys", true)?; + kwargs.set_item("separators", (",", ":"))?; + kwargs.set_item("default", imported(py, "builtins", "str")?)?; + metadata = json.getattr("dumps")?.call((payload,), Some(&kwargs))?; + } + let destination = model.getattr("destination")?; + replace( + fingerprint, + &[ + ("node_name", model.getattr("name")?), + ("target_database", destination.getattr("database")?), + ("target_schema", destination.getattr("schema")?), + ("target_name", destination.getattr("name")?), + ( + "version_hash", + if version.is_truthy()? { + version.clone() + } else { + fingerprint.getattr("version_hash")? + }, + ), + ("metadata_json", metadata), + ], + ) +} + +pub(super) fn equivalent<'py>( + model: &Bound<'py, PyAny>, + handover: Bound<'py, PyAny>, + destinations: &Bound<'py, PyAny>, + dialect: &Bound<'py, PyAny>, +) -> PyResult> { + let py = model.py(); + if handover.is_none() + || !destinations.contains( + imported(py, "sqlbuild._native", "stored_migration_fingerprint")? + .call1((&handover,))?, + )? + { + return Ok(handover); + } + replace( + &handover, + &[ + ("definition", model.getattr("query_sql")?), + ( + "definition_hash", + native( + py, + "compute_query_hash", + &[ + ("query_sql", model.getattr("query_sql")?), + ("dialect", dialect.clone()), + ], + )?, + ), + ], + ) +} + +pub(super) fn empty_version<'py>( + model: &Bound<'py, PyAny>, + fingerprint: &Bound<'py, PyAny>, +) -> PyResult> { + handover(model, fingerprint, &text(model.py(), "")) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs new file mode 100644 index 0000000000..b3f2430eb5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs @@ -0,0 +1,134 @@ +//! Project inherited fingerprints and live relation facts before normal planning. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_migrations::column_objects::{config, model, replace}; +use crate::bindings::_helpers::planner_migrations::constants::PLANNING; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; + +pub(super) fn overlay<'py>( + args: &KeywordArguments<'py>, + state: &Bound<'py, PyAny>, + entries: &[Bound<'py, PyAny>], + handovers: &Bound<'py, PyDict>, +) -> PyResult> { + let snapshot = args.value("snapshot")?; + if handovers.is_empty() { + return Ok(snapshot); + } + let runtime = args.value("runtime")?; + let py = runtime.py(); + let relations = snapshot + .getattr("existing_relations")? + .cast_into::()? + .copy()?; + let columns = snapshot + .getattr("existing_columns")? + .cast_into::()? + .copy()?; + let fingerprints = snapshot.getattr("fingerprints")?; + let models = fingerprints + .getattr("models")? + .cast_into::()? + .copy()?; + let redirected = PyDict::new(py); + for entry in entries { + let name = entry.getattr("model_name")?; + let Some(handover) = handovers.get_item(&name)? else { + continue; + }; + if handover.is_none() { + models.call_method1("pop", (&name, py.None()))?; + } else { + models.set_item(&name, handover)?; + } + if !entry + .getattr("decision")? + .getattr("moves_data")? + .is_truthy()? + { + continue; + } + let origin = entry.getattr("origin")?; + let origin_relation = state.call_method1("relation", (&origin,))?; + if origin_relation.is_none() { + continue; + } + let destination = entry.getattr("destination")?; + let transient = entry.getattr("stage_is_transient")?; + relations.set_item( + &name, + replace( + &origin_relation, + &[ + ("database", destination.getattr("database")?), + ("schema", destination.getattr("schema")?), + ("name", destination.getattr("name")?), + ( + "is_transient", + if transient.is_none() { + origin_relation.getattr("is_transient")? + } else { + transient + }, + ), + ], + )?, + )?; + columns.set_item(&name, state.call_method1("relation_columns", (&origin,))?)?; + let qualified = origin.getattr("qualified_name")?; + if !qualified.is_none() && cursor_enabled(&runtime, &name)? { + redirected.set_item(name, qualified)?; + } + } + let cursors = snapshot + .getattr("cursor_snapshots")? + .cast_into::()? + .copy()?; + let gathered = call_with( + &imported(py, PLANNING, "redirected_cursor_snapshots")?, + &[ + ("runtime", runtime), + ("scope", args.value("scope")?), + ("existing_relations", relations.clone().into_any()), + ("target_relations", redirected.into_any()), + ("overrides", args.value("overrides")?), + ("deferral", args.value("deferral")?), + ], + )?; + cursors.call_method1("update", (gathered,))?; + replace( + &snapshot, + &[ + ("existing_relations", relations.into_any()), + ("existing_columns", columns.into_any()), + ( + "fingerprints", + model( + py, + "WarehouseFingerprints", + &[ + ("models", models.into_any()), + ("functions", fingerprints.getattr("functions")?), + ("seeds", fingerprints.getattr("seeds")?), + ("python_nodes", fingerprints.getattr("python_nodes")?), + ], + )?, + ), + ("cursor_snapshots", cursors.into_any()), + ], + ) +} + +fn cursor_enabled(runtime: &Bound<'_, PyAny>, name: &Bound<'_, PyAny>) -> PyResult { + for item in runtime.getattr("project")?.getattr("models")?.try_iter()? { + let item = item?; + if item.getattr("name")?.eq(name)? { + return Ok(config(&item, "materialized")?.eq("incremental")? + && !config(&item, "cursor")?.is_none()); + } + } + Ok(false) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs new file mode 100644 index 0000000000..25253f3b49 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -0,0 +1,367 @@ +//! Model migration orchestration over Python-owned live inspection. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_migrations::column_objects::{ + config, model, native, replace, +}; +use crate::bindings::_helpers::planner_migrations::constants::{MIGRATION_TYPES, PLANNING}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyString, PyTuple, +}; +use pyo3::{pyfunction, wrap_pyfunction}; + +fn declared<'py>(item: &Bound<'py, PyAny>) -> PyResult>> { + let py = item.py(); + let values = item.getattr("config")?.getattr("values")?; + let origin = values.call_method1("get", ("migrate_from",))?; + if !origin.is_instance_of::() { + return Ok(None); + } + let materialization = config(item, "materialized")?; + Ok(Some(model( + py, + "ModelMigrationRequest", + &[ + ("model", item.clone()), + ( + "discovery", + imported(py, MIGRATION_TYPES, "MigrationDiscovery")?.getattr("MANUAL")?, + ), + ("raw_origin", origin.call_method0("strip")?), + ( + "force", + PyBool::new( + py, + values + .call_method1("get", ("migrate_force",))? + .is(PyBool::new(py, true)), + ) + .to_owned() + .into_any(), + ), + ( + "identity_only", + PyBool::new( + py, + materialization.eq("table")? || materialization.eq("view")?, + ) + .to_owned() + .into_any(), + ), + ], + )?)) +} + +#[pyfunction] +#[pyo3(signature = (*, scope))] +fn manual_migration_requests<'py>(scope: &Bound<'py, PyAny>) -> PyResult> { + let mut requests: Vec> = Vec::new(); + for key in scope.getattr("execution_order")?.try_iter()? { + let key = key?; + if !scope.getattr("selected_keys")?.contains(&key)? + || !key.getattr("resource_type")?.eq("model")? + { + continue; + } + let item = scope + .getattr("models_by_name")? + .call_method1("get", (key.getattr("name")?,))?; + if !item.is_none() + && let Some(request) = declared(&item)? + { + requests.push(request); + } + } + Ok(PyTuple::new(scope.py(), requests)?.into_any()) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +fn decide_model_migration<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + super::model_entries::decide(&KeywordArguments::read( + py, + "decide_model_migration", + options, + ( + &[ + "runtime", + "request", + "origin", + "origin_model", + "snapshot", + "state", + "snapshots_config", + ], + &[], + ), + )?) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +fn plan_model_migrations<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "plan_model_migrations", + options, + ( + &[ + "runtime", + "scope", + "snapshot", + "overrides", + "deferral", + "fingerprints", + ], + &[], + ), + )?; + let runtime = args.value("runtime")?; + let snapshot = args.value("snapshot")?; + let schemas = call_with( + &imported(py, PLANNING, "project_schemas")?, + &[("runtime", runtime.clone())], + )?; + let state = call_with( + &imported( + py, + "sqlbuild.compiler.planner.classes.migration_state_inspection", + "MigrationStateInspection", + )?, + &[ + ("adapter", runtime.getattr("adapter")?), + ("connection", runtime.getattr("connection")?), + ( + "database", + call_with( + &imported(py, PLANNING, "planning_database")?, + &[("runtime", runtime.clone())], + )?, + ), + ("known_fingerprints", snapshot.getattr("fingerprints")?), + ], + )?; + let discovery = discover(&args, &state, &schemas)?; + finish(&args, &state, &schemas, &discovery) +} + +fn discover<'py>( + args: &KeywordArguments<'py>, + state: &Bound<'py, PyAny>, + schemas: &Bound<'py, PyAny>, +) -> PyResult> { + let runtime = args.value("runtime")?; + let py = runtime.py(); + let scope = args.value("scope")?; + let snapshot = args.value("snapshot")?; + let mut declared_requests: Vec> = Vec::new(); + for item in runtime.getattr("project")?.getattr("models")?.try_iter()? { + if let Some(request) = declared(&item?)? { + declared_requests.push(request); + } + } + if !declared_requests.is_empty() { + call_with( + &state.getattr("inspect_schemas")?, + &[("schemas", schemas.clone())], + )?; + } + let mut declarations: Vec> = Vec::new(); + for request in declared_requests { + let resolved = call_with( + &imported(py, PLANNING, "_resolve_origin")?, + &[ + ("request", request.clone()), + ("runtime", runtime.clone()), + ("state", state.clone()), + ], + )?; + declarations.push(model( + py, + "ModelMigrationDeclaration", + &[ + ("model_name", request.getattr("model")?.getattr("name")?), + ("origin_location", resolved.get_item(0)?), + ("origin_model", resolved.get_item(1)?), + ], + )?); + } + call_with( + &imported( + py, + "sqlbuild.compiler.planner._helpers.migrations.discovery", + "discover_model_migrations", + )?, + &[ + ("runtime", runtime.clone()), + ("scope", scope.clone()), + ("snapshot", snapshot.clone()), + ("manual_requests", manual_migration_requests(&scope)?), + ("declarations", PyTuple::new(py, declarations)?.into_any()), + ("state", state.clone()), + ("project_schemas", schemas.clone()), + ("fingerprints", args.value("fingerprints")?), + ], + ) +} + +fn finish<'py>( + args: &KeywordArguments<'py>, + state: &Bound<'py, PyAny>, + schemas: &Bound<'py, PyAny>, + discovery: &Bound<'py, PyAny>, +) -> PyResult> { + let runtime = args.value("runtime")?; + let py = runtime.py(); + let scope = args.value("scope")?; + let snapshot = args.value("snapshot")?; + let mut resolved: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); + for request in discovery.getattr("requests")?.try_iter()? { + let request = request?; + let origin = call_with( + &imported(py, PLANNING, "_resolve_origin")?, + &[ + ("request", request.clone()), + ("runtime", runtime.clone()), + ("state", state.clone()), + ], + )?; + resolved.push((request, origin.get_item(0)?, origin.get_item(1)?)); + } + let mut entries: Vec> = Vec::new(); + let handovers = PyDict::new(py); + if !resolved.is_empty() { + let origin_schemas = PySet::empty(py)?; + let mut origins: Vec> = Vec::new(); + for (_, origin, _) in &resolved { + call_with( + &imported(py, PLANNING, "_require_same_database")?, + &[("state", state.clone()), ("location", origin.clone())], + )?; + let schema = origin.getattr("schema")?; + if !schema.is_none() { + origin_schemas.add(schema)?; + } + origins.push(origin.clone()); + } + call_with( + &state.getattr("inspect_schemas")?, + &[("schemas", origin_schemas.into_any())], + )?; + call_with( + &state.getattr("inspect_relations")?, + &[("locations", PyTuple::new(py, origins)?.into_any())], + )?; + let project_config = runtime.getattr("project_config")?; + let snapshots_config = if project_config.is_none() { + imported(py, "sqlbuild.spec.contracts.models", "SnapshotsConfig")?.call0()? + } else { + project_config.getattr("snapshots")? + }; + let policy = call_with( + &imported(py, PLANNING, "missing_origin_policy")?, + &[("runtime", runtime.clone())], + )?; + for (request, origin, origin_model) in resolved { + let item = request.getattr("model")?; + let name = item.getattr("name")?; + let decided = native( + py, + "decide_model_migration", + &[ + ("runtime", runtime.clone()), + ("request", request), + ("origin", origin), + ("origin_model", origin_model), + ("snapshot", snapshot.clone()), + ("state", state.clone()), + ("snapshots_config", snapshots_config.clone()), + ], + )?; + entries.push(replace( + &decided.get_item(0)?, + &[("missing_origin_policy", policy.clone())], + )?); + if decided.get_item(2)?.is_truthy()? { + let destinations = discovery + .getattr("destination_fingerprints")? + .call_method1("get", (&name, PyTuple::empty(py)))?; + handovers.set_item( + name, + super::model_handover::equivalent( + &item, + decided.get_item(1)?, + &destinations, + &snapshot.getattr("column_dialect")?, + )?, + )?; + } + } + } + let planned = PyDict::new(py); + let mut renamed: Vec> = Vec::new(); + let mut warnings: Vec> = Vec::new(); + for warning in discovery.getattr("warnings")?.try_iter()? { + warnings.push(warning?); + } + for entry in &entries { + let name = entry.getattr("model_name")?; + if let Some(handover) = handovers.get_item(&name)? { + if !handover.is_none() && entry.getattr("decision")?.eq("renamed")? { + renamed.push(name.clone()); + } + let origin_model = entry.getattr("origin_model")?; + if !origin_model.is_none() { + planned.set_item(&name, origin_model)?; + } + } + if let Some(warning) = super::model_warnings::entry_warning(entry)? { + warnings.push(warning); + } + } + let overlaid = if entries.is_empty() { + snapshot.clone() + } else { + super::model_overlay::overlay(args, state, &entries, &handovers)? + }; + let reference_renames = native( + py, + "plan_reference_renames", + &[ + ("runtime", runtime), + ("scope", scope), + ("snapshot", snapshot), + ("state", state.clone()), + ("schemas", schemas.clone()), + ("planned", planned.into_any()), + ], + )?; + let mut fields = vec![("reference_renames", reference_renames)]; + if !entries.is_empty() { + fields.push(("renamed_models", PyFrozenSet::new(py, renamed)?.into_any())); + } + model( + py, + "ModelMigrationPlanning", + &[ + ("snapshot", replace(&overlaid, &fields)?), + ("entries", PyTuple::new(py, entries)?.into_any()), + ("warnings", PyTuple::new(py, warnings)?.into_any()), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(manual_migration_requests, module)?)?; + module.add_function(wrap_pyfunction!(decide_model_migration, module)?)?; + module.add_function(wrap_pyfunction!(plan_model_migrations, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs new file mode 100644 index 0000000000..ddae3fd5a5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs @@ -0,0 +1,68 @@ +//! Adapt diagnostic display facts while preserving policy and code ordering. + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::planner_migrations::column_warnings::warning; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyString; +use sqlbuild_analysis::planner_migrations::main::model_warning::model_warning; +use sqlbuild_analysis::planner_migrations::models::ModelWarning; + +pub(super) fn entry_warning<'py>(entry: &Bound<'py, PyAny>) -> PyResult>> { + let py = entry.py(); + let name: String = entry.getattr("model_name")?.extract()?; + let origin = display(&entry.getattr("origin")?)?; + let destination = display(&entry.getattr("destination")?)?; + let decision: String = entry.getattr("decision")?.extract()?; + let completed = entry.getattr("completed_at")?; + let completed: Option = if completed.is_none() { + None + } else { + Some(completed.call_method0("isoformat")?.extract()?) + }; + let target: String = entry.getattr("target_label")?.extract()?; + let policy = entry.getattr("missing_origin_policy")?; + let remedy: String = imported( + py, + "sqlbuild.compiler.planner.constants", + "HIDDEN_ORIGIN_REMEDY", + )? + .extract()?; + let outcome: String = imported( + py, + "sqlbuild.compiler.planner.constants", + "MISSING_ORIGIN_OUTCOMES", + )? + .get_item(&policy)? + .extract()?; + let findings: String = PyString::new(py, "; ") + .call_method1("join", (entry.getattr("compatibility_findings")?,))? + .extract()?; + let facts = ModelWarning { + name: &name, + origin: &origin, + destination: &destination, + decision: &decision, + manual: entry.getattr("discovery")?.eq("manual")?, + completed: completed.as_deref(), + target: &target, + hidden: entry.getattr("origin_hidden")?.is_truthy()?, + missing: entry.getattr("origin_missing")?.is_truthy()?, + deny: policy.eq("deny")?, + blocked: entry.getattr("blocks_build")?.is_truthy()?, + remedy: &remedy, + outcome: &outcome, + findings: &findings, + }; + model_warning(&facts) + .map(|(code, message, error)| warning(entry, code, &message, error)) + .transpose() +} + +fn display(location: &Bound<'_, PyAny>) -> PyResult { + let qualified = location.getattr("qualified_name")?; + if qualified.is_truthy()? { + qualified.extract() + } else { + location.getattr("name")?.extract() + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs index 00b53a7bab..5c1b369b29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs @@ -8,6 +8,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::old_names::register(module)?; super::columns::register(module)?; super::column_planning::register(module)?; + super::model_planning::register(module)?; super::projections::register(module)?; super::fingerprint::register(module) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index c12fd82743..c00e0d0371 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -220,6 +220,8 @@ from sqlbuild.compiler.planner.models import ( MigrationCompatibilityResult, MissingUpstream, ModelCursorSnapshot, + ModelMigrationPlanEntry, + ModelMigrationPlanning, ModelMigrationRequest, ModelPlanContext, ModelPlanEntry, @@ -4577,6 +4579,26 @@ def migration_destination_fingerprints( ) -> dict[str, tuple[str, ...]]: ... def stored_migration_fingerprint(fingerprint: Fingerprint) -> str | None: ... def declared_column_aliases(model: CompiledModelView) -> tuple[tuple[str, str], ...]: ... +def manual_migration_requests(*, scope: PlannerScope) -> tuple[ModelMigrationRequest, ...]: ... +def decide_model_migration( + *, + runtime: PlannerRuntime, + request: ModelMigrationRequest, + origin: CompiledRelationLocation, + origin_model: str | None, + snapshot: WarehouseSnapshot, + state: MigrationStateInspection, + snapshots_config: SnapshotsConfig, +) -> tuple[ModelMigrationPlanEntry, Fingerprint | None, bool]: ... +def plan_model_migrations( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + overrides: PlannerOverrides, + deferral: DeferralInputs, + fingerprints: MigrationFingerprintCache, +) -> ModelMigrationPlanning: ... def plan_column_migrations( *, runtime: PlannerRuntime, diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index 193e9791cc..caed78e3b2 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -1,36 +1,24 @@ -"""Per-run decisions for direct-mode model migrations.""" +"""Warehouse inspection and adapter boundaries for native model migration planning.""" from __future__ import annotations -import json from dataclasses import replace -from datetime import UTC, datetime from sqlbuild._native import ( - check_migration_compatibility, - compute_query_hash, - get_config_str, - plan_reference_renames, - stored_migration_fingerprint, + manual_migration_requests as manual_migration_requests, ) -from sqlbuild.adapter.contract.models import ColumnInfo, MigrationStagePlan, RelationInfo +from sqlbuild._native import ( + plan_model_migrations as plan_model_migrations, +) +from sqlbuild.adapter.contract.models import MigrationStagePlan, RelationInfo from sqlbuild.adapter.contract.types import BuiltinAdapter, MigrationTransfer -from sqlbuild.compiler.compile.constants import MIGRATE_FORCE_CONFIG_KEY, MIGRATE_FROM_CONFIG_KEY from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.migrations.main._newest_event import newest_migration_event_mentioning -from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location -from sqlbuild.compiler.migrations.models import MigrationEvent from sqlbuild.compiler.migrations.types import ( - MigrationCompatibility, MigrationDecision, - MigrationDiscovery, MigrationPromotion, ) -from sqlbuild.compiler.planner._helpers.migrations.discovery import ( - discover_model_migrations, -) from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( effectively_full_refreshed_model_names, ) @@ -38,257 +26,21 @@ from sqlbuild.compiler.planner._helpers.warehouse.snapshot import ( gather_redirected_cursor_snapshots, ) -from sqlbuild.compiler.planner.classes.migration_fingerprint_cache import ( - MigrationFingerprintCache, -) -from sqlbuild.compiler.planner.classes.migration_state_inspection import ( - MigrationStateInspection, -) -from sqlbuild.compiler.planner.constants import ( - HIDDEN_ORIGIN_REMEDY, - MIGRATION_MODEL_NAME_METADATA_KEY, - MISSING_ORIGIN_OUTCOMES, - QUALIFIED_RELATION_MAX_PARTS, -) +from sqlbuild.compiler.planner.classes.migration_state_inspection import MigrationStateInspection +from sqlbuild.compiler.planner.constants import QUALIFIED_RELATION_MAX_PARTS from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( CursorSnapshotScope, DeferralInputs, - MigrationCompatibilityResult, ModelCursorSnapshot, - ModelMigrationDeclaration, - ModelMigrationDiscovery, ModelMigrationPlanEntry, - ModelMigrationPlanning, ModelMigrationRequest, PlannerOverrides, PlannerRuntime, PlannerScope, - PlanWarning, - WarehouseFingerprints, - WarehouseSnapshot, ) -from sqlbuild.compiler.planner.types import MaterializationType, WarningSeverity -from sqlbuild.spec.contracts.models import SnapshotsConfig from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy, TableType -_IDENTITY_MATERIALIZATIONS: frozenset[str] = frozenset( - {MaterializationType.TABLE, MaterializationType.VIEW} -) - - -def manual_migration_requests(*, scope: PlannerScope) -> tuple[ModelMigrationRequest, ...]: - """Collect selected models that declare migrate_from.""" - - requests: list[ModelMigrationRequest] = [] - key: object - for key in scope.execution_order: - if key not in scope.selected_keys or getattr(key, "resource_type", None) != ( - CompiledResourceType.MODEL - ): - continue - model: CompiledModelView | None = scope.models_by_name.get(getattr(key, "name", "")) - request: ModelMigrationRequest | None = None if model is None else _declared_request(model) - if request is not None: - requests.append(request) - return tuple(requests) - - -def _declared_request(model: CompiledModelView) -> ModelMigrationRequest | None: - raw_origin: object | None = model.config.values.get(MIGRATE_FROM_CONFIG_KEY) - if not isinstance(raw_origin, str): - return None - return ModelMigrationRequest( - model=model, - discovery=MigrationDiscovery.MANUAL, - raw_origin=raw_origin.strip(), - force=model.config.values.get(MIGRATE_FORCE_CONFIG_KEY) is True, - identity_only=get_config_str(values=model.config.values, key="materialized") - in _IDENTITY_MATERIALIZATIONS, - ) - - -def _project_declarations( - *, runtime: PlannerRuntime, state: MigrationStateInspection -) -> tuple[ModelMigrationDeclaration, ...]: - """Resolve every project model's migrate_from, selected or not.""" - - declarations: list[ModelMigrationDeclaration] = [] - model: CompiledModelView - for model in runtime.project.models: - request: ModelMigrationRequest | None = _declared_request(model) - if request is None: - continue - origin: CompiledRelationLocation - origin_model: str | None - origin, origin_model = _resolve_origin(request=request, runtime=runtime, state=state) - declarations.append( - ModelMigrationDeclaration( - model_name=model.name, origin_location=origin, origin_model=origin_model - ) - ) - return tuple(declarations) - - -def plan_model_migrations( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - overrides: PlannerOverrides, - deferral: DeferralInputs, - fingerprints: MigrationFingerprintCache, -) -> ModelMigrationPlanning: - """Decide every declared or discovered migration and project the post-migration snapshot.""" - - manual: tuple[ModelMigrationRequest, ...] = manual_migration_requests(scope=scope) - schemas: set[str] = project_schemas(runtime=runtime) - state: MigrationStateInspection = MigrationStateInspection( - adapter=runtime.adapter, - connection=runtime.connection, - database=planning_database(runtime=runtime), - known_fingerprints=snapshot.fingerprints, - ) - declares_migrations: bool = any( - _declared_request(model) is not None for model in runtime.project.models - ) - if declares_migrations: - state.inspect_schemas(schemas=schemas) - discovery: ModelMigrationDiscovery = discover_model_migrations( - runtime=runtime, - scope=scope, - snapshot=snapshot, - manual_requests=manual, - declarations=_project_declarations(runtime=runtime, state=state), - state=state, - project_schemas=schemas, - fingerprints=fingerprints, - ) - if not discovery.requests: - return ModelMigrationPlanning( - snapshot=replace( - snapshot, - reference_renames=plan_reference_renames( - runtime=runtime, - scope=scope, - snapshot=snapshot, - state=state, - schemas=schemas, - planned={}, - ), - ), - warnings=discovery.warnings, - ) - resolved: list[tuple[ModelMigrationRequest, CompiledRelationLocation, str | None]] = [ - (request, *_resolve_origin(request=request, runtime=runtime, state=state)) - for request in discovery.requests - ] - origin: CompiledRelationLocation - for _, origin, _ in resolved: - _require_same_database(state=state, location=origin) - state.inspect_schemas( - schemas={location.schema for _, location, _ in resolved if location.schema is not None} - ) - state.inspect_relations(locations=tuple(location for _, location, _ in resolved)) - snapshots_config: SnapshotsConfig = ( - runtime.project_config.snapshots - if runtime.project_config is not None - else SnapshotsConfig() - ) - origin_policy: MissingMigrationOriginPolicy = missing_origin_policy(runtime=runtime) - entries: list[ModelMigrationPlanEntry] = [] - handovers: dict[str, Fingerprint | None] = {} - request: ModelMigrationRequest - origin: CompiledRelationLocation - origin_model: str | None - for request, origin, origin_model in resolved: - entry: ModelMigrationPlanEntry - handover: Fingerprint | None - applies: bool - entry, handover, applies = _decide( - runtime=runtime, - request=request, - origin=origin, - origin_model=origin_model, - snapshot=snapshot, - state=state, - snapshots_config=snapshots_config, - ) - entries.append(replace(entry, missing_origin_policy=origin_policy)) - if applies: - handovers[request.model.name] = _equivalent_definition( - model=request.model, - handover=handover, - dialect=snapshot.column_dialect, - destination_fingerprints=discovery.destination_fingerprints.get( - request.model.name, () - ), - ) - renamed: frozenset[str] = _renamed_models(entries=tuple(entries), handovers=handovers) - return ModelMigrationPlanning( - snapshot=replace( - _overlay_snapshot( - runtime=runtime, - scope=scope, - snapshot=snapshot, - entries=tuple(entries), - handovers=handovers, - state=state, - overrides=overrides, - deferral=deferral, - ), - renamed_models=renamed, - reference_renames=plan_reference_renames( - runtime=runtime, - scope=scope, - snapshot=snapshot, - state=state, - schemas=schemas, - planned={ - entry.model_name: entry.origin_model - for entry in entries - if entry.model_name in handovers and entry.origin_model is not None - }, - ), - ), - entries=tuple(entries), - warnings=( - *discovery.warnings, - *(warning for entry in entries if (warning := _entry_warning(entry)) is not None), - ), - ) - - -def _renamed_models( - *, - entries: tuple[ModelMigrationPlanEntry, ...], - handovers: dict[str, Fingerprint | None], -) -> frozenset[str]: - """Return renamed tables and views that inherit their predecessor's identity.""" - - return frozenset( - entry.model_name - for entry in entries - if entry.decision == MigrationDecision.RENAMED - and handovers.get(entry.model_name) is not None - ) - - -def _equivalent_definition( - *, - model: CompiledModelView, - handover: Fingerprint | None, - destination_fingerprints: tuple[str, ...], - dialect: str | None, -) -> Fingerprint | None: - if handover is None or stored_migration_fingerprint(handover) not in destination_fingerprints: - return handover - return replace( - handover, - definition=model.query_sql, - definition_hash=compute_query_hash(query_sql=model.query_sql, dialect=dialect), - ) - def planning_database(*, runtime: PlannerRuntime) -> str | None: """Return the single database that direct planning inspects for migration state.""" @@ -335,10 +87,6 @@ def _require_same_database( ) -def _relation_key(location: CompiledRelationLocation) -> tuple[str, str]: - return ((location.schema or "").lower(), location.name.lower()) - - def _resolve_origin( *, request: ModelMigrationRequest, runtime: PlannerRuntime, state: MigrationStateInspection ) -> tuple[CompiledRelationLocation, str | None]: @@ -410,157 +158,6 @@ def _location( ) -def _decide( - *, - runtime: PlannerRuntime, - request: ModelMigrationRequest, - origin: CompiledRelationLocation, - origin_model: str | None, - snapshot: WarehouseSnapshot, - state: MigrationStateInspection, - snapshots_config: SnapshotsConfig, -) -> tuple[ModelMigrationPlanEntry, Fingerprint | None, bool]: - model: CompiledModelView = request.model - destination: CompiledRelationLocation = model.destination - if _relation_key(origin) == _relation_key(destination): - raise PlannerInputError( - f"model '{model.name}': migrate_from names the model's own relation" - ) - newest: MigrationEvent | None = newest_migration_event_mentioning( - events=state.events, relation=migration_relation_for_location(destination) - ) - origin_relation: RelationInfo | None = state.relation(origin) - origin_fingerprint: Fingerprint | None = state.model_fingerprint( - location=origin, model_name=origin_model - ) - base: ModelMigrationPlanEntry = ModelMigrationPlanEntry( - model_name=model.name, - discovery=request.discovery, - decision=MigrationDecision.MIGRATE, - compatibility=MigrationCompatibility.NOT_CHECKED, - origin_model=origin_model, - origin=origin, - destination=destination, - target_name=runtime.project.effective_target_name, - origin_version_hash=( - origin_fingerprint.version_hash if origin_fingerprint is not None else "" - ), - origin_is_transient=bool(origin_relation is not None and origin_relation.is_transient), - origin_tracked=origin_fingerprint is not None, - ) - if request.identity_only: - return _decide_rename( - model=model, - base=base, - newest=newest, - origin=origin, - origin_fingerprint=origin_fingerprint, - origin_exists=origin_relation is not None, - manual=request.discovery == MigrationDiscovery.MANUAL, - snapshot=snapshot, - ) - if ( - newest is not None - and newest.destination.matches(migration_relation_for_location(destination)) - and newest.origin.matches(migration_relation_for_location(origin)) - ): - return ( - replace( - base, - decision=MigrationDecision.DONE, - completed_at=newest.created_at, - origin_version_hash=newest.origin_version_hash, - target_name=newest.target_name, - ), - _done_handover( - model=model, - event=newest, - snapshot=snapshot, - origin_fingerprint=origin_fingerprint, - ), - _done_handover_applies(model=model, event=newest, snapshot=snapshot), - ) - if origin_relation is None: - return replace(base, decision=MigrationDecision.ORIGIN_MISSING), None, False - compatibility: MigrationCompatibilityResult = check_migration_compatibility( - model=model, - origin_relation=origin_relation, - origin_columns=state.relation_columns(origin), - sql_analysis_enabled=runtime.project.settings.sql_analysis, - column_dialect=snapshot.column_dialect, - snapshots_config=snapshots_config, - ) - decision: MigrationDecision - if model.name not in snapshot.existing_relations: - decision = MigrationDecision.MIGRATE - elif newest is not None and newest.origin.matches(migration_relation_for_location(destination)): - decision = MigrationDecision.SUPERSEDED_REPLACE - elif newest is None and model.name not in snapshot.fingerprints.models: - decision = MigrationDecision.REDO - elif request.force: - decision = MigrationDecision.FORCED_REPLACE - else: - decision = MigrationDecision.CONFLICT - entry: ModelMigrationPlanEntry = replace( - base, - decision=decision, - compatibility=compatibility.status, - compatibility_findings=compatibility.findings, - ) - if not decision.moves_data or entry.blocks_build: - return entry, None, False - return ( - _with_execution(runtime=runtime, model=model, entry=entry), - _handover_fingerprint( - model=model, - fingerprint=origin_fingerprint, - version_hash=entry.origin_version_hash, - ), - True, - ) - - -def _decide_rename( - *, - model: CompiledModelView, - base: ModelMigrationPlanEntry, - newest: MigrationEvent | None, - origin: CompiledRelationLocation, - origin_fingerprint: Fingerprint | None, - origin_exists: bool, - manual: bool, - snapshot: WarehouseSnapshot, -) -> tuple[ModelMigrationPlanEntry, Fingerprint | None, bool]: - """Hand a renamed table or view's identity to its successor; no data moves.""" - - recorded: bool = ( - newest is not None - and newest.decision == MigrationDecision.RENAMED - and newest.destination.matches(migration_relation_for_location(model.destination)) - and newest.origin.matches(migration_relation_for_location(origin)) - ) - if manual and not recorded and not origin_exists: - return replace(base, decision=MigrationDecision.ORIGIN_MISSING), None, False - built_since_rename: bool = ( - recorded - and newest is not None - and not _done_handover_applies(model=model, event=newest, snapshot=snapshot) - ) - return ( - replace( - base, - decision=MigrationDecision.RENAMED, - completed_at=newest.created_at if recorded and newest is not None else None, - ), - ( - snapshot.fingerprints.models.get(model.name) - if built_since_rename - else _handover_fingerprint(model=model, fingerprint=origin_fingerprint, version_hash="") - ), - True, - ) - - def _with_execution( *, runtime: PlannerRuntime, model: CompiledModelView, entry: ModelMigrationPlanEntry ) -> ModelMigrationPlanEntry: @@ -596,129 +193,6 @@ def _promotion(*, runtime: PlannerRuntime, decision: MigrationDecision) -> Migra return MigrationPromotion.RENAME -def _done_handover_applies( - *, model: CompiledModelView, event: MigrationEvent, snapshot: WarehouseSnapshot -) -> bool: - own: Fingerprint | None = snapshot.fingerprints.models.get(model.name) - return own is None or _utc(own.ts) < _utc(event.created_at) - - -def _utc(value: datetime) -> datetime: - if value.tzinfo is None or value.utcoffset() is None: - return value.replace(tzinfo=UTC) - return value.astimezone(UTC) - - -def _done_handover( - *, - model: CompiledModelView, - event: MigrationEvent, - snapshot: WarehouseSnapshot, - origin_fingerprint: Fingerprint | None, -) -> Fingerprint | None: - if not _done_handover_applies(model=model, event=event, snapshot=snapshot): - return snapshot.fingerprints.models.get(model.name) - return _handover_fingerprint( - model=model, fingerprint=origin_fingerprint, version_hash=event.origin_version_hash - ) - - -def _handover_fingerprint( - *, model: CompiledModelView, fingerprint: Fingerprint | None, version_hash: str -) -> Fingerprint | None: - if fingerprint is None: - return None - metadata_json: str = fingerprint.metadata_json - try: - payload: object = json.loads(metadata_json) - except json.JSONDecodeError: - payload = None - if isinstance(payload, dict) and MIGRATION_MODEL_NAME_METADATA_KEY in payload: - payload[MIGRATION_MODEL_NAME_METADATA_KEY] = model.name - metadata_json = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str) - return replace( - fingerprint, - node_name=model.name, - target_database=model.destination.database, - target_schema=model.destination.schema, - target_name=model.destination.name, - version_hash=version_hash or fingerprint.version_hash, - metadata_json=metadata_json, - ) - - -def _overlay_snapshot( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - entries: tuple[ModelMigrationPlanEntry, ...], - handovers: dict[str, Fingerprint | None], - state: MigrationStateInspection, - overrides: PlannerOverrides, - deferral: DeferralInputs, -) -> WarehouseSnapshot: - if not handovers: - return snapshot - relations: dict[str, RelationInfo] = dict(snapshot.existing_relations) - columns: dict[str, tuple[ColumnInfo, ...]] = dict(snapshot.existing_columns) - model_fingerprints: dict[str, Fingerprint] = dict(snapshot.fingerprints.models) - redirected: dict[str, str] = {} - entry: ModelMigrationPlanEntry - for entry in entries: - if entry.model_name not in handovers: - continue - handover: Fingerprint | None = handovers[entry.model_name] - if handover is None: - _ = model_fingerprints.pop(entry.model_name, None) - else: - model_fingerprints[entry.model_name] = handover - if not entry.decision.moves_data: - continue - origin_relation: RelationInfo | None = state.relation(entry.origin) - if origin_relation is None: - continue - relations[entry.model_name] = replace( - origin_relation, - database=entry.destination.database, - schema=entry.destination.schema, - name=entry.destination.name, - is_transient=( - origin_relation.is_transient - if entry.stage_is_transient is None - else entry.stage_is_transient - ), - ) - columns[entry.model_name] = state.relation_columns(entry.origin) - if entry.origin.qualified_name is not None and _has_cursor( - runtime=runtime, model_name=entry.model_name - ): - redirected[entry.model_name] = entry.origin.qualified_name - cursor_snapshots: dict[str, ModelCursorSnapshot] = { - **snapshot.cursor_snapshots, - **redirected_cursor_snapshots( - runtime=runtime, - scope=scope, - existing_relations=relations, - target_relations=redirected, - overrides=overrides, - deferral=deferral, - ), - } - return replace( - snapshot, - existing_relations=relations, - existing_columns=columns, - fingerprints=WarehouseFingerprints( - models=model_fingerprints, - functions=snapshot.fingerprints.functions, - seeds=snapshot.fingerprints.seeds, - python_nodes=snapshot.fingerprints.python_nodes, - ), - cursor_snapshots=cursor_snapshots, - ) - - def redirected_cursor_snapshots( *, runtime: PlannerRuntime, @@ -753,83 +227,3 @@ def redirected_cursor_snapshots( on_progress=runtime.on_progress, origin_cursor_columns=origin_cursor_columns, ) - - -def _has_cursor(*, runtime: PlannerRuntime, model_name: str) -> bool: - model: CompiledModelView | None = next( - (model for model in runtime.project.models if model.name == model_name), None - ) - return ( - model is not None - and get_config_str(values=model.config.values, key="materialized") - == MaterializationType.INCREMENTAL - and get_config_str(values=model.config.values, key="cursor") is not None - ) - - -def _entry_warning(entry: ModelMigrationPlanEntry) -> PlanWarning | None: - origin: str = entry.origin.qualified_name or entry.origin.name - destination: str = entry.destination.qualified_name or entry.destination.name - if entry.decision == MigrationDecision.DONE or ( - entry.decision == MigrationDecision.RENAMED and entry.completed_at is not None - ): - if entry.discovery != MigrationDiscovery.MANUAL: - return None - completed: str = entry.completed_at.isoformat() if entry.completed_at else "unknown time" - return PlanWarning( - model_name=entry.model_name, - severity=WarningSeverity.WARNING, - message=( - f"migration from {origin} completed at {completed} on target " - f"'{entry.target_label}'; migrate_from can be removed from '{entry.model_name}'" - ), - code="M101", - ) - if entry.origin_hidden: - return PlanWarning( - model_name=entry.model_name, - severity=WarningSeverity.ERROR, - message=( - f"model '{entry.model_name}': migrate_from origin {origin} was built in this " - f"target but is not visible; {HIDDEN_ORIGIN_REMEDY}" - ), - code="M102", - ) - if entry.origin_missing: - return PlanWarning( - model_name=entry.model_name, - severity=( - WarningSeverity.ERROR - if entry.missing_origin_policy == MissingMigrationOriginPolicy.DENY - else WarningSeverity.WARNING - ), - message=( - f"model '{entry.model_name}': migrate_from origin {origin} does not exist in " - "this target and no recorded migration into it was found; " - + MISSING_ORIGIN_OUTCOMES[entry.missing_origin_policy] - ), - code="M102", - ) - if entry.decision == MigrationDecision.CONFLICT: - return PlanWarning( - model_name=entry.model_name, - severity=WarningSeverity.ERROR, - message=( - f"migration conflict: {destination} already exists with its own build history " - f"and has no recorded migration from {origin}; set migrate_force true to " - "replace it (the replaced table is kept under a _sqb_archive__ name until " - "janitor expires it)" - ), - code="M103", - ) - if entry.blocks_build: - return PlanWarning( - model_name=entry.model_name, - severity=WarningSeverity.ERROR, - message=( - f"migration from {origin} is incompatible with '{entry.model_name}': " - + "; ".join(entry.compatibility_findings) - ), - code="M104", - ) - return None From 02fbd8f9543535190256406b261f2f954931d67a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:57:28 +0100 Subject: [PATCH 20/29] chore: update Python boundary after model migration port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index df759ce888..ebb21977cc 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 35602 +remaining_lines = 34996 [[stays]] path = "executor" From 49077724e378964dacfc2eb4af63bbb2030674c5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:25:42 +0100 Subject: [PATCH 21/29] refactor: move fixture schema and type planning into native code --- .../planner_fixtures/main/fixture_schema.rs | 22 +++ .../src/planner_fixtures/main/mod.rs | 4 + .../planner_fixtures/main/propagate_types.rs | 39 ++++ .../main/requires_analysis.rs | 17 ++ .../planner_fixtures/main/unknown_columns.rs | 22 +++ .../src/planner_fixtures/models.rs | 37 ++++ .../src/planner_fixtures/tests.rs | 4 + .../src/planner_fixtures/tests/helpers.rs | 20 ++ .../tests/test_propagation.rs | 51 ++++++ .../src/planner_fixtures/tests/test_schema.rs | 88 +++++++++ .../src/planner_fixtures/tests/test_types.rs | 16 ++ .../planner_strategy/fixture_completion.rs | 37 ++-- .../planner_strategy/fixture_context.rs | 117 ++++++++++++ .../planner_strategy/fixture_metadata.rs | 4 +- .../planner_strategy/fixture_selection.rs | 66 +++++++ .../planner_strategy/fixture_types.rs | 90 +++++++++ .../bindings/_helpers/planner_strategy/mod.rs | 3 + .../_helpers/planner_strategy/registration.rs | 3 + src/sqlbuild/_native.pyi | 27 +++ .../planner/_helpers/fixtures/completion.py | 173 +----------------- .../compiler/planner/_helpers/_test_types.py | 10 + .../planner/_helpers/test_fixture_planning.py | 74 +++++++- .../_helpers/test_fixture_planning_work.py | 5 +- 23 files changed, 740 insertions(+), 189 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_schema.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/main/propagate_types.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/main/requires_analysis.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/main/unknown_columns.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/tests/test_propagation.rs create mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/tests/test_schema.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_schema.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_schema.rs new file mode 100644 index 0000000000..d5e1de8904 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_schema.rs @@ -0,0 +1,22 @@ +//! Derive fixture schema facts from relation metadata. + +use crate::planner_fixtures::models::{FixtureColumn, FixtureSchema}; + +/// Collect authoritative folded names and last-declared known column types. +pub fn fixture_schema(columns: &[FixtureColumn], authoritative: bool) -> FixtureSchema { + let mut expected_types: Vec<(String, String)> = Vec::new(); + for column in columns { + if let Some(sql_type) = &column.sql_type { + expected_types.push((column.folded_name.clone(), sql_type.clone())); + } + } + FixtureSchema { + authoritative_names: authoritative.then(|| { + columns + .iter() + .map(|column| column.folded_name.clone()) + .collect() + }), + expected_types, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs index 1bc35b463a..e07b486003 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs @@ -1,6 +1,10 @@ pub mod align_columns; pub mod empty_fixture_error; pub mod empty_fixture_marker; +pub mod fixture_schema; pub mod merge_metadata; +pub mod propagate_types; pub mod render_completed; pub mod render_empty; +pub mod requires_analysis; +pub mod unknown_columns; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/propagate_types.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/propagate_types.rs new file mode 100644 index 0000000000..b1c6f50a48 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/propagate_types.rs @@ -0,0 +1,39 @@ +//! Ordered direct-lineage fixture type propagation. + +use std::collections::BTreeSet; + +use crate::planner_fixtures::models::{FixtureTypeModel, FixtureTypeRead, FixtureTypeState}; + +/// Infer a missing output type only when known upstream types agree exactly. +pub fn propagate_types( + mut state: FixtureTypeState, + models: &[FixtureTypeModel], +) -> FixtureTypeState { + for model in models { + for projection in &model.projections { + let upstream_types: BTreeSet = projection + .upstream + .iter() + .filter_map(|read| state.known.get(&read.key)?.get(&read.column).cloned()) + .collect(); + if upstream_types.len() != 1 { + continue; + } + let Some(sql_type) = upstream_types.into_iter().next() else { + continue; + }; + let model_types = state.known.entry(model.key.clone()).or_default(); + if !model_types.contains_key(&projection.output_column) { + model_types.insert(projection.output_column.clone(), sql_type.clone()); + state.additions.push(( + FixtureTypeRead { + key: model.key.clone(), + column: projection.output_column.clone(), + }, + sql_type, + )); + } + } + } + state +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/requires_analysis.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/requires_analysis.rs new file mode 100644 index 0000000000..c8a26b65bc --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/requires_analysis.rs @@ -0,0 +1,17 @@ +//! Select incomplete or open fixture schemas for read analysis. + +use std::collections::BTreeSet; + +use crate::planner_fixtures::models::FixtureColumn; + +/// Require analysis unless every authoritative name is supplied. +pub fn requires_analysis( + supplied: &[FixtureColumn], + authoritative: Option<&BTreeSet>, +) -> bool { + authoritative.is_none_or(|expected| { + let supplied_names: BTreeSet<_> = + supplied.iter().map(|column| &column.folded_name).collect(); + expected.iter().any(|name| !supplied_names.contains(name)) + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/unknown_columns.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/unknown_columns.rs new file mode 100644 index 0000000000..59dd4ad08b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/unknown_columns.rs @@ -0,0 +1,22 @@ +//! Diagnose supplied names outside a closed relation schema. + +use std::collections::BTreeSet; + +use crate::planner_fixtures::models::FixtureColumn; + +/// Report sorted unknown names while preserving duplicate supplied names. +pub fn unknown_columns( + label: &str, + supplied: &[FixtureColumn], + authoritative: Option<&BTreeSet>, +) -> Option { + let expected = authoritative?; + let mut unknown: Vec<_> = supplied + .iter() + .filter(|column| !expected.contains(&column.folded_name)) + .map(|column| column.name.as_str()) + .collect(); + unknown.sort_unstable(); + (!unknown.is_empty()) + .then(|| format!("{label} supplies unknown columns: {}", unknown.join(", "))) +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/models.rs b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs index 3e33082170..2fbeba43f8 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/models.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs @@ -1,5 +1,42 @@ //! Passive fixture schema and alignment facts. +use std::collections::{BTreeSet, HashMap}; + +/// One upstream typed column identified by its resource and folded name. +#[derive(Debug)] +pub struct FixtureTypeRead { + pub key: (String, String), + pub column: String, +} + +/// Direct lineage facts used to propagate fixture types. +#[derive(Debug)] +pub struct FixtureTypeProjection { + pub output_column: String, + pub upstream: Vec, +} + +/// One model's ordered direct projections and initial fixture types. +#[derive(Debug)] +pub struct FixtureTypeModel { + pub key: (String, String), + pub projections: Vec, +} + +/// A known relation type map and the ordered additions inferred from lineage. +#[derive(Debug, Default)] +pub struct FixtureTypeState { + pub known: HashMap<(String, String), HashMap>, + pub additions: Vec<(FixtureTypeRead, String)>, +} + +/// Derived column names and known types for one fixture relation. +#[derive(Debug)] +pub struct FixtureSchema { + pub authoritative_names: Option>, + pub expected_types: Vec<(String, String)>, +} + /// One authoritative or inferred fixture column. #[derive(Clone, Debug)] pub struct FixtureColumn { diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs index bcfaf02203..a1512cca60 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs @@ -2,5 +2,9 @@ mod helpers; #[path = "tests/test_alignment.rs"] mod test_alignment; +#[path = "tests/test_propagation.rs"] +mod test_propagation; +#[path = "tests/test_schema.rs"] +mod test_schema; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/helpers.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/helpers.rs index 51ca2f5bc9..4afe32de53 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/helpers.rs @@ -1,4 +1,24 @@ use crate::planner_fixtures::models::FixtureColumn; +use crate::planner_fixtures::models::{FixtureTypeModel, FixtureTypeProjection, FixtureTypeRead}; + +pub(super) fn type_model() -> FixtureTypeModel { + FixtureTypeModel { + key: ("model".into(), "combined".into()), + projections: vec![FixtureTypeProjection { + output_column: "id".into(), + upstream: vec![ + FixtureTypeRead { + key: ("source".into(), "orders".into()), + column: "order_id".into(), + }, + FixtureTypeRead { + key: ("source".into(), "customers".into()), + column: "customer_id".into(), + }, + ], + }], + } +} pub(super) fn column(name: &str, sql_type: &str, nullable: bool) -> FixtureColumn { FixtureColumn { diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_propagation.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_propagation.rs new file mode 100644 index 0000000000..37cb82077b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_propagation.rs @@ -0,0 +1,51 @@ +use std::collections::HashMap; + +use crate::planner_fixtures::main::propagate_types::propagate_types; +use crate::planner_fixtures::models::FixtureTypeState; +use crate::planner_fixtures::tests::{helpers::type_model, test_types::PropagationTestCase}; + +#[test] +fn given_upstream_types_when_propagating_then_only_agreed_known_types_are_used() { + let test_cases = [ + PropagationTestCase { + description: "matching known types", + upstream_type: Some("INTEGER"), + expected_type: Some("INTEGER"), + }, + PropagationTestCase { + description: "conflicting known types", + upstream_type: Some("VARCHAR"), + expected_type: None, + }, + PropagationTestCase { + description: "unknown type ignored", + upstream_type: None, + expected_type: Some("INTEGER"), + }, + ]; + for test_case in test_cases { + let mut state = FixtureTypeState::default(); + state.known.insert( + ("source".into(), "orders".into()), + HashMap::from([("order_id".into(), "INTEGER".into())]), + ); + state.known.insert( + ("source".into(), "customers".into()), + test_case + .upstream_type + .into_iter() + .map(|sql_type| ("customer_id".into(), sql_type.into())) + .collect(), + ); + let state = propagate_types(state, &[type_model()]); + assert_eq!( + state + .additions + .first() + .map(|(_, sql_type)| sql_type.as_str()), + test_case.expected_type, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_schema.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_schema.rs new file mode 100644 index 0000000000..bce18ee592 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_schema.rs @@ -0,0 +1,88 @@ +use crate::planner_fixtures::main::{ + fixture_schema::fixture_schema, requires_analysis::requires_analysis, + unknown_columns::unknown_columns, +}; +use crate::planner_fixtures::tests::{helpers::column, test_types::SchemaTestCase}; + +#[test] +fn given_closed_and_open_schemas_when_selecting_then_names_and_diagnostics_agree() { + let test_cases = [ + SchemaTestCase { + description: "closed complete", + authoritative: true, + supplied: &["order_id", "status"], + expected_analysis: false, + expected_diagnostic: None, + }, + SchemaTestCase { + description: "closed incomplete", + authoritative: true, + supplied: &["order_id"], + expected_analysis: true, + expected_diagnostic: None, + }, + SchemaTestCase { + description: "closed unknown preserves duplicates", + authoritative: true, + supplied: &["z_extra", "a_extra", "a_extra"], + expected_analysis: true, + expected_diagnostic: Some( + "mock source 'orders' supplies unknown columns: a_extra, a_extra, z_extra", + ), + }, + SchemaTestCase { + description: "open complete", + authoritative: false, + supplied: &["order_id", "status"], + expected_analysis: true, + expected_diagnostic: None, + }, + SchemaTestCase { + description: "open unknown", + authoritative: false, + supplied: &["extra"], + expected_analysis: true, + expected_diagnostic: None, + }, + ]; + for test_case in test_cases { + let schema = fixture_schema( + &[ + column("order_id", "INTEGER", false), + column("status", "VARCHAR", true), + ], + test_case.authoritative, + ); + let supplied: Vec<_> = test_case + .supplied + .iter() + .map(|name| column(name, "INTEGER", true)) + .collect(); + assert_eq!( + requires_analysis(&supplied, schema.authoritative_names.as_ref()), + test_case.expected_analysis, + "{}", + test_case.description + ); + assert_eq!( + unknown_columns( + "mock source 'orders'", + &supplied, + schema.authoritative_names.as_ref() + ) + .as_deref(), + test_case.expected_diagnostic, + "{}", + test_case.description + ); + assert_eq!( + schema.expected_types, + vec![ + ("order_id".into(), "INTEGER".into()), + ("status".into(), "VARCHAR".into()) + ], + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs index 5aed7a8a9f..0b02326180 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs @@ -1,5 +1,21 @@ //! Fixture alignment regression inputs. +#[derive(Debug)] +pub(crate) struct PropagationTestCase { + pub(crate) description: &'static str, + pub(crate) upstream_type: Option<&'static str>, + pub(crate) expected_type: Option<&'static str>, +} + +#[derive(Debug)] +pub(crate) struct SchemaTestCase { + pub(crate) description: &'static str, + pub(crate) authoritative: bool, + pub(crate) supplied: &'static [&'static str], + pub(crate) expected_analysis: bool, + pub(crate) expected_diagnostic: Option<&'static str>, +} + #[derive(Debug)] pub(crate) struct AlignmentTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs index 811ad0edea..488767569c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs @@ -7,7 +7,11 @@ use sqlbuild_analysis::planner_fixtures::main::align_columns::align_columns; use sqlbuild_analysis::planner_fixtures::models::{AlignmentInput, RequiredColumn}; use crate::bindings::_helpers::planner_strategy::fixture_preparation::{PreparedFixtures, prepare}; +use crate::bindings::_helpers::planner_strategy::fixture_selection::{ + fixture_keys_requiring_analysis, unknown_fixture_column_diagnostics, +}; use crate::bindings::_helpers::planner_strategy::fixture_sql::{CompletionSql, render_completed}; +use crate::bindings::_helpers::planner_strategy::fixture_types::expected_fixture_types; use crate::bindings::_helpers::planner_strategy::fixture_values::{ COMPLETION, columns, diagnostic, folded, label, record, strings, }; @@ -45,13 +49,12 @@ fn build_relation_fixture_completion<'py>( adapter, &context.getattr("fixture_inferences")?, )?; - let kwargs = PyDict::new(py); - kwargs.set_item("inferred_by_fixture", &prepared.inferred)?; - kwargs.set_item( - "authoritative_columns", - context.getattr("authoritative_columns")?, + let fixture_keys = fixture_keys_requiring_analysis( + &prepared.inferred, + &context + .getattr("authoritative_columns")? + .cast_into::()?, )?; - let fixture_keys = helper(py, "_fixture_keys_requiring_analysis", &kwargs)?; let kwargs = PyDict::new(py); kwargs.set_item("ordered_model_names", &ordered_model_names)?; kwargs.set_item("model_map", context.getattr("models_by_name")?)?; @@ -65,12 +68,17 @@ fn build_relation_fixture_completion<'py>( let required_and_stars = helper(py, "_required_fixture_columns", &kwargs)?; let required = required_and_stars.get_item(0)?.cast_into::()?; let stars = required_and_stars.get_item(1)?; - let kwargs = PyDict::new(py); - kwargs.set_item("base_expected_types", context.getattr("expected_types")?)?; - kwargs.set_item("ordered_model_names", &ordered_model_names)?; - kwargs.set_item("model_map", context.getattr("models_by_name")?)?; - let expected = helper(py, "_expected_fixture_types", &kwargs)?; - complete_prepared(prepared, (relations, required, stars), expected, adapter) + let expected = expected_fixture_types( + &context.getattr("expected_types")?.cast_into::()?, + ordered_model_names.extract()?, + &context.getattr("models_by_name")?.cast_into::()?, + )?; + complete_prepared( + prepared, + (relations, required, stars), + expected.into_any(), + adapter, + ) } fn complete_prepared<'py>( @@ -96,10 +104,7 @@ fn complete_prepared<'py>( typed_nulls.set_item(key, selected)?; } } - let kwargs = PyDict::new(py); - kwargs.set_item("inferred_by_fixture", &prepared.inferred)?; - kwargs.set_item("relations", &relations)?; - for value in helper(py, "_unknown_fixture_column_diagnostics", &kwargs)?.try_iter()? { + for value in unknown_fixture_column_diagnostics(&prepared.inferred, &relations)?.try_iter()? { prepared.diagnostics.push(value?); } let completed_keys = PySet::empty(py)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs new file mode 100644 index 0000000000..d6509e6ecf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs @@ -0,0 +1,117 @@ +//! Project fixture context materialization and schema conversions. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_fixtures::main::fixture_schema::fixture_schema; + +use crate::bindings::_helpers::planner_strategy::fixture_metadata::model_fixture_metadata; +use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, metadata, record}; + +#[pyfunction] +#[pyo3(signature = (*, project, models_by_name = None))] +pub(crate) fn build_relation_fixture_context<'py>( + project: &Bound<'py, PyAny>, + models_by_name: Option>, +) -> PyResult> { + let py = project.py(); + let model_map = match models_by_name { + Some(models) => models, + None => { + let models = PyDict::new(py); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + models.set_item(model.getattr("name")?, &model)?; + } + models + } + }; + let relations = relation_metadata(project, &model_map)?; + let kwargs = PyDict::new(py); + kwargs.set_item("models_by_name", model_map)?; + kwargs.set_item( + "authoritative_columns", + authoritative_fixture_columns(&relations)?, + )?; + kwargs.set_item("expected_types", base_expected_fixture_types(&relations)?)?; + kwargs.set_item("relations", relations)?; + record(py, "RelationFixturePlanningContext", &kwargs) +} + +fn relation_metadata<'py>( + project: &Bound<'py, PyAny>, + models: &Bound<'py, PyDict>, +) -> PyResult> { + let py = project.py(); + let kinds = py + .import("sqlbuild.compiler.compile.types")? + .getattr("CompiledResourceType")?; + let relations = PyDict::new(py); + for (name, model) in models.iter() { + relations.set_item( + (kinds.getattr("MODEL")?, name), + model_fixture_metadata(&model)?, + )?; + } + for (collection, entry, kind, source) in [ + ("sources", "source_entry", "SOURCE", true), + ("seeds", "schema_entry", "SEED", false), + ] { + for resource in project.getattr(collection)?.try_iter()? { + let resource = resource?; + let entry = resource.getattr(entry)?; + let columns = columns(&entry.getattr("columns")?, false)?; + let authoritative = if source { + entry.getattr("contract")?.eq("enforced")? + } else { + !columns.is_empty() + }; + relations.set_item( + (kinds.getattr(kind)?, resource.getattr("name")?), + metadata(py, &columns, authoritative)?, + )?; + } + } + Ok(relations) +} + +#[pyfunction] +#[pyo3(signature = (*, relations))] +pub(crate) fn authoritative_fixture_columns<'py>( + relations: &Bound<'py, PyDict>, +) -> PyResult> { + let result = PyDict::new(relations.py()); + for (key, relation) in relations.iter() { + let schema = fixture_schema( + &columns(&relation.getattr("columns")?, false)?, + relation.getattr("authoritative_names")?.extract()?, + ); + if let Some(names) = schema.authoritative_names { + result.set_item(key, PyFrozenSet::new(relations.py(), names)?)?; + } + } + Ok(result) +} + +#[pyfunction] +#[pyo3(signature = (*, relations))] +pub(crate) fn base_expected_fixture_types<'py>( + relations: &Bound<'py, PyDict>, +) -> PyResult> { + let result = PyDict::new(relations.py()); + for (key, relation) in relations.iter() { + let schema = fixture_schema(&columns(&relation.getattr("columns")?, false)?, false); + let types = PyDict::new(relations.py()); + for (name, sql_type) in schema.expected_types { + types.set_item(name, sql_type)?; + } + result.set_item(key, types)?; + } + Ok(result) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_relation_fixture_context, module)?)?; + module.add_function(wrap_pyfunction!(authoritative_fixture_columns, module)?)?; + module.add_function(wrap_pyfunction!(base_expected_fixture_types, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs index f5801b126d..8839867676 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs @@ -9,7 +9,9 @@ use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, metad #[pyfunction] #[pyo3(signature = (*, model))] -fn model_fixture_metadata<'py>(model: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn model_fixture_metadata<'py>( + model: &Bound<'py, PyAny>, +) -> PyResult> { let py = model.py(); let inferred = model.getattr("inferred_columns")?; let schema = model.getattr("schema_entry")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs new file mode 100644 index 0000000000..c21012b96e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs @@ -0,0 +1,66 @@ +//! Fixture analysis selection and diagnostic boundary conversions. + +use std::collections::BTreeSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_fixtures::main::{ + fixture_schema::fixture_schema, requires_analysis::requires_analysis, + unknown_columns::unknown_columns, +}; + +use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, diagnostic, label}; + +#[pyfunction] +#[pyo3(signature = (*, inferred_by_fixture, authoritative_columns))] +pub(crate) fn fixture_keys_requiring_analysis<'py>( + inferred_by_fixture: &Bound<'py, PyDict>, + authoritative_columns: &Bound<'py, PyDict>, +) -> PyResult> { + let mut selected: Vec> = Vec::new(); + for (key, inferred) in inferred_by_fixture.iter() { + let expected: Option> = authoritative_columns + .get_item(&key)? + .map(|names| names.extract()) + .transpose()?; + if requires_analysis(&columns(&inferred, true)?, expected.as_ref()) { + selected.push(key); + } + } + PyFrozenSet::new(inferred_by_fixture.py(), selected) +} + +#[pyfunction] +#[pyo3(signature = (*, inferred_by_fixture, relations))] +pub(crate) fn unknown_fixture_column_diagnostics<'py>( + inferred_by_fixture: &Bound<'py, PyDict>, + relations: &Bound<'py, PyDict>, +) -> PyResult> { + let mut diagnostics = Vec::>::new(); + for (key, inferred) in inferred_by_fixture.iter() { + let Some(relation) = relations.get_item(&key)? else { + continue; + }; + let schema = fixture_schema( + &columns(&relation.getattr("columns")?, false)?, + relation.getattr("authoritative_names")?.extract()?, + ); + if let Some(message) = unknown_columns( + &label(&key)?, + &columns(&inferred, true)?, + schema.authoritative_names.as_ref(), + ) { + diagnostics.push(diagnostic(&key, &message)?); + } + } + PyTuple::new(inferred_by_fixture.py(), diagnostics) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(fixture_keys_requiring_analysis, module)?)?; + module.add_function(wrap_pyfunction!( + unknown_fixture_column_diagnostics, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs new file mode 100644 index 0000000000..03a9894c8d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs @@ -0,0 +1,90 @@ +//! Expected fixture type propagation boundary conversions. + +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_fixtures::main::propagate_types::propagate_types; +use sqlbuild_analysis::planner_fixtures::models::{ + FixtureTypeModel, FixtureTypeProjection, FixtureTypeRead, FixtureTypeState, +}; + +use crate::bindings::_helpers::planner_strategy::fixture_values::folded; + +#[pyfunction] +#[pyo3(signature = (*, base_expected_types, ordered_model_names, model_map))] +pub(crate) fn expected_fixture_types<'py>( + base_expected_types: &Bound<'py, PyDict>, + ordered_model_names: Vec, + model_map: &Bound<'py, PyDict>, +) -> PyResult> { + let state = FixtureTypeState { + known: base_expected_types.extract()?, + ..FixtureTypeState::default() + }; + let result = base_expected_types.copy()?; + let mut models: Vec = Vec::new(); + for name in ordered_model_names { + let Some(model) = model_map.get_item(&name)? else { + continue; + }; + let key = ("model".to_owned(), name); + let types = result + .get_item(&key)? + .ok_or_else(|| PyKeyError::new_err(key.clone()))? + .cast_into::()? + .copy()?; + result.set_item(&key, types)?; + models.push(FixtureTypeModel { + key, + projections: projections(&model)?, + }); + } + let state = propagate_types(state, &models); + for (read, sql_type) in state.additions { + result + .get_item(read.key)? + .ok_or_else(|| PyKeyError::new_err("fixture types"))? + .set_item(read.column, sql_type)?; + } + Ok(result) +} + +fn projections(model: &Bound<'_, PyAny>) -> PyResult> { + let lineage = model.getattr("fast_lineage_columns")?; + let mut projections: Vec = Vec::new(); + if lineage.is_none() { + return Ok(projections); + } + let direct = model + .py() + .import("sqlbuild.compiler.lineage.types")? + .getattr("ColumnTransformKind")? + .getattr("DIRECT")?; + for column in lineage.try_iter()? { + let column = column?; + if !column.getattr("transform_kind")?.eq(&direct)? { + continue; + } + let mut upstream: Vec = Vec::new(); + for source in column.getattr("upstream_columns")?.try_iter()? { + let source = source?; + upstream.push(FixtureTypeRead { + key: ( + source.getattr("resource_type")?.extract()?, + source.getattr("resource_name")?.extract()?, + ), + column: folded(&source.getattr("column_name")?)?, + }); + } + projections.push(FixtureTypeProjection { + output_column: folded(&column.getattr("output_column")?)?, + upstream, + }); + } + Ok(projections) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(expected_fixture_types, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs index 0099559538..d1023a4c18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs @@ -1,9 +1,12 @@ pub(crate) mod audits; pub(crate) mod cursor_types; pub(crate) mod fixture_completion; +pub(crate) mod fixture_context; pub(crate) mod fixture_metadata; pub(crate) mod fixture_preparation; +pub(crate) mod fixture_selection; pub(crate) mod fixture_sql; +pub(crate) mod fixture_types; pub(crate) mod fixture_values; pub(crate) mod lineage; pub(crate) mod microbatches; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs index d0332ef440..c04fcde07b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -13,6 +13,9 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { fixture_completion::register(module)?; fixture_metadata::register(module)?; fixture_sql::register(module)?; + super::fixture_context::register(module)?; + super::fixture_selection::register(module)?; + super::fixture_types::register(module)?; lineage::register(module)?; microbatches::register(module)?; strategy::register(module) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index c00e0d0371..904686eb5b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -130,6 +130,7 @@ from sqlbuild.compiler.compile.types import ( CompiledFunctionView, CompiledModelView, CompiledProjectView, + CompiledResourceType, CompiledSeedView, CompiledSourceView, CompileInputsView, @@ -238,6 +239,7 @@ from sqlbuild.compiler.planner.models import ( QueryShape, ReferenceRename, RelationFixtureCompletion, + RelationFixtureDiagnostic, RelationFixturePlanningContext, SchemaAction, SchemaFinding, @@ -2457,6 +2459,31 @@ def complete_typed_null_columns( adapter: BaseAdapter, ) -> str: ... def model_fixture_metadata(*, model: CompiledModelView) -> FixtureRelationMetadata: ... +def expected_fixture_types( + *, + base_expected_types: dict[tuple[CompiledResourceType, str], dict[str, str]], + ordered_model_names: tuple[str, ...], + model_map: dict[str, CompiledModelView], +) -> dict[tuple[CompiledResourceType, str], dict[str, str]]: ... +def build_relation_fixture_context( + *, project: CompiledProjectView, models_by_name: dict[str, CompiledModelView] | None = None +) -> RelationFixturePlanningContext: ... +def authoritative_fixture_columns( + *, relations: dict[tuple[CompiledResourceType, str], FixtureRelationMetadata] +) -> dict[tuple[CompiledResourceType, str], frozenset[str]]: ... +def base_expected_fixture_types( + *, relations: dict[tuple[CompiledResourceType, str], FixtureRelationMetadata] +) -> dict[tuple[CompiledResourceType, str], dict[str, str]]: ... +def fixture_keys_requiring_analysis( + *, + inferred_by_fixture: dict[tuple[CompiledResourceType, str], tuple[InferredColumn, ...]], + authoritative_columns: dict[tuple[CompiledResourceType, str], frozenset[str]], +) -> frozenset[tuple[CompiledResourceType, str]]: ... +def unknown_fixture_column_diagnostics( + *, + inferred_by_fixture: dict[tuple[CompiledResourceType, str], tuple[InferredColumn, ...]], + relations: dict[tuple[CompiledResourceType, str], FixtureRelationMetadata], +) -> tuple[RelationFixtureDiagnostic, ...]: ... def build_relation_fixture_completion( *, project: CompiledProjectView, diff --git a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py b/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py index fd24ef88cc..68711fc1df 100644 --- a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py +++ b/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py @@ -3,8 +3,7 @@ from __future__ import annotations import re -from collections import ChainMap -from collections.abc import Mapping, Sequence +from collections.abc import Sequence from functools import lru_cache from typing import Any @@ -15,13 +14,13 @@ build_relation_fixture_completion as _native_fixture_completion, ) from sqlbuild._native import ( - complete_empty_fixture_sql as complete_empty_fixture_sql, + build_relation_fixture_context as build_relation_fixture_context, ) from sqlbuild._native import ( - complete_typed_null_columns as complete_typed_null_columns, + complete_empty_fixture_sql as complete_empty_fixture_sql, ) from sqlbuild._native import ( - model_fixture_metadata as _model_fixture_metadata, + complete_typed_null_columns as complete_typed_null_columns, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( @@ -29,7 +28,6 @@ CompiledLineageSourceFact, CompileSqlReference, DynamicColumnContractProof, - InferredColumn, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, @@ -38,16 +36,13 @@ ) from sqlbuild.compiler.lineage.types import ( ColumnLineageConfidence, - ColumnTransformKind, ) from sqlbuild.compiler.planner.models import ( - FixtureColumnMetadata, FixtureRelationMetadata, RelationFixtureCompletion, - RelationFixtureDiagnostic, RelationFixturePlanningContext, ) -from sqlbuild.compiler.planner.types import ContractPolicy, FixtureGroups, FixtureKey +from sqlbuild.compiler.planner.types import FixtureGroups, FixtureKey from sqlbuild.compiler.references.constants import REF_PATTERN, SEED_PATTERN, SOURCE_PATTERN from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql @@ -73,57 +68,6 @@ def build_relation_fixture_completion( ) -def _fixture_keys_requiring_analysis( - *, - inferred_by_fixture: dict[FixtureKey, tuple[InferredColumn, ...]], - authoritative_columns: dict[FixtureKey, frozenset[str]], -) -> frozenset[FixtureKey]: - required: set[FixtureKey] = set() - for key, inferred in inferred_by_fixture.items(): - expected_names: frozenset[str] | None = authoritative_columns.get(key) - supplied_names: frozenset[str] = frozenset(column.name.casefold() for column in inferred) - if expected_names is None or not expected_names.issubset(supplied_names): - required.add(key) - return frozenset(required) - - -def build_relation_fixture_context( - *, - project: CompiledProjectView, - models_by_name: dict[str, CompiledModelView] | None = None, -) -> RelationFixturePlanningContext: - """Build immutable project-wide fixture metadata once per planning invocation.""" - - model_map: dict[str, CompiledModelView] = ( - {model.name: model for model in project.models} - if models_by_name is None - else models_by_name - ) - relations: dict[FixtureKey, FixtureRelationMetadata] = _fixture_relation_metadata( - project=project, - model_map=model_map, - ) - authoritative_columns: dict[FixtureKey, frozenset[str]] = _authoritative_fixture_columns( - relations=relations - ) - return RelationFixturePlanningContext( - models_by_name=model_map, - relations=relations, - authoritative_columns=authoritative_columns, - expected_types=_base_expected_fixture_types(relations=relations), - ) - - -def _authoritative_fixture_columns( - *, relations: dict[FixtureKey, FixtureRelationMetadata] -) -> dict[FixtureKey, frozenset[str]]: - authoritative: dict[FixtureKey, frozenset[str]] = {} - for key, relation in relations.items(): - if relation.authoritative_names: - authoritative[key] = frozenset(column.name.casefold() for column in relation.columns) - return authoritative - - def _required_fixture_columns( *, ordered_model_names: tuple[str, ...], @@ -318,110 +262,3 @@ def _replace_relation_markers_with_stubs(query_sql: str) -> str: result = SEED_PATTERN.sub(r"\1", result) result = SOURCE_PATTERN.sub(r"\1", result) return _DBT_REF_PATTERN.sub(r"\2", result) - - -def _fixture_relation_metadata( - *, project: CompiledProjectView, model_map: dict[str, CompiledModelView] -) -> dict[FixtureKey, FixtureRelationMetadata]: - relations: dict[FixtureKey, FixtureRelationMetadata] = {} - for model_name, model in model_map.items(): - relations[(CompiledResourceType.MODEL, model_name)] = _model_fixture_metadata(model=model) - for source in project.sources: - relations[(CompiledResourceType.SOURCE, source.name)] = FixtureRelationMetadata( - columns=tuple( - FixtureColumnMetadata( - name=column.name, - type=column.type, - nullable=column.nullable, - ) - for column in source.source_entry.columns - ), - authoritative_names=source.source_entry.contract == ContractPolicy.ENFORCED, - ) - for seed in project.seeds: - seed_columns: tuple[FixtureColumnMetadata, ...] = tuple( - FixtureColumnMetadata( - name=column.name, - type=column.type, - nullable=column.nullable, - ) - for column in seed.schema_entry.columns - ) - relations[(CompiledResourceType.SEED, seed.name)] = FixtureRelationMetadata( - columns=seed_columns, - authoritative_names=bool(seed_columns), - ) - return relations - - -def _expected_fixture_types( - *, - base_expected_types: dict[FixtureKey, dict[str, str]], - ordered_model_names: tuple[str, ...], - model_map: dict[str, CompiledModelView], -) -> Mapping[FixtureKey, dict[str, str]]: - expected_types: ChainMap[FixtureKey, dict[str, str]] = ChainMap({}, base_expected_types) - for model_name in ordered_model_names: - model: CompiledModelView | None = model_map.get(model_name) - if model is None: - continue - model_key: FixtureKey = (CompiledResourceType.MODEL, model_name) - model_types: dict[str, str] = dict(expected_types[model_key]) - expected_types[model_key] = model_types - for lineage_column in model.fast_lineage_columns or (): - if lineage_column.transform_kind != ColumnTransformKind.DIRECT: - continue - upstream_types: set[str] = set() - for upstream in lineage_column.upstream_columns: - upstream_type: str | None = expected_types.get( - (CompiledResourceType(upstream.resource_type), upstream.resource_name), {} - ).get(upstream.column_name.casefold()) - if upstream_type is not None: - upstream_types.add(upstream_type) - if len(upstream_types) == 1: - model_types.setdefault( - lineage_column.output_column.casefold(), upstream_types.pop() - ) - return expected_types - - -def _base_expected_fixture_types( - *, relations: dict[FixtureKey, FixtureRelationMetadata] -) -> dict[FixtureKey, dict[str, str]]: - expected_types: dict[FixtureKey, dict[str, str]] = {} - for key, relation in relations.items(): - column_types: dict[str, str] = {} - for column in relation.columns: - if column.type is not None: - column_types[column.name.casefold()] = column.type - expected_types[key] = column_types - return expected_types - - -def _unknown_fixture_column_diagnostics( - *, - inferred_by_fixture: dict[FixtureKey, tuple[InferredColumn, ...]], - relations: dict[FixtureKey, FixtureRelationMetadata], -) -> tuple[RelationFixtureDiagnostic, ...]: - diagnostics: list[RelationFixtureDiagnostic] = [] - for key, inferred in inferred_by_fixture.items(): - relation: FixtureRelationMetadata | None = relations.get(key) - if relation is None or not relation.authoritative_names: - continue - expected_names: set[str] = {column.name.casefold() for column in relation.columns} - unknown: tuple[str, ...] = tuple( - sorted( - column.name for column in inferred if column.name.casefold() not in expected_names - ) - ) - if unknown: - diagnostics.append( - RelationFixtureDiagnostic( - key=key, - message=( - f"mock {key[0].value} '{key[1]}' supplies unknown columns: " - f"{', '.join(unknown)}" - ), - ) - ) - return tuple(diagnostics) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 0e4a35dbd4..c7e1899610 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -119,3 +119,13 @@ class ExecuteAuditTestCase: model_locations: dict[str, str] source_map: dict[str, SourceEntry] expected_row_count: int + + +@dataclass(frozen=True) +class FixtureSchemaIntegrationTestCase: + """Closed and open source schemas through compilation and fixture planning.""" + + description: str + contract: str + expected_authoritative: bool + expected_diagnostics: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index eafda6ddd7..14ea786712 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -7,12 +7,22 @@ from sqlbuild._native import build_compiled_project, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.planner._helpers.fixtures.completion import ( + build_relation_fixture_completion, + build_relation_fixture_context, +) from sqlbuild.compiler.planner._helpers.output.plan_output import build_selected_test_entries -from sqlbuild.compiler.planner.models import SqlTestPlanEntry, SqlTestSelection +from sqlbuild.compiler.planner.models import ( + RelationFixtureCompletion, + RelationFixturePlanningContext, + SqlTestPlanEntry, + SqlTestSelection, +) from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( FixturePlanningIntegrationTestCase, + FixtureSchemaIntegrationTestCase, ) @@ -151,5 +161,65 @@ def test_given_untyped_null_fixture_when_planning_then_applies_authoritative_typ assert test_case.expected_sql_fragment in entry.chain[0].resolved_sql +@pytest.mark.parametrize( + "test_case", + ( + FixtureSchemaIntegrationTestCase( + description="enforced source rejects unknown supplied columns", + contract=" contract: enforced\n", + expected_authoritative=True, + expected_diagnostics=("mock source 'raw_orders' supplies unknown columns: extra",), + ), + FixtureSchemaIntegrationTestCase( + description="open source permits unknown supplied columns", + contract="", + expected_authoritative=False, + expected_diagnostics=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_source_contract_when_completing_fixture_then_respects_authoritative_names( + test_case: FixtureSchemaIntegrationTestCase, + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], +) -> None: + write_repo_files( + tmp_path, + { + "sqlbuild_project.toml": 'name = "fixture_schema_demo"\nadapter = "duckdb"\n[connection]\ndatabase = ":memory:"\n', + "models/orders.sql": 'MODEL (description "Order facts."); SELECT order_id FROM __source("raw_orders")', + "sources/raw_orders.yml": ( + "sources:\n - name: raw_orders\n description: Raw order facts.\n" + " expression: SELECT 1 AS order_id\n" + f"{test_case.contract}" + " columns:\n - name: order_id\n type: INTEGER\n" + ), + }, + ) + adapter: DuckDbAdapter = DuckDbAdapter() + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=adapter + ) + context: RelationFixturePlanningContext = build_relation_fixture_context(project=project) + key: tuple[CompiledResourceType, str] = (CompiledResourceType.SOURCE, "raw_orders") + completed: RelationFixtureCompletion = build_relation_fixture_completion( + project=project, + adapter=adapter, + ordered_model_names=("orders",), + fixture_groups=( + (CompiledResourceType.SOURCE, {"raw_orders": "SELECT 1 AS ORDER_ID, 2 AS extra"}), + ), + planning_context=context, + ) + assert context.relations[key].authoritative_names is test_case.expected_authoritative + assert context.expected_types[key] == {"order_id": "INTEGER"} + assert ( + tuple(diagnostic.message for diagnostic in completed.diagnostics) + == test_case.expected_diagnostics + ) + assert completed.fixture_sql_by_key[key] == "SELECT 1 AS ORDER_ID, 2 AS extra" + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index a789428dd2..8616d21334 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -20,6 +20,7 @@ CompiledResourceType, ) from sqlbuild.compiler.planner._helpers.fixtures import completion as fixture_completion +from sqlbuild.compiler.planner._helpers.output import plan_output from sqlbuild.compiler.planner._helpers.output.plan_output import build_selected_test_entries from sqlbuild.compiler.planner._helpers.sql_tests import assembly as sql_test_assembly from sqlbuild.compiler.planner.classes import fixture_column_inferences as fixture_inferences @@ -99,8 +100,8 @@ def test_given_many_tests_when_building_entries_then_project_fixture_metadata_is sql_tests=tests, settings=SettingsConfig(sql_analysis=True), ) - metadata_builder: Mock = Mock(wraps=fixture_completion._model_fixture_metadata) - monkeypatch.setattr(fixture_completion, "_model_fixture_metadata", metadata_builder) + metadata_builder: Mock = Mock(wraps=fixture_completion.build_relation_fixture_context) + monkeypatch.setattr(plan_output, "build_relation_fixture_context", metadata_builder) fixture_completion._cached_resolved_column_reads.cache_clear() fallback_analyzer: Mock = Mock(wraps=fixture_completion.analyze_resolved_column_reads) monkeypatch.setattr( From 9a64326eac28add75c005728a9e6b9df902571d9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:27:12 +0100 Subject: [PATCH 22/29] chore: update Python boundary after fixture type planning port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index ebb21977cc..98a81181ea 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 34996 +remaining_lines = 34833 [[stays]] path = "executor" From 4623db23c7eb8129813877629e1e84f867bdc6ea Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:28:01 +0100 Subject: [PATCH 23/29] refactor: move scenario graph and relation planning to Rust --- crates/sqlbuild-analysis/src/lib.rs | 1 + .../src/planner_scenario/main/graph.rs | 119 ++++++ .../src/planner_scenario/main/mod.rs | 3 + .../src/planner_scenario/mod.rs | 7 + .../src/planner_scenario/models.rs | 34 ++ .../src/planner_scenario/tests.rs | 6 + .../src/planner_scenario/tests/test_graph.rs | 115 ++++++ .../src/planner_scenario/tests/test_types.rs | 12 + .../src/bindings/_helpers/mod.rs | 1 + .../bindings/_helpers/planner_scenario/cli.rs | 179 +++++++++ .../_helpers/planner_scenario/constants.rs | 4 + .../_helpers/planner_scenario/graph.rs | 112 ++++++ .../bindings/_helpers/planner_scenario/mod.rs | 8 + .../_helpers/planner_scenario/registration.rs | 15 + .../_helpers/planner_scenario/relations.rs | 161 ++++++++ .../_helpers/planner_scenario/values.rs | 36 ++ .../src/bindings/main/register.rs | 1 + src/sqlbuild/_native.pyi | 28 ++ .../compiler/planner/_helpers/scenario/cli.py | 137 ------- .../planner/_helpers/scenario/graph.py | 350 ------------------ .../planner/_helpers/scenario/relations.py | 151 -------- .../planner/main/scenarios/scenario.py | 2 +- .../planner/_helpers/test_scenario_cli.py | 2 +- .../planner/_helpers/test_scenario_graph.py | 2 +- .../_helpers/test_scenario_relations.py | 2 +- 25 files changed, 846 insertions(+), 642 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_scenario/main/graph.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scenario/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scenario/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scenario/models.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scenario/tests.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scenario/tests/test_graph.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scenario/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/scenario/cli.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/scenario/graph.py diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 4cf17bc857..4d8753e629 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -23,6 +23,7 @@ pub mod planner_changes; pub mod planner_fixtures; pub mod planner_identity; pub mod planner_migrations; +pub mod planner_scenario; pub mod planner_scope; pub mod planner_selection; pub mod planner_strategy; diff --git a/crates/sqlbuild-analysis/src/planner_scenario/main/graph.rs b/crates/sqlbuild-analysis/src/planner_scenario/main/graph.rs new file mode 100644 index 0000000000..6413b4ddeb --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/main/graph.rs @@ -0,0 +1,119 @@ +//! Infer scenario closure while preserving dependency and diagnostic order. + +use crate::planner_scenario::models::{ScenarioGraphInput, ScenarioGraphOutput}; + +/// Infer required relations, cutting traversal at authored reference fixtures. +pub fn graph(input: &ScenarioGraphInput) -> ScenarioGraphOutput { + let mut output = ScenarioGraphOutput::default(); + for (names, label) in [ + (&input.expected, "expects unknown model"), + (&input.assertions, "assertion references unknown model"), + ] { + for name in names { + if !input.models.contains_key(name) { + output + .warnings + .push((format!("Scenario '{}' {label} '{name}'.", input.name), true)); + } + } + } + if input.targets.is_empty() { + output.warnings.push((format!( + "Scenario '{}' has no target models. Add __expected__ or reference a model with __ref(...) inside an __assert__ CTE.", input.name + ), true)); + } + for (names, kind) in [ + (&input.refs, "model"), + (&input.sources, "source"), + (&input.seeds, "seed"), + ] { + for name in names { + let exists = match kind { + "model" => input.models.contains_key(name), + "source" => input.project_sources.contains(name), + _ => input.project_seeds.contains(name), + }; + if !exists { + output.warnings.push(( + format!( + "Scenario '{}' provides fixture for unknown {kind} '{name}'.", + input.name + ), + true, + )); + } + } + } + for target in &input.targets { + output = walk(target, input, output); + } + for name in &output.sources { + if !input.sources.contains(name) { + output.warnings.push((format!( + "Scenario '{}' requires source '{name}', but no fixture was provided. Add a CTE named __source__{} AS (...).", + input.name, name.replace('.', "__") + ), false)); + } + } + for name in &output.dbt_refs { + if !input.dbt_refs.contains(name) { + output.warnings.push((format!( + "Scenario '{}' requires dbt ref '{name}', but no fixture was provided. Add a CTE named __dbt_ref__{name} AS (...).", input.name + ), false)); + } + } + output.seed_fixtures = output + .seeds + .iter() + .filter(|name| input.seeds.contains(name)) + .cloned() + .collect(); + output +} + +fn walk( + name: &str, + input: &ScenarioGraphInput, + mut output: ScenarioGraphOutput, +) -> ScenarioGraphOutput { + if input.refs.iter().any(|fixture| fixture == name) { + output.refs.insert(name.to_owned()); + return output; + } + if output.models.contains(name) { + return output; + } + let Some(deps) = input.models.get(name) else { + return output; + }; + output.models.insert(name.to_owned()); + for dep in deps { + match dep.0.as_str() { + "model" => { + if input.refs.contains(&dep.1) { + output.refs.insert(dep.1.clone()); + } else if !input.models.contains_key(&dep.1) { + output.warnings.push((format!( + "Scenario '{}' requires model '{}', but it does not exist and no __ref__ fixture was provided.", input.name, dep.1 + ), true)); + } else { + output = walk(&dep.1, input, output); + } + } + "source" => { + output.sources.insert(dep.1.clone()); + } + "seed" => { + output.seeds.insert(dep.1.clone()); + } + "dbt_ref" => { + output.dbt_refs.insert(dep.1.replace('.', "__")); + } + "udf" | "table_fn" if !output.functions.contains(dep) => { + output.functions.push(dep.clone()); + } + _ => {} + } + } + output +} diff --git a/crates/sqlbuild-analysis/src/planner_scenario/main/mod.rs b/crates/sqlbuild-analysis/src/planner_scenario/main/mod.rs new file mode 100644 index 0000000000..97c2acc407 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/main/mod.rs @@ -0,0 +1,3 @@ +//! Scenario planning entry points. + +pub mod graph; diff --git a/crates/sqlbuild-analysis/src/planner_scenario/mod.rs b/crates/sqlbuild-analysis/src/planner_scenario/mod.rs new file mode 100644 index 0000000000..2c241db51f --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/mod.rs @@ -0,0 +1,7 @@ +//! Scenario graph decisions independent of adapters and execution. + +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/planner_scenario/models.rs b/crates/sqlbuild-analysis/src/planner_scenario/models.rs new file mode 100644 index 0000000000..fea389e82d --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/models.rs @@ -0,0 +1,34 @@ +//! Scenario graph input and output values. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use crate::assembly::project::types::ObjectKey; + +/// Authored scenario names and project dependency facts in declaration order. +#[derive(Debug, Default)] +pub struct ScenarioGraphInput { + pub name: String, + pub expected: Vec, + pub assertions: Vec, + pub targets: Vec, + pub refs: Vec, + pub sources: Vec, + pub seeds: Vec, + pub dbt_refs: Vec, + pub models: HashMap>, + pub project_sources: HashSet, + pub project_seeds: HashSet, +} + +/// Required scenario relations and diagnostics in observable order. +#[derive(Debug, Default)] +pub struct ScenarioGraphOutput { + pub models: BTreeSet, + pub refs: BTreeSet, + pub sources: BTreeSet, + pub seeds: BTreeSet, + pub seed_fixtures: BTreeSet, + pub dbt_refs: BTreeSet, + pub functions: Vec, + pub warnings: Vec<(String, bool)>, +} diff --git a/crates/sqlbuild-analysis/src/planner_scenario/tests.rs b/crates/sqlbuild-analysis/src/planner_scenario/tests.rs new file mode 100644 index 0000000000..59d9eea16f --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/tests.rs @@ -0,0 +1,6 @@ +//! Scenario graph regression coverage. + +#[path = "tests/test_graph.rs"] +mod test_graph; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_scenario/tests/test_graph.rs b/crates/sqlbuild-analysis/src/planner_scenario/tests/test_graph.rs new file mode 100644 index 0000000000..5d51593ed2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/tests/test_graph.rs @@ -0,0 +1,115 @@ +//! Validate fixture cuts, shared dependencies, and diagnostic ordering. + +use crate::planner_scenario::main::graph::graph; +use crate::planner_scenario::models::ScenarioGraphInput; +use crate::planner_scenario::tests::test_types::GraphTestCase; + +#[test] +fn given_scenario_dependencies_when_planning_then_preserves_closure_and_warning_order() { + let test_cases = [ + GraphTestCase { + description: "fixture cut and shared function dependency", + input: ScenarioGraphInput { + name: "orders".into(), + targets: vec!["summary".into()], + refs: vec!["staging".into()], + models: [ + ( + "summary".into(), + vec![ + ("model".into(), "staging".into()), + ("udf".into(), "total".into()), + ("model".into(), "details".into()), + ], + ), + ("staging".into(), vec![("source".into(), "hidden".into())]), + ( + "details".into(), + vec![ + ("udf".into(), "total".into()), + ("table_fn".into(), "items".into()), + ("model".into(), "summary".into()), + ], + ), + ] + .into(), + ..ScenarioGraphInput::default() + }, + expected_models: vec!["details", "summary"], + expected_refs: vec!["staging"], + expected_functions: vec![("udf", "total"), ("table_fn", "items")], + expected_warnings: vec![], + }, + GraphTestCase { + description: "declared and upstream diagnostic ordering", + input: ScenarioGraphInput { + name: "orders".into(), + targets: vec!["summary".into()], + expected: vec!["unknown".into()], + models: [( + "summary".into(), + vec![ + ("source".into(), "raw.orders".into()), + ("model".into(), "missing".into()), + ("dbt_ref".into(), "catalog.items".into()), + ], + )] + .into(), + ..ScenarioGraphInput::default() + }, + expected_models: vec!["summary"], + expected_refs: vec![], + expected_functions: vec![], + expected_warnings: vec![ + ("Scenario 'orders' expects unknown model 'unknown'.", true), + ( + "Scenario 'orders' requires model 'missing', but it does not exist and no __ref__ fixture was provided.", + true, + ), + ( + "Scenario 'orders' requires source 'raw.orders', but no fixture was provided. Add a CTE named __source__raw__orders AS (...).", + false, + ), + ( + "Scenario 'orders' requires dbt ref 'catalog__items', but no fixture was provided. Add a CTE named __dbt_ref__catalog__items AS (...).", + false, + ), + ], + }, + ]; + for test_case in test_cases { + let result = graph(&test_case.input); + assert_eq!( + result.models.iter().map(String::as_str).collect::>(), + test_case.expected_models, + "{}", + test_case.description + ); + assert_eq!( + result.refs.iter().map(String::as_str).collect::>(), + test_case.expected_refs, + "{}", + test_case.description + ); + assert_eq!( + result + .functions + .iter() + .map(|(kind, name)| (kind.as_str(), name.as_str())) + .collect::>(), + test_case.expected_functions, + "{}", + test_case.description + ); + assert_eq!( + result + .warnings + .iter() + .map(|(message, code)| (message.as_str(), *code)) + .collect::>(), + test_case.expected_warnings, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_scenario/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_scenario/tests/test_types.rs new file mode 100644 index 0000000000..eb99591ff8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/tests/test_types.rs @@ -0,0 +1,12 @@ +//! Scenario graph cases. + +use crate::planner_scenario::models::ScenarioGraphInput; + +pub(super) struct GraphTestCase { + pub description: &'static str, + pub input: ScenarioGraphInput, + pub expected_models: Vec<&'static str>, + pub expected_refs: Vec<&'static str>, + pub expected_functions: Vec<(&'static str, &'static str)>, + pub expected_warnings: Vec<(&'static str, bool)>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index b7f4e1e29d..2f8f0ebb07 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -49,6 +49,7 @@ pub(crate) mod planner_changes; pub(crate) mod planner_identity; pub(crate) mod planner_migrations; pub(crate) mod planner_resolution; +pub(crate) mod planner_scenario; pub(crate) mod planner_scope; pub(crate) mod planner_selection; pub(crate) mod planner_strategy; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs new file mode 100644 index 0000000000..e07ff15c71 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -0,0 +1,179 @@ +//! Assemble CLI scenario plans while retaining adapter and executor boundaries. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; +use crate::bindings::_helpers::planner_scenario::relations::build_scenario_relation_plan; +use crate::bindings::_helpers::planner_scenario::values::{CONSTANTS, MODELS, TYPES, string}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +const ARTIFACTS: &str = "sqlbuild.compiler.planner._helpers.scenario.artifacts"; + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn build_cli_scenario_plan<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_cli_scenario_plan", + kwargs, + ( + &[ + "scenario", + "pipeline_result", + "adapter", + "project_name", + "source_lexical_syntax", + ], + &[], + ), + )?; + let scenario = args.value("scenario")?; + let project = args.value("pipeline_result")?.getattr("project")?; + let adapter = args.value("adapter")?; + let (graph, warnings) = plan_scenario_graph(&scenario, &project, true, None)?; + raise_for_warnings(warnings.as_any())?; + let hashes = call_with( + &imported(py, ARTIFACTS, "build_scenario_hash_index")?, + &[ + ("project_name", args.value("project_name")?), + ("scenarios", project.getattr("sql_scenarios")?), + ( + "run_namespace", + project.getattr("scenario")?.getattr("run_namespace")?, + ), + ], + )?; + let mut identities: Vec> = Vec::new(); + for (field, kind) in [ + ("source_fixture_names", "SOURCE"), + ("ref_fixture_names", "REF"), + ("dbt_ref_fixture_names", "DBT_REF"), + ("seed_names", "SEED"), + ("model_names", "MODEL"), + ] { + for name in graph.getattr(field)?.try_iter()? { + identities.push(call_with( + &imported(py, MODELS, "ScenarioArtifactIdentity")?, + &[ + ( + "kind", + imported(py, TYPES, "ScenarioArtifactKind")?.getattr(kind)?, + ), + ("logical_name", name?), + ], + )?); + } + } + let map = call_with( + &imported(py, ARTIFACTS, "build_scenario_relation_map")?, + &[ + ("scenario_name", scenario.getattr("name")?), + ("hash_prefix", hashes.get_item(scenario.getattr("name")?)?), + ("artifacts", PyTuple::new(py, identities)?.into_any()), + ( + "identifier_limit", + adapter.call_method0("maximum_identifier_length")?, + ), + ], + )?; + let (database, schema) = namespace(&project, &graph)?; + let relation_args = PyDict::new(py); + for (name, value) in [ + ("project", project.clone()), + ("graph_plan", graph.clone()), + ("relation_map", map), + ( + "render_qualified_name", + adapter.getattr("render_qualified_name")?, + ), + ("database", database), + ("schema", schema), + ] { + relation_args.set_item(name, value)?; + } + let relations = build_scenario_relation_plan(py, Some(relation_args))?; + let result = call_with( + &imported( + py, + "sqlbuild.compiler.planner._helpers.scenario.relations", + "build_scenario_execution_plan", + )?, + &[ + ("scenario", scenario), + ("project", project), + ("adapter", adapter), + ("graph_plan", graph), + ("relation_plan", relations), + ( + "source_lexical_syntax", + args.value("source_lexical_syntax")?, + ), + ], + )?; + raise_for_warnings(&result.get_item(1)?)?; + result.get_item(0) +} + +fn namespace<'py>( + project: &Bound<'py, PyAny>, + graph: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py = project.py(); + let models = PyDict::new(py); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + models.set_item(model.getattr("name")?, model)?; + } + for field in ["target_model_names", "model_names"] { + for name in graph.getattr(field)?.try_iter()? { + if let Some(model) = models.get_item(name?)? { + let destination = model.getattr("destination")?; + return Ok(( + destination.getattr("database")?, + destination.getattr("schema")?, + )); + } + } + } + Ok((py.None().into_bound(py), py.None().into_bound(py))) +} + +fn raise_for_warnings(warnings: &Bound<'_, PyAny>) -> PyResult<()> { + let py = warnings.py(); + let severity = imported(py, TYPES, "WarningSeverity")?.getattr("ERROR")?; + let mut messages: Vec = Vec::new(); + let mut code = None; + for warning in warnings.try_iter()? { + let warning = warning?; + if warning.getattr("severity")?.eq(&severity)? { + if code.is_none() { + let value = warning.getattr("code")?; + code = Some(if value.is_truthy()? { + value + } else { + imported(py, CONSTANTS, "SCENARIO_PLAN_GRAPH_VALIDATION")? + }); + } + messages.push(warning.getattr("message")?.extract::()?); + } + } + if let Some(code) = code { + return Err(PyErr::from_value(call_with( + &imported( + py, + "sqlbuild.compiler.planner.exceptions", + "PlannerInputError", + )?, + &[ + ("message", string(py, &messages.join("\n"))), + ("code", code), + ], + )?)); + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs new file mode 100644 index 0000000000..bdb5f14345 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs @@ -0,0 +1,4 @@ +//! Relation categories with additional scenario binding semantics. + +pub(super) const REF: &str = "ref"; +pub(super) const SOURCE: &str = "source"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs new file mode 100644 index 0000000000..fe9dbb626a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs @@ -0,0 +1,112 @@ +//! Convert project views into pure scenario dependency facts. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; +use crate::bindings::_helpers::planner_scenario::values::{CONSTANTS, MODELS, TYPES, string}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, pyfunction}; +use pyo3::types::PyTuple; +use sqlbuild_analysis::planner_scenario::main::graph::graph; +use sqlbuild_analysis::planner_scenario::models::ScenarioGraphInput; + +#[pyfunction] +#[pyo3(signature = (*, scenario, project, sql_analysis_enabled=true, sql_analysis_dialect=None))] +pub(crate) fn plan_scenario_graph<'py>( + scenario: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, + sql_analysis_enabled: bool, + sql_analysis_dialect: Option<&str>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyTuple>)> { + let _ = (sql_analysis_enabled, sql_analysis_dialect); + let py = scenario.py(); + let mut input = ScenarioGraphInput { + name: scenario.getattr("name")?.extract()?, + expected: scenario.getattr("expected_model_names")?.extract()?, + assertions: scenario + .getattr("assertion_target_model_names")? + .extract()?, + targets: scenario.getattr("target_model_names")?.extract()?, + refs: scenario.getattr("ref_fixture_names")?.extract()?, + sources: scenario.getattr("source_fixture_names")?.extract()?, + seeds: scenario.getattr("seed_fixture_names")?.extract()?, + dbt_refs: scenario.getattr("dbt_ref_fixture_names")?.extract()?, + ..ScenarioGraphInput::default() + }; + for model in project.getattr("models")?.try_iter()? { + let model = model?; + let deps = model + .getattr("deps")? + .try_iter()? + .map(|dep| native_key(&dep?)) + .collect::>()?; + input.models.insert(model.getattr("name")?.extract()?, deps); + } + for source in project.getattr("sources")?.try_iter()? { + input + .project_sources + .insert(source?.getattr("name")?.extract()?); + } + for seed in project.getattr("seeds")?.try_iter()? { + input + .project_seeds + .insert(seed?.getattr("name")?.extract()?); + } + let output = graph(&input); + input.targets.sort(); + input.assertions.sort(); + let plan = call_with( + &imported(py, MODELS, "ScenarioGraphPlan")?, + &[ + ("key", scenario.getattr("key")?), + ("name", scenario.getattr("name")?), + ( + "target_model_names", + PyTuple::new(py, input.targets)?.into_any(), + ), + ( + "assertion_target_model_names", + PyTuple::new(py, input.assertions)?.into_any(), + ), + ("model_names", PyTuple::new(py, output.models)?.into_any()), + ( + "source_fixture_names", + PyTuple::new(py, output.sources)?.into_any(), + ), + ( + "ref_fixture_names", + PyTuple::new(py, output.refs)?.into_any(), + ), + ("seed_names", PyTuple::new(py, output.seeds)?.into_any()), + ( + "seed_fixture_names", + PyTuple::new(py, output.seed_fixtures)?.into_any(), + ), + ( + "dbt_ref_fixture_names", + PyTuple::new(py, output.dbt_refs)?.into_any(), + ), + ( + "function_deps", + PyTuple::new(py, KeyCache::new(py)?.keys(&output.functions)?)?.into_any(), + ), + ], + )?; + let mut warnings: Vec> = Vec::new(); + for (message, coded) in output.warnings { + let mut values = vec![ + ("model_name", py.None().into_bound(py)), + ( + "severity", + imported(py, TYPES, "WarningSeverity")?.getattr("ERROR")?, + ), + ("message", string(py, &message)), + ]; + if coded { + values.push(( + "code", + imported(py, CONSTANTS, "SCENARIO_PLAN_GRAPH_VALIDATION")?, + )); + } + warnings.push(call_with(&imported(py, MODELS, "PlanWarning")?, &values)?); + } + Ok((plan, PyTuple::new(py, warnings)?)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs new file mode 100644 index 0000000000..05781c3595 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs @@ -0,0 +1,8 @@ +//! Python boundaries for native scenario planning. + +mod cli; +mod constants; +mod graph; +pub(crate) mod registration; +mod relations; +mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs new file mode 100644 index 0000000000..c7c943ec48 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs @@ -0,0 +1,15 @@ +//! Register scenario planning bindings. + +use crate::bindings::_helpers::planner_scenario::{cli, graph, relations}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(graph::plan_scenario_graph, module)?)?; + module.add_function(wrap_pyfunction!( + relations::build_scenario_relation_plan, + module + )?)?; + module.add_function(wrap_pyfunction!(cli::build_cli_scenario_plan, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs new file mode 100644 index 0000000000..13763c301f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs @@ -0,0 +1,161 @@ +//! Build scenario relation locations through adapter qualification callbacks. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_scenario::constants::{REF, SOURCE}; +use crate::bindings::_helpers::planner_scenario::values::{MODELS, error}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::{PyDict, PyDictMethods}; + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn build_scenario_relation_plan<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_scenario_relation_plan", + kwargs, + ( + &[ + "project", + "graph_plan", + "relation_map", + "render_qualified_name", + ], + &["database", "schema"], + ), + )?; + let graph = args.value("graph_plan")?; + let artifacts = PyDict::new(py); + for artifact in args + .value("relation_map")? + .getattr("artifacts")? + .try_iter()? + { + let artifact = artifact?; + let identity = artifact.getattr("identity")?; + artifacts.set_item( + (identity.getattr("kind")?, identity.getattr("logical_name")?), + artifact.getattr("physical_name")?, + )?; + } + let sources = PyDict::new(py); + for source in args.value("project")?.getattr("sources")?.try_iter()? { + let source = source?; + sources.set_item(source.getattr("name")?, source.getattr("source_entry")?)?; + } + let models = PyDict::new(py); + let refs = PyDict::new(py); + let source_fixtures = PyDict::new(py); + let dbt_refs = PyDict::new(py); + let seeds = PyDict::new(py); + let seed_fixtures = PyDict::new(py); + let source_map = PyDict::new(py); + for (field, kind, locations) in [ + ("model_names", "model", &models), + ("ref_fixture_names", "ref", &refs), + ("source_fixture_names", "source", &source_fixtures), + ("dbt_ref_fixture_names", "dbt_ref", &dbt_refs), + ("seed_names", "seed", &seeds), + ] { + for name in graph.getattr(field)?.try_iter()? { + let name: String = name?.extract()?; + let target = target_for_artifact(&args, &artifacts, kind, &name)?; + locations.set_item(&name, &target)?; + if kind == REF { + models.set_item(&name, &target)?; + } else if kind == SOURCE { + let entry = sources + .get_item(&name)? + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(name.clone()))?; + let changes = PyDict::new(py); + for field in ["database", "schema"] { + changes.set_item(field, target.getattr(field)?)?; + } + changes.set_item("table", target.getattr("name")?)?; + changes.set_item("expression", py.None())?; + changes.set_item("type_enforcement", false)?; + let replacement = + imported(py, "dataclasses", "replace")?.call((entry,), Some(&changes))?; + source_map.set_item(&name, replacement)?; + } + } + } + for name in graph.getattr("seed_fixture_names")?.try_iter()? { + let name: String = name?.extract()?; + let Some(target) = seeds.get_item(&name)? else { + return Err(error( + py, + &format!("Scenario relation plan is missing seed target '{name}'"), + "SCENARIO_PLAN_MISSING_RELATION_TARGET", + Some("This is likely a SQLBuild bug. Please file an issue with the scenario name."), + )?); + }; + seed_fixtures.set_item(name, target)?; + } + call_with( + &imported(py, MODELS, "ScenarioRelationPlan")?, + &[ + ("scenario_name", graph.getattr("name")?), + ("relation_map", args.value("relation_map")?), + ("model_locations", models.into_any()), + ("seed_locations", seeds.into_any()), + ("project_source_map", sources.into_any()), + ("source_map", source_map.into_any()), + ("source_fixture_locations", source_fixtures.into_any()), + ("ref_fixture_locations", refs.into_any()), + ("dbt_ref_fixture_locations", dbt_refs.into_any()), + ("seed_fixture_locations", seed_fixtures.into_any()), + ], + ) +} + +fn target_for_artifact<'py>( + args: &KeywordArguments<'py>, + artifacts: &Bound<'py, PyDict>, + kind: &str, + name: &str, +) -> PyResult> { + let py = artifacts.py(); + let Some(physical) = artifacts.get_item((kind, name))? else { + return Err(error( + py, + &format!("Scenario relation map is missing {kind} artifact '{name}'"), + "SCENARIO_PLAN_INTERNAL", + Some("This is likely a SQLBuild bug. Please file an issue with the scenario name."), + )?); + }; + let database = args.value("database")?; + let schema = args.value("schema")?; + let qualified = call_with( + &args.value("render_qualified_name")?, + &[ + ("database", database.clone()), + ("schema", schema.clone()), + ("name", physical.clone()), + ], + )?; + call_with( + &imported( + py, + "sqlbuild.compiler.compile.models", + "CompiledRelationLocation", + )?, + &[ + ("database", database), + ("schema", schema), + ("name", physical.clone()), + ( + "qualified_name", + if qualified.is_none() { + physical + } else { + qualified + }, + ), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs new file mode 100644 index 0000000000..c51cc792de --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs @@ -0,0 +1,36 @@ +//! Existing Python model constructors and scenario diagnostics. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use pyo3::prelude::{Bound, PyAny, PyErr, PyResult, Python}; +use pyo3::types::PyString; + +pub(super) const MODELS: &str = "sqlbuild.compiler.planner.models"; +pub(super) const TYPES: &str = "sqlbuild.compiler.planner.types"; +pub(super) const CONSTANTS: &str = "sqlbuild.compiler.planner.constants"; + +pub(super) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { + PyString::new(py, text).into_any() +} + +pub(super) fn error( + py: Python<'_>, + message: &str, + code: &str, + help: Option<&str>, +) -> PyResult { + let mut args = vec![ + ("message", string(py, message)), + ("code", imported(py, CONSTANTS, code)?), + ]; + if let Some(help) = help { + args.push(("help", string(py, help))); + } + Ok(PyErr::from_value(call_with( + &imported( + py, + "sqlbuild.compiler.planner.exceptions", + "PlannerInputError", + )?, + &args, + )?)) +} diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 87a62e7eed..d141f8a041 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -51,6 +51,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { planner_identity::registration::register(module)?; planner_selection::registration::register(module)?; planner_scope::registration::register(module)?; + crate::bindings::_helpers::planner_scenario::registration::register(module)?; planner_strategy::registration::register(module)?; crate::bindings::_helpers::planner_resolution::registration::register(module)?; references::registration::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 904686eb5b..aed229a053 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -241,6 +241,10 @@ from sqlbuild.compiler.planner.models import ( RelationFixtureCompletion, RelationFixtureDiagnostic, RelationFixturePlanningContext, + ScenarioExecutionPlan, + ScenarioGraphPlan, + ScenarioRelationMap, + ScenarioRelationPlan, SchemaAction, SchemaFinding, SelectionStalenessGraph, @@ -3895,6 +3899,30 @@ def run_dag_command( json_output: bool = False, cli_vars: dict[str, object] | None = None, ) -> int: ... +def plan_scenario_graph( + *, + scenario: CompiledSqlScenario, + project: CompiledProjectView, + sql_analysis_enabled: bool = True, + sql_analysis_dialect: str | None = None, +) -> tuple[ScenarioGraphPlan, tuple[PlanWarning, ...]]: ... +def build_scenario_relation_plan( + *, + project: CompiledProjectView, + graph_plan: ScenarioGraphPlan, + relation_map: ScenarioRelationMap, + render_qualified_name: Callable[..., str | None], + database: str | None = None, + schema: str | None = None, +) -> ScenarioRelationPlan: ... +def build_cli_scenario_plan( + *, + scenario: CompiledSqlScenario, + pipeline_result: CompilePipelineResult, + adapter: BaseAdapter, + project_name: str, + source_lexical_syntax: SqlLexicalSyntax, +) -> ScenarioExecutionPlan: ... def run_scenario_test_command(*, request: ScenarioTestCommandRequest) -> int: ... def run_scenario_capture_command(*, request: ScenarioCaptureCommandRequest) -> int: ... def sql_analysis_off_guidance( diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/cli.py b/src/sqlbuild/compiler/planner/_helpers/scenario/cli.py deleted file mode 100644 index 4a762c8440..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/cli.py +++ /dev/null @@ -1,137 +0,0 @@ -"""CLI-facing scenario planning helper.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( - CompiledSqlScenario, -) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner._helpers.scenario.artifacts import ( - build_scenario_hash_index, - build_scenario_relation_map, -) -from sqlbuild.compiler.planner._helpers.scenario.graph import plan_scenario_graph -from sqlbuild.compiler.planner._helpers.scenario.relations import ( - build_scenario_execution_plan, - build_scenario_relation_plan, -) -from sqlbuild.compiler.planner.constants import SCENARIO_PLAN_GRAPH_VALIDATION -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - PlanWarning, - ScenarioArtifactIdentity, - ScenarioExecutionPlan, - ScenarioGraphPlan, - ScenarioRelationMap, - ScenarioRelationPlan, -) -from sqlbuild.compiler.planner.types import ScenarioArtifactKind, WarningSeverity -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -def build_cli_scenario_plan( - *, - scenario: CompiledSqlScenario, - pipeline_result: CompilePipelineResult, - adapter: BaseAdapter, - project_name: str, - source_lexical_syntax: SqlLexicalSyntax, -) -> ScenarioExecutionPlan: - """Build a scenario execution plan for CLI execution.""" - - graph_plan: ScenarioGraphPlan - graph_warnings: tuple[PlanWarning, ...] - graph_plan, graph_warnings = plan_scenario_graph( - scenario=scenario, - project=pipeline_result.project, - ) - _raise_for_error_warnings(graph_warnings) - hash_index: dict[str, str] = build_scenario_hash_index( - project_name=project_name, - scenarios=pipeline_result.project.sql_scenarios, - run_namespace=pipeline_result.project.scenario.run_namespace, - ) - relation_map: ScenarioRelationMap = build_scenario_relation_map( - scenario_name=scenario.name, - hash_prefix=hash_index[scenario.name], - artifacts=_scenario_artifacts(graph_plan), - identifier_limit=adapter.maximum_identifier_length(), - ) - database: str | None - schema: str | None - database, schema = _scenario_namespace(project=pipeline_result.project, graph_plan=graph_plan) - relation_plan: ScenarioRelationPlan = build_scenario_relation_plan( - project=pipeline_result.project, - graph_plan=graph_plan, - relation_map=relation_map, - render_qualified_name=adapter.render_qualified_name, - database=database, - schema=schema, - ) - scenario_plan: ScenarioExecutionPlan - execution_warnings: tuple[PlanWarning, ...] - scenario_plan, execution_warnings = build_scenario_execution_plan( - scenario=scenario, - project=pipeline_result.project, - adapter=adapter, - graph_plan=graph_plan, - relation_plan=relation_plan, - source_lexical_syntax=source_lexical_syntax, - ) - _raise_for_error_warnings(execution_warnings) - return scenario_plan - - -def _scenario_artifacts(graph_plan: ScenarioGraphPlan) -> tuple[ScenarioArtifactIdentity, ...]: - artifacts: list[ScenarioArtifactIdentity] = [] - source_name: str - for source_name in graph_plan.source_fixture_names: - artifacts.append( - ScenarioArtifactIdentity(kind=ScenarioArtifactKind.SOURCE, logical_name=source_name) - ) - ref_name: str - for ref_name in graph_plan.ref_fixture_names: - artifacts.append( - ScenarioArtifactIdentity(kind=ScenarioArtifactKind.REF, logical_name=ref_name) - ) - dbt_ref_name: str - for dbt_ref_name in graph_plan.dbt_ref_fixture_names: - artifacts.append( - ScenarioArtifactIdentity(kind=ScenarioArtifactKind.DBT_REF, logical_name=dbt_ref_name) - ) - seed_name: str - for seed_name in graph_plan.seed_names: - artifacts.append( - ScenarioArtifactIdentity(kind=ScenarioArtifactKind.SEED, logical_name=seed_name) - ) - model_name: str - for model_name in graph_plan.model_names: - artifacts.append( - ScenarioArtifactIdentity(kind=ScenarioArtifactKind.MODEL, logical_name=model_name) - ) - return tuple(artifacts) - - -def _scenario_namespace( - *, project: CompiledProjectView, graph_plan: ScenarioGraphPlan -) -> tuple[str | None, str | None]: - models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} - model_name: str - for model_name in (*graph_plan.target_model_names, *graph_plan.model_names): - model: CompiledModelView | None = models_by_name.get(model_name) - if model is not None: - return model.destination.database, model.destination.schema - return None, None - - -def _raise_for_error_warnings(warnings: tuple[PlanWarning, ...]) -> None: - error_warnings: tuple[PlanWarning, ...] = tuple( - warning for warning in warnings if warning.severity == WarningSeverity.ERROR - ) - if error_warnings: - raise PlannerInputError( - "\n".join(warning.message for warning in error_warnings), - code=error_warnings[0].code or SCENARIO_PLAN_GRAPH_VALIDATION, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/graph.py b/src/sqlbuild/compiler/planner/_helpers/scenario/graph.py deleted file mode 100644 index 543cb3791b..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/graph.py +++ /dev/null @@ -1,350 +0,0 @@ -"""Graph inference and validation for SQL-native scenarios.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledSqlScenario, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.planner.constants import SCENARIO_PLAN_GRAPH_VALIDATION -from sqlbuild.compiler.planner.models import PlanWarning, ScenarioGraphPlan -from sqlbuild.compiler.planner.types import WarningSeverity - - -@dataclass(frozen=True) -class _UpstreamRequirements: - """Upstream closure requirements collected for one scenario.""" - - model_names: frozenset[str] - required_ref_fixture_names: frozenset[str] - required_source_names: frozenset[str] - required_seed_names: frozenset[str] - required_dbt_ref_fixture_names: frozenset[str] - function_deps: tuple[CompiledObjectKey, ...] - warnings: tuple[PlanWarning, ...] - - -def plan_scenario_graph( - *, - scenario: CompiledSqlScenario, - project: CompiledProjectView, - sql_analysis_enabled: bool = True, - sql_analysis_dialect: str | None = None, -) -> tuple[ScenarioGraphPlan, tuple[PlanWarning, ...]]: - """Infer scenario target models, build closure, and required fixtures.""" - - model_map: dict[str, CompiledModelView] = {model.name: model for model in project.models} - source_names: frozenset[str] = frozenset(source.name for source in project.sources) - seed_names: frozenset[str] = frozenset(seed.name for seed in project.seeds) - del sql_analysis_enabled, sql_analysis_dialect - assertion_target_names: tuple[str, ...] = scenario.assertion_target_model_names - target_model_names: tuple[str, ...] = scenario.target_model_names - - warnings: list[PlanWarning] = [] - warnings.extend( - _declared_name_warnings( - scenario=scenario, - target_model_names=target_model_names, - assertion_target_names=assertion_target_names, - model_map=model_map, - source_names=source_names, - seed_names=seed_names, - ) - ) - - ref_fixture_names: frozenset[str] = frozenset(scenario.ref_fixture_names) - source_fixture_names: frozenset[str] = frozenset(scenario.source_fixture_names) - seed_fixture_names: frozenset[str] = frozenset(scenario.seed_fixture_names) - dbt_ref_fixture_names: frozenset[str] = frozenset(scenario.dbt_ref_fixture_names) - - requirements: _UpstreamRequirements = _collect_upstream_requirements( - target_model_names=target_model_names, - scenario_name=scenario.name, - model_map=model_map, - ref_fixture_names=ref_fixture_names, - ) - warnings.extend(requirements.warnings) - model_names: frozenset[str] = requirements.model_names - required_ref_fixture_names: frozenset[str] = requirements.required_ref_fixture_names - required_source_names: frozenset[str] = requirements.required_source_names - required_seed_names: frozenset[str] = requirements.required_seed_names - required_dbt_ref_fixture_names: frozenset[str] = requirements.required_dbt_ref_fixture_names - function_deps: tuple[CompiledObjectKey, ...] = requirements.function_deps - - missing_source_names: tuple[str, ...] = tuple( - sorted(required_source_names - source_fixture_names) - ) - source_name: str - for source_name in missing_source_names: - warnings.append( - PlanWarning( - model_name=None, - severity=WarningSeverity.ERROR, - message=( - f"Scenario '{scenario.name}' requires source '{source_name}', " - "but no fixture was provided. Add a CTE named " - f"__source__{_fixture_name_for(source_name)} AS (...)." - ), - ) - ) - - dbt_ref_fixture_name: str - for dbt_ref_fixture_name in sorted(required_dbt_ref_fixture_names - dbt_ref_fixture_names): - warnings.append( - PlanWarning( - model_name=None, - severity=WarningSeverity.ERROR, - message=( - f"Scenario '{scenario.name}' requires dbt ref '{dbt_ref_fixture_name}', " - "but no fixture was provided. Add a CTE named " - f"__dbt_ref__{dbt_ref_fixture_name} AS (...)." - ), - ) - ) - - plan: ScenarioGraphPlan = ScenarioGraphPlan( - key=scenario.key, - name=scenario.name, - target_model_names=tuple(sorted(target_model_names)), - assertion_target_model_names=tuple(sorted(assertion_target_names)), - model_names=tuple(sorted(model_names)), - source_fixture_names=tuple(sorted(required_source_names)), - ref_fixture_names=tuple(sorted(required_ref_fixture_names)), - seed_names=tuple(sorted(required_seed_names)), - seed_fixture_names=tuple(sorted(required_seed_names & seed_fixture_names)), - dbt_ref_fixture_names=tuple(sorted(required_dbt_ref_fixture_names)), - function_deps=tuple(function_deps), - ) - return plan, tuple(warnings) - - -def _declared_name_warnings( - *, - scenario: CompiledSqlScenario, - target_model_names: tuple[str, ...], - assertion_target_names: tuple[str, ...], - model_map: dict[str, CompiledModelView], - source_names: frozenset[str], - seed_names: frozenset[str], -) -> tuple[PlanWarning, ...]: - warnings: list[PlanWarning] = [] - expected_model_name: str - for expected_model_name in scenario.expected_model_names: - if expected_model_name not in model_map: - warnings.append( - _error( - message=( - f"Scenario '{scenario.name}' expects unknown model '{expected_model_name}'." - ) - ) - ) - - assertion_target_name: str - for assertion_target_name in assertion_target_names: - if assertion_target_name not in model_map: - warnings.append( - _error( - message=f"Scenario '{scenario.name}' assertion references unknown model " - f"'{assertion_target_name}'." - ) - ) - - if not target_model_names: - warnings.append( - _error( - message=f"Scenario '{scenario.name}' has no target models. Add __expected__ " - "or reference a model with __ref(...) inside an __assert__ CTE." - ) - ) - - ref_fixture_name: str - for ref_fixture_name in scenario.ref_fixture_names: - if ref_fixture_name not in model_map: - warnings.append( - _error( - message=f"Scenario '{scenario.name}' provides fixture for unknown model " - f"'{ref_fixture_name}'." - ) - ) - - source_fixture_name: str - for source_fixture_name in scenario.source_fixture_names: - if source_fixture_name not in source_names: - warnings.append( - _error( - message=f"Scenario '{scenario.name}' provides fixture for unknown source " - f"'{source_fixture_name}'." - ) - ) - - seed_fixture_name: str - for seed_fixture_name in scenario.seed_fixture_names: - if seed_fixture_name not in seed_names: - warnings.append( - _error( - message=f"Scenario '{scenario.name}' provides fixture for unknown seed " - f"'{seed_fixture_name}'." - ) - ) - return tuple(warnings) - - -def _collect_upstream_requirements( - *, - target_model_names: tuple[str, ...], - scenario_name: str, - model_map: dict[str, CompiledModelView], - ref_fixture_names: frozenset[str], -) -> _UpstreamRequirements: - """Walk target model upstreams and collect required fixtures and functions.""" - - def _walk( - *, - model_name: str, - requirements: _UpstreamRequirements, - ) -> _UpstreamRequirements: - if model_name in ref_fixture_names: - return _UpstreamRequirements( - model_names=requirements.model_names, - required_ref_fixture_names=requirements.required_ref_fixture_names | {model_name}, - required_source_names=requirements.required_source_names, - required_seed_names=requirements.required_seed_names, - required_dbt_ref_fixture_names=requirements.required_dbt_ref_fixture_names, - function_deps=requirements.function_deps, - warnings=requirements.warnings, - ) - if model_name in requirements.model_names: - return requirements - - model: CompiledModelView | None = model_map.get(model_name) - if model is None: - return requirements - requirements = _UpstreamRequirements( - model_names=requirements.model_names | {model_name}, - required_ref_fixture_names=requirements.required_ref_fixture_names, - required_source_names=requirements.required_source_names, - required_seed_names=requirements.required_seed_names, - required_dbt_ref_fixture_names=requirements.required_dbt_ref_fixture_names, - function_deps=requirements.function_deps, - warnings=requirements.warnings, - ) - - dep_key: CompiledObjectKey - for dep_key in model.deps: - if dep_key.resource_type == CompiledResourceType.MODEL: - if dep_key.name in ref_fixture_names: - requirements = _UpstreamRequirements( - model_names=requirements.model_names, - required_ref_fixture_names=requirements.required_ref_fixture_names - | {dep_key.name}, - required_source_names=requirements.required_source_names, - required_seed_names=requirements.required_seed_names, - required_dbt_ref_fixture_names=requirements.required_dbt_ref_fixture_names, - function_deps=requirements.function_deps, - warnings=requirements.warnings, - ) - continue - if dep_key.name not in model_map: - requirements = _UpstreamRequirements( - model_names=requirements.model_names, - required_ref_fixture_names=requirements.required_ref_fixture_names, - required_source_names=requirements.required_source_names, - required_seed_names=requirements.required_seed_names, - required_dbt_ref_fixture_names=requirements.required_dbt_ref_fixture_names, - function_deps=requirements.function_deps, - warnings=( - *requirements.warnings, - _error( - message=( - f"Scenario '{scenario_name}' requires model " - f"'{dep_key.name}', " - "but it does not exist and no __ref__ fixture was provided." - ) - ), - ), - ) - continue - requirements = _walk(model_name=dep_key.name, requirements=requirements) - continue - if dep_key.resource_type == CompiledResourceType.SOURCE: - requirements = _UpstreamRequirements( - model_names=requirements.model_names, - required_ref_fixture_names=requirements.required_ref_fixture_names, - required_source_names=requirements.required_source_names | {dep_key.name}, - required_seed_names=requirements.required_seed_names, - required_dbt_ref_fixture_names=requirements.required_dbt_ref_fixture_names, - function_deps=requirements.function_deps, - warnings=requirements.warnings, - ) - continue - if dep_key.resource_type == CompiledResourceType.SEED: - requirements = _UpstreamRequirements( - model_names=requirements.model_names, - required_ref_fixture_names=requirements.required_ref_fixture_names, - required_source_names=requirements.required_source_names, - required_seed_names=requirements.required_seed_names | {dep_key.name}, - required_dbt_ref_fixture_names=requirements.required_dbt_ref_fixture_names, - function_deps=requirements.function_deps, - warnings=requirements.warnings, - ) - continue - if dep_key.resource_type == CompiledResourceType.DBT_REF: - requirements = _UpstreamRequirements( - model_names=requirements.model_names, - required_ref_fixture_names=requirements.required_ref_fixture_names, - required_source_names=requirements.required_source_names, - required_seed_names=requirements.required_seed_names, - required_dbt_ref_fixture_names=requirements.required_dbt_ref_fixture_names - | {dep_key.name.replace(".", "__")}, - function_deps=requirements.function_deps, - warnings=requirements.warnings, - ) - continue - if dep_key.resource_type in {CompiledResourceType.UDF, CompiledResourceType.TABLE_FN}: - if dep_key in requirements.function_deps: - continue - requirements = _UpstreamRequirements( - model_names=requirements.model_names, - required_ref_fixture_names=requirements.required_ref_fixture_names, - required_source_names=requirements.required_source_names, - required_seed_names=requirements.required_seed_names, - required_dbt_ref_fixture_names=requirements.required_dbt_ref_fixture_names, - function_deps=(*requirements.function_deps, dep_key), - warnings=requirements.warnings, - ) - return requirements - - requirements: _UpstreamRequirements = _UpstreamRequirements( - model_names=frozenset(), - required_ref_fixture_names=frozenset(), - required_source_names=frozenset(), - required_seed_names=frozenset(), - required_dbt_ref_fixture_names=frozenset(), - function_deps=(), - warnings=(), - ) - target_model_name: str - for target_model_name in target_model_names: - requirements = _walk(model_name=target_model_name, requirements=requirements) - - return requirements - - -def _fixture_name_for(name: str) -> str: - return name.replace(".", "__") - - -def _error(*, message: str, code: str = SCENARIO_PLAN_GRAPH_VALIDATION) -> PlanWarning: - return PlanWarning( - model_name=None, - severity=WarningSeverity.ERROR, - message=message, - code=code, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py index 9b7e1d46ad..da8e69b766 100644 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py +++ b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py @@ -45,7 +45,6 @@ ) from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_function_sql from sqlbuild.compiler.planner.constants import ( - SCENARIO_PLAN_INTERNAL, SCENARIO_PLAN_INVALID_FIXTURE, SCENARIO_PLAN_MISSING_FIXTURE_SQL, SCENARIO_PLAN_MISSING_RELATION_TARGET, @@ -60,13 +59,11 @@ ModelPlanEntry, PlanWarning, RelationFixtureCompletion, - ScenarioArtifactIdentity, ScenarioAssertionExpectationPlan, ScenarioExecutionPlan, ScenarioExpectedExpectationPlan, ScenarioFixturePlan, ScenarioGraphPlan, - ScenarioRelationMap, ScenarioRelationPlan, SeedPlanEntry, WarehouseSnapshot, @@ -93,121 +90,6 @@ ) -def build_scenario_relation_plan( - *, - project: CompiledProjectView, - graph_plan: ScenarioGraphPlan, - relation_map: ScenarioRelationMap, - render_qualified_name: Callable[..., str | None], - database: str | None = None, - schema: str | None = None, -) -> ScenarioRelationPlan: - """Build scenario-scoped relation locations and source entries.""" - - artifacts: dict[ScenarioArtifactIdentity, str] = { - artifact.identity: artifact.physical_name for artifact in relation_map.artifacts - } - source_entries: dict[str, SourceEntry] = { - source.name: source.source_entry for source in project.sources - } - - model_locations: dict[str, CompiledRelationLocation] = {} - source_fixture_locations: dict[str, CompiledRelationLocation] = {} - ref_fixture_locations: dict[str, CompiledRelationLocation] = {} - dbt_ref_fixture_locations: dict[str, CompiledRelationLocation] = {} - seed_fixture_locations: dict[str, CompiledRelationLocation] = {} - seed_locations: dict[str, CompiledRelationLocation] = {} - source_map: dict[str, SourceEntry] = {} - - model_name: str - for model_name in graph_plan.model_names: - model_locations[model_name] = _target_for_artifact( - artifacts=artifacts, - kind=ScenarioArtifactKind.MODEL, - logical_name=model_name, - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - ) - - ref_name: str - for ref_name in graph_plan.ref_fixture_names: - target: CompiledRelationLocation = _target_for_artifact( - artifacts=artifacts, - kind=ScenarioArtifactKind.REF, - logical_name=ref_name, - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - ) - ref_fixture_locations[ref_name] = target - model_locations[ref_name] = target - - source_name: str - for source_name in graph_plan.source_fixture_names: - target = _target_for_artifact( - artifacts=artifacts, - kind=ScenarioArtifactKind.SOURCE, - logical_name=source_name, - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - ) - source_fixture_locations[source_name] = target - source_entry: SourceEntry = source_entries[source_name] - source_map[source_name] = replace( - source_entry, - database=target.database, - schema=target.schema, - table=target.name, - expression=None, - type_enforcement=False, - ) - - dbt_ref_name: str - for dbt_ref_name in graph_plan.dbt_ref_fixture_names: - dbt_ref_fixture_locations[dbt_ref_name] = _target_for_artifact( - artifacts=artifacts, - kind=ScenarioArtifactKind.DBT_REF, - logical_name=dbt_ref_name, - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - ) - - seed_name: str - for seed_name in graph_plan.seed_names: - target = _target_for_artifact( - artifacts=artifacts, - kind=ScenarioArtifactKind.SEED, - logical_name=seed_name, - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - ) - seed_locations[seed_name] = target - - for seed_name in graph_plan.seed_fixture_names: - seed_fixture_locations[seed_name] = _required_target( - targets=seed_locations, - name=seed_name, - kind=ScenarioArtifactKind.SEED, - ) - - return ScenarioRelationPlan( - scenario_name=graph_plan.name, - relation_map=relation_map, - model_locations=model_locations, - seed_locations=seed_locations, - project_source_map=source_entries, - source_map=source_map, - source_fixture_locations=source_fixture_locations, - ref_fixture_locations=ref_fixture_locations, - dbt_ref_fixture_locations=dbt_ref_fixture_locations, - seed_fixture_locations=seed_fixture_locations, - ) - - def resolve_scenario_check_sql( *, sql: str, @@ -860,36 +742,3 @@ def _dbt_ref_fixture_name(match: re.Match[str]) -> str: if second is None: return first return f"{first}__{second}" - - -def _target_for_artifact( - *, - artifacts: dict[ScenarioArtifactIdentity, str], - kind: ScenarioArtifactKind, - logical_name: str, - database: str | None, - schema: str | None, - render_qualified_name: Callable[..., str | None], -) -> CompiledRelationLocation: - identity: ScenarioArtifactIdentity = ScenarioArtifactIdentity( - kind=kind, - logical_name=logical_name, - ) - physical_name: str | None = artifacts.get(identity) - if physical_name is None: - raise PlannerInputError( - f"Scenario relation map is missing {kind.value} artifact '{logical_name}'", - code=SCENARIO_PLAN_INTERNAL, - help="This is likely a SQLBuild bug. Please file an issue with the scenario name.", - ) - qualified_name: str | None = render_qualified_name( - database=database, - schema=schema, - name=physical_name, - ) - return CompiledRelationLocation( - database=database, - schema=schema, - name=physical_name, - qualified_name=qualified_name if qualified_name is not None else physical_name, - ) diff --git a/src/sqlbuild/compiler/planner/main/scenarios/scenario.py b/src/sqlbuild/compiler/planner/main/scenarios/scenario.py index c56dc7f37d..b37b5a2a73 100644 --- a/src/sqlbuild/compiler/planner/main/scenarios/scenario.py +++ b/src/sqlbuild/compiler/planner/main/scenarios/scenario.py @@ -2,10 +2,10 @@ from __future__ import annotations +from sqlbuild._native import build_cli_scenario_plan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner._helpers.scenario.cli import build_cli_scenario_plan from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py index 27fe0d89d3..94e928157e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py @@ -5,9 +5,9 @@ import pytest +from sqlbuild._native import build_cli_scenario_plan from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner._helpers.scenario.cli import build_cli_scenario_plan from sqlbuild.compiler.planner.models import ScenarioArtifactName, ScenarioExecutionPlan from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py index b66b33bf92..0491153fb4 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py @@ -4,12 +4,12 @@ import pytest +from sqlbuild._native import plan_scenario_graph from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledSqlScenario, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner._helpers.scenario.graph import plan_scenario_graph from sqlbuild.compiler.planner.models import ScenarioGraphPlan from sqlbuild.compiler.planner.types import WarningSeverity from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py index 587ed8102f..e307c9bdea 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ( + build_scenario_relation_plan, project_with, ) from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -24,7 +25,6 @@ from sqlbuild.compiler.planner._helpers.scenario.relations import ( build_scenario_execution_plan, build_scenario_fixture_plans, - build_scenario_relation_plan, resolve_scenario_check_sql, ) from sqlbuild.compiler.planner.exceptions import PlannerInputError From 331948d80c2bce6d7a27859cc4c86d767ae809f0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:29:07 +0100 Subject: [PATCH 24/29] chore: update Python boundary after scenario planning port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 98a81181ea..c108635c75 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 34833 +remaining_lines = 34195 [[stays]] path = "executor" From 490654ae5028e7ea4d264721847a960c4c53c95e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:32:41 +0100 Subject: [PATCH 25/29] refactor: host migration planning boundaries natively --- .../planner_migrations/column_overlay.rs | 7 +- .../planner_migrations/column_planning.rs | 5 +- .../_helpers/planner_migrations/constants.rs | 1 - .../_helpers/planner_migrations/mod.rs | 3 + .../planner_migrations/model_entries.rs | 4 +- .../planner_migrations/model_overlay.rs | 7 +- .../planner_migrations/model_planning.rs | 14 +- .../planner_migrations/planning_boundary.rs | 114 +++++++++ .../planner_migrations/planning_execution.rs | 95 ++++++++ .../planner_migrations/planning_origin.rs | 206 ++++++++++++++++ .../planner_migrations/registration.rs | 3 + src/sqlbuild/_native.pyi | 33 ++- .../planner/_helpers/migrations/old_names.py | 10 +- .../planner/_helpers/migrations/planning.py | 229 ------------------ .../_helpers/planning/warehouse_state.py | 3 +- 15 files changed, 481 insertions(+), 253 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/migrations/planning.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs index 9ba85bcbb2..62862ac921 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs @@ -6,7 +6,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::planner_migrations::column_objects::{ config, definition_hash, lower, model, native, replace, }; -use crate::bindings::_helpers::planner_migrations::constants::PLANNING; use crate::bindings::_helpers::planner_migrations::models::ColumnCandidate; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -161,7 +160,11 @@ pub(super) fn cursors<'py>( targets.set_item(name, target)?; } let redirected = call_with( - &imported(py, PLANNING, "redirected_cursor_snapshots")?, + &imported( + py, + "sqlbuild._native", + "migration_redirected_cursor_snapshots", + )?, &[ ("runtime", args.value("runtime")?), ("scope", scope), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index 9fcb001fd2..6753114e68 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -4,7 +4,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::planner_migrations::column_objects::{model, replace}; -use crate::bindings::_helpers::planner_migrations::constants::PLANNING; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PySet, PySetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -45,7 +44,7 @@ fn plan_column_migrations<'py>( return model(py, "ColumnMigrationPlanning", &[("snapshot", snapshot)]); } let database = call_with( - &imported(py, PLANNING, "planning_database")?, + &imported(py, "sqlbuild._native", "migration_planning_database")?, &[("runtime", runtime.clone())], )?; let state = call_with( @@ -103,7 +102,7 @@ fn plan_column_migrations<'py>( warnings.extend(checked.warnings); } let policy = call_with( - &imported(py, PLANNING, "missing_origin_policy")?, + &imported(py, "sqlbuild._native", "migration_missing_origin_policy")?, &[("runtime", runtime)], )?; for entry in &mut entries { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs index e7ce1e1bdd..1715fd4546 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs @@ -1,5 +1,4 @@ pub(super) const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; pub(super) const MIGRATION_TYPES: &str = "sqlbuild.compiler.migrations.types"; -pub(super) const PLANNING: &str = "sqlbuild.compiler.planner._helpers.migrations.planning"; pub(super) const AUTOMATIC: &str = "automatic"; pub(super) const DONE: &str = "done"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index 626c5b1a8a..96a4f5cffc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -17,5 +17,8 @@ mod model_planning; mod model_warnings; mod models; mod old_names; +mod planning_boundary; +mod planning_execution; +mod planning_origin; mod projections; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs index 81178554ef..d426b7231b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs @@ -6,7 +6,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::planner_migrations::column_objects::{ lower, model, native, replace, text, }; -use crate::bindings::_helpers::planner_migrations::constants::{MIGRATION_TYPES, PLANNING}; +use crate::bindings::_helpers::planner_migrations::constants::MIGRATION_TYPES; use crate::bindings::_helpers::planner_migrations::model_handover::{ applies, empty_version, handover, }; @@ -284,7 +284,7 @@ pub(super) fn decide<'py>(args: &KeywordArguments<'py>) -> PyResult( .cast_into::()? .copy()?; let gathered = call_with( - &imported(py, PLANNING, "redirected_cursor_snapshots")?, + &imported( + py, + "sqlbuild._native", + "migration_redirected_cursor_snapshots", + )?, &[ ("runtime", runtime), ("scope", args.value("scope")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index 25253f3b49..96bc3ee46e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -6,7 +6,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::planner_migrations::column_objects::{ config, model, native, replace, }; -use crate::bindings::_helpers::planner_migrations::constants::{MIGRATION_TYPES, PLANNING}; +use crate::bindings::_helpers::planner_migrations::constants::MIGRATION_TYPES; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{ PyBool, PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyString, PyTuple, @@ -128,7 +128,7 @@ fn plan_model_migrations<'py>( let runtime = args.value("runtime")?; let snapshot = args.value("snapshot")?; let schemas = call_with( - &imported(py, PLANNING, "project_schemas")?, + &imported(py, "sqlbuild._native", "migration_project_schemas")?, &[("runtime", runtime.clone())], )?; let state = call_with( @@ -143,7 +143,7 @@ fn plan_model_migrations<'py>( ( "database", call_with( - &imported(py, PLANNING, "planning_database")?, + &imported(py, "sqlbuild._native", "migration_planning_database")?, &[("runtime", runtime.clone())], )?, ), @@ -178,7 +178,7 @@ fn discover<'py>( let mut declarations: Vec> = Vec::new(); for request in declared_requests { let resolved = call_with( - &imported(py, PLANNING, "_resolve_origin")?, + &imported(py, "sqlbuild._native", "migration_resolve_origin")?, &[ ("request", request.clone()), ("runtime", runtime.clone()), @@ -228,7 +228,7 @@ fn finish<'py>( for request in discovery.getattr("requests")?.try_iter()? { let request = request?; let origin = call_with( - &imported(py, PLANNING, "_resolve_origin")?, + &imported(py, "sqlbuild._native", "migration_resolve_origin")?, &[ ("request", request.clone()), ("runtime", runtime.clone()), @@ -244,7 +244,7 @@ fn finish<'py>( let mut origins: Vec> = Vec::new(); for (_, origin, _) in &resolved { call_with( - &imported(py, PLANNING, "_require_same_database")?, + &imported(py, "sqlbuild._native", "migration_require_same_database")?, &[("state", state.clone()), ("location", origin.clone())], )?; let schema = origin.getattr("schema")?; @@ -268,7 +268,7 @@ fn finish<'py>( project_config.getattr("snapshots")? }; let policy = call_with( - &imported(py, PLANNING, "missing_origin_policy")?, + &imported(py, "sqlbuild._native", "migration_missing_origin_policy")?, &[("runtime", runtime.clone())], )?; for (request, origin, origin_model) in resolved { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs new file mode 100644 index 0000000000..b97b34d114 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs @@ -0,0 +1,114 @@ +//! Assemble migration policy and redirected live cursor inspection requests. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_migrations::column_objects::model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; + +#[pyfunction] +#[pyo3(signature = (*, runtime))] +fn migration_missing_origin_policy<'py>( + runtime: &Bound<'py, PyAny>, +) -> PyResult> { + call_with( + &imported( + runtime.py(), + "sqlbuild.compiler.planner._helpers.planning.retention", + "effective_target_config", + )?, + &[("runtime", runtime.clone())], + )? + .getattr("missing_migration_origin") +} + +#[pyfunction] +#[pyo3(signature = (**options))] +fn migration_redirected_cursor_snapshots<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "migration_redirected_cursor_snapshots", + options, + ( + &[ + "runtime", + "scope", + "existing_relations", + "target_relations", + "overrides", + "deferral", + ], + &["origin_cursor_columns"], + ), + )?; + let runtime = args.value("runtime")?; + let scope = args.value("scope")?; + let overrides = args.value("overrides")?; + let project = runtime.getattr("project")?; + let config = runtime.getattr("project_config")?; + let start = if config.is_none() { + py.None().into_bound(py) + } else { + config.getattr("cursors")?.getattr("start")? + }; + let cursor_scope = model( + py, + "CursorSnapshotScope", + &[ + ("model_keys", scope.getattr("selected_keys")?), + ("runtime_producer_keys", scope.getattr("selected_keys")?), + ("invocation_time", runtime.getattr("invocation_time")?), + ("start_cursor_config", start), + ("cursor_overrides", overrides.getattr("cursor_overrides")?), + ], + )?; + let full_refresh = call_with( + &imported( + py, + "sqlbuild.compiler.planner._helpers.planning.full_refresh", + "effectively_full_refreshed_model_names", + )?, + &[ + ("project", project.clone()), + ("cli_full_refresh", overrides.getattr("full_refresh")?), + ], + )?; + call_with( + &imported( + py, + "sqlbuild.compiler.planner._helpers.warehouse.snapshot", + "gather_redirected_cursor_snapshots", + )?, + &[ + ("project", project), + ("adapter", runtime.getattr("adapter")?), + ("connection", runtime.getattr("connection")?), + ("existing_relations", args.value("existing_relations")?), + ("target_relations", args.value("target_relations")?), + ("cursor_scope", cursor_scope), + ("full_refresh_model_names", full_refresh), + ( + "deferred_locations", + args.value("deferral")?.getattr("deferred_locations")?, + ), + ("on_progress", runtime.getattr("on_progress")?), + ( + "origin_cursor_columns", + args.value("origin_cursor_columns")?, + ), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(migration_missing_origin_policy, module)?)?; + module.add_function(wrap_pyfunction!( + migration_redirected_cursor_snapshots, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs new file mode 100644 index 0000000000..f0c0e34069 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs @@ -0,0 +1,95 @@ +//! Preserve adapter overrides when describing migration execution. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_migrations::{ + column_objects::replace, constants::MIGRATION_TYPES, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::PyBool; +use pyo3::{pyfunction, wrap_pyfunction}; + +fn label<'py>(location: &Bound<'py, PyAny>) -> PyResult> { + let qualified = location.getattr("qualified_name")?; + if qualified.is_truthy()? { + Ok(qualified) + } else { + location.getattr("name") + } +} + +#[pyfunction] +#[pyo3(signature = (*, runtime, decision))] +fn migration_promotion<'py>( + runtime: &Bound<'py, PyAny>, + decision: &Bound<'py, PyAny>, +) -> PyResult> { + let adapter = runtime.getattr("adapter")?; + let promotion = imported(runtime.py(), MIGRATION_TYPES, "MigrationPromotion")?; + if adapter + .call_method0("supports_transactional_ddl")? + .is_truthy()? + { + return promotion.getattr("TRANSACTIONAL_RENAME"); + } + if !decision.eq("migrate")? && adapter.getattr("adapter_name")?.eq("snowflake")? { + return promotion.getattr("SWAP"); + } + promotion.getattr("RENAME") +} + +#[pyfunction] +#[pyo3(signature = (*, runtime, model, entry))] +fn migration_with_execution<'py>( + runtime: &Bound<'py, PyAny>, + model: &Bound<'py, PyAny>, + entry: &Bound<'py, PyAny>, +) -> PyResult> { + let py = runtime.py(); + let adapter = runtime.getattr("adapter")?; + let transient = if adapter.getattr("adapter_name")?.eq("snowflake")? { + PyBool::new( + py, + model + .getattr("config")? + .getattr("table_type")? + .getattr("value")? + .eq("transient")?, + ) + .to_owned() + .into_any() + } else { + py.None().into_bound(py) + }; + let stage = call_with( + &adapter.getattr("render_migration_stage")?, + &[ + ("origin", label(&entry.getattr("origin")?)?), + ("stage", label(&entry.getattr("destination")?)?), + ("origin_is_transient", entry.getattr("origin_is_transient")?), + ("stage_is_transient", transient.clone()), + ], + )?; + let transfer = stage.getattr("transfer")?; + let fallback = if stage.getattr("fallback_statements")?.is_truthy()? { + imported(py, "sqlbuild.adapter.contract.types", "MigrationTransfer")?.getattr("COPY")? + } else { + py.None().into_bound(py) + }; + replace( + entry, + &[ + ("stage_is_transient", transient), + ("transfer", transfer), + ("transfer_fallback", fallback), + ( + "promotion", + migration_promotion(runtime, &entry.getattr("decision")?)?, + ), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(migration_promotion, module)?)?; + module.add_function(wrap_pyfunction!(migration_with_execution, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs new file mode 100644 index 0000000000..2b66c81b70 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs @@ -0,0 +1,206 @@ +//! Resolve authored migration origins at the adapter and inspection boundaries. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PySet, PySetMethods, PyTuple}; +use pyo3::{PyErr, pyfunction, wrap_pyfunction}; + +fn location<'py>( + runtime: &Bound<'py, PyAny>, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let qualified = call_with( + &runtime + .getattr("adapter")? + .getattr("render_qualified_name")?, + fields, + )?; + let mut fields = fields.to_vec(); + fields.push(("qualified_name", qualified)); + call_with( + &imported( + runtime.py(), + "sqlbuild.compiler.compile.models", + "CompiledRelationLocation", + )?, + &fields, + ) +} + +fn named<'py>( + name: &Bound<'py, PyAny>, + destination: &Bound<'py, PyAny>, + runtime: &Bound<'py, PyAny>, + state: &Bound<'py, PyAny>, +) -> PyResult> { + for item in runtime.getattr("project")?.getattr("models")?.try_iter()? { + let item = item?; + if item.getattr("name")?.eq(name)? { + return Ok( + PyTuple::new(runtime.py(), [item.getattr("destination")?, name.clone()])? + .into_any(), + ); + } + } + let fingerprint = call_with( + &state.getattr("named_fingerprint")?, + &[("model_name", name.clone())], + )?; + let fields = if !fingerprint.is_none() && !fingerprint.getattr("target_name")?.is_none() { + [ + ("database", fingerprint.getattr("target_database")?), + ("schema", fingerprint.getattr("target_schema")?), + ("name", fingerprint.getattr("target_name")?), + ] + } else { + [ + ("database", destination.getattr("database")?), + ("schema", destination.getattr("schema")?), + ("name", name.clone()), + ] + }; + Ok(PyTuple::new(runtime.py(), [location(runtime, &fields)?, name.clone()])?.into_any()) +} + +fn input_error(py: Python<'_>, message: String) -> PyResult { + Ok(PyErr::from_value( + imported( + py, + "sqlbuild.compiler.planner.exceptions", + "PlannerInputError", + )? + .call1((message,))?, + )) +} + +#[pyfunction] +#[pyo3(signature = (*, runtime))] +fn migration_planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { + let project = runtime.getattr("project")?; + let fallback = project.getattr("effective_target_database")?; + for item in project.getattr("models")?.try_iter()? { + let database = item?.getattr("destination")?.getattr("database")?; + if !database.is_none() { + return Ok(database); + } + } + Ok(fallback) +} + +#[pyfunction] +#[pyo3(signature = (*, runtime))] +fn migration_project_schemas<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { + let schemas = PySet::empty(runtime.py())?; + for item in runtime.getattr("project")?.getattr("models")?.try_iter()? { + let schema = item?.getattr("destination")?.getattr("schema")?; + if !schema.is_none() { + schemas.add(schema)?; + } + } + Ok(schemas.into_any()) +} + +#[pyfunction] +#[pyo3(signature = (*, state, location))] +fn migration_require_same_database( + state: &Bound<'_, PyAny>, + location: &Bound<'_, PyAny>, +) -> PyResult<()> { + let database = location.getattr("database")?; + if database.is_none() { + return Ok(()); + } + let inspected = state.getattr("database")?; + if inspected.is_none() + || database + .call_method0("lower")? + .eq(inspected.call_method0("lower")?)? + { + return Ok(()); + } + let qualified = location.getattr("qualified_name")?; + let label = if qualified.is_truthy()? { + qualified + } else { + location.getattr("name")? + }; + Err(input_error( + state.py(), + format!( + "migration origin {label} is in database '{database}', but direct planning inspects '{inspected}'; cross-database migrations are not supported" + ), + )?) +} + +#[pyfunction] +#[pyo3(signature = (*, request, runtime, state))] +fn migration_resolve_origin<'py>( + request: &Bound<'py, PyAny>, + runtime: &Bound<'py, PyAny>, + state: &Bound<'py, PyAny>, +) -> PyResult> { + let py = runtime.py(); + let explicit = request.getattr("origin_location")?; + if !explicit.is_none() { + return Ok(PyTuple::new(py, [explicit, request.getattr("origin_model")?])?.into_any()); + } + let raw = request.getattr("raw_origin")?; + let raw = if raw.is_truthy()? { + raw + } else { + super::column_objects::text(py, "") + }; + let item = request.getattr("model")?; + let destination = item.getattr("destination")?; + let mut parts: Vec> = Vec::new(); + for part in raw.call_method1("split", (".",))?.try_iter()? { + parts.push( + part? + .call_method0("strip")? + .call_method1("strip", ("\"`",))?, + ); + } + let maximum: usize = imported( + py, + "sqlbuild.compiler.planner.constants", + "QUALIFIED_RELATION_MAX_PARTS", + )? + .extract()?; + let mut invalid = parts.len() > maximum; + for part in &parts { + invalid |= !part.is_truthy()?; + } + if invalid { + return Err(input_error( + py, + format!( + "model '{}': migrate_from '{raw}' is not a model name or a qualified relation", + item.getattr("name")? + ), + )?); + } + if parts.len() == 1 { + return named(&parts[0], &destination, runtime, state); + } + let database = if parts.len() == maximum { + parts[0].clone() + } else { + destination.getattr("database")? + }; + let origin = location( + runtime, + &[ + ("database", database), + ("schema", parts[parts.len() - 2].clone()), + ("name", parts[parts.len() - 1].clone()), + ], + )?; + Ok(PyTuple::new(py, [origin, py.None().into_bound(py)])?.into_any()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(migration_planning_database, module)?)?; + module.add_function(wrap_pyfunction!(migration_project_schemas, module)?)?; + module.add_function(wrap_pyfunction!(migration_require_same_database, module)?)?; + module.add_function(wrap_pyfunction!(migration_resolve_origin, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs index 5c1b369b29..7119c0a7bc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs @@ -9,6 +9,9 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::columns::register(module)?; super::column_planning::register(module)?; super::model_planning::register(module)?; + super::planning_boundary::register(module)?; + super::planning_origin::register(module)?; + super::planning_execution::register(module)?; super::projections::register(module)?; super::fingerprint::register(module) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index aed229a053..73caf02168 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -183,7 +183,11 @@ from sqlbuild.compiler.lineage.models import ( ) from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from sqlbuild.compiler.migrations.models import MigrationRelation -from sqlbuild.compiler.migrations.types import OldNameViewEventType +from sqlbuild.compiler.migrations.types import ( + MigrationDecision, + MigrationPromotion, + OldNameViewEventType, +) from sqlbuild.compiler.pipeline.models import ( ClonePipelineConnection, ClonePipelineOptions, @@ -432,6 +436,7 @@ from sqlbuild.spec.contracts.models import ( ) from sqlbuild.spec.contracts.types import ( MicrobatchLimitAction, + MissingMigrationOriginPolicy, SourceFreshnessStrategy, SourceFreshnessValueKind, WarehouseGroup, @@ -442,6 +447,32 @@ from sqlbuild.sql_values.types import CollectionRendering, StateSqlValueType BUILD_IDENTITY: str STAGE_CAPTURE_OMITTED_ATTRIBUTES: dict[str, frozenset[str]] +def migration_planning_database(*, runtime: PlannerRuntime) -> str | None: ... +def migration_project_schemas(*, runtime: PlannerRuntime) -> set[str]: ... +def migration_missing_origin_policy(*, runtime: PlannerRuntime) -> MissingMigrationOriginPolicy: ... +def migration_require_same_database( + *, state: MigrationStateInspection, location: CompiledRelationLocation +) -> None: ... +def migration_resolve_origin( + *, request: ModelMigrationRequest, runtime: PlannerRuntime, state: MigrationStateInspection +) -> tuple[CompiledRelationLocation, str | None]: ... +def migration_promotion( + *, runtime: PlannerRuntime, decision: MigrationDecision +) -> MigrationPromotion: ... +def migration_with_execution( + *, runtime: PlannerRuntime, model: CompiledModelView, entry: ModelMigrationPlanEntry +) -> ModelMigrationPlanEntry: ... +def migration_redirected_cursor_snapshots( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + existing_relations: dict[str, RelationInfo], + target_relations: dict[str, str], + overrides: PlannerOverrides, + deferral: DeferralInputs, + origin_cursor_columns: dict[str, str] | None = None, +) -> dict[str, ModelCursorSnapshot]: ... + class CompileInputReads: """Environment, settings, and run-identity reads observed during one compile.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py b/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py index 2823838847..81fda8457b 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py @@ -15,6 +15,12 @@ old_name_schemas_to_read, old_name_view_skip_reason, ) +from sqlbuild._native import ( + migration_planning_database as planning_database, +) +from sqlbuild._native import ( + migration_project_schemas as project_schemas, +) from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import RelationType from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type @@ -31,10 +37,6 @@ OldNameViewAction, OldNameViewStatus, ) -from sqlbuild.compiler.planner._helpers.migrations.planning import ( - planning_database, - project_schemas, -) from sqlbuild.compiler.planner.classes.migration_state_inspection import ( MigrationStateInspection, ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py deleted file mode 100644 index caed78e3b2..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ /dev/null @@ -1,229 +0,0 @@ -"""Warehouse inspection and adapter boundaries for native model migration planning.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild._native import ( - manual_migration_requests as manual_migration_requests, -) -from sqlbuild._native import ( - plan_model_migrations as plan_model_migrations, -) -from sqlbuild.adapter.contract.models import MigrationStagePlan, RelationInfo -from sqlbuild.adapter.contract.types import BuiltinAdapter, MigrationTransfer -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.migrations.types import ( - MigrationDecision, - MigrationPromotion, -) -from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( - effectively_full_refreshed_model_names, -) -from sqlbuild.compiler.planner._helpers.planning.retention import effective_target_config -from sqlbuild.compiler.planner._helpers.warehouse.snapshot import ( - gather_redirected_cursor_snapshots, -) -from sqlbuild.compiler.planner.classes.migration_state_inspection import MigrationStateInspection -from sqlbuild.compiler.planner.constants import QUALIFIED_RELATION_MAX_PARTS -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - CursorSnapshotScope, - DeferralInputs, - ModelCursorSnapshot, - ModelMigrationPlanEntry, - ModelMigrationRequest, - PlannerOverrides, - PlannerRuntime, - PlannerScope, -) -from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy, TableType - - -def planning_database(*, runtime: PlannerRuntime) -> str | None: - """Return the single database that direct planning inspects for migration state.""" - - return next( - ( - model.destination.database - for model in runtime.project.models - if model.destination.database is not None - ), - runtime.project.effective_target_database, - ) - - -def missing_origin_policy(*, runtime: PlannerRuntime) -> MissingMigrationOriginPolicy: - """Return how the effective target treats a declared migration origin it cannot find.""" - - return effective_target_config(runtime=runtime).missing_migration_origin - - -def project_schemas(*, runtime: PlannerRuntime) -> set[str]: - """Return every destination schema owned by project models.""" - - return { - model.destination.schema - for model in runtime.project.models - if model.destination.schema is not None - } - - -def _require_same_database( - *, state: MigrationStateInspection, location: CompiledRelationLocation -) -> None: - if ( - location.database is None - or state.database is None - or location.database.lower() == state.database.lower() - ): - return - raise PlannerInputError( - f"migration origin {location.qualified_name or location.name} is in database " - f"'{location.database}', but direct planning inspects '{state.database}'; " - "cross-database migrations are not supported" - ) - - -def _resolve_origin( - *, request: ModelMigrationRequest, runtime: PlannerRuntime, state: MigrationStateInspection -) -> tuple[CompiledRelationLocation, str | None]: - if request.origin_location is not None: - return request.origin_location, request.origin_model - raw: str = request.raw_origin or "" - destination: CompiledRelationLocation = request.model.destination - parts: list[str] = [part.strip().strip('"`') for part in raw.split(".")] - if any(not part for part in parts) or len(parts) > QUALIFIED_RELATION_MAX_PARTS: - raise PlannerInputError( - f"model '{request.model.name}': migrate_from '{raw}' is not a model name or a " - "qualified relation" - ) - if len(parts) == 1: - return _resolve_named_origin( - name=parts[0], destination=destination, runtime=runtime, state=state - ) - database: str | None = ( - parts[0] if len(parts) == QUALIFIED_RELATION_MAX_PARTS else destination.database - ) - schema: str = parts[-2] - return ( - _location(runtime=runtime, database=database, schema=schema, name=parts[-1]), - None, - ) - - -def _resolve_named_origin( - *, - name: str, - destination: CompiledRelationLocation, - runtime: PlannerRuntime, - state: MigrationStateInspection, -) -> tuple[CompiledRelationLocation, str | None]: - project_model: CompiledModelView | None = next( - (model for model in runtime.project.models if model.name == name), None - ) - if project_model is not None: - return project_model.destination, name - fingerprint: Fingerprint | None = state.named_fingerprint(model_name=name) - if fingerprint is not None and fingerprint.target_name is not None: - return ( - _location( - runtime=runtime, - database=fingerprint.target_database, - schema=fingerprint.target_schema, - name=fingerprint.target_name, - ), - name, - ) - return ( - _location( - runtime=runtime, database=destination.database, schema=destination.schema, name=name - ), - name, - ) - - -def _location( - *, runtime: PlannerRuntime, database: str | None, schema: str | None, name: str -) -> CompiledRelationLocation: - return CompiledRelationLocation( - database=database, - schema=schema, - name=name, - qualified_name=runtime.adapter.render_qualified_name( - database=database, schema=schema, name=name - ), - ) - - -def _with_execution( - *, runtime: PlannerRuntime, model: CompiledModelView, entry: ModelMigrationPlanEntry -) -> ModelMigrationPlanEntry: - """Describe how the executor will stage and promote this migration.""" - - stage_is_transient: bool | None = ( - model.config.table_type.value == TableType.TRANSIENT - if runtime.adapter.adapter_name == BuiltinAdapter.SNOWFLAKE - else None - ) - stage: MigrationStagePlan = runtime.adapter.render_migration_stage( - origin=entry.origin.qualified_name or entry.origin.name, - stage=entry.destination.qualified_name or entry.destination.name, - origin_is_transient=entry.origin_is_transient, - stage_is_transient=stage_is_transient, - ) - return replace( - entry, - stage_is_transient=stage_is_transient, - transfer=stage.transfer, - transfer_fallback=MigrationTransfer.COPY if stage.fallback_statements else None, - promotion=_promotion(runtime=runtime, decision=entry.decision), - ) - - -def _promotion(*, runtime: PlannerRuntime, decision: MigrationDecision) -> MigrationPromotion: - if runtime.adapter.supports_transactional_ddl(): - return MigrationPromotion.TRANSACTIONAL_RENAME - if decision != MigrationDecision.MIGRATE and ( - runtime.adapter.adapter_name == BuiltinAdapter.SNOWFLAKE - ): - return MigrationPromotion.SWAP - return MigrationPromotion.RENAME - - -def redirected_cursor_snapshots( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - existing_relations: dict[str, RelationInfo], - target_relations: dict[str, str], - overrides: PlannerOverrides, - deferral: DeferralInputs, - origin_cursor_columns: dict[str, str] | None = None, -) -> dict[str, ModelCursorSnapshot]: - """Re-read target cursors from where history lives before this run's migrations.""" - - return gather_redirected_cursor_snapshots( - project=runtime.project, - adapter=runtime.adapter, - connection=runtime.connection, - existing_relations=existing_relations, - target_relations=target_relations, - cursor_scope=CursorSnapshotScope( - model_keys=scope.selected_keys, - runtime_producer_keys=scope.selected_keys, - invocation_time=runtime.invocation_time, - start_cursor_config=( - runtime.project_config.cursors.start if runtime.project_config is not None else None - ), - cursor_overrides=overrides.cursor_overrides, - ), - full_refresh_model_names=effectively_full_refreshed_model_names( - project=runtime.project, cli_full_refresh=overrides.full_refresh - ), - deferred_locations=deferral.deferred_locations, - on_progress=runtime.on_progress, - origin_cursor_columns=origin_cursor_columns, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py index c01d8ea5fc..51064cf363 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py @@ -4,9 +4,8 @@ import time -from sqlbuild._native import plan_column_migrations +from sqlbuild._native import plan_column_migrations, plan_model_migrations from sqlbuild.compiler.planner._helpers.migrations.old_names import plan_old_name_views -from sqlbuild.compiler.planner._helpers.migrations.planning import plan_model_migrations from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( effectively_full_refreshed_model_names, From 09b19cdf7b8be45f4b9b3b454349f607b23026d1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:34:21 +0100 Subject: [PATCH 26/29] chore: update Python boundary after migration glue port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index c108635c75..f6cbe30234 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 34195 +remaining_lines = 33967 [[stays]] path = "executor" From 958ca5e6081a3de581f4a3521d189f215ff592f0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:08:45 +0100 Subject: [PATCH 27/29] refactor: host migration discovery orchestration natively --- .../main/discovery_materialization.rs | 18 + .../main/discovery_order.rs | 31 ++ .../src/planner_migrations/main/mod.rs | 2 + .../_helpers/planner_migrations/discovery.rs | 156 ++++++ .../planner_migrations/discovery_events.rs | 166 +++++++ .../planner_migrations/discovery_models.rs | 117 +++++ .../planner_migrations/discovery_requests.rs | 195 ++++++++ .../_helpers/planner_migrations/mod.rs | 4 + .../planner_migrations/model_planning.rs | 6 +- .../planner_migrations/planning_origin.rs | 2 +- .../planner_migrations/registration.rs | 1 + src/sqlbuild/_native.pyi | 13 + .../planner/_helpers/migrations/discovery.py | 461 ------------------ 13 files changed, 705 insertions(+), 467 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/main/discovery_materialization.rs create mode 100644 crates/sqlbuild-analysis/src/planner_migrations/main/discovery_order.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/discovery_materialization.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/discovery_materialization.rs new file mode 100644 index 0000000000..92ea7b7030 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/discovery_materialization.rs @@ -0,0 +1,18 @@ +//! Stored materialization ownership for automatic migration requests. + +/// Check whether a stored definition can transfer history or identity to the current model. +pub fn discovery_materialization(current: &str, metadata: &str) -> bool { + let stored: serde_json::Value = match serde_json::from_str(metadata) { + Ok(payload) => payload, + Err(_) => return false, + }; + let previous = stored + .get("config") + .and_then(|config| config.get("materialized")) + .and_then(serde_json::Value::as_str); + matches!( + (current, previous), + ("incremental" | "snapshot", Some("incremental" | "snapshot")) + | ("table" | "view", Some("table" | "view")) + ) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/discovery_order.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/discovery_order.rs new file mode 100644 index 0000000000..04c8d5fa44 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/discovery_order.rs @@ -0,0 +1,31 @@ +//! Stable depth-first ordering for migration identity propagation. + +use std::collections::{BTreeMap, BTreeSet}; + +/// Order models after their model dependencies using lexical traversal ties. +pub fn discovery_order(dependencies: &BTreeMap>) -> Vec { + let mut pending: Vec<(String, bool)> = dependencies + .keys() + .rev() + .map(|name| (name.clone(), false)) + .collect(); + let mut visited: BTreeSet = BTreeSet::new(); + let mut ordered: Vec = Vec::new(); + while let Some((name, expanded)) = pending.pop() { + if expanded { + ordered.push(name); + continue; + } + if !visited.insert(name.clone()) { + continue; + } + pending.push((name.clone(), true)); + let mut upstream = dependencies[&name].clone(); + upstream.sort(); + pending.extend(upstream.into_iter().rev().filter_map(|dependency| { + (dependencies.contains_key(&dependency) && !visited.contains(&dependency)) + .then_some((dependency, false)) + })); + } + ordered +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs index d984291613..2197b6fda1 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs @@ -2,6 +2,8 @@ pub mod column_decision; pub mod column_warning; pub mod compatibility_findings; pub mod declared_column_aliases; +pub mod discovery_materialization; +pub mod discovery_order; pub mod identical_renames; pub mod match_migrations; pub mod migration_fingerprint; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs new file mode 100644 index 0000000000..32c1eb2f2b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs @@ -0,0 +1,156 @@ +//! Native migration discovery over existing inspection and fingerprint cache boundaries. + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::planner_migrations::column_objects::{model, native}; +use crate::bindings::_helpers::planner_migrations::{ + discovery_events, discovery_models, discovery_requests, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +#[pyfunction] +#[pyo3(signature = (**options))] +fn discover_model_migrations<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "discover_model_migrations", + options, + ( + &[ + "runtime", + "scope", + "snapshot", + "manual_requests", + "declarations", + "state", + "project_schemas", + "fingerprints", + ], + &[], + ), + )?; + let runtime = args.value("runtime")?; + let scope = args.value("scope")?; + let state = args.value("state")?; + let manual = args.value("manual_requests")?; + let declarations = args.value("declarations")?; + let models = discovery_models::ordered(&runtime)?; + let declared_names = PyDict::new(py); + for declaration in declarations.try_iter()? { + declared_names.set_item(declaration?.getattr("model_name")?, py.None())?; + } + let triggered = + discovery_models::triggered(&models, &scope, &args.value("snapshot")?, &declared_names)?; + if !triggered && !manual.is_truthy()? { + return model(py, "ModelMigrationDiscovery", &[]); + } + let metadata = discovery_models::metadata(&runtime, &models)?; + let dialect = runtime + .getattr("adapter")? + .call_method0("sql_analysis_dialect")?; + call_with( + &state.getattr("inspect_schemas")?, + &[("schemas", args.value("project_schemas")?)], + )?; + let declared = discovery_events::declared(&declarations, &state)?; + let names = PyDict::new(py); + for item in &models { + names.set_item(item.getattr("name")?, item)?; + } + let event_renames = discovery_events::renames(&runtime, &names, &state)?; + for (name, origin) in declared.iter() { + event_renames.set_item(name, origin)?; + } + let seeds = PyTuple::new(py, [event_renames, declared])?; + let cache = args.value("fingerprints")?; + if !triggered { + let hashes = native( + py, + "migration_destination_fingerprints", + &[ + ("requests", manual.clone()), + ("variants", seeds.into_any()), + ("metadata_jsons", metadata.into_any()), + ("dialect", dialect), + ("fingerprints", cache), + ], + )?; + return model( + py, + "ModelMigrationDiscovery", + &[("requests", manual), ("destination_fingerprints", hashes)], + ); + } + let unbuilt = PyDict::new(py); + for item in &models { + let name = item.getattr("name")?; + if !declared_names.contains(&name)? + && call_with( + &state.getattr("named_fingerprint")?, + &[("model_name", name.clone())], + )? + .is_none() + { + unbuilt.set_item(name, item)?; + } + } + let resumed = discovery_events::resumed(&runtime, &models, &unbuilt, &state)?; + let candidates = + discovery_requests::candidates(&runtime, &state, &names, (&declarations, &resumed))?; + for (name, _) in resumed.iter() { + unbuilt.del_item(name)?; + } + let matched = native( + py, + "match_migration_fingerprints", + &[ + ("models", PyTuple::new(py, &models)?.into_any()), + ( + "new_names", + PyFrozenSet::new(py, unbuilt.keys().iter())?.into_any(), + ), + ("seed_variants", seeds.into_any()), + ("candidates", candidates.into_any()), + ("metadata_jsons", metadata.clone().into_any()), + ("dialect", dialect.clone()), + ("fingerprints", cache.clone()), + ], + )?; + let mut requests: Vec> = manual.try_iter()?.collect::>()?; + requests.extend(discovery_requests::automatic( + &runtime, + &scope, + &models, + (&matched.get_item(1)?, &resumed), + )?); + let requests = PyTuple::new(py, requests)?; + let warnings = discovery_requests::warnings(&models, &scope, &matched.get_item(2)?)?; + let hashes = native( + py, + "migration_destination_fingerprints", + &[ + ("requests", requests.clone().into_any()), + ("variants", matched.get_item(0)?), + ("metadata_jsons", metadata.into_any()), + ("dialect", dialect), + ("fingerprints", cache), + ], + )?; + model( + py, + "ModelMigrationDiscovery", + &[ + ("requests", requests.into_any()), + ("warnings", warnings.into_any()), + ("destination_fingerprints", hashes), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(discover_model_migrations, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs new file mode 100644 index 0000000000..649ca5f5ed --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs @@ -0,0 +1,166 @@ +//! Preserve event and declared-origin precedence at the inspection boundary. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_migrations::column_objects::config; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; + +pub(super) fn identity(item: &Bound<'_, PyAny>) -> PyResult { + let materialized = config(item, "materialized")?; + Ok(materialized.eq("table")? || materialized.eq("view")?) +} + +pub(super) fn history(item: &Bound<'_, PyAny>) -> PyResult { + let materialized = config(item, "materialized")?; + Ok(materialized.eq("incremental")? || materialized.eq("snapshot")?) +} + +pub(super) fn newest<'py>( + events: &Bound<'py, PyAny>, + location: &Bound<'py, PyAny>, +) -> PyResult> { + let relation = imported( + location.py(), + "sqlbuild.compiler.migrations.main.relation_for_location", + "migration_relation_for_location", + )? + .call1((location,))?; + call_with( + &imported( + location.py(), + "sqlbuild.compiler.migrations.main._newest_event", + "newest_migration_event_mentioning", + )?, + &[("events", events.clone()), ("relation", relation)], + ) +} + +pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { + imported( + location.py(), + "sqlbuild.compiler.migrations.main.relation_for_location", + "migration_relation_for_location", + )? + .call1((location,)) +} + +pub(super) fn declared<'py>( + declarations: &Bound<'py, PyAny>, + state: &Bound<'py, PyAny>, +) -> PyResult> { + let result = PyDict::new(state.py()); + for item in declarations.try_iter()? { + let item = item?; + let fingerprint = call_with( + &state.getattr("model_fingerprint")?, + &[ + ("location", item.getattr("origin_location")?), + ("model_name", item.getattr("origin_model")?), + ], + )?; + let explicit = item.getattr("origin_model")?; + let origin = if explicit.is_truthy()? { + explicit + } else if !fingerprint.is_none() { + fingerprint.getattr("node_name")? + } else { + state.py().None().into_bound(state.py()) + }; + if !origin.is_none() { + result.set_item(item.getattr("model_name")?, origin)?; + } + } + Ok(result) +} + +pub(super) fn renames<'py>( + runtime: &Bound<'py, PyAny>, + names: &Bound<'py, PyDict>, + state: &Bound<'py, PyAny>, +) -> PyResult> { + let py = runtime.py(); + let target = runtime + .getattr("project")? + .getattr("effective_target_name")?; + let ordered = PyList::empty(py); + let events: Vec> = state + .getattr("events")? + .try_iter()? + .collect::>()?; + for (index, event) in events.iter().enumerate() { + ordered.append(( + event.getattr("created_at")?, + event.getattr("event_id")?, + index, + ))?; + } + ordered.call_method0("sort")?; + let latest = PyDict::new(py); + for item in ordered.iter() { + let index: usize = item.get_item(2)?.extract()?; + let event = &events[index]; + let event_target = event.getattr("target_name")?; + if event_target.is_none() || event_target.eq(&target)? { + latest.set_item(event.getattr("destination_model")?, event)?; + } + } + let result = PyDict::new(py); + for (name, event) in latest.iter() { + let origin = event.getattr("origin_model")?; + if names.contains(&name)? && !origin.is_none() && !names.contains(&origin)? { + result.set_item(name, origin)?; + } + } + Ok(result) +} + +pub(super) fn resumed<'py>( + runtime: &Bound<'py, PyAny>, + models: &[Bound<'py, PyAny>], + unbuilt: &Bound<'py, PyDict>, + state: &Bound<'py, PyAny>, +) -> PyResult> { + let result = PyDict::new(runtime.py()); + let target = runtime + .getattr("project")? + .getattr("effective_target_name")?; + let events = state.getattr("events")?; + for item in models { + let name = item.getattr("name")?; + if !unbuilt.contains(&name)? || !(history(item)? || identity(item)?) { + continue; + } + let destination = item.getattr("destination")?; + let event = newest(&events, &destination)?; + if event.is_none() { + continue; + } + let event_target = event.getattr("target_name")?; + if event + .getattr("destination")? + .call_method1("matches", (relation(&destination)?,))? + .is_truthy()? + && (event_target.is_none() || event_target.eq(&target)?) + && event.getattr("decision")?.eq("renamed")? == identity(item)? + { + result.set_item(name, event)?; + } + } + Ok(result) +} + +pub(super) fn key<'py>(location: &Bound<'py, PyAny>) -> PyResult> { + let schema = location.getattr("schema")?; + let schema = if schema.is_truthy()? { + schema + } else { + pyo3::types::PyString::new(location.py(), "").into_any() + }; + PyTuple::new( + location.py(), + [ + schema.call_method0("lower")?, + location.getattr("name")?.call_method0("lower")?, + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs new file mode 100644 index 0000000000..3542ad0358 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs @@ -0,0 +1,117 @@ +//! Model ordering, selection and metadata at the migration discovery boundary. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_migrations::column_objects::native; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods}; +use sqlbuild_analysis::planner_migrations::main::discovery_order::discovery_order; +use std::collections::BTreeMap; + +pub(super) fn ordered<'py>(runtime: &Bound<'py, PyAny>) -> PyResult>> { + let mut models: BTreeMap> = BTreeMap::new(); + let mut dependencies: BTreeMap> = BTreeMap::new(); + for item in runtime.getattr("project")?.getattr("models")?.try_iter()? { + let item = item?; + let name: String = item.getattr("name")?.extract()?; + let mut upstream: Vec = Vec::new(); + for dependency in item.getattr("deps")?.try_iter()? { + let dependency = dependency?; + if dependency.getattr("resource_type")?.eq("model")? { + upstream.push(dependency.getattr("name")?.extract()?); + } + } + dependencies.insert(name.clone(), upstream); + models.insert(name, item); + } + Ok(discovery_order(&dependencies) + .iter() + .map(|name| models[name].clone()) + .collect()) +} + +pub(super) fn triggered( + models: &[Bound<'_, PyAny>], + scope: &Bound<'_, PyAny>, + snapshot: &Bound<'_, PyAny>, + declared: &Bound<'_, PyDict>, +) -> PyResult { + let py = scope.py(); + let selected: Vec> = scope + .getattr("selected_keys")? + .try_iter()? + .collect::>()?; + let relevant = PySet::new(py, &selected)?; + let mut frontier = selected; + while let Some(key) = frontier.pop() { + let upstream = scope + .getattr("upstream_deps")? + .call_method1("get", (key, pyo3::types::PyTuple::empty(py)))?; + for dependency in upstream.try_iter()? { + let dependency = dependency?; + if !relevant.contains(&dependency)? { + relevant.add(&dependency)?; + frontier.push(dependency); + } + } + } + let fingerprints = snapshot.getattr("fingerprints")?.getattr("models")?; + for item in models { + let name = item.getattr("name")?; + if relevant.contains(item.getattr("key")?)? + && !fingerprints.contains(&name)? + && !declared.contains(&name)? + { + return Ok(true); + } + } + Ok(false) +} + +pub(super) fn metadata<'py>( + runtime: &Bound<'py, PyAny>, + models: &[Bound<'py, PyAny>], +) -> PyResult> { + let py = runtime.py(); + let project = runtime.getattr("project")?; + let functions = native( + py, + "build_function_local_hashes", + &[ + ("functions", project.getattr("functions")?), + ("dialect", project.getattr("sql_analysis_dialect")?), + ], + )?; + let identities = imported( + py, + "sqlbuild.compiler.python_nodes.main._hook_identities", + "build_hook_identities", + )? + .call1((project.getattr("hook_functions")?,))?; + let hooks = PyDict::new(py); + for pair in identities.call_method0("items")?.try_iter()? { + let pair = pair?; + hooks.set_item( + pair.get_item(0)?, + pair.get_item(1)?.getattr("version_hash")?, + )?; + } + let result = PyDict::new(py); + for item in models { + result.set_item( + item.getattr("name")?, + call_with( + &imported( + py, + "sqlbuild._native", + "build_model_version_identity_metadata_json", + )?, + &[ + ("model", item.clone()), + ("function_local_hashes", functions.clone()), + ("hook_version_hashes", hooks.clone().into_any()), + ], + )?, + )?; + } + Ok(result) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs new file mode 100644 index 0000000000..d678acd4f0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs @@ -0,0 +1,195 @@ +//! Live candidate inspection and automatic request materialization. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_migrations::column_objects::{config, model}; +use crate::bindings::_helpers::planner_migrations::discovery_events::{ + identity, key, newest, relation, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PySet, PySetMethods, PyString, + PyTuple, +}; +use sqlbuild_analysis::planner_migrations::main::discovery_materialization::discovery_materialization; +use sqlbuild_analysis::planner_migrations::main::stored_migration_fingerprint::stored_migration_fingerprint; +use std::collections::BTreeMap; + +fn location<'py>( + runtime: &Bound<'py, PyAny>, + item: &Bound<'py, PyAny>, + event: bool, +) -> PyResult> { + let fields = if event { + [ + ("database", item.getattr("database")?), + ("schema", item.getattr("schema")?), + ("name", item.getattr("name")?), + ] + } else { + let target = item.getattr("target_name")?; + let name = if target.is_truthy()? { + target + } else { + item.getattr("node_name")? + }; + [ + ("database", item.getattr("target_database")?), + ("schema", item.getattr("target_schema")?), + ("name", name), + ] + }; + super::planning_origin::location(runtime, &fields) +} + +pub(super) fn candidates<'py>( + runtime: &Bound<'py, PyAny>, + state: &Bound<'py, PyAny>, + names: &Bound<'py, PyDict>, + claims: (&Bound<'py, PyAny>, &Bound<'py, PyDict>), +) -> PyResult> { + let py = runtime.py(); + let excluded = PySet::empty(py)?; + for declaration in claims.0.try_iter()? { + excluded.add(key(&declaration?.getattr("origin_location")?)?)?; + } + for (_, event) in claims.1.iter() { + excluded.add(key(&event.getattr("origin")?)?)?; + } + let orphans = call_with( + &state.getattr("orphan_fingerprints")?, + &[( + "project_model_names", + PyFrozenSet::new(py, names.keys().iter())?.into_any(), + )], + )?; + let mut fingerprints: Vec> = Vec::new(); + let mut locations: Vec> = Vec::new(); + for item in orphans.try_iter()? { + let item = item?; + let metadata: String = item.getattr("metadata_json")?.extract()?; + if stored_migration_fingerprint(&metadata).is_some() + && !item.getattr("target_name")?.is_none() + { + locations.push(location(runtime, &item, false)?); + fingerprints.push(item); + } + } + call_with( + &state.getattr("inspect_relations")?, + &[("locations", PyTuple::new(py, &locations)?.into_any())], + )?; + let mut result: Vec> = Vec::new(); + for (fingerprint, location) in fingerprints.into_iter().zip(locations) { + if state.call_method1("relation", (&location,))?.is_none() + || excluded.contains(key(&location)?)? + { + continue; + } + let event = newest(&state.getattr("events")?, &location)?; + if !event.is_none() + && event + .getattr("origin")? + .call_method1("matches", (relation(&location)?,))? + .is_truthy()? + { + continue; + } + result.push(fingerprint); + } + PyTuple::new(py, result) +} + +pub(super) fn automatic<'py>( + runtime: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, + models: &[Bound<'py, PyAny>], + matches: (&Bound<'py, PyAny>, &Bound<'py, PyDict>), +) -> PyResult>> { + let py = runtime.py(); + let mut requests: Vec> = Vec::new(); + for item in models { + if !scope + .getattr("selected_keys")? + .contains(item.getattr("key")?)? + { + continue; + } + let name = item.getattr("name")?; + let event = matches.1.get_item(&name)?; + let candidate = matches.0.call_method1("get", (&name,))?; + let origin = if let Some(event) = event { + Some(( + location(runtime, &event.getattr("origin")?, true)?, + event.getattr("origin_model")?, + )) + } else if !candidate.is_none() { + let materialized: Option = config(item, "materialized")?.extract()?; + let metadata: String = candidate.getattr("metadata_json")?.extract()?; + if discovery_materialization(materialized.as_deref().unwrap_or_default(), &metadata) { + Some(( + location(runtime, &candidate, false)?, + candidate.getattr("node_name")?, + )) + } else { + None + } + } else { + None + }; + if let Some((location, origin_model)) = origin { + requests.push(model( + py, + "ModelMigrationRequest", + &[ + ("model", item.clone()), + ( + "discovery", + imported( + py, + "sqlbuild.compiler.migrations.types", + "MigrationDiscovery", + )? + .getattr("AUTOMATIC")?, + ), + ("origin_location", location), + ("origin_model", origin_model), + ( + "identity_only", + PyBool::new(py, identity(item)?).to_owned().into_any(), + ), + ], + )?); + } + } + Ok(requests) +} + +pub(super) fn warnings<'py>( + models: &[Bound<'py, PyAny>], + scope: &Bound<'py, PyAny>, + ambiguous: &Bound<'py, PyAny>, +) -> PyResult> { + let py = scope.py(); + let selected = PySet::empty(py)?; + for item in models { + if scope + .getattr("selected_keys")? + .contains(item.getattr("key")?)? + { + selected.add(item.getattr("name")?)?; + } + } + let ambiguous: BTreeMap = ambiguous.extract()?; + let mut warnings: Vec> = Vec::new(); + for (name, detail) in ambiguous { + if selected.contains(&name)? { + warnings.push(model(py, "PlanWarning", &[ + ("model_name", PyString::new(py, &name).into_any()), + ("severity", imported(py, "sqlbuild.compiler.planner.types", "WarningSeverity")?.getattr("WARNING")?), + ("message", PyString::new(py, &format!("'{name}' matches {detail}; no automatic migration was inferred. Add migrate_from to choose one.")).into_any()), + ("code", PyString::new(py, "M107").into_any()), + ])?); + } + } + PyTuple::new(py, warnings) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index 96a4f5cffc..18301e2647 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -7,6 +7,10 @@ mod column_warnings; pub(crate) mod columns; mod compatibility; mod constants; +mod discovery; +mod discovery_events; +mod discovery_models; +mod discovery_requests; mod errors; mod fingerprint; mod matching; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index 96bc3ee46e..2a2da8815b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -196,11 +196,7 @@ fn discover<'py>( )?); } call_with( - &imported( - py, - "sqlbuild.compiler.planner._helpers.migrations.discovery", - "discover_model_migrations", - )?, + &imported(py, "sqlbuild._native", "discover_model_migrations")?, &[ ("runtime", runtime.clone()), ("scope", scope.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs index 2b66c81b70..509346b2a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs @@ -5,7 +5,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PySet, PySetMethods, PyTuple}; use pyo3::{PyErr, pyfunction, wrap_pyfunction}; -fn location<'py>( +pub(super) fn location<'py>( runtime: &Bound<'py, PyAny>, fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs index 7119c0a7bc..b2850d000d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs @@ -3,6 +3,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + super::discovery::register(module)?; super::compatibility::register(module)?; super::matching::register(module)?; super::old_names::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 73caf02168..98fcfc3d8a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -225,6 +225,8 @@ from sqlbuild.compiler.planner.models import ( MigrationCompatibilityResult, MissingUpstream, ModelCursorSnapshot, + ModelMigrationDeclaration, + ModelMigrationDiscovery, ModelMigrationPlanEntry, ModelMigrationPlanning, ModelMigrationRequest, @@ -4666,6 +4668,17 @@ def migration_destination_fingerprints( def stored_migration_fingerprint(fingerprint: Fingerprint) -> str | None: ... def declared_column_aliases(model: CompiledModelView) -> tuple[tuple[str, str], ...]: ... def manual_migration_requests(*, scope: PlannerScope) -> tuple[ModelMigrationRequest, ...]: ... +def discover_model_migrations( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + manual_requests: tuple[ModelMigrationRequest, ...], + declarations: tuple[ModelMigrationDeclaration, ...], + state: MigrationStateInspection, + project_schemas: set[str], + fingerprints: MigrationFingerprintCache, +) -> ModelMigrationDiscovery: ... def decide_model_migration( *, runtime: PlannerRuntime, diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py deleted file mode 100644 index daa83a0aad..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py +++ /dev/null @@ -1,461 +0,0 @@ -"""Fingerprint-based discovery of renamed models within the project's schemas.""" - -from __future__ import annotations - -import json -from typing import Any - -from sqlbuild._native import ( - build_function_local_hashes, - build_model_version_identity_metadata_json, - get_config_str, - match_migration_fingerprints, - migration_destination_fingerprints, - stored_migration_fingerprint, -) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.migrations.main._newest_event import newest_migration_event_mentioning -from sqlbuild.compiler.migrations.main.relation_for_location import ( - migration_relation_for_location, -) -from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation -from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery -from sqlbuild.compiler.planner.classes.migration_fingerprint_cache import ( - MigrationFingerprintCache, -) -from sqlbuild.compiler.planner.classes.migration_state_inspection import ( - MigrationStateInspection, -) -from sqlbuild.compiler.planner.models import ( - ModelMigrationDeclaration, - ModelMigrationDiscovery, - ModelMigrationRequest, - PlannerRuntime, - PlannerScope, - PlanWarning, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import MaterializationType, WarningSeverity -from sqlbuild.compiler.python_nodes.main._hook_identities import build_hook_identities - -_HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( - {MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT} -) -_IDENTITY_MATERIALIZATIONS: frozenset[str] = frozenset( - {MaterializationType.TABLE, MaterializationType.VIEW} -) -_CONFIG_KEY: str = "config" -_MATERIALIZED_KEY: str = "materialized" - - -def discover_model_migrations( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - manual_requests: tuple[ModelMigrationRequest, ...], - declarations: tuple[ModelMigrationDeclaration, ...], - state: MigrationStateInspection, - project_schemas: set[str], - fingerprints: MigrationFingerprintCache, -) -> ModelMigrationDiscovery: - """Add unique one-to-one fingerprint matches to the explicit migration requests.""" - - models: tuple[CompiledModelView, ...] = _topological_models(runtime=runtime) - declared: frozenset[str] = frozenset(item.model_name for item in declarations) - triggered: bool = bool( - _new_model_names(models=models, scope=scope, snapshot=snapshot) - declared - ) - if not triggered and not manual_requests: - return ModelMigrationDiscovery() - metadata_jsons: dict[str, str] = _metadata_jsons(runtime=runtime, models=models) - dialect: str | None = runtime.adapter.sql_analysis_dialect() - state.inspect_schemas(schemas=project_schemas) - declared_renames: dict[str, str] = _declared_renames(declarations=declarations, state=state) - event_renames: dict[str, str] = _event_renames(runtime=runtime, models=models, state=state) - seed_variants: tuple[dict[str, str], ...] = ( - {**event_renames, **declared_renames}, - dict(declared_renames), - ) - if not triggered: - return ModelMigrationDiscovery( - requests=manual_requests, - destination_fingerprints=migration_destination_fingerprints( - requests=manual_requests, - variants=seed_variants, - metadata_jsons=metadata_jsons, - dialect=dialect, - fingerprints=fingerprints, - ), - ) - unbuilt: frozenset[str] = frozenset( - model.name - for model in models - if model.name not in declared and state.named_fingerprint(model_name=model.name) is None - ) - resumed: dict[str, MigrationEvent] = _resumed_events( - runtime=runtime, models=models, unbuilt=unbuilt, state=state - ) - candidates: tuple[Fingerprint, ...] = _origin_candidates( - runtime=runtime, - state=state, - models=models, - excluded=_claimed_origins(declarations=declarations, resumed=resumed), - ) - variants: tuple[dict[str, str], ...] - matches: dict[str, Fingerprint] - ambiguous: dict[str, str] - variants, matches, ambiguous = match_migration_fingerprints( - models=models, - new_names=unbuilt - frozenset(resumed), - seed_variants=seed_variants, - candidates=candidates, - metadata_jsons=metadata_jsons, - dialect=dialect, - fingerprints=fingerprints, - ) - requests: tuple[ModelMigrationRequest, ...] = ( - *manual_requests, - *_automatic_requests( - runtime=runtime, scope=scope, models=models, matches=matches, resumed=resumed - ), - ) - return ModelMigrationDiscovery( - requests=requests, - warnings=_ambiguity_warnings(models=models, scope=scope, ambiguous=ambiguous), - destination_fingerprints=migration_destination_fingerprints( - requests=requests, - variants=variants, - metadata_jsons=metadata_jsons, - dialect=dialect, - fingerprints=fingerprints, - ), - ) - - -def _declared_renames( - *, declarations: tuple[ModelMigrationDeclaration, ...], state: MigrationStateInspection -) -> dict[str, str]: - """Map each explicitly migrated model to the model name its origin was built as.""" - - renames: dict[str, str] = {} - declaration: ModelMigrationDeclaration - for declaration in declarations: - origin_fingerprint: Fingerprint | None = state.model_fingerprint( - location=declaration.origin_location, model_name=declaration.origin_model - ) - origin_model: str | None = declaration.origin_model or ( - origin_fingerprint.node_name if origin_fingerprint is not None else None - ) - if origin_model is not None: - renames[declaration.model_name] = origin_model - return renames - - -def _event_renames( - *, - runtime: PlannerRuntime, - models: tuple[CompiledModelView, ...], - state: MigrationStateInspection, -) -> dict[str, str]: - """Map project models to the removed model their latest recorded move came from.""" - - project_names: frozenset[str] = frozenset(model.name for model in models) - target_name: str | None = runtime.project.effective_target_name - latest: dict[str, MigrationEvent] = {} - event: MigrationEvent - for event in sorted(state.events, key=lambda item: (item.created_at, item.event_id)): - if event.target_name in (None, target_name): - latest[event.destination_model] = event - return { - name: event.origin_model - for name, event in latest.items() - if name in project_names - and event.origin_model is not None - and event.origin_model not in project_names - } - - -def _resumed_events( - *, - runtime: PlannerRuntime, - models: tuple[CompiledModelView, ...], - unbuilt: frozenset[str], - state: MigrationStateInspection, -) -> dict[str, MigrationEvent]: - """Return recorded moves or renames into models that have not been built since.""" - - target_name: str | None = runtime.project.effective_target_name - resumed: dict[str, MigrationEvent] = {} - model: CompiledModelView - for model in models: - if model.name not in unbuilt or not (_stores_history(model) or _hands_over_identity(model)): - continue - destination: MigrationRelation = migration_relation_for_location(model.destination) - newest: MigrationEvent | None = newest_migration_event_mentioning( - events=state.events, relation=destination - ) - if ( - newest is not None - and newest.destination.matches(destination) - and newest.target_name in (None, target_name) - and (newest.decision == MigrationDecision.RENAMED) is _hands_over_identity(model) - ): - resumed[model.name] = newest - return resumed - - -def _claimed_origins( - *, - declarations: tuple[ModelMigrationDeclaration, ...], - resumed: dict[str, MigrationEvent], -) -> frozenset[tuple[str, str]]: - """Return relation keys already claimed by explicit or recorded migrations.""" - - declared: set[tuple[str, str]] = { - _relation_key(schema=item.origin_location.schema, name=item.origin_location.name) - for item in declarations - } - recorded: set[tuple[str, str]] = { - _relation_key(schema=event.origin.schema, name=event.origin.name) - for event in resumed.values() - } - return frozenset(declared | recorded) - - -def _relation_key(*, schema: str | None, name: str) -> tuple[str, str]: - return ((schema or "").lower(), name.lower()) - - -def _automatic_requests( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - models: tuple[CompiledModelView, ...], - matches: dict[str, Fingerprint], - resumed: dict[str, MigrationEvent], -) -> tuple[ModelMigrationRequest, ...]: - """Return automatic requests for selected destinations only.""" - - requests: list[ModelMigrationRequest] = [] - model: CompiledModelView - for model in models: - if model.key not in scope.selected_keys: - continue - event: MigrationEvent | None = resumed.get(model.name) - match: Fingerprint | None = matches.get(model.name) - if event is not None: - requests.append( - ModelMigrationRequest( - model=model, - discovery=MigrationDiscovery.AUTOMATIC, - origin_location=_event_origin_location(runtime=runtime, event=event), - origin_model=event.origin_model, - identity_only=_hands_over_identity(model), - ) - ) - elif match is not None and ( - _moves_history(model=model, origin=match) - or _renames_identity(model=model, origin=match) - ): - requests.append( - ModelMigrationRequest( - model=model, - discovery=MigrationDiscovery.AUTOMATIC, - origin_location=_fingerprint_location(runtime=runtime, fingerprint=match), - origin_model=match.node_name, - identity_only=_hands_over_identity(model), - ) - ) - return tuple(requests) - - -def _event_origin_location( - *, runtime: PlannerRuntime, event: MigrationEvent -) -> CompiledRelationLocation: - return CompiledRelationLocation( - database=event.origin.database, - schema=event.origin.schema, - name=event.origin.name, - qualified_name=runtime.adapter.render_qualified_name( - database=event.origin.database, schema=event.origin.schema, name=event.origin.name - ), - ) - - -def _ambiguity_warnings( - *, models: tuple[CompiledModelView, ...], scope: PlannerScope, ambiguous: dict[str, str] -) -> tuple[PlanWarning, ...]: - selected: frozenset[str] = frozenset( - model.name for model in models if model.key in scope.selected_keys - ) - return tuple( - PlanWarning( - model_name=model_name, - severity=WarningSeverity.WARNING, - message=( - f"'{model_name}' matches {detail}; no automatic migration was inferred. " - "Add migrate_from to choose one." - ), - code="M107", - ) - for model_name, detail in sorted(ambiguous.items()) - if model_name in selected - ) - - -def _topological_models(*, runtime: PlannerRuntime) -> tuple[CompiledModelView, ...]: - by_name: dict[str, CompiledModelView] = {model.name: model for model in runtime.project.models} - ordered: list[CompiledModelView] = [] - visited: set[str] = set() - pending: list[tuple[str, bool]] = [(name, False) for name in sorted(by_name, reverse=True)] - while pending: - name: str - expanded: bool - name, expanded = pending.pop() - if expanded: - ordered.append(by_name[name]) - continue - if name in visited: - continue - visited.add(name) - pending.append((name, True)) - pending.extend( - (dep.name, False) - for dep in sorted(by_name[name].deps, key=lambda key: key.name, reverse=True) - if dep.resource_type == CompiledResourceType.MODEL - and dep.name in by_name - and dep.name not in visited - ) - return tuple(ordered) - - -def _new_model_names( - *, models: tuple[CompiledModelView, ...], scope: PlannerScope, snapshot: WarehouseSnapshot -) -> frozenset[str]: - relevant: set[CompiledObjectKey] = set(scope.selected_keys) - frontier: list[CompiledObjectKey] = list(scope.selected_keys) - while frontier: - key: CompiledObjectKey = frontier.pop() - upstream: CompiledObjectKey - for upstream in scope.upstream_deps.get(key, ()): - if upstream not in relevant: - relevant.add(upstream) - frontier.append(upstream) - return frozenset( - model.name - for model in models - if model.key in relevant and model.name not in snapshot.fingerprints.models - ) - - -def _metadata_jsons( - *, runtime: PlannerRuntime, models: tuple[CompiledModelView, ...] -) -> dict[str, str]: - function_hashes: dict[str, str] = build_function_local_hashes( - functions=runtime.project.functions, dialect=runtime.project.sql_analysis_dialect - ) - hook_hashes: dict[str, str] = { - name: identity.version_hash - for name, identity in build_hook_identities(runtime.project.hook_functions).items() - } - return { - model.name: build_model_version_identity_metadata_json( - model=model, function_local_hashes=function_hashes, hook_version_hashes=hook_hashes - ) - for model in models - } - - -def _origin_candidates( - *, - runtime: PlannerRuntime, - state: MigrationStateInspection, - models: tuple[CompiledModelView, ...], - excluded: frozenset[tuple[str, str]], -) -> tuple[Fingerprint, ...]: - project_model_names: frozenset[str] = frozenset(model.name for model in models) - orphans: tuple[Fingerprint, ...] = tuple( - fingerprint - for fingerprint in state.orphan_fingerprints(project_model_names=project_model_names) - if stored_migration_fingerprint(fingerprint) is not None - and fingerprint.target_name is not None - ) - locations: tuple[CompiledRelationLocation, ...] = tuple( - _fingerprint_location(runtime=runtime, fingerprint=fingerprint) for fingerprint in orphans - ) - state.inspect_relations(locations=locations) - return tuple( - fingerprint - for fingerprint, location in zip(orphans, locations, strict=True) - if state.relation(location) is not None - and _relation_key(schema=location.schema, name=location.name) not in excluded - and not _superseded(events=state.events, location=location) - ) - - -def _superseded(*, events: tuple[MigrationEvent, ...], location: CompiledRelationLocation) -> bool: - relation: MigrationRelation = migration_relation_for_location(location) - newest: MigrationEvent | None = newest_migration_event_mentioning( - events=events, relation=relation - ) - return newest is not None and newest.origin.matches(relation) - - -def _moves_history(*, model: CompiledModelView, origin: Fingerprint) -> bool: - return _stores_history(model) and _stored_materialization(origin) in _HISTORY_MATERIALIZATIONS - - -def _renames_identity(*, model: CompiledModelView, origin: Fingerprint) -> bool: - return ( - _hands_over_identity(model) - and _stored_materialization(origin) in _IDENTITY_MATERIALIZATIONS - ) - - -def _hands_over_identity(model: CompiledModelView) -> bool: - return ( - get_config_str(values=model.config.values, key=_MATERIALIZED_KEY) - in _IDENTITY_MATERIALIZATIONS - ) - - -def _stores_history(model: CompiledModelView) -> bool: - return ( - get_config_str(values=model.config.values, key=_MATERIALIZED_KEY) - in _HISTORY_MATERIALIZATIONS - ) - - -def _stored_materialization(fingerprint: Fingerprint) -> str | None: - payload: Any = _metadata_payload(fingerprint) - config: Any = payload.get(_CONFIG_KEY) if payload else None - value: Any = config.get(_MATERIALIZED_KEY) if isinstance(config, dict) else None - return value if isinstance(value, str) else None - - -def _metadata_payload(fingerprint: Fingerprint) -> dict[str, Any] | None: - try: - payload: Any = json.loads(fingerprint.metadata_json) - except json.JSONDecodeError: - return None - return payload if isinstance(payload, dict) else None - - -def _fingerprint_location( - *, runtime: PlannerRuntime, fingerprint: Fingerprint -) -> CompiledRelationLocation: - name: str = fingerprint.target_name or fingerprint.node_name - return CompiledRelationLocation( - database=fingerprint.target_database, - schema=fingerprint.target_schema, - name=name, - qualified_name=runtime.adapter.render_qualified_name( - database=fingerprint.target_database, schema=fingerprint.target_schema, name=name - ), - ) From 901acd06d06d1ec1b14c251924e35573ee932707 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:10:15 +0100 Subject: [PATCH 28/29] chore: update Python boundary after migration discovery port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index f6cbe30234..e8fb54d60b 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 33967 +remaining_lines = 33506 [[stays]] path = "executor" From b21843528235b7f0d2d013c5ff4e91a0a1276432 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:34:27 +0100 Subject: [PATCH 29/29] fix: scope fixture type conversion to selected models --- .../planner_strategy/fixture_types.rs | 40 ++++++-- src/sqlbuild/_native.pyi | 2 +- .../planner/_helpers/fixtures/__init__.py | 0 .../planner/_helpers/fixtures/_test_types.py | 9 ++ .../_helpers/fixtures/test_type_scope.py | 99 +++++++++++++++++++ 5 files changed, 141 insertions(+), 9 deletions(-) create mode 100644 tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/__init__.py create mode 100644 tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/_test_types.py create mode 100644 tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs index 03a9894c8d..482e9284a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs @@ -17,29 +17,49 @@ pub(crate) fn expected_fixture_types<'py>( base_expected_types: &Bound<'py, PyDict>, ordered_model_names: Vec, model_map: &Bound<'py, PyDict>, -) -> PyResult> { - let state = FixtureTypeState { - known: base_expected_types.extract()?, - ..FixtureTypeState::default() - }; - let result = base_expected_types.copy()?; +) -> PyResult> { + let mut state = FixtureTypeState::default(); + let result = PyDict::new(base_expected_types.py()); let mut models: Vec = Vec::new(); for name in ordered_model_names { let Some(model) = model_map.get_item(&name)? else { continue; }; let key = ("model".to_owned(), name); - let types = result + let types = base_expected_types .get_item(&key)? .ok_or_else(|| PyKeyError::new_err(key.clone()))? .cast_into::()? .copy()?; + state.known.insert(key.clone(), types.extract()?); result.set_item(&key, types)?; models.push(FixtureTypeModel { key, projections: projections(&model)?, }); } + for model in &models { + for projection in &model.projections { + for read in &projection.upstream { + if state + .known + .get(&read.key) + .is_some_and(|types| types.contains_key(&read.column)) + { + continue; + } + if let Some(types) = base_expected_types.get_item(&read.key)? + && let Some(value) = types.cast_into::()?.get_item(&read.column)? + { + state + .known + .entry(read.key.clone()) + .or_default() + .insert(read.column.clone(), value.extract()?); + } + } + } + } let state = propagate_types(state, &models); for (read, sql_type) in state.additions { result @@ -47,7 +67,11 @@ pub(crate) fn expected_fixture_types<'py>( .ok_or_else(|| PyKeyError::new_err("fixture types"))? .set_item(read.column, sql_type)?; } - Ok(result) + base_expected_types + .py() + .import("collections")? + .getattr("ChainMap")? + .call1((result, base_expected_types)) } fn projections(model: &Bound<'_, PyAny>) -> PyResult> { diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 98fcfc3d8a..2d2fc3b2f9 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2501,7 +2501,7 @@ def expected_fixture_types( base_expected_types: dict[tuple[CompiledResourceType, str], dict[str, str]], ordered_model_names: tuple[str, ...], model_map: dict[str, CompiledModelView], -) -> dict[tuple[CompiledResourceType, str], dict[str, str]]: ... +) -> Mapping[tuple[CompiledResourceType, str], dict[str, str]]: ... def build_relation_fixture_context( *, project: CompiledProjectView, models_by_name: dict[str, CompiledModelView] | None = None ) -> RelationFixturePlanningContext: ... diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/__init__.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/_test_types.py new file mode 100644 index 0000000000..e0cdaac8a4 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/_test_types.py @@ -0,0 +1,9 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class FixtureTypeScopeTestCase: + description: str + unrelated_relations: int + expected_eager_relations: int = 1 + expected_eager_columns: int = 2 diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py new file mode 100644 index 0000000000..4627afef02 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py @@ -0,0 +1,99 @@ +from collections import ChainMap +from collections.abc import Callable, Mapping +from copy import deepcopy +from pathlib import Path + +import pytest + +from sqlbuild._native import ( + build_compiled_project, + build_relation_fixture_completion, + build_relation_fixture_context, + discover_project_inputs, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.planner.models import ( + RelationFixtureCompletion, + RelationFixturePlanningContext, +) +from tests.integration.src.sqlbuild.compiler.planner._helpers.fixtures._test_types import ( + FixtureTypeScopeTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + ( + FixtureTypeScopeTestCase(description="small metadata map", unrelated_relations=10), + FixtureTypeScopeTestCase(description="large metadata map", unrelated_relations=1000), + FixtureTypeScopeTestCase(description="very large metadata map", unrelated_relations=10000), + ), + ids=lambda case: case.description, +) +def test_given_unrelated_project_types_when_completing_fixture_then_eager_work_is_chain_local( + test_case: FixtureTypeScopeTestCase, + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], +) -> None: + write_repo_files( + tmp_path, + { + "sqlbuild_project.toml": 'name = "fixture_scope"\nadapter = "duckdb"\n', + "models/orders.sql": ( + 'MODEL (description "Order facts.");\n' + 'SELECT order_id, status FROM __source("raw_orders")' + ), + "sources/raw_orders.yml": ( + "sources:\n - name: raw_orders\n description: Order input.\n" + " expression: SELECT 1 AS order_id, 'open' AS status\n" + " contract: enforced\n columns:\n" + " - name: order_id\n type: INTEGER\n nullable: false\n" + " - name: status\n type: VARCHAR\n nullable: true\n" + ), + }, + ) + adapter: DuckDbAdapter = DuckDbAdapter() + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=adapter + ) + context: RelationFixturePlanningContext = build_relation_fixture_context(project=project) + # Model the shared project metadata supplied to every individual SQL-test plan. + for index in range(test_case.unrelated_relations): + context.expected_types[(CompiledResourceType.SOURCE, f"unrelated_orders_{index}")] = { + "order_id": "INTEGER", + "status": "VARCHAR", + } + before: dict[tuple[CompiledResourceType, str], dict[str, str]] = deepcopy( + context.expected_types + ) + fixture_key: tuple[CompiledResourceType, str] = (CompiledResourceType.SOURCE, "raw_orders") + completed: RelationFixtureCompletion = build_relation_fixture_completion( + project=project, + adapter=adapter, + request=( + ("orders",), + ((CompiledResourceType.SOURCE, {"raw_orders": "SELECT 1 AS order_id"}),), + ), + planning_context=context, + ) + assert completed.diagnostics == () + assert 'CAST(NULL AS VARCHAR) AS "status"' in completed.fixture_sql_by_key[fixture_key] + assert dict(completed.expected_types) == before + assert list(completed.expected_types) == list(before) + assert context.expected_types == before + # Count eagerly materialized relation/type entries, independent of machine timing. + eager: Mapping[tuple[CompiledResourceType, str], dict[str, str]] = getattr( + completed.expected_types, "maps", (completed.expected_types,) + )[0] + assert len(eager) == test_case.expected_eager_relations + assert sum(len(columns) for columns in eager.values()) == test_case.expected_eager_columns + assert isinstance(completed.expected_types, ChainMap) + assert completed.expected_types.maps[-1] is context.expected_types + assert completed.expected_types[fixture_key] is context.expected_types[fixture_key] + model_key: tuple[CompiledResourceType, str] = (CompiledResourceType.MODEL, "orders") + assert completed.expected_types[model_key] is not context.expected_types[model_key] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"])