diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 33bb966429..71b65946ea 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -412,7 +412,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.format_performance == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 10 + timeout-minutes: 30 steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 @@ -430,6 +430,7 @@ jobs: - name: Install dependencies run: uv sync --locked --all-groups --all-extras --python 3.12 - name: Run formatter performance guards + timeout-minutes: 10 run: >- uv run pytest tests/e2e/src/sqlbuild/cli/commands/main/format/test_format_performance.py @@ -659,7 +660,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.performance == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -681,6 +682,7 @@ jobs: - name: Install dependencies run: uv sync --locked --all-groups --all-extras - name: Run fresh-process compile cache guard under 4 GiB + timeout-minutes: 15 env: BENCHMARK_MODELS: ${{ matrix.models }} MEMORY_LIMIT_BYTES: "4294967296" @@ -703,6 +705,7 @@ jobs: - name: Run varied 1k warm-cache guard under 4 GiB if: matrix.models == 3000 + timeout-minutes: 15 env: MEMORY_LIMIT_BYTES: "4294967296" run: | diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 7046ea142a..dac314a359 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -21,6 +21,7 @@ pub mod lineage_graph; pub mod manifest; pub mod planner_changes; pub mod planner_identity; +pub mod planner_migrations; pub mod planner_scope; pub mod planner_selection; pub mod planner_strategy; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/ast.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/ast.rs new file mode 100644 index 0000000000..547296b1ed --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/ast.rs @@ -0,0 +1,109 @@ +//! Canonical migration AST values. + +use crate::planner_migrations::constants::{HAVING_MAX_KEY, OPAQUE_TUPLE_FIELDS}; +use serde_json::Value; +use std::collections::BTreeSet; + +/// Tuple-valued AST fields are opaque to the historical Python dict/list walkers. +pub(crate) fn opaque_field(key: &str, value: &Value) -> bool { + key == HAVING_MAX_KEY + || (OPAQUE_TUPLE_FIELDS.contains(&key) + && value + .as_array() + .is_some_and(|items| !items.is_empty() && items.iter().all(Value::is_array))) +} + +pub(crate) fn strip_formatting(node: &mut Value) { + match node { + Value::Object(object) => { + object.shift_remove("span"); + for (key, value) in object { + if opaque_field(key, value) { + continue; + } + if key.ends_with("comments") { + *value = Value::Array(Vec::new()); + } else { + strip_formatting(value); + } + } + } + Value::Array(items) => items.iter_mut().for_each(strip_formatting), + _ => {} + } +} + +pub(crate) fn canonical(node: &mut Value) { + match node { + Value::Array(items) => items.iter_mut().for_each(canonical), + Value::Object(object) => { + for (key, value) in object.iter_mut() { + if !opaque_field(key, value) { + canonical(value); + } + } + if object.get("quoted") == Some(&Value::Bool(false)) + && let Some(Value::String(name)) = object.get_mut("name") + { + *name = name.to_lowercase(); + } + } + _ => {} + } +} + +pub(crate) fn identifier(node: &Value) -> Option<&str> { + node.get("name")?.as_str() +} + +pub(crate) fn identifiers(node: &Value) -> Vec { + items(node) + .iter() + .filter_map(identifier) + .map(str::to_lowercase) + .collect() +} + +pub(crate) fn items(node: &Value) -> &[Value] { + node.as_array().map_or(&[], Vec::as_slice) +} + +pub(crate) fn clause_items(node: &Value) -> &[Value] { + if node.is_array() { + items(node) + } else { + items(&node["expressions"]) + } +} + +pub(crate) fn references(node: &Value, unqualified: bool) -> BTreeSet { + matching_fields(node, "column") + .into_iter() + .filter(|column| !unqualified || column["table"].is_null()) + .filter_map(|column| identifier(&column["name"])) + .map(str::to_lowercase) + .collect() +} + +pub(crate) fn matching_fields<'a>(node: &'a Value, key: &str) -> Vec<&'a Value> { + let mut found: Vec<&Value> = Vec::new(); + let mut pending: Vec<&Value> = vec![node]; + while let Some(current) = pending.pop() { + match current { + Value::Array(items) => pending.extend(items), + Value::Object(object) => { + if let Some(value) = object.get(key) { + found.push(value); + } + pending.extend( + object + .iter() + .filter(|(key, value)| !opaque_field(key, value)) + .map(|(_, value)| value), + ); + } + _ => {} + } + } + found +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/binding_proof.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/binding_proof.rs new file mode 100644 index 0000000000..912ab4e69a --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/binding_proof.rs @@ -0,0 +1,76 @@ +//! Conservative alias resolution in unchanged and renamed projections. + +use crate::planner_migrations::_helpers::ast::identifier; +use crate::planner_migrations::models::QueryShape; +use serde_json::Value; +use std::collections::BTreeSet; + +pub(crate) fn other_references(shape: &QueryShape, excluded: &str) -> BTreeSet { + shape + .projections + .iter() + .filter(|item| item.name.to_lowercase() != excluded) + .flat_map(|item| item.references.iter().cloned()) + .collect() +} + +pub(crate) fn binding_proven( + shapes: (&QueryShape, &QueryShape), + names: (&str, &str), + input_columns: &mut impl FnMut(&str, &str) -> Result>, E>, +) -> Result { + let (previous, current) = shapes; + let (origin, destination) = names; + let mut referenced = previous.clause_references.clone(); + referenced.extend(current.clause_references.iter().cloned()); + referenced.extend(other_references(previous, origin)); + referenced.extend(other_references(current, destination)); + if !referenced.contains(origin) && !referenced.contains(destination) { + return Ok(true); + } + let Some(relations) = ¤t.input_relations else { + return Ok(false); + }; + let mut inputs = current.local_input_columns.clone(); + for (kind, name) in relations { + let Some(columns) = input_columns(kind, name)? else { + return Ok(false); + }; + inputs.extend(columns); + } + if inputs.contains(destination) { + return Ok(false); + } + if inputs.contains(origin) { + return Ok(previous + .projections + .iter() + .any(|item| item.name.to_lowercase() == origin && item.passthrough)); + } + Ok(!current.clause_references.contains(origin) + && !other_references(current, "").contains(origin)) +} + +pub(crate) fn substitute(node: &mut Value, names: &[(String, String)]) { + match node { + Value::Array(items) => items.iter_mut().for_each(|item| substitute(item, names)), + Value::Object(object) => { + if object.contains_key("select") { + return; + } + if let Some(column) = object.get_mut("column").filter(|column| column.is_object()) + && column["table"].is_null() + && column["name"]["quoted"] == false + && let Some(name) = identifier(&column["name"]) + && let Some((_, replacement)) = names.iter().rev().find(|(key, _)| key == name) + { + column["name"]["name"] = Value::String(replacement.clone()); + return; + } + object + .values_mut() + .for_each(|value| substitute(value, names)); + } + _ => {} + } +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/clauses.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/clauses.rs new file mode 100644 index 0000000000..e605497b23 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/clauses.rs @@ -0,0 +1,37 @@ +//! Non-projection clauses and positional references of a SELECT. + +use crate::planner_migrations::_helpers::ast::{clause_items, items, references}; +use crate::planner_migrations::constants::NUMBER_LITERAL; +use serde_json::Value; +use std::collections::BTreeSet; + +pub(crate) fn clause_references(select: &Value) -> BTreeSet { + let clauses: Vec = select + .as_object() + .into_iter() + .flat_map(|object| object.iter()) + .filter(|(key, _)| !["expressions", "with", "from", "joins"].contains(&key.as_str())) + .map(|(_, value)| value.clone()) + .chain( + items(&select["joins"]) + .iter() + .filter(|join| join.is_object()) + .map(|join| join["on"].clone()), + ) + .collect(); + references(&Value::Array(clauses), true) +} + +pub(crate) fn positional(select: &Value) -> bool { + clause_items(&select["group_by"]) + .iter() + .chain(clause_items(&select["order_by"]).iter().map(|item| { + if item.is_object() { + &item["this"] + } else { + item + } + })) + .chain(clause_items(&select["distinct_on"])) + .any(|item| item["literal"]["literal_type"] == NUMBER_LITERAL) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/encoding.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/encoding.rs new file mode 100644 index 0000000000..000dabb842 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/encoding.rs @@ -0,0 +1,42 @@ +//! Python-compatible compact sorted JSON for migration proofs. + +use serde_json::Value; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; + +pub(crate) fn encode(value: &Value) -> Result { + dumps( + &converted(value)?, + &JsonDialect::Stdlib(StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }), + ) + .map_err(|error| format!("{error:?}")) +} + +fn converted(value: &Value) -> Result { + Ok(match value { + Value::Null => JsonValue::Null, + Value::Bool(flag) => JsonValue::Bool(*flag), + Value::String(text) => JsonValue::String(text.clone()), + Value::Number(number) => { + if !number.is_f64() { + JsonValue::Integer( + JsonInteger::parse(&number.to_string()).ok_or("invalid integer")?, + ) + } else { + JsonValue::Float(number.as_f64().ok_or("invalid float")?) + } + } + Value::Array(items) => { + JsonValue::Array(items.iter().map(converted).collect::>()?) + } + Value::Object(object) => JsonValue::Object( + object + .iter() + .map(|(key, value)| Ok((key.clone(), converted(value)?))) + .collect::>()?, + ), + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/fingerprint.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/fingerprint.rs new file mode 100644 index 0000000000..ff72274bee --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/fingerprint.rs @@ -0,0 +1,84 @@ +//! Normalize migration metadata and SQL without changing the persisted format. + +use crate::planner_changes::main::version_identity_payload::version_identity_payload; +use crate::planner_changes::models::IdentityPayload; +use crate::planner_migrations::_helpers::ast::strip_formatting; +use crate::planner_migrations::_helpers::local_names::{local_names, rewrite, unambiguous}; +use crate::planner_migrations::_helpers::parse::parse; +use crate::planner_migrations::constants::{EXCLUDED_CONFIG, REF_MARKER}; +use polyglot_sql::{Expression, dialects::Dialect}; +use regex::Regex; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; + +pub(crate) fn metadata(text: &str) -> Value { + match version_identity_payload(Some(text), false) { + IdentityPayload::Absent => Value::Null, + IdentityPayload::Raw(text) => Value::String(text), + IdentityPayload::Object(object) => Value::Object(object), + } +} + +pub(crate) fn identity_metadata(text: &str, identities: &BTreeMap) -> Value { + let mut payload = metadata(text); + if let Some(object) = payload.as_object_mut() { + object.shift_remove("model_name"); + if let Some(config) = object.get_mut("config").and_then(Value::as_object_mut) { + config.retain(|key, _| !EXCLUDED_CONFIG.contains(&key.as_str())); + if let Some(Value::Object(inputs)) = config.get_mut("cursor_inputs") { + *inputs = std::mem::take(inputs) + .into_iter() + .map(|(name, value)| (identities.get(&name).cloned().unwrap_or(name), value)) + .collect(); + } + } + } + payload +} + +pub(crate) fn normalized( + sql: &str, + identities: &BTreeMap, + dialect: &str, +) -> Result, String> { + let marker = Regex::new(REF_MARKER).map_err(|error| error.to_string())?; + let placeholder = marker.replace_all(sql, |captures: ®ex::Captures<'_>| { + let name = captures + .get(1) + .or_else(|| captures.get(2)) + .map_or("", |value| value.as_str()); + let identity = identities.get(name).map_or(name, String::as_str); + let digest = format!("{:x}", Sha256::digest(identity.as_bytes())); + format!("__sqb_migration_ref__{}", &digest[..16]) + }); + let Some(mut parsed) = parse(&placeholder, dialect)? else { + return Ok(None); + }; + strip_formatting(&mut parsed); + let definitions = local_names(&parsed); + if !unambiguous(&parsed, &definitions) { + return Ok(None); + } + let mut names: BTreeMap = BTreeMap::new(); + for (name, _) in &definitions { + let next = names.len() + 1; + names + .entry(name.clone()) + .or_insert_with(|| format!("__sqb_local_{next}")); + } + let mut ctes: BTreeMap = BTreeMap::new(); + for (name, is_cte) in &definitions { + if *is_cte && let Some(value) = names.get(name) { + ctes.insert(name.clone(), value.clone()); + } + } + rewrite(&mut parsed, &names, &ctes); + let expression: Expression = + serde_json::from_value(parsed).map_err(|error| error.to_string())?; + Dialect::get_by_name(dialect) + .ok_or_else(|| format!("unknown dialect {dialect}"))? + .generate(&expression) + .map(Some) + .map_err(|error| error.to_string()) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/inputs.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/inputs.rs new file mode 100644 index 0000000000..e548568c3a --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/inputs.rs @@ -0,0 +1,115 @@ +//! Resolve only inputs whose columns can be proved from the query shape. + +use crate::planner_migrations::_helpers::ast::{clause_items, identifier, identifiers, items}; +use crate::planner_migrations::_helpers::projection::projection; +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet}; + +type Columns = BTreeSet; +type Input = (Option<(String, String)>, Option); +type Inputs = (Option>, Columns); + +pub(crate) fn select_inputs(select: &Value) -> Result { + if items(&select["lateral_views"]).iter().any(Value::is_object) { + return Ok((None, Columns::new())); + } + let mut ctes: BTreeMap> = BTreeMap::new(); + for cte in items(&select["with"]["ctes"]) + .iter() + .filter(|item| item.is_object()) + { + if let Some(name) = identifier(&cte["alias"]) { + let declared = identifiers(&cte["columns"]); + ctes.insert( + name.to_owned(), + if declared.is_empty() { + output_names(&cte["this"])? + } else { + Some(declared.into_iter().collect()) + }, + ); + } + } + let mut relations: Vec<(String, String)> = Vec::new(); + let mut local = Columns::new(); + for item in clause_items(&select["from"]).iter().chain( + items(&select["joins"]) + .iter() + .filter(|join| join.is_object()) + .map(|join| &join["this"]), + ) { + match input_item(item, &ctes)? { + (Some(relation), _) => relations.push(relation), + (_, Some(columns)) => local.extend(columns), + _ => return Ok((None, Columns::new())), + } + } + Ok((Some(relations), local)) +} + +fn input_item(item: &Value, ctes: &BTreeMap>) -> Result { + let Some(object) = item.as_object().filter(|object| object.len() == 1) else { + return Ok((None, None)); + }; + if let Some(alias) = object.get("alias") { + if !alias.is_object() { + return Ok((None, None)); + } + let renamed = identifiers(&alias["column_aliases"]); + let inner = input_item(&alias["this"], ctes)?; + return Ok( + if !renamed.is_empty() && (inner.0.is_some() || inner.1.is_some()) { + (None, Some(renamed.into_iter().collect())) + } else { + inner + }, + ); + } + if let Some(function) = object.get("function").filter(|value| value.is_object()) { + let kind = function["name"].as_str().unwrap_or_default().to_lowercase(); + let args = items(&function["args"]); + if !["__source", "__ref"].contains(&kind.as_str()) || args.len() != 1 { + return Ok((None, None)); + } + return Ok(( + identifier(&args[0]["column"]["name"]).map(|name| (kind, name.to_owned())), + None, + )); + } + if let Some(table) = object.get("table").filter(|value| value.is_object()) { + if !table["schema"].is_null() || !table["catalog"].is_null() { + return Ok((None, None)); + } + return Ok(( + None, + identifier(&table["name"]) + .and_then(|name| ctes.get(name)) + .cloned() + .flatten(), + )); + } + if let Some(subquery) = object.get("subquery").filter(|value| value.is_object()) { + return Ok((None, output_names(&subquery["this"])?)); + } + Ok((None, None)) +} + +fn output_names(query: &Value) -> Result, String> { + if !query + .as_object() + .is_some_and(|object| object.len() == 1 && object.contains_key("select")) + { + return Ok(None); + } + let Some(expressions) = query["select"]["expressions"].as_array() else { + return Ok(None); + }; + let mut names = Columns::new(); + for expression in expressions { + let Some(projection) = projection(expression)? else { + return Ok(None); + }; + names.insert(projection.name.to_lowercase()); + } + Ok(Some(names)) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/local_names.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/local_names.rs new file mode 100644 index 0000000000..d002926aae --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/local_names.rs @@ -0,0 +1,169 @@ +//! CTE and relation-alias identity with conservative scope rejection. + +use crate::planner_migrations::_helpers::ast::{identifier, items, matching_fields, opaque_field}; +use crate::planner_migrations::constants::{TABLE_KEY, WITH_KEY}; +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet}; + +fn is_identifier(node: &Value) -> bool { + identifier(node).is_some() && node.get("quoted").is_some() +} + +fn is_relation(node: &Value) -> bool { + node.is_object() && !is_identifier(node) +} + +fn unqualified(node: &Value) -> bool { + node["schema"].is_null() && node["catalog"].is_null() +} + +fn names(node: &Value) -> Vec { + items(node) + .iter() + .filter(|item| is_identifier(item)) + .filter_map(identifier) + .map(str::to_lowercase) + .collect() +} + +pub(crate) fn local_names(node: &Value) -> Vec<(String, bool)> { + let mut found: Vec<(String, bool)> = Vec::new(); + match node { + Value::Array(items) => { + for item in items { + found.extend(local_names(item)); + } + } + Value::Object(object) => { + if let Some(with) = object.get("with").filter(|with| with.is_object()) { + for cte in items(&with["ctes"]).iter().filter(|cte| cte.is_object()) { + if is_identifier(&cte["alias"]) + && let Some(name) = identifier(&cte["alias"]) + { + found.push((name.to_lowercase(), true)); + } + } + } + for (key, value) in object { + if opaque_field(key, value) { + continue; + } + if ["table", "subquery"].contains(&key.as_str()) + && is_relation(value) + && is_identifier(&value["alias"]) + && let Some(name) = identifier(&value["alias"]) + { + found.push((name.to_lowercase(), false)); + } + found.extend(local_names(value)); + } + } + _ => {} + } + found +} + +fn with_clauses(node: &Value) -> Vec<&Value> { + matching_fields(node, WITH_KEY) + .into_iter() + .filter(|value| value.is_object()) + .collect() +} + +fn relation_names(node: &Value) -> BTreeSet { + matching_fields(node, TABLE_KEY) + .into_iter() + .filter(|value| is_relation(value) && unqualified(value) && is_identifier(&value["name"])) + .filter_map(|value| identifier(&value["name"])) + .map(str::to_lowercase) + .collect() +} + +pub(crate) fn unambiguous(root: &Value, definitions: &[(String, bool)]) -> bool { + let top_level = root + .as_object() + .filter(|object| object.len() == 1) + .and_then(|object| object.values().next()) + .map(|statement| &statement["with"]); + let clauses = with_clauses(root); + if !clauses.is_empty() && (!top_level.is_some_and(Value::is_object) || clauses.len() > 1) { + return false; + } + let top_level = top_level.unwrap_or(&Value::Null); + if top_level["recursive"].as_bool() == Some(true) { + return false; + } + let ctes: Vec<&Value> = items(&top_level["ctes"]) + .iter() + .filter(|cte| cte.is_object()) + .collect(); + let cte_names = names(&Value::Array( + ctes.iter().map(|cte| cte["alias"].clone()).collect(), + )); + if cte_names.len() != ctes.len() + || cte_names.iter().collect::>().len() != ctes.len() + { + return false; + } + for (index, cte) in ctes.iter().enumerate() { + let references = relation_names(&cte["this"]); + if cte_names[index..] + .iter() + .any(|name| references.contains(name)) + { + return false; + } + } + let physical = relation_names(root); + !definitions + .iter() + .any(|(name, is_cte)| !is_cte && physical.contains(name) && !cte_names.contains(name)) +} + +fn renamed(mut node: Value, names: &BTreeMap) -> Value { + if is_identifier(&node) + && let Some(name) = identifier(&node) + && let Some(replacement) = names.get(&name.to_lowercase()) + { + node["name"] = Value::String(replacement.clone()); + node["quoted"] = Value::Bool(false); + } + node +} + +pub(crate) fn rewrite( + node: &mut Value, + names: &BTreeMap, + cte_names: &BTreeMap, +) { + match node { + Value::Array(items) => items + .iter_mut() + .for_each(|item| rewrite(item, names, cte_names)), + Value::Object(object) => { + for (key, value) in object { + if opaque_field(key, value) { + continue; + } + let original = value.clone(); + rewrite(value, names, cte_names); + if ["column", "star"].contains(&key.as_str()) && original.is_object() { + value["table"] = renamed(original["table"].clone(), names); + } else if ["table", "subquery"].contains(&key.as_str()) && is_relation(&original) { + value["alias"] = renamed(original["alias"].clone(), names); + if unqualified(&original) && is_identifier(&original["name"]) { + value["name"] = renamed(original["name"].clone(), cte_names); + } + } else if key == WITH_KEY + && original.is_object() + && let Some(ctes) = value["ctes"].as_array_mut() + { + for cte in ctes { + cte["alias"] = renamed(cte["alias"].clone(), names); + } + } + } + } + _ => {} + } +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/mod.rs new file mode 100644 index 0000000000..9e34a2d27c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod ast; +pub(crate) mod binding_proof; +pub(crate) mod clauses; +pub(crate) mod encoding; +pub(crate) mod fingerprint; +pub(crate) mod inputs; +pub(crate) mod local_names; +pub(crate) mod parse; +pub(crate) mod projection; +pub(crate) mod removed_and_added; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/parse.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/parse.rs new file mode 100644 index 0000000000..a84b3ee534 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/parse.rs @@ -0,0 +1,26 @@ +//! Parse migration SQL under the compiler's complexity limit. + +use crate::lineage::constants::MAX_FUNCTION_CALL_DEPTH; +use polyglot_sql::dialects::Dialect; +use polyglot_sql::{ComplexityGuardOptions, ParseOptions}; +use serde_json::{Value, json}; + +pub(crate) fn parse(sql: &str, dialect: &str) -> Result, String> { + let dialect = + Dialect::get_by_name(dialect).ok_or_else(|| format!("unknown dialect {dialect}"))?; + let guard: ComplexityGuardOptions = + serde_json::from_value(json!({"maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH})) + .map_err(|error| error.to_string())?; + let options = ParseOptions { + complexity_guard: Some(guard), + }; + let Ok(mut statements) = dialect.parse_with_options(sql, &options) else { + return Ok(None); + }; + if statements.len() != 1 { + return Ok(None); + } + serde_json::to_value(statements.remove(0)) + .map(Some) + .map_err(|error| error.to_string()) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/projection.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/projection.rs new file mode 100644 index 0000000000..e46bf22ec0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/projection.rs @@ -0,0 +1,53 @@ +//! A named output and its expression proof. + +use crate::planner_migrations::_helpers::ast::{canonical, identifier, references}; +use crate::planner_migrations::_helpers::encoding::encode; +use crate::planner_migrations::models::QueryProjection; +use serde_json::Value; +use std::collections::BTreeSet; + +pub(crate) fn named_projections(select: &Value) -> Result>, String> { + let Some(expressions) = select["expressions"].as_array() else { + return Ok(None); + }; + let mut projections: Vec = Vec::new(); + let mut keys: BTreeSet = BTreeSet::new(); + for expression in expressions { + let Some(projection) = projection(expression)? else { + return Ok(None); + }; + if !keys.insert(projection.name.to_lowercase()) { + return Ok(None); + } + projections.push(projection); + } + Ok(Some(projections)) +} + +pub(crate) fn projection(expression: &Value) -> Result, String> { + let Some(object) = expression.as_object().filter(|object| object.len() == 1) else { + return Ok(None); + }; + let (name, body) = if let Some(alias) = object.get("alias") { + (identifier(&alias["alias"]), &alias["this"]) + } else if let Some(column) = object.get("column") { + (identifier(&column["name"]), expression) + } else { + return Ok(None); + }; + let Some(name) = name.filter(|_| !body.is_null()) else { + return Ok(None); + }; + let mut canonical_body = body.clone(); + canonical(&mut canonical_body); + let column = &body["column"]; + Ok(Some(QueryProjection { + name: name.to_owned(), + expression: encode(&canonical_body)?, + references: references(body, false), + passthrough: column.is_object() + && column["table"].is_null() + && identifier(&column["name"]) + .is_some_and(|reference| reference.to_lowercase() == name.to_lowercase()), + })) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/removed_and_added.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/removed_and_added.rs new file mode 100644 index 0000000000..8d868f5167 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/removed_and_added.rs @@ -0,0 +1,23 @@ +//! Match output membership without changing projection order. + +use std::collections::BTreeSet; + +use crate::planner_migrations::models::Projection; + +/// Previous-only and current-only projections, excluding declared migrations. +pub(crate) fn removed_and_added<'a>( + (previous, current): (&'a [Projection], &'a [Projection]), + excluded: &BTreeSet, +) -> (Vec<&'a Projection>, Vec<&'a Projection>) { + let previous_names: BTreeSet<&str> = previous.iter().map(|item| item.key.as_str()).collect(); + let current_names: BTreeSet<&str> = current.iter().map(|item| item.key.as_str()).collect(); + let removed: Vec<&Projection> = previous + .iter() + .filter(|item| !current_names.contains(item.key.as_str()) && !excluded.contains(&item.key)) + .collect(); + let added: Vec<&Projection> = current + .iter() + .filter(|item| !previous_names.contains(item.key.as_str()) && !excluded.contains(&item.key)) + .collect(); + (removed, added) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/constants.rs b/crates/sqlbuild-analysis/src/planner_migrations/constants.rs new file mode 100644 index 0000000000..37daafa4a4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/constants.rs @@ -0,0 +1,44 @@ +//! Persisted migration identity constants. + +/// Python tuple fields that the historical dict/list walkers leave opaque. +pub(crate) const OPAQUE_TUPLE_FIELDS: &[&str] = &[ + "whens", + "rename", + "partition", + "partitions", + "set", + "credentials", + "define", + "fields", + "search_results", + "pairs", + "path_values", + "with_properties", + "mysql_table_options", + "with_options", + "options", + "with_params", + "properties", + "expressions", + "tags", +]; +pub(crate) const HAVING_MAX_KEY: &str = "having_max"; +pub(crate) const WITH_KEY: &str = "with"; +pub(crate) const TABLE_KEY: &str = "table"; +pub(crate) const NUMBER_LITERAL: &str = "number"; + +pub(crate) const REF_MARKER: &str = r#"__ref\(\s*(?:"([^"]+)"|'([^']+)')\s*\)"#; +pub(crate) const EXCLUDED_CONFIG: &[&str] = &[ + "batch_size", + "full_refresh", + "incremental_mode", + "incremental_strategy", + "lookback", + "materialized", + "max_microbatches", + "microbatch_limit", + "microbatch_strategy", + "on_schema_change", + "snapshot_full_refresh", + "snapshot_schema_change", +]; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/identical_renames.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/identical_renames.rs new file mode 100644 index 0000000000..d19a3ad0a0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/identical_renames.rs @@ -0,0 +1,40 @@ +//! Select unique expression matches, then prove the complete query change. + +use std::collections::BTreeSet; + +use crate::planner_migrations::_helpers::removed_and_added::removed_and_added; +use crate::planner_migrations::models::Projection; +use crate::planner_migrations::types::RenameProof; + +/// Unique old/new names in current projection order, only when the complete rename proof succeeds. +pub fn identical_renames( + shapes: (&[Projection], &[Projection]), + excluded: &BTreeSet, + explains_change: &mut RenameProof<'_, E>, +) -> Result, E> { + let (removed, added) = removed_and_added(shapes, excluded); + let mut pairs: Vec<(String, String)> = Vec::new(); + for column in &added { + let mut matches = removed + .iter() + .filter(|origin| origin.expression == column.expression); + let Some(origin) = matches.next() else { + continue; + }; + if matches.next().is_some() { + continue; + } + if added + .iter() + .filter(|other| other.expression == origin.expression) + .count() + == 1 + { + pairs.push((origin.name.clone(), column.name.clone())); + } + } + if pairs.is_empty() || !explains_change(&pairs)? { + return Ok(Vec::new()); + } + Ok(pairs) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/migration_fingerprint.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/migration_fingerprint.rs new file mode 100644 index 0000000000..b9c48ce968 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/migration_fingerprint.rs @@ -0,0 +1,22 @@ +//! Persisted rename identity of a model. + +use crate::planner_migrations::_helpers::encoding::encode; +use crate::planner_migrations::_helpers::fingerprint::{identity_metadata, normalized}; +use serde_json::json; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; + +/// Hash normalized SQL and metadata exactly as migration-fingerprint-v1. +pub fn migration_fingerprint( + sql: &str, + metadata: &str, + identities: &BTreeMap, + dialect: &str, +) -> Result, String> { + let Some(sql) = normalized(sql, identities, dialect)? else { + return Ok(None); + }; + let payload = + encode(&json!({"metadata": identity_metadata(metadata, identities), "sql": sql}))?; + Ok(Some(format!("{:x}", Sha256::digest(payload.as_bytes())))) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/migration_ref_names.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/migration_ref_names.rs new file mode 100644 index 0000000000..0c1e66a937 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/migration_ref_names.rs @@ -0,0 +1,21 @@ +//! Model names that participate in a migration identity. + +use crate::planner_migrations::_helpers::fingerprint::metadata; +use crate::planner_migrations::constants::REF_MARKER; +use regex::Regex; +use std::collections::BTreeSet; + +/// Every query reference and configured cursor-input name. +pub fn migration_ref_names(sql: &str, metadata_json: &str) -> Result, String> { + let marker = Regex::new(REF_MARKER).map_err(|error| error.to_string())?; + let mut names: BTreeSet = BTreeSet::new(); + for captures in marker.captures_iter(sql) { + if let Some(value) = captures.get(1).or_else(|| captures.get(2)) { + names.insert(value.as_str().to_owned()); + } + } + if let Some(inputs) = metadata(metadata_json)["config"]["cursor_inputs"].as_object() { + names.extend(inputs.keys().cloned()); + } + Ok(names) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs new file mode 100644 index 0000000000..5c524b7ad7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/mod.rs @@ -0,0 +1,6 @@ +pub mod identical_renames; +pub mod migration_fingerprint; +pub mod migration_ref_names; +pub mod query_shape; +pub mod rename_hints; +pub mod renames_explain_change; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/query_shape.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/query_shape.rs new file mode 100644 index 0000000000..5d8dfac8b0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/query_shape.rs @@ -0,0 +1,47 @@ +//! Parse the conservative shape used for column migration proofs. + +use crate::planner_migrations::_helpers::ast::{canonical, strip_formatting}; +use crate::planner_migrations::_helpers::clauses::{clause_references, positional}; +use crate::planner_migrations::_helpers::encoding::encode; +use crate::planner_migrations::_helpers::inputs::select_inputs; +use crate::planner_migrations::_helpers::parse::parse; +use crate::planner_migrations::_helpers::projection::named_projections; +use crate::planner_migrations::models::QueryShape; +use serde_json::json; + +/// Named projections of exactly one plain SELECT, or None when unknowable. +pub fn query_shape(sql: &str, dialect: &str) -> Result, String> { + let Some(mut root) = parse(sql, dialect)? else { + return Ok(None); + }; + strip_formatting(&mut root); + if !root + .as_object() + .is_some_and(|object| object.len() == 1 && object.contains_key("select")) + { + return Ok(None); + } + let select = &mut root["select"]; + let Some(projections) = named_projections(select)? else { + return Ok(None); + }; + canonical(select); + let mut body = select.clone(); + if let Some(object) = body.as_object_mut() { + for key in ["expressions", "order_by", "qualify"] { + object.shift_remove(key); + } + } + let (input_relations, local_input_columns) = select_inputs(select)?; + Ok(Some(QueryShape { + projections, + body: encode(&body)?, + alias_clauses: encode( + &json!({"order_by": select["order_by"], "qualify": select["qualify"]}), + )?, + positional: positional(select), + input_relations, + local_input_columns, + clause_references: clause_references(select), + })) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/rename_hints.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/rename_hints.rs new file mode 100644 index 0000000000..8ad01abc49 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/rename_hints.rs @@ -0,0 +1,50 @@ +//! Suggest declarations for output columns resembling removed live columns. + +use std::collections::BTreeSet; + +use crate::planner_migrations::_helpers::removed_and_added::removed_and_added; +use crate::planner_migrations::models::{Projection, RenameHint}; + +/// Hints ordered by added projection, preferring identical expressions over shared references. +pub fn rename_hints( + shapes: (&[Projection], &[Projection]), + excluded: &BTreeSet, + live_columns: &BTreeSet, + same_binding: bool, +) -> Vec { + let (removed, added) = removed_and_added(shapes, excluded); + let live_removed: Vec<&Projection> = removed + .into_iter() + .filter(|origin| live_columns.contains(&origin.key)) + .collect(); + let mut hints: Vec = Vec::new(); + for column in added { + if live_columns.contains(&column.key) { + continue; + } + let identical: Vec = live_removed + .iter() + .filter(|origin| same_binding && origin.expression == column.expression) + .map(|origin| origin.name.clone()) + .collect(); + let similar: Vec = live_removed + .iter() + .filter(|origin| { + !origin.references.is_empty() && origin.references == column.references + }) + .map(|origin| origin.name.clone()) + .collect(); + if !identical.is_empty() || !similar.is_empty() { + hints.push(RenameHint { + added_column: column.name.clone(), + identical: !identical.is_empty(), + candidate_columns: if identical.is_empty() { + similar + } else { + identical + }, + }); + } + } + hints +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/renames_explain_change.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/renames_explain_change.rs new file mode 100644 index 0000000000..7bafbf7db4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/renames_explain_change.rs @@ -0,0 +1,75 @@ +//! Prove that output renames explain the entire query change. + +use crate::planner_migrations::_helpers::binding_proof::{binding_proven, substitute}; +use crate::planner_migrations::_helpers::encoding::encode; +use crate::planner_migrations::models::QueryShape; +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet}; + +/// Whether names and alias bindings are the only change, preserving callback failures. +pub fn renames_explain_change>( + shapes: (&QueryShape, &QueryShape), + renames: &[(String, String)], + mut input_columns: impl FnMut(&str, &str) -> Result>, E>, +) -> Result { + let (previous, current) = shapes; + if renames.is_empty() || previous.body != current.body { + return Ok(false); + } + let mut lowered: Vec<(String, String)> = Vec::new(); + for (old, new) in renames { + let old = old.to_lowercase(); + let new = new.to_lowercase(); + if let Some(pair) = lowered.iter_mut().find(|pair| pair.0 == old) { + pair.1 = new; + } else { + lowered.push((old, new)); + } + } + let key = |name: &str| { + let key = name.to_lowercase(); + lowered + .iter() + .find(|pair| pair.0 == key) + .map_or(key.clone(), |pair| pair.1.clone()) + }; + let renamed: BTreeMap = previous + .projections + .iter() + .map(|item| (key(&item.name), item.expression.as_str())) + .collect(); + let desired: BTreeMap = current + .projections + .iter() + .map(|item| (item.name.to_lowercase(), item.expression.as_str())) + .collect(); + if renamed.len() != previous.projections.len() || renamed != desired { + return Ok(false); + } + let reordered = previous + .projections + .iter() + .map(|item| key(&item.name)) + .ne(current + .projections + .iter() + .map(|item| item.name.to_lowercase())); + if reordered && (previous.positional || current.positional) { + return Ok(false); + } + for (origin, destination) in &lowered { + if !binding_proven(shapes, (origin, destination), &mut input_columns)? { + return Ok(false); + } + } + let mut restored: Value = + serde_json::from_str(¤t.alias_clauses).map_err(|error| E::from(error.to_string()))?; + substitute( + &mut restored, + &lowered + .into_iter() + .map(|(old, new)| (new, old)) + .collect::>(), + ); + Ok(encode(&restored).map_err(E::from)? == previous.alias_clauses) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/mod.rs b/crates/sqlbuild-analysis/src/planner_migrations/mod.rs new file mode 100644 index 0000000000..bedb9c0f5e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/mod.rs @@ -0,0 +1,9 @@ +//! Pure column migration candidate selection. + +mod _helpers; +mod constants; +pub mod main; +pub mod models; +#[cfg(test)] +mod tests; +pub mod types; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/models.rs b/crates/sqlbuild-analysis/src/planner_migrations/models.rs new file mode 100644 index 0000000000..2649924a27 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/models.rs @@ -0,0 +1,41 @@ +//! Data used to match output columns across a migration. + +use std::collections::BTreeSet; + +/// A named projection with canonical expression and reference names. +#[derive(Debug)] +pub struct Projection { + pub name: String, + pub key: String, + pub expression: String, + pub references: BTreeSet, +} + +/// A suggestion to declare an explicit column migration. +#[derive(Debug)] +pub struct RenameHint { + pub added_column: String, + pub candidate_columns: Vec, + pub identical: bool, +} + +/// One output expression and its alias binding facts. +#[derive(Debug)] +pub struct QueryProjection { + pub name: String, + pub expression: String, + pub references: BTreeSet, + pub passthrough: bool, +} + +/// A plain SELECT with canonical expressions and input binding facts. +#[derive(Debug)] +pub struct QueryShape { + pub projections: Vec, + pub body: String, + pub alias_clauses: String, + pub positional: bool, + pub input_relations: Option>, + pub local_input_columns: BTreeSet, + pub clause_references: BTreeSet, +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/tests.rs b/crates/sqlbuild-analysis/src/planner_migrations/tests.rs new file mode 100644 index 0000000000..44054f0428 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_column_renames.rs"] +mod test_column_renames; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/tests/helpers.rs b/crates/sqlbuild-analysis/src/planner_migrations/tests/helpers.rs new file mode 100644 index 0000000000..4ae2c47ebd --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/tests/helpers.rs @@ -0,0 +1,12 @@ +//! Synthetic projection facts. + +use crate::planner_migrations::models::Projection; + +pub(super) fn projection(name: &str) -> Projection { + Projection { + name: name.to_owned(), + key: name.to_lowercase(), + expression: "amount".to_owned(), + references: ["amount".to_owned()].into(), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/tests/test_column_renames.rs b/crates/sqlbuild-analysis/src/planner_migrations/tests/test_column_renames.rs new file mode 100644 index 0000000000..2759c35627 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/tests/test_column_renames.rs @@ -0,0 +1,60 @@ +//! Proof callbacks fail transparently and are skipped for ambiguous candidates. + +use std::collections::BTreeSet; + +use crate::planner_migrations::main::identical_renames::identical_renames; +use crate::planner_migrations::tests::helpers::projection; +use crate::planner_migrations::tests::test_types::ProofTestCase; + +#[test] +fn given_candidates_when_proving_then_propagates_failures_and_skips_ambiguity() { + let test_cases = [ + ProofTestCase { + description: "lookup error propagates", + added_names: &["revenue"], + proof_result: Err("input lookup failed"), + expected_pairs: Err("input lookup failed"), + expected_calls: 1, + }, + ProofTestCase { + description: "ambiguous pairs skip proof", + added_names: &["revenue", "total"], + proof_result: Err("must not call"), + expected_pairs: Ok(vec![]), + expected_calls: 0, + }, + ProofTestCase { + description: "failed proof rejects pairs", + added_names: &["revenue"], + proof_result: Ok(false), + expected_pairs: Ok(vec![]), + expected_calls: 1, + }, + ProofTestCase { + description: "successful proof keeps pairs", + added_names: &["revenue"], + proof_result: Ok(true), + expected_pairs: Ok(vec![("amount".to_owned(), "revenue".to_owned())]), + expected_calls: 1, + }, + ]; + for test_case in test_cases { + let previous = [projection("amount")]; + let current: Vec<_> = test_case + .added_names + .iter() + .map(|name| projection(name)) + .collect(); + let mut calls = 0; + let result = identical_renames((&previous, ¤t), &BTreeSet::new(), &mut |_| { + calls += 1; + test_case.proof_result + }); + assert_eq!( + result, test_case.expected_pairs, + "{}", + test_case.description + ); + assert_eq!(calls, test_case.expected_calls, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_migrations/tests/test_types.rs new file mode 100644 index 0000000000..bba7f61a2c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/tests/test_types.rs @@ -0,0 +1,9 @@ +//! Cases for proof failure and ambiguity handling. + +pub(super) struct ProofTestCase { + pub(super) description: &'static str, + pub(super) added_names: &'static [&'static str], + pub(super) proof_result: Result, + pub(super) expected_pairs: Result, &'static str>, + pub(super) expected_calls: usize, +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/types.rs b/crates/sqlbuild-analysis/src/planner_migrations/types.rs new file mode 100644 index 0000000000..fa987b6932 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_migrations/types.rs @@ -0,0 +1,4 @@ +//! Query proof boundary for candidate selection. + +/// Prove the entire query change using the proposed column rename pairs. +pub type RenameProof<'a, E> = dyn FnMut(&[(String, String)]) -> Result + 'a; diff --git a/crates/sqlbuild-core/src/dbt_cli/constants.rs b/crates/sqlbuild-core/src/dbt_cli/constants.rs index 09bd04e0cb..4ccf2bf8eb 100644 --- a/crates/sqlbuild-core/src/dbt_cli/constants.rs +++ b/crates/sqlbuild-core/src/dbt_cli/constants.rs @@ -36,3 +36,4 @@ pub const EVENT_TIME_CURSOR_CONFLICT_HELP: &str = /// Help for a missing dbt project directory. pub const MISSING_PROJECT_DIR_HELP: &str = "to point SQLBuild at the dbt project, set this in sqlbuild_project.toml:\n [dbt]\n project_dir = \"path/to/dbt_project\"\n = help: or pass `--project-dir` to `sqb dbt`"; pub const MISSING_PROJECT_DIR_MESSAGE: &str = "dbt project directory is not configured"; +pub(crate) const DBT_NO_CONNECTION_FLAG: &str = "--no-connection"; diff --git a/crates/sqlbuild-core/src/dbt_cli/main/dbt_debug_args.rs b/crates/sqlbuild-core/src/dbt_cli/main/dbt_debug_args.rs new file mode 100644 index 0000000000..43e0940dd2 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/main/dbt_debug_args.rs @@ -0,0 +1,22 @@ +//! Strip SQLBuild-owned debug and configuration flags before invoking dbt. + +use crate::dbt_cli::constants::DBT_NO_CONNECTION_FLAG; + +/// Preserve dbt arguments while removing the configuration flags already resolved by SQLBuild. +pub fn dbt_debug_args(args: &[String]) -> Vec { + let mut stripped: Vec = Vec::new(); + let mut skip_next = false; + for arg in args { + if skip_next { + skip_next = false; + } else if matches!( + arg.as_str(), + "--project-dir" | "--profiles-dir" | "--target" | "--target-path" + ) { + skip_next = true; + } else if arg != DBT_NO_CONNECTION_FLAG { + stripped.push(arg.clone()); + } + } + stripped +} diff --git a/crates/sqlbuild-core/src/dbt_cli/main/mod.rs b/crates/sqlbuild-core/src/dbt_cli/main/mod.rs index bb436c444f..c7e471d790 100644 --- a/crates/sqlbuild-core/src/dbt_cli/main/mod.rs +++ b/crates/sqlbuild-core/src/dbt_cli/main/mod.rs @@ -1,4 +1,5 @@ pub mod build_dbt_argv; pub mod choose_dbt_config; +pub mod dbt_debug_args; pub mod resolve_dbt_executable; pub mod route_dbt_interop_args; diff --git a/crates/sqlbuild-core/src/dbt_cli/tests/test_dbt_cli.rs b/crates/sqlbuild-core/src/dbt_cli/tests/test_dbt_cli.rs index a5be010e1d..184632ce9a 100644 --- a/crates/sqlbuild-core/src/dbt_cli/tests/test_dbt_cli.rs +++ b/crates/sqlbuild-core/src/dbt_cli/tests/test_dbt_cli.rs @@ -1,5 +1,6 @@ use crate::dbt_cli::main::build_dbt_argv::build_dbt_argv; use crate::dbt_cli::main::choose_dbt_config::choose_dbt_config; +use crate::dbt_cli::main::dbt_debug_args::dbt_debug_args; use crate::dbt_cli::main::resolve_dbt_executable::resolve_dbt_executable; use crate::dbt_cli::main::route_dbt_interop_args::route_dbt_interop_args; use crate::dbt_cli::models::{ @@ -8,6 +9,41 @@ use crate::dbt_cli::models::{ use crate::dbt_cli::tests::helpers::{buckets, failure, full_options, text, texts}; use crate::dbt_cli::tests::test_types::{ArgvTestCase, ConfigTestCase, RouteTestCase}; +#[test] +fn given_debug_flags_when_routing_then_preserves_only_dbt_owned_arguments() { + let test_cases = [ + ArgvTestCase { + description: "strip local flags and separate configuration values", + argv: dbt_debug_args(&texts(&[ + "--project-dir", + "project", + "--profiles-dir", + "profiles", + "--target", + "dev", + "--target-path", + "target", + "--no-connection", + "--connection", + "--debug", + ])), + expected_argv: vec!["--connection", "--debug"], + }, + ArgvTestCase { + description: "preserve equals forms for dbt and tolerate a trailing value flag", + argv: dbt_debug_args(&texts(&["--target=dev", "--no-connection", "--target"])), + expected_argv: vec!["--target=dev"], + }, + ]; + for test_case in test_cases { + assert_eq!( + test_case.argv, test_case.expected_argv, + "{}", + test_case.description + ); + } +} + #[test] fn given_options_when_building_argv_then_orders_flags_like_dbt() { let test_cases = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs index eef0e1eae4..c32e8340ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs @@ -15,8 +15,8 @@ use crate::bindings::_helpers::dbt_command::support::{color_enabled, effective_d const NATIVE_MODULE: &str = "sqlbuild._native"; const VALIDATE_ARGS_MODULE: &str = "sqlbuild.integrations.dbt.main.cli._validate_execution_args"; -const PLAN_MODULE: &str = "sqlbuild.integrations.dbt.main.pipeline.plan"; -const EXECUTE_MODULE: &str = "sqlbuild.integrations.dbt.main.pipeline.execute"; +const PLAN_MODULE: &str = "sqlbuild._native"; +const EXECUTE_MODULE: &str = "sqlbuild._native"; const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; const PRESENTATION_MODELS: &str = "sqlbuild.presentation.models"; const PLAN_COMMAND: &str = "plan"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/debug.rs index d2b5733349..5146169ccc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/debug.rs @@ -8,7 +8,7 @@ use crate::bindings::_helpers::dbt_command::auto_init::args_tuple; use crate::bindings::_helpers::dbt_command::support::{color_enabled, flag, status, stream}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const DEBUG_MODULE: &str = "sqlbuild.integrations.dbt.main.pipeline.debug"; +const DEBUG_MODULE: &str = "sqlbuild._native"; /// Run `body` under a stderr status line that is always closed; returns its exit code. fn with_status( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/constants.rs new file mode 100644 index 0000000000..5ffb936dad --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/constants.rs @@ -0,0 +1,4 @@ +//! dbt runtime argument boundaries. + +pub(crate) const COMMAND_AND_SUBCOMMAND_ARGUMENT_COUNT: usize = 2; +pub(crate) const PLAN_DISPLAY_LIMIT: usize = 10; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs index 7fde8df5ca..c512d186d8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs @@ -1,8 +1,16 @@ pub(crate) mod cli; +pub(crate) mod constants; pub(crate) mod execution; pub(crate) mod graph; pub(crate) mod manifest; pub(crate) mod orchestration; +pub(crate) mod pipeline; +pub(crate) mod pipeline_phases; +pub(crate) mod pipeline_prologue; pub(crate) mod plan; +pub(crate) mod plan_output; pub(crate) mod profile; pub(crate) mod registration; +pub(crate) mod runtime_execution; +pub(crate) mod runtime_results; +pub(crate) mod sqlbuild_work; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs new file mode 100644 index 0000000000..4212c4663c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs @@ -0,0 +1,389 @@ +//! Public project pipelines for dbt plan, execution and debug. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::dbt_cli::main::dbt_debug_args::dbt_debug_args; +use std::time::Instant; + +use crate::bindings::_helpers::build_command::support::{boolean, none, text}; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::dbt_interop::pipeline_prologue::progress; + +const MODELS: &str = "sqlbuild.integrations.dbt.models"; +const PLAN_OPTIONS: (&[&str], &[&str]) = ( + &["project_dir", "args"], + &[ + "dbt_runner", + "dbt_executable", + "sqlbuild_executable", + "no_sql_validation", + "on_progress", + "progress_stream", + "use_color", + ], +); +const DEBUG_OPTIONS: (&[&str], &[&str]) = ( + &["project_dir", "args", "stdout_stream", "stderr_stream"], + &["dbt_runner", "dbt_executable"], +); + +fn native<'py>( + py: Python<'py>, + name: &str, + pairs: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with(&imported(py, "sqlbuild._native", name)?, pairs) +} + +fn prepare<'py>( + request: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py = request.py(); + let invocation = + imported(py, "sqlbuild._native", "resolve_dbt_execution_invocation")?.call1((request,))?; + let manifest = native( + py, + "load_compiled_dbt_manifest", + &[ + ("runner", invocation.getattr("runner")?), + ("dbt_options", invocation.getattr("dbt_options")?), + ("full_refresh", boolean(py, false)), + ("on_progress", request.getattr("on_progress")?), + ], + )?; + let compiled = native( + py, + "compile_dbt_interop_project", + &[ + ("project_dir", request.getattr("project_dir")?), + ( + "discovered_inputs", + invocation.getattr("discovered_inputs")?, + ), + ("manifest", manifest.clone()), + ("dbt_vars", invocation.getattr("dbt_vars")?), + ("no_sql_validation", request.getattr("no_sql_validation")?), + ("on_progress", request.getattr("on_progress")?), + ], + )?; + Ok((invocation, manifest, compiled)) +} + +fn resolve_plan<'py>( + request: &Bound<'py, PyAny>, + invocation: &Bound<'py, PyAny>, + compiled: &Bound<'py, PyAny>, + manifest: &Bound<'py, PyAny>, +) -> PyResult> { + native( + request.py(), + "resolve_dbt_interop_plan", + &[ + ("command", request.getattr("command")?), + ("invocation", invocation.clone()), + ("compiled", compiled.clone()), + ("manifest", manifest.clone()), + ( + "sqlbuild_executable", + request.getattr("sqlbuild_executable")?, + ), + ("on_progress", request.getattr("on_progress")?), + ], + ) +} + +/// Execute dbt first and then the SQLBuild work selected by the combined graph. +#[pyfunction] +fn execute_dbt_interop_from_project(request: &Bound<'_, PyAny>) -> PyResult { + let py = request.py(); + let command = request.getattr("command")?; + let kind = imported(py, "sqlbuild.integrations.dbt.types", "DbtInteropCommand")?; + if !command.eq(kind.getattr("RUN")?)? && !command.eq(kind.getattr("BUILD")?)? { + return Err(PyErr::from_value( + imported( + py, + "sqlbuild.integrations.dbt.exceptions", + "DbtInteropArgumentError", + )? + .call1((format!( + "unsupported dbt interop execution command: {}", + command.str()? + ),))?, + )); + } + let (invocation, manifest, compiled) = prepare(request)?; + let resolution = resolve_plan(request, &invocation, &compiled, &manifest)?; + let plan = resolution.getattr("plan")?; + let merged = native( + py, + "build_merged_dbt_execution_argv", + &[ + ("command", command), + ("options", invocation.getattr("dbt_options")?), + ( + "routed_args", + invocation.getattr("routed")?.getattr("dbt_args")?, + ), + ("plan", plan.clone()), + ], + )?; + native( + py, + "write_dbt_execution_plan_text", + &[ + ("request", request.clone()), + ("invocation", invocation.clone()), + ("plan", plan.clone()), + ("merged_dbt_argv", merged.clone()), + ], + )?; + let execution = native( + py, + "execute_dbt_without_state_tracking", + &[ + ("request", request.clone()), + ("invocation", invocation.clone()), + ("merged_dbt_argv", merged), + ], + )?; + native( + py, + "write_dbt_execution_summary", + &[ + ("request", request.clone()), + ("invocation", invocation.clone()), + ("execution", execution.clone()), + ], + )?; + let failed = native( + py, + "build_failed_sqlbuild_model_names", + &[ + ("graph", resolution.getattr("graph")?), + ("manifest", manifest.clone()), + ("node_results", execution.getattr("node_results")?), + ], + )?; + let code: i64 = execution.getattr("returncode")?.extract()?; + if code != 0 && !execution.getattr("node_results")?.is_truthy()? { + return Ok(code); + } + let output = native( + py, + "resolve_sqlbuild_execution_plan_output", + &[ + ("request", request.clone()), + ("invocation", invocation.clone()), + ("compiled", compiled.clone()), + ("manifest", manifest), + ("plan", plan.clone()), + ("failed_sqlbuild_model_names", failed), + ], + )?; + let skip = if output.is_none() { + if plan.getattr("sqlbuild_skip_reason")?.is_none() { + return Ok(code); + } + Some("No SQLBuild work selected.") + } else if !native(py, "plan_has_executable_work", &[("plan", output.clone())])?.is_truthy()? { + Some("Skipping SQLBuild: no executable work selected.") + } else { + None + }; + if let Some(message) = skip { + native( + py, + "write_sqlbuild_skip_notice", + &[ + ("request", request.clone()), + ("invocation", invocation), + ("message", text(py, message)), + ], + )?; + return Ok(code); + } + let connection = native( + py, + "resolve_dbt_connection_config", + &[ + ("request", request.clone()), + ("invocation", invocation.clone()), + ("compiled", compiled.clone()), + ], + )?; + let result = native( + py, + "run_dbt_sqlbuild_work", + &[ + ("request", request.clone()), + ("invocation", invocation), + ("compiled", compiled), + ("plan_output", output), + ("connection_config", connection), + ], + )?; + Ok(code.max(result.extract()?)) +} + +/// Build an interop plan from actual project files, dbt artifacts and warehouse facts. +#[pyfunction] +#[pyo3(signature = (**options))] +fn plan_dbt_interop_from_project<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read(py, "plan_dbt_interop_from_project", options, PLAN_OPTIONS)?; + let request = call_with( + &imported(py, MODELS, "DbtInteropExecutionRequest")?, + &[ + ( + "command", + imported(py, "sqlbuild.integrations.dbt.types", "DbtInteropCommand")? + .getattr("PLAN")?, + ), + ("project_dir", args.value("project_dir")?), + ("args", args.value("args")?), + ("dbt_runner", args.value("dbt_runner")?), + ("dbt_executable", args.value("dbt_executable")?), + ( + "sqlbuild_executable", + if args.passed("sqlbuild_executable")? { + args.value("sqlbuild_executable")? + } else { + text(py, "sqb") + }, + ), + ( + "no_sql_validation", + args.value_or_false("no_sql_validation")?, + ), + ("on_progress", args.value("on_progress")?), + ("progress_stream", args.value("progress_stream")?), + ("use_color", args.value_or_false("use_color")?), + ], + )?; + let (invocation, manifest, compiled) = prepare(&request)?; + let stream = args.value("progress_stream")?; + let connection = if stream.is_none() { + none(py) + } else { + native( + py, + "ConnectionProgressReporter", + &[ + ("adapter_name", compiled.getattr("adapter_name")?), + ("stream", stream), + ("use_color", args.value_or_false("use_color")?), + ], + )? + }; + let start = Instant::now(); + let resolution = resolve_plan(&request, &invocation, &compiled, &manifest)?; + let plan = native( + py, + "attach_sqlbuild_plan_output", + &[ + ("plan", resolution.getattr("plan")?), + ("project_dir", args.value("project_dir")?), + ( + "discovered_inputs", + invocation.getattr("discovered_inputs")?, + ), + ("compiled", compiled), + ( + "sqlbuild_args", + invocation.getattr("effective_sqlbuild_args")?, + ), + ("connection_progress", connection), + ("on_progress", args.value("on_progress")?), + ], + )?; + progress( + &args.value("on_progress")?, + &format!( + "Generated dbt interop plan. ({:.2}s)", + start.elapsed().as_secs_f64() + ), + )?; + Ok(plan) +} + +/// Run dbt debug using SQLBuild's resolved dbt configuration. +#[pyfunction] +#[pyo3(signature = (**options))] +fn debug_dbt_from_project(py: Python<'_>, options: Option>) -> PyResult { + let args = KeywordArguments::read(py, "debug_dbt_from_project", options, DEBUG_OPTIONS)?; + let discovered = native( + py, + "discover_project_inputs", + &[("project_dir", args.value("project_dir")?)], + )?; + let overrides = imported( + py, + "sqlbuild.integrations.dbt.main.config._parse_config_overrides", + "parse_dbt_config_overrides", + )? + .call1((args.value("args")?,))?; + let resolved = native( + py, + "resolve_dbt_config", + &[ + ("project_root", args.value("project_dir")?), + ( + "config", + discovered.getattr("project_config")?.getattr("dbt")?, + ), + ( + "local_config", + discovered.getattr("local_config")?.getattr("dbt")?, + ), + ("overrides", overrides), + ("require_project_dir", boolean(py, true)), + ], + )?; + let dbt_options = call_with( + &imported(py, MODELS, "DbtCliOptions")?, + &[ + ("project_dir", resolved.getattr("project_dir")?), + ("profiles_dir", resolved.getattr("profiles_dir")?), + ("target", resolved.getattr("target")?), + ], + )?; + let mut runner = args.value("dbt_runner")?; + if !runner.is_truthy()? { + runner = call_with( + &imported( + py, + "sqlbuild.integrations.dbt.classes.dbt_runner", + "DbtRunner", + )?, + &[("dbt_executable", args.value("dbt_executable")?)], + )?; + } + let argv: Vec = args.value("args")?.extract()?; + let result = call_with( + &runner.getattr("debug")?, + &[ + ("options", dbt_options), + ("args", PyTuple::new(py, dbt_debug_args(&argv))?.into_any()), + ], + )?; + let stdout = args.value("stdout_stream")?; + let stderr = args.value("stderr_stream")?; + stdout.call_method1("write", (result.getattr("stdout")?,))?; + stderr.call_method1("write", (result.getattr("stderr")?,))?; + stdout.call_method0("flush")?; + stderr.call_method0("flush")?; + result.getattr("returncode")?.extract() +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(execute_dbt_interop_from_project, module)?)?; + module.add_function(wrap_pyfunction!(plan_dbt_interop_from_project, module)?)?; + module.add_function(wrap_pyfunction!(debug_dbt_from_project, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs new file mode 100644 index 0000000000..8a0d48b7df --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs @@ -0,0 +1,324 @@ +//! dbt execution phases, live SQLBuild planning and executor handoff. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::build_command::support::{boolean, emit, none, text, tuple}; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, +}; +use crate::bindings::_helpers::dbt_interop::pipeline_prologue::progress; +use crate::bindings::_helpers::dbt_interop::plan_output::{connection_config, connection_hooks}; + +const MODELS: &str = "sqlbuild.integrations.dbt.models"; +const OPTIONS: (&[&str], &[&str]) = ( + &["request", "invocation"], + &[ + "compiled", + "plan", + "merged_dbt_argv", + "execution", + "manifest", + "failed_sqlbuild_model_names", + "plan_output", + "connection_config", + "message", + ], +); + +fn environment<'py>(args: &KeywordArguments<'py>) -> PyResult> { + let request = args.value("request")?; + let compiled = args.value("compiled")?; + call_with( + &imported(request.py(), MODELS, "DbtPlanEnvironment")?, + &[ + ("project_dir", request.getattr("project_dir")?), + ( + "discovered_inputs", + args.value("invocation")?.getattr("discovered_inputs")?, + ), + ("project", compiled.getattr("project")?), + ("adapter", compiled.getattr("adapter")?), + ("adapter_name", compiled.getattr("adapter_name")?), + ], + ) +} + +/// Resolve the selected SQLBuild work's effective connection configuration. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_dbt_connection_config<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read(py, "resolve_dbt_connection_config", options, OPTIONS)?; + connection_config(&environment(&args)?) +} + +/// Render the merged selection unless the command is producing JSON. +#[pyfunction] +#[pyo3(signature = (**options))] +fn write_dbt_execution_plan_text( + py: Python<'_>, + options: Option>, +) -> PyResult<()> { + let args = KeywordArguments::read(py, "write_dbt_execution_plan_text", options, OPTIONS)?; + let request = args.value("request")?; + if request.getattr("json_output")?.is_truthy()? { + return Ok(()); + } + let plan = args.value("plan")?; + let merged = args.value("merged_dbt_argv")?; + let display = if merged.is_none() { + plan + } else { + imported(py, "dataclasses", "replace")?.call( + (plan,), + Some(&keywords( + py, + &[ + ("dbt_command_argv", merged), + ("supplemental_dbt_command_argvs", tuple(py, &[])?), + ], + )?), + )? + }; + let limit = if request.getattr("verbose")?.is_truthy()? { + none(py) + } else { + pyo3::types::PyInt::new(py, super::constants::PLAN_DISPLAY_LIMIT).into_any() + }; + let rendered = call_with( + &imported(py, "sqlbuild._native", "render_dbt_interop_plan")?, + &[ + ("plan", display), + ("json_output", boolean(py, false)), + ("use_color", request.getattr("use_color")?), + ( + "display_options", + call_with( + &imported(py, "sqlbuild.presentation.models", "DisplayOptions")?, + &[("max_entries_per_section", limit)], + )?, + ), + ], + )?; + emit( + &args.value("invocation")?.getattr("output_stream")?, + &format!("{}\n\n", rendered.extract::()?), + ) +} + +/// Execute dbt without introducing any SQLBuild state callbacks. +#[pyfunction] +#[pyo3(signature = (**options))] +fn execute_dbt_without_state_tracking<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read(py, "execute_dbt_without_state_tracking", options, OPTIONS)?; + let request = args.value("request")?; + let invocation = args.value("invocation")?; + call_with( + &imported(py, "sqlbuild._native", "execute_dbt_commands")?, + &[ + ("runner", invocation.getattr("runner")?), + ("options", invocation.getattr("dbt_options")?), + ("merged_argv", args.value("merged_dbt_argv")?), + ("progress_stream", invocation.getattr("output_stream")?), + ("stdout_stream", invocation.getattr("dbt_output_stream")?), + ("use_color", request.getattr("use_color")?), + ( + "skip_message", + text(py, "Skipping dbt: no dbt work selected."), + ), + ("on_progress", request.getattr("on_progress")?), + ], + ) +} + +/// Write the summary of actual dbt execution events. +#[pyfunction] +#[pyo3(signature = (**options))] +fn write_dbt_execution_summary(py: Python<'_>, options: Option>) -> PyResult<()> { + let args = KeywordArguments::read(py, "write_dbt_execution_summary", options, OPTIONS)?; + let footer = call_with( + &imported( + py, + "sqlbuild._native", + "render_dbt_execution_summary_footer", + )?, + &[ + ( + "node_results", + args.value("execution")?.getattr("node_results")?, + ), + ("use_color", args.value("request")?.getattr("use_color")?), + ], + )?; + if !footer.is_none() { + emit( + &args.value("invocation")?.getattr("output_stream")?, + &format!("\n{}\n", footer.extract::()?), + )?; + } + Ok(()) +} + +/// Plan SQLBuild work after dbt using actual dbt failures as external blockers. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_sqlbuild_execution_plan_output<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "resolve_sqlbuild_execution_plan_output", + options, + OPTIONS, + )?; + let plan = args.value("plan")?; + if !plan.getattr("sqlbuild_skip_reason")?.is_none() { + return Ok(none(py)); + } + let selected = plan.getattr("selection")?.getattr("sqlbuild_model_names")?; + if !selected.is_truthy()? { + return Ok(none(py)); + } + let invocation = args.value("invocation")?; + let stream = invocation.getattr("output_stream")?; + emit(&stream, "\n")?; + let reporter = call_with( + &imported(py, "sqlbuild._native", "ConnectionProgressReporter")?, + &[ + ( + "adapter_name", + args.value("compiled")?.getattr("adapter_name")?, + ), + ("stream", stream), + ("blank_line_after_complete", boolean(py, true)), + ("use_color", args.value("request")?.getattr("use_color")?), + ], + )?; + let request = call_with( + &imported(py, MODELS, "DbtSqlbuildPlanRequest")?, + &[ + ("selected_model_names", selected), + ( + "sqlbuild_args", + invocation.getattr("effective_sqlbuild_args")?, + ), + ( + "external_blocked_model_names", + args.value("failed_sqlbuild_model_names")?, + ), + ], + )?; + call_with( + &imported(py, "sqlbuild._native", "build_sqlbuild_plan_output")?, + &[ + ("environment", environment(&args)?), + ("request", request), + ("hooks", connection_hooks(&reporter, none(py))?), + ], + ) +} + +fn execution_limits<'py>( + discovered: &Bound<'py, PyAny>, + target: Bound<'py, PyAny>, +) -> PyResult> { + let py = discovered.py(); + if target.is_none() { + return imported( + py, + "sqlbuild.spec.contracts.models", + "ExecutionLimitsConfig", + )? + .call0(); + } + call_with( + &imported(py, "sqlbuild._native", "resolve_target_config")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ("target_name", target), + ], + )? + .getattr("execution_limits") +} + +/// Execute the planned SQLBuild phase with the selected target's execution limits. +#[pyfunction] +#[pyo3(signature = (**options))] +fn run_dbt_sqlbuild_work<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read(py, "run_dbt_sqlbuild_work", options, OPTIONS)?; + let request = args.value("request")?; + let invocation = args.value("invocation")?; + let compiled = args.value("compiled")?; + let project = compiled.getattr("project")?; + let discovered = invocation.getattr("discovered_inputs")?; + let context = call_with( + &imported(py, "sqlbuild.cli.commands.models", "DbtSqlbuildWorkContext")?, + &[ + ("plan_output", args.value("plan_output")?), + ("connection_config", args.value("connection_config")?), + ("adapter", compiled.getattr("adapter")?), + ("adapter_name", compiled.getattr("adapter_name")?), + ("output_stream", invocation.getattr("output_stream")?), + ("use_color", request.getattr("use_color")?), + ( + "snapshots_config", + discovered.getattr("project_config")?.getattr("snapshots")?, + ), + ( + "execution_limits", + execution_limits(&discovered, project.getattr("effective_target_name")?)?, + ), + ], + )?; + call_with( + &imported(py, "sqlbuild._native", "execute_dbt_sqlbuild_work")?, + &[ + ("context", context), + ("command", request.getattr("command")?), + ("project", project), + ("project_dir", request.getattr("project_dir")?), + ("fail_fast", request.getattr("fail_fast")?), + ("verbose", request.getattr("verbose")?), + ], + ) +} + +/// Report a SQLBuild skip transition without writing machine output to stdout. +#[pyfunction] +#[pyo3(signature = (**options))] +fn write_sqlbuild_skip_notice(py: Python<'_>, options: Option>) -> PyResult<()> { + let args = KeywordArguments::read(py, "write_sqlbuild_skip_notice", options, OPTIONS)?; + emit(&args.value("invocation")?.getattr("output_stream")?, "\n")?; + progress( + &args.value("request")?.getattr("on_progress")?, + &args.value("message")?.extract::()?, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_dbt_connection_config, module)?)?; + module.add_function(wrap_pyfunction!(write_dbt_execution_plan_text, module)?)?; + module.add_function(wrap_pyfunction!( + execute_dbt_without_state_tracking, + module + )?)?; + module.add_function(wrap_pyfunction!(write_dbt_execution_summary, module)?)?; + module.add_function(wrap_pyfunction!( + resolve_sqlbuild_execution_plan_output, + module + )?)?; + module.add_function(wrap_pyfunction!(run_dbt_sqlbuild_work, module)?)?; + module.add_function(wrap_pyfunction!(write_sqlbuild_skip_notice, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs new file mode 100644 index 0000000000..f1c0637a8d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs @@ -0,0 +1,333 @@ +//! Discover, compile and resolve the combined dbt pipeline through its existing boundaries. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; +use std::time::Instant; + +const MODELS: &str = "sqlbuild.integrations.dbt.models"; +const COMPILE_OPTIONS: (&[&str], &[&str]) = ( + &[ + "project_dir", + "discovered_inputs", + "manifest", + "dbt_vars", + "no_sql_validation", + "on_progress", + ], + &[], +); +const MANIFEST_OPTIONS: (&[&str], &[&str]) = ( + &["runner", "dbt_options", "full_refresh", "on_progress"], + &[], +); +const PLAN_OPTIONS: (&[&str], &[&str]) = ( + &[ + "command", + "invocation", + "compiled", + "manifest", + "sqlbuild_executable", + "on_progress", + ], + &[], +); + +pub(crate) fn progress(callback: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { + if !callback.is_none() { + callback.call1((message,))?; + } + Ok(()) +} + +fn completed(callback: &Bound<'_, PyAny>, message: &str, start: Instant) -> PyResult<()> { + progress( + callback, + &format!("{message} ({:.2}s)", start.elapsed().as_secs_f64()), + ) +} + +/// Resolve the execution invocation without invoking dbt or SQLBuild work. +#[pyfunction] +pub(crate) fn resolve_dbt_execution_invocation<'py>( + request: &Bound<'py, PyAny>, +) -> PyResult> { + let py = request.py(); + let mut executable = request.getattr("dbt_executable")?; + if !executable.is_truthy()? { + executable = imported(py, "sqlbuild._native", "resolve_dbt_executable")?.call0()?; + } + let mut stream = request.getattr("progress_stream")?; + if !stream.is_truthy()? { + stream = py.import("sys")?.getattr("stdout")?; + } + let parsed = call_with( + &imported( + py, + "sqlbuild.integrations.dbt.main.cli._parse_execution_args", + "parse_dbt_execution_args", + )?, + &[ + ("command", request.getattr("command")?), + ("args", request.getattr("args")?), + ], + )?; + let routed = call_with( + &imported(py, "sqlbuild._native", "route_dbt_interop_args")?, + &[("command", request.getattr("command")?), ("parsed", parsed)], + )?; + let discovered = call_with( + &imported(py, "sqlbuild._native", "discover_project_inputs")?, + &[("project_dir", request.getattr("project_dir")?)], + )?; + let options = call_with( + &imported( + py, + "sqlbuild.integrations.dbt.main.config._resolve_plan_options", + "resolve_dbt_plan_options", + )?, + &[ + ("project_dir", request.getattr("project_dir")?), + ("discovered_inputs", discovered.clone()), + ("dbt_args", routed.getattr("dbt_args")?), + ], + )?; + let vars = call_with( + &imported( + py, + "sqlbuild.integrations.dbt.main.config._resolve_vars_mapping", + "resolve_dbt_vars_mapping", + )?, + &[ + ( + "project_config", + discovered.getattr("project_config")?.getattr("dbt")?, + ), + ( + "local_config", + discovered.getattr("local_config")?.getattr("dbt")?, + ), + ("dbt_args", routed.getattr("dbt_args")?), + ], + )?; + let mut runner = request.getattr("dbt_runner")?; + if !runner.is_truthy()? { + runner = call_with( + &imported( + py, + "sqlbuild.integrations.dbt.classes.dbt_runner", + "DbtRunner", + )?, + &[("dbt_executable", executable.clone())], + )?; + } + let dbt_stream = request.getattr("dbt_stdout_stream")?; + call_with( + &imported(py, MODELS, "DbtInteropInvocation")?, + &[ + ("dbt_executable", executable), + ("output_stream", stream.clone()), + ( + "dbt_output_stream", + if dbt_stream.is_truthy()? { + dbt_stream + } else { + stream + }, + ), + ("effective_sqlbuild_args", routed.getattr("sqlbuild_args")?), + ("routed", routed), + ("discovered_inputs", discovered), + ("dbt_options", options), + ("dbt_vars", vars), + ("runner", runner), + ], + ) +} + +/// Compile dbt and load the manifest after a successful subprocess result. +#[pyfunction] +#[pyo3(signature = (**options))] +fn load_compiled_dbt_manifest<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read(py, "load_compiled_dbt_manifest", options, MANIFEST_OPTIONS)?; + let callback = args.value("on_progress")?; + let start = Instant::now(); + progress(&callback, "Compiling dbt project...")?; + let result = call_with( + &args.value("runner")?.getattr("compile")?, + &[ + ("options", args.value("dbt_options")?), + ("full_refresh", args.value("full_refresh")?), + ], + )?; + if result.getattr("returncode")?.extract::()? != 0 { + let detail = imported(py, "sqlbuild._native", "dbt_failure_detail")?.call1((result,))?; + let error = imported( + py, + "sqlbuild.integrations.dbt.exceptions", + "DbtInteropRuntimeError", + )? + .call( + ("dbt compile failed",), + Some( + &crate::bindings::_helpers::compile_driver::python_calls::keywords( + py, + &[("help", detail)], + )?, + ), + )?; + return Err(PyErr::from_value(error)); + } + completed(&callback, "Compiled dbt project.", start)?; + let start = Instant::now(); + progress(&callback, "Loading dbt manifest...")?; + let path = call_with( + &imported( + py, + "sqlbuild.integrations.dbt.main.manifest._resolve_manifest_path", + "resolve_dbt_manifest_path", + )?, + &[("options", args.value("dbt_options")?)], + )?; + let manifest = call_with( + &imported(py, "sqlbuild._native", "load_dbt_manifest_index")?, + &[("manifest_path", path)], + )?; + completed(&callback, "Loaded dbt manifest.", start)?; + Ok(manifest) +} + +/// Compile SQLBuild models against dbt references using the existing adapter boundary. +#[pyfunction] +#[pyo3(signature = (**options))] +fn compile_dbt_interop_project<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read(py, "compile_dbt_interop_project", options, COMPILE_OPTIONS)?; + let callback = args.value("on_progress")?; + let start = Instant::now(); + progress(&callback, "Compiling SQLBuild project...")?; + let discovered = args.value("discovered_inputs")?; + let name = call_with( + &imported(py, "sqlbuild._native", "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + let adapter = call_with( + &imported( + py, + "sqlbuild.integrations.dbt.main.runtime._resolve_interop_adapter", + "resolve_dbt_interop_adapter", + )?, + &[ + ("adapter_name", name.clone()), + ("project_dir", args.value("project_dir")?), + ], + )?; + let resolver = call_with( + &imported( + py, + "sqlbuild.integrations.dbt.classes.dbt_compile_reference_resolver", + "DbtCompileReferenceResolver", + )?, + &[("dbt_manifest", args.value("manifest")?)], + )?; + let project = call_with( + &imported(py, "sqlbuild._native", "build_compiled_project")?, + &[ + ("discovered_inputs", discovered), + ("adapter", adapter.clone()), + ("no_sql_validation", args.value("no_sql_validation")?), + ("cli_vars", args.value("dbt_vars")?), + ("external_sql_reference_resolver", resolver), + ], + )?; + completed(&callback, "Compiled SQLBuild project.", start)?; + call_with( + &imported(py, MODELS, "DbtInteropCompiledProject")?, + &[ + ("adapter_name", name), + ("adapter", adapter), + ("project", project), + ], + ) +} + +/// Build the combined graph and resolve its dbt and SQLBuild selections. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_dbt_interop_plan<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read(py, "resolve_dbt_interop_plan", options, PLAN_OPTIONS)?; + let callback = args.value("on_progress")?; + let start = Instant::now(); + progress(&callback, "Building dbt interop graph...")?; + let project = args.value("compiled")?.getattr("project")?; + let graph = call_with( + &imported(py, "sqlbuild._native", "build_dbt_combined_graph")?, + &[ + ("manifest", args.value("manifest")?), + ("project", project.clone()), + ], + )?; + completed(&callback, "Built dbt interop graph.", start)?; + let start = Instant::now(); + progress(&callback, "Resolving dbt and SQLBuild selection...")?; + let invocation = args.value("invocation")?; + let routed = invocation.getattr("routed")?; + let tuple = py.import("builtins")?.getattr("tuple")?; + let command_args = call_with( + &imported(py, MODELS, "DbtInteropCommandArgs")?, + &[ + ("select", tuple.call1((routed.getattr("select")?,))?), + ("exclude", tuple.call1((routed.getattr("exclude")?,))?), + ( + "dbt_command_args", + tuple.call1((routed.getattr("dbt_args")?,))?, + ), + ( + "sqlbuild_command_args", + tuple.call1((invocation.getattr("effective_sqlbuild_args")?,))?, + ), + ("dbt_executable", invocation.getattr("dbt_executable")?), + ("sqlbuild_executable", args.value("sqlbuild_executable")?), + ], + )?; + let plan = call_with( + &imported(py, "sqlbuild._native", "plan_dbt_interop_command")?, + &[ + ("command", args.value("command")?), + ("project", project), + ("manifest", args.value("manifest")?), + ("graph", graph.clone()), + ("dbt_runner", invocation.getattr("runner")?), + ("dbt_options", invocation.getattr("dbt_options")?), + ("command_args", command_args), + ], + )?; + completed(&callback, "Resolved dbt and SQLBuild selection.", start)?; + call_with( + &imported(py, MODELS, "DbtInteropPlanResolution")?, + &[("graph", graph), ("plan", plan)], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_dbt_execution_invocation, module)?)?; + module.add_function(wrap_pyfunction!(load_compiled_dbt_manifest, module)?)?; + module.add_function(wrap_pyfunction!(compile_dbt_interop_project, module)?)?; + module.add_function(wrap_pyfunction!(resolve_dbt_interop_plan, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs new file mode 100644 index 0000000000..897a2de7c8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -0,0 +1,323 @@ +//! Plan selected SQLBuild work against the live adapter and attach it to the interop plan. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::build_command::support::{boolean, keyword_relay, none}; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, +}; + +const MODELS: &str = "sqlbuild.integrations.dbt.models"; +const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +const HOOKS: &str = "sqlbuild.runtime.contracts.models"; +const BUILD_OPTIONS: (&[&str], &[&str]) = (&["environment", "request", "hooks"], &[]); +const ATTACH_OPTIONS: (&[&str], &[&str]) = ( + &[ + "plan", + "project_dir", + "discovered_inputs", + "compiled", + "sqlbuild_args", + "connection_progress", + "on_progress", + ], + &[], +); + +pub(crate) fn connection_config<'py>( + environment: &Bound<'py, PyAny>, +) -> PyResult> { + let py = environment.py(); + let discovered = environment.getattr("discovered_inputs")?; + let raw = call_with( + &imported(py, "sqlbuild._native", "build_effective_connection_config")?, + &[("discovered_inputs", discovered.clone())], + )?; + let config = call_with( + &imported( + py, + "sqlbuild.integrations.dbt.main.profile._resolve_connection_config", + "resolve_connection_config", + )?, + &[ + ("raw_config", raw), + ("project_dir", environment.getattr("project_dir")?), + ("adapter_name", environment.getattr("adapter_name")?), + ("discovered_inputs", discovered.clone()), + ], + )?; + call_with( + &imported(py, "sqlbuild._native", "apply_command_warehouse")?, + &[ + ("config", config), + ("adapter_name", environment.getattr("adapter_name")?), + ("discovered_inputs", discovered), + ("project_dir", environment.getattr("project_dir")?), + ("selected_target", none(py)), + ], + ) +} + +fn flag_value<'py>(args: &Bound<'py, PyAny>, flag: &str) -> PyResult> { + if !args.contains(flag)? { + return Ok(none(args.py())); + } + let index: usize = args.call_method1("index", (flag,))?.extract()?; + if index + 1 < args.len()? { + args.get_item(index + 1) + } else { + Ok(none(args.py())) + } +} + +pub(crate) fn connection_hooks<'py>( + reporter: &Bound<'py, PyAny>, + callback: Bound<'py, PyAny>, +) -> PyResult> { + let py = reporter.py(); + let mut pairs = vec![("on_progress", callback)]; + if !reporter.is_none() { + pairs.push(( + "on_connection_start", + reporter.getattr("on_connection_start")?, + )); + for name in ["on_connection_complete", "on_connection_error"] { + pairs.push(( + name, + keyword_relay( + reporter.getattr(name)?, + &["connection_count", "elapsed_seconds"], + )?, + )); + } + } + call_with(&imported(py, HOOKS, "ConnectionHooks")?, &pairs) +} + +/// Open through the shared adapter boundary while retaining keyword-only timing callbacks. +fn open_connection<'py>( + environment: &Bound<'py, PyAny>, + config: Bound<'py, PyAny>, + hooks: &Bound<'py, PyAny>, +) -> PyResult> { + let py = environment.py(); + let mut pairs = vec![("on_connection_start", hooks.getattr("on_connection_start")?)]; + for name in ["on_connection_complete", "on_connection_error"] { + let callback = hooks.getattr(name)?; + pairs.push(( + name, + if callback.is_none() { + callback + } else { + keyword_relay(callback, &["connection_count", "elapsed_seconds"])? + }, + )); + } + let relayed = call_with(&imported(py, HOOKS, "ConnectionHooks")?, &pairs)?; + call_with( + &imported( + py, + "sqlbuild.runtime.contracts.main.open_connection", + "open_connection_with_hooks", + )?, + &[ + ("adapter", environment.getattr("adapter")?), + ("connection_config", config), + ("hooks", relayed), + ], + ) +} + +/// Build the SQLBuild plan from current warehouse facts, closing the custom adapter connection. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_sqlbuild_plan_output<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments = + KeywordArguments::read(py, "build_sqlbuild_plan_output", options, BUILD_OPTIONS)?; + let request = arguments.value("request")?; + let selected = request.getattr("selected_model_names")?; + if !selected.is_truthy()? { + return Ok(none(py)); + } + let environment = arguments.value("environment")?; + let hooks = arguments.value("hooks")?; + let connection = open_connection(&environment, connection_config(&environment)?, &hooks)?; + let planned = (|| { + let args = request.getattr("sqlbuild_args")?; + let deferral = call_with( + &imported(py, "sqlbuild._native", "resolve_model_deferral_inputs")?, + &[ + ("project", environment.getattr("project")?), + ( + "discovered_inputs", + environment.getattr("discovered_inputs")?, + ), + ("adapter", environment.getattr("adapter")?), + ("connection", connection.clone()), + ("defer_to", flag_value(&args, "--defer-to")?), + ], + )?; + let cursor = call_with( + &imported(py, PLANNER_MODELS, "CursorOverrides")?, + &[ + ("start_ts", flag_value(&args, "--start-cursor-ts")?), + ("end_ts", flag_value(&args, "--end-cursor-ts")?), + ("start_int", flag_value(&args, "--start-cursor-int")?), + ("end_int", flag_value(&args, "--end-cursor-int")?), + ], + )?; + let full_refresh = boolean(py, args.contains("--full-refresh")?); + let selection = call_with( + &imported(py, PLANNER_MODELS, "PlannerSelection")?, + &[("select", selected.clone())], + )?; + let overrides = call_with( + &imported(py, PLANNER_MODELS, "PlannerOverrides")?, + &[ + ("cursor_overrides", cursor), + ("full_refresh", full_refresh.clone()), + ( + "external_blocked_model_names", + request.getattr("external_blocked_model_names")?, + ), + ], + )?; + let discovered = environment.getattr("discovered_inputs")?; + let result = call_with( + &imported( + py, + "sqlbuild.compiler.planner.main.execution.execution", + "build_execution_plan", + )?, + &[ + ("project", environment.getattr("project")?), + ("adapter", environment.getattr("adapter")?), + ("connection", connection.clone()), + ("selection", selection), + ("overrides", overrides), + ("deferral", deferral), + ( + "policies", + imported(py, PLANNER_MODELS, "PlannerPolicies")?.call0()?, + ), + ("on_progress", hooks.getattr("on_progress")?), + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + ); + match result { + Err(error) + if error.is_instance( + py, + &imported( + py, + "sqlbuild.compiler.planner.exceptions", + "PlannerInputError", + )?, + ) => + { + call_with( + &imported( + py, + "sqlbuild.compiler.planner.main.execution.display_plan", + "build_display_only_sqlbuild_plan", + )?, + &[ + ("project", environment.getattr("project")?), + ("selected_model_names", selected), + ("full_refresh", full_refresh), + ], + ) + } + result => result, + } + })(); + environment + .getattr("adapter")? + .call_method1("close", (connection,))?; + planned +} + +/// Attach live SQLBuild planning when the selection contains SQLBuild models. +#[pyfunction] +#[pyo3(signature = (**options))] +fn attach_sqlbuild_plan_output<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read(py, "attach_sqlbuild_plan_output", options, ATTACH_OPTIONS)?; + let compiled = args.value("compiled")?; + let plan = args.value("plan")?; + let environment = call_with( + &imported(py, MODELS, "DbtPlanEnvironment")?, + &[ + ("project_dir", args.value("project_dir")?), + ("discovered_inputs", args.value("discovered_inputs")?), + ("project", compiled.getattr("project")?), + ("adapter", compiled.getattr("adapter")?), + ("adapter_name", compiled.getattr("adapter_name")?), + ], + )?; + let request = call_with( + &imported(py, MODELS, "DbtSqlbuildPlanRequest")?, + &[ + ( + "selected_model_names", + plan.getattr("selection")?.getattr("sqlbuild_model_names")?, + ), + ("sqlbuild_args", args.value("sqlbuild_args")?), + ], + )?; + let output = build_sqlbuild_plan_output( + py, + Some(keywords( + py, + &[ + ("environment", environment), + ("request", request), + ( + "hooks", + connection_hooks( + &args.value("connection_progress")?, + args.value("on_progress")?, + )?, + ), + ], + )?), + )?; + if output.is_none() { + return Ok(plan); + } + imported(py, "dataclasses", "replace")?.call( + (plan,), + Some(&keywords(py, &[("sqlbuild_plan_output", output)])?), + ) +} + +/// Prefer stderr for dbt subprocess failure details, retaining Python whitespace semantics. +#[pyfunction] +fn dbt_failure_detail<'py>(result: &Bound<'py, PyAny>) -> PyResult> { + let stderr = result.getattr("stderr")?; + let detail = if stderr.is_truthy()? { + stderr + } else { + result.getattr("stdout")? + } + .call_method0("strip")?; + if detail.is_truthy()? { + Ok(detail) + } else { + Ok(none(result.py())) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_sqlbuild_plan_output, module)?)?; + module.add_function(wrap_pyfunction!(attach_sqlbuild_plan_output, module)?)?; + module.add_function(wrap_pyfunction!(dbt_failure_detail, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs index edbec1eca6..fc5cb1f0ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs @@ -13,5 +13,11 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { manifest::register(module)?; orchestration::register(module)?; plan::register(module)?; - profile::register(module) + profile::register(module)?; + super::runtime_execution::register(module)?; + super::pipeline_prologue::register(module)?; + super::plan_output::register(module)?; + super::sqlbuild_work::register(module)?; + super::pipeline_phases::register(module)?; + super::pipeline::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs new file mode 100644 index 0000000000..75905f28f0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs @@ -0,0 +1,258 @@ +//! Execute dbt through the subprocess boundary and finalize results in artifact order. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::dbt_interop::runtime_results::{append_missing, result_ids}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyInt, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use std::collections::HashSet; +use std::time::Instant; + +const OPTIONS: (&[&str], &[&str]) = ( + &[ + "runner", + "options", + "merged_argv", + "progress_stream", + "stdout_stream", + "use_color", + ], + &[ + "skip_message", + "expected_nodes", + "on_node_result", + "on_progress", + ], +); + +fn write(stream: &Bound<'_, PyAny>, text: &str) -> PyResult<()> { + stream.call_method1("write", (text,))?; + stream.call_method0("flush")?; + Ok(()) +} + +fn expected_total<'py>( + arguments: &KeywordArguments<'py>, + argv: &Bound<'py, PyAny>, +) -> PyResult> { + let py = argv.py(); + let ls = call_with( + &imported(py, "sqlbuild._native", "dbt_execution_ls_argv")?, + &[("argv", argv.clone())], + )?; + if ls.is_none() { + return Ok(py.None().into_bound(py)); + } + let status = call_with( + &imported(py, "sqlbuild._native", "TransientStatusReporter")?, + &[ + ("stream", arguments.value("progress_stream")?), + ("use_color", arguments.value("use_color")?), + ], + )?; + status.call_method1("start", ("Resolving dbt execution selection...",))?; + let start = Instant::now(); + let result = call_with( + &arguments.value("runner")?.getattr("invoke")?, + &[ + ( + "argv", + py.import("builtins")?.getattr("tuple")?.call1((ls,))?, + ), + ("cwd", arguments.value("options")?.getattr("project_dir")?), + ], + ); + call_with( + &status.getattr("complete")?, + &[( + "message", + PyString::new( + py, + &format!( + "Resolved dbt execution selection. ({:.2}s)", + start.elapsed().as_secs_f64() + ), + ) + .into_any(), + )], + )?; + let result = result?; + if result.getattr("returncode")?.extract::()? != 0 { + return Ok(py.None().into_bound(py)); + } + let nodes = call_with( + &imported(py, "sqlbuild._native", "parse_dbt_ls_json_lines")?, + &[("stdout", result.getattr("stdout")?)], + )?; + Ok(PyInt::new(py, nodes.len()?).into_any()) +} + +fn progress(arguments: &KeywordArguments<'_>, message: &str) -> PyResult<()> { + let callback = arguments.value("on_progress")?; + if !callback.is_none() { + callback.call1((message,))?; + } + Ok(()) +} + +/// Execute selected dbt work and backfill results absent from its JSON event stream. +#[pyfunction] +#[pyo3(signature = (**options))] +fn execute_dbt_commands<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments = KeywordArguments::read(py, "execute_dbt_commands", options, OPTIONS)?; + let argv = arguments.value("merged_argv")?; + let output = arguments.value("progress_stream")?; + let style = call_with( + &imported(py, "sqlbuild.presentation.classes.cli_style", "CliStyle")?, + &[("use_color", arguments.value("use_color")?)], + )?; + let result_type = imported( + py, + "sqlbuild.integrations.dbt.models", + "DbtCommandExecutionResult", + )?; + if argv.is_none() { + let message = arguments.value("skip_message")?; + let message = if message.is_none() { + PyString::new(py, "Skipping dbt: no dbt work selected.").into_any() + } else { + message + }; + write( + &output, + &format!( + "{}\n", + style + .call_method1("muted", (message,))? + .extract::()? + ), + )?; + return call_with( + &result_type, + &[("returncode", PyInt::new(py, 0).into_any())], + ); + } + let name: String = py + .import("pathlib")? + .getattr("Path")? + .call1((argv.get_item(0)?,))? + .getattr("name")? + .extract()?; + let detail = if argv.len()? >= super::constants::COMMAND_AND_SUBCOMMAND_ARGUMENT_COUNT { + format!("{name} {}", argv.get_item(1)?.extract::()?) + } else { + name + }; + write( + &output, + &format!( + "{} {}\n\n", + style + .call_method1("dbt_execution_label", ("dbt execution",))? + .extract::()?, + style + .call_method1("muted", (detail,))? + .extract::()? + ), + )?; + let expected_nodes = arguments.value("expected_nodes")?; + let expected: Option> = if expected_nodes.is_none() { + None + } else { + Some(result_ids(&expected_nodes)?) + }; + let total = if let Some(ids) = &expected { + PyInt::new(py, ids.len()).into_any() + } else { + expected_total(&arguments, &argv)? + }; + let dbt_options = arguments.value("options")?; + let mut target = dbt_options.getattr("target_path")?; + let project_dir = dbt_options.getattr("project_dir")?; + if target.is_none() && !project_dir.is_none() { + target = project_dir.call_method1("__truediv__", ("target",))?; + } + let (returncode, streamed): (Bound<'py, PyAny>, Bound<'py, PyAny>) = call_with( + &imported( + py, + "sqlbuild.integrations.dbt.main.runtime._execute_json_event_stream", + "execute_dbt_json_event_stream", + )?, + &[ + ("argv", argv), + ("cwd", project_dir), + ("stream", arguments.value("stdout_stream")?), + ("use_color", arguments.value("use_color")?), + ("target_path", target.clone()), + ("display_total", total.clone()), + ("on_node_result", arguments.value("on_node_result")?), + ], + )? + .extract()?; + progress(&arguments, "Finalizing dbt run...")?; + let streamed_ids = result_ids(&streamed)?; + let mut filtered: Vec> = Vec::new(); + for result in streamed.try_iter()? { + let result = result?; + let id: String = result.getattr("unique_id")?.extract()?; + if expected.as_ref().is_none_or(|ids| ids.contains(&id)) { + filtered.push(result); + } + } + let results = append_missing( + &PyTuple::new(py, filtered)?.into_any(), + &target, + expected.as_ref(), + )?; + let mut appended: Vec> = Vec::new(); + for result in results.try_iter()? { + let result = result?; + if !streamed_ids.contains(&result.getattr("unique_id")?.extract::()?) { + appended.push(result); + } + } + let count = results.len()? - appended.len(); + for (offset, result) in appended.into_iter().enumerate() { + let rendered = call_with( + &imported(py, "sqlbuild._native", "render_dbt_node_result")?, + &[ + ("result", result.clone()), + ("use_color", style.getattr("use_color")?), + ( + "display_index", + PyInt::new(py, count + offset + 1).into_any(), + ), + ( + "display_total", + if total.is_none() { + PyInt::new(py, results.len()?).into_any() + } else { + total.clone() + }, + ), + ], + )?; + write( + &arguments.value("stdout_stream")?, + &rendered.extract::()?, + )?; + let callback = arguments.value("on_node_result")?; + if !callback.is_none() { + callback.call1((result,))?; + } + } + progress(&arguments, "Finalized dbt run.")?; + call_with( + &result_type, + &[("returncode", returncode), ("node_results", results)], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(execute_dbt_commands, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs new file mode 100644 index 0000000000..8aaa940d20 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs @@ -0,0 +1,145 @@ +//! Backfill streamed dbt results using Python's JSON and UTF-8 artifact contract. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use pyo3::exceptions::PyOSError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{ + PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyListMethods, PyString, PyTuple, +}; +use std::collections::HashSet; + +const MODELS: &str = "sqlbuild.integrations.dbt.models"; + +pub(crate) fn result_ids(results: &Bound<'_, PyAny>) -> PyResult> { + results + .try_iter()? + .map(|result| result?.getattr("unique_id")?.extract()) + .collect() +} + +fn messages<'py>(raw: &Bound<'py, PyDict>) -> PyResult> { + let py = raw.py(); + let text = if let Some(message) = raw + .get_item("message")? + .filter(|value| value.is_instance_of::()) + { + let stripped = message.call_method0("strip")?; + stripped.is_truthy()?.then_some(stripped) + } else { + None + }; + let text = if text.is_some() { + text + } else if let Some(failures) = raw + .get_item("failures")? + .filter(|value| value.is_instance_of::()) + { + if failures.gt(0)? { + Some(PyString::new(py, &format!("{} failure(s)", failures.str()?)).into_any()) + } else { + None + } + } else { + None + }; + let Some(text) = text else { + return PyTuple::new(py, Vec::>::new()); + }; + PyTuple::new( + py, + [call_with( + &imported(py, MODELS, "DbtNodeMessage")?, + &[ + ("level", PyString::new(py, "error").into_any()), + ("message", text), + ], + )?], + ) +} + +pub(crate) fn append_missing<'py>( + results: &Bound<'py, PyAny>, + target: &Bound<'py, PyAny>, + expected: Option<&HashSet>, +) -> PyResult> { + let py = results.py(); + if target.is_none() { + return Ok(results.clone()); + } + let path = target.call_method1("__truediv__", ("run_results.json",))?; + if !path.call_method0("exists")?.is_truthy()? { + return Ok(results.clone()); + } + let existing = result_ids(results)?; + let json = py.import("json")?; + let payload = (|| { + let text = call_with( + &path.getattr("read_text")?, + &[("encoding", PyString::new(py, "utf-8").into_any())], + )?; + json.getattr("loads")?.call1((text,)) + })(); + let payload = match payload { + Ok(payload) => payload, + Err(error) + if error.is_instance_of::(py) + || error.is_instance(py, &json.getattr("JSONDecodeError")?) => + { + return Ok(results.clone()); + } + Err(error) => return Err(error), + }; + let Ok(payload) = payload.cast::() else { + return Ok(results.clone()); + }; + let Some(raw_results) = payload.get_item("results")? else { + return Ok(results.clone()); + }; + let Ok(raw_results) = raw_results.cast::() else { + return Ok(results.clone()); + }; + let mut appended = results.try_iter()?.collect::>>()?; + for (index, raw) in raw_results.iter().enumerate() { + let Ok(raw) = raw.cast::() else { + continue; + }; + let Some(unique_id) = raw + .get_item("unique_id")? + .filter(|value| value.is_instance_of::()) + else { + continue; + }; + let id: String = unique_id.extract()?; + if existing.contains(&id) || expected.is_some_and(|ids| !ids.contains(&id)) { + continue; + } + let resource_type = id.split_once('.').map_or("node", |(kind, _)| kind); + let name = id.rsplit('.').next().unwrap_or(&id); + let status = raw + .get_item("status")? + .filter(|value| value.is_instance_of::()) + .unwrap_or_else(|| PyString::new(py, "unknown").into_any()); + let time = if let Some(value) = raw + .get_item("execution_time")? + .filter(|value| value.is_instance_of::() || value.is_instance_of::()) + { + py.import("builtins")?.getattr("float")?.call1((value,))? + } else { + py.None().into_bound(py) + }; + appended.push(call_with( + &imported(py, MODELS, "DbtNodeExecutionResult")?, + &[ + ("unique_id", unique_id), + ("resource_type", PyString::new(py, resource_type).into_any()), + ("node_name", PyString::new(py, name).into_any()), + ("status", status), + ("index", PyInt::new(py, index + 1).into_any()), + ("total", PyInt::new(py, raw_results.len()).into_any()), + ("execution_time", time), + ("messages", messages(raw)?.into_any()), + ], + )?); + } + Ok(PyTuple::new(py, appended)?.into_any()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs new file mode 100644 index 0000000000..2c82dfb7d8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs @@ -0,0 +1,227 @@ +//! Enforce interop plan safety before entering the Python build executor. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::build_command::support::{ + boolean, emit, joined, keyword_relay, none, text, tuple, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const WORK_OPTIONS: (&[&str], &[&str]) = ( + &[ + "context", + "command", + "project", + "project_dir", + "fail_fast", + "verbose", + ], + &[], +); +const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; +const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const COMMAND_CONSTANTS: &str = "sqlbuild.cli.commands.constants"; + +/// Interop has no allow flags; destructive work uses the ordinary interactive safety gates. +#[pyfunction] +fn dbt_interop_plan_safety_gates(py: Python<'_>) -> PyResult> { + call_with( + &imported(py, COMMAND_MODELS, "PlanSafetyGates")?, + &[ + ("allow_missing_migration_origin", boolean(py, false)), + ("allow_snapshot_full_refresh", boolean(py, false)), + ("allow_table_type_downgrade", boolean(py, false)), + ("allow_retention_decrease", boolean(py, false)), + ( + "missing_origin_help", + imported(py, COMMAND_CONSTANTS, "MISSING_ORIGIN_DBT_HELP")?, + ), + ( + "snapshot_full_refresh_help", + imported(py, COMMAND_CONSTANTS, "SNAPSHOT_FULL_REFRESH_DBT_HELP")?, + ), + ( + "table_type_downgrade_help", + imported(py, COMMAND_CONSTANTS, "TABLE_TYPE_DOWNGRADE_DBT_HELP")?, + ), + ( + "retention_decrease_help", + imported(py, COMMAND_CONSTANTS, "RETENTION_DECREASE_DBT_HELP")?, + ), + ( + "execution_limit_note", + imported(py, COMMAND_CONSTANTS, "EXECUTION_LIMIT_DBT_NOTE")?, + ), + ], + ) +} + +/// Execute SQLBuild work only after its plan safety policies succeed. +#[pyfunction] +#[pyo3(signature = (**options))] +fn execute_dbt_sqlbuild_work<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let args = KeywordArguments::read(py, "execute_dbt_sqlbuild_work", options, WORK_OPTIONS)?; + let context = args.value("context")?; + let project = args.value("project")?; + let plan = context.getattr("plan_output")?; + let output = context.getattr("output_stream")?; + let color = context.getattr("use_color")?; + call_with( + &imported(py, "sqlbuild._native", "enforce_plan_safety_policies")?, + &[ + ("plan", plan.clone()), + ("gates", dbt_interop_plan_safety_gates(py)?), + ("snapshots_config", context.getattr("snapshots_config")?), + ("target_name", project.getattr("effective_target_name")?), + ("execution_limits", context.getattr("execution_limits")?), + ( + "streams", + tuple(py, &[py.import("sys")?.getattr("stdin")?, output.clone()])?, + ), + ], + )?; + let callbacks = call_with( + &imported(py, "sqlbuild._native", "BuildProgressCallbacks")?, + &[ + ("plan", plan.clone()), + ("use_color", color.clone()), + ("verbose", args.value("verbose")?), + ("debug", boolean(py, false)), + ], + )?; + let command = args.value("command")?; + let kinds = imported(py, "sqlbuild.integrations.dbt.types", "DbtInteropCommand")?; + let display = if command.eq(kinds.getattr("RUN")?)? { + "sqb build --no-tests --no-audits".to_owned() + } else { + format!("sqb {}", command.getattr("value")?.extract::()?) + }; + let concurrency = project.getattr("settings")?.getattr("concurrency")?; + let header = call_with( + &imported(py, "sqlbuild._native", "format_execution_header")?, + &[ + ("command", text(py, &display)), + ("target", none(py)), + ("concurrency", concurrency.clone()), + ], + )?; + let style = call_with( + &imported(py, "sqlbuild.presentation.classes.cli_style", "CliStyle")?, + &[("use_color", color.clone())], + )?; + emit( + &output, + &format!( + "{} {}\n\n", + style + .call_method1("object_name", ("SQLBuild execution",))? + .extract::()?, + style + .call_method1("muted", (header,))? + .extract::()? + ), + )?; + let connection = call_with( + &imported(py, "sqlbuild._native", "ConnectionProgressReporter")?, + &[ + ("adapter_name", context.getattr("adapter_name")?), + ("stream", output.clone()), + ("blank_line_after_complete", boolean(py, true)), + ("use_color", color.clone()), + ], + )?; + let build = command.eq(kinds.getattr("BUILD")?)?; + let runtime = call_with( + &imported(py, BUILD_MODELS, "BuildRuntimeParams")?, + &[ + ("run_id", project.getattr("run_id")?), + ("run_tests", boolean(py, build)), + ("run_audits", boolean(py, build)), + ("fail_fast", args.value("fail_fast")?), + ("max_concurrency", concurrency), + ("use_color", color.clone()), + ], + )?; + let hooks = call_with( + &imported(py, BUILD_MODELS, "BuildCallbacks")?, + &[ + ( + "on_node_start", + keyword_relay( + callbacks.getattr("on_node_start")?, + &["name", "resource_kind"], + )?, + ), + ("on_node_complete", callbacks.getattr("on_node_complete")?), + ("on_sub_progress", callbacks.getattr("on_sub_progress")?), + ( + "on_connection_start", + connection.getattr("on_connection_start")?, + ), + ( + "on_connection_complete", + keyword_relay( + connection.getattr("on_connection_complete")?, + &["connection_count", "elapsed_seconds"], + )?, + ), + ( + "on_connection_error", + keyword_relay( + connection.getattr("on_connection_error")?, + &["connection_count", "elapsed_seconds"], + )?, + ), + ], + )?; + let result = call_with( + &imported( + py, + "sqlbuild.executor.pipeline.main.run", + "run_build_pipeline", + )?, + &[ + ("plan", plan.clone()), + ("connection_config", context.getattr("connection_config")?), + ("adapter", context.getattr("adapter")?), + ("settings", project.getattr("settings")?), + ("runtime", runtime), + ("callbacks", hooks), + ], + )?; + call_with( + &imported(py, "sqlbuild._native", "write_runtime_target")?, + &[ + ( + "target_dir", + joined(&args.value("project_dir")?, &["target"])?, + ), + ("plan_output", plan), + ("result", result.clone()), + ], + )?; + let footer = call_with( + &imported(py, "sqlbuild._native", "format_build_footer")?, + &[ + ("result", result.clone()), + ("elapsed", callbacks.getattr("elapsed")?), + ("use_color", color), + ], + )?; + emit(&output, &format!("\n{}\n", footer.extract::()?))?; + Ok(i64::from(!result.getattr("status")?.eq( + imported(py, "sqlbuild.executor.build.types", "BuildStatus")?.getattr("SUCCESS")?, + )?)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(dbt_interop_plan_safety_gates, module)?)?; + module.add_function(wrap_pyfunction!(execute_dbt_sqlbuild_work, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs index 5b9d3a77aa..d4356bf2a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::format_command::files::{ }; const NATIVE_MODULE: &str = "sqlbuild._native"; -const FIXES_MODULE: &str = "sqlbuild.lint._helpers.fixes"; +const FIXES_MODULE: &str = "sqlbuild._native"; const FIXTURE_AUTOFIX_MODULE: &str = "sqlbuild.compiler.planner.classes.fixture_null_autofix"; const LINT_ERRORS_MODULE: &str = "sqlbuild.lint.exceptions"; const DESCRIPTION_PRESENT: &str = "description-present"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 08ef0a6c01..f7989db28e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -47,6 +47,7 @@ pub(crate) mod plan_command; pub(crate) mod plan_output; pub(crate) mod planner_changes; pub(crate) mod planner_identity; +pub(crate) mod planner_migrations; pub(crate) mod planner_scope; pub(crate) mod planner_selection; pub(crate) mod planner_strategy; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs new file mode 100644 index 0000000000..a48b0a41be --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs @@ -0,0 +1,148 @@ +//! Convert projection facts and construct the existing Python hint models. + +use std::collections::BTreeSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_migrations::main::{ + identical_renames as native_renames, rename_hints as native_hints, +}; +use sqlbuild_analysis::planner_migrations::models::Projection; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; + +fn projections(shape: &Bound<'_, PyAny>) -> PyResult> { + let mut result: Vec = Vec::new(); + for item in shape.getattr("projections")?.try_iter()? { + let item = item?; + result.push(Projection { + name: item.getattr("name")?.extract()?, + key: item.getattr("key")?.extract()?, + expression: item.getattr("expression")?.extract()?, + references: item.getattr("references")?.extract()?, + }); + } + Ok(result) +} + +/// Unique automatic column renames whose full query change is explained. +#[pyfunction] +#[pyo3(signature = (**options))] +fn identical_renames<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments = KeywordArguments::read( + py, + "identical_renames", + options, + ( + &[ + "previous", + "current", + "excluded", + "declared", + "input_columns", + ], + &[], + ), + )?; + let previous = arguments.value("previous")?; + let current = arguments.value("current")?; + let excluded: BTreeSet = arguments.value("excluded")?.extract()?; + let previous_projections = projections(&previous)?; + let current_projections = projections(¤t)?; + let mut proof = |pairs: &[(String, String)]| -> PyResult { + let renames = PyDict::new(py); + for item in arguments + .value("declared")? + .call_method0("items")? + .try_iter()? + { + let item = item?; + renames.set_item(item.get_item(0)?, item.get_item(1)?)?; + } + for (old, new) in pairs { + renames.set_item(old, new)?; + } + super::projections::explain( + &previous, + ¤t, + &renames.into_any(), + &arguments.value("input_columns")?, + ) + }; + PyTuple::new( + py, + native_renames::identical_renames( + (&previous_projections, ¤t_projections), + &excluded, + &mut proof, + )?, + ) +} + +/// Explicit migration hints for newly added columns resembling removed live columns. +#[pyfunction] +#[pyo3(signature = (**options))] +fn rename_hints<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments = KeywordArguments::read( + py, + "rename_hints", + options, + ( + &[ + "model_name", + "previous", + "current", + "excluded", + "live_columns", + ], + &[], + ), + )?; + let previous = arguments.value("previous")?; + let current = arguments.value("current")?; + let hints = native_hints::rename_hints( + (&projections(&previous)?, &projections(¤t)?), + &arguments.value("excluded")?.extract()?, + &arguments.value("live_columns")?.extract()?, + previous.getattr("body")?.eq(current.getattr("body")?)?, + ); + let mut result: Vec> = Vec::new(); + for hint in hints { + result.push(call_with( + &imported(py, MODELS_MODULE, "ColumnRenameHint")?, + &[ + ("model_name", arguments.value("model_name")?), + ( + "added_column", + hint.added_column.into_pyobject(py)?.into_any(), + ), + ( + "candidate_columns", + PyTuple::new(py, hint.candidate_columns)?.into_any(), + ), + ( + "identical", + hint.identical.into_pyobject(py)?.to_owned().into_any(), + ), + ], + )?); + } + PyTuple::new(py, result) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(identical_renames, module)?)?; + module.add_function(wrap_pyfunction!(rename_hints, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/errors.rs new file mode 100644 index 0000000000..2f26486686 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/errors.rs @@ -0,0 +1,11 @@ +//! Preserve input lookup exceptions across the pure proof callback. + +use pyo3::{PyErr, exceptions::PyValueError}; + +pub(crate) struct ProofError(pub(crate) PyErr); + +impl From for ProofError { + fn from(message: String) -> Self { + Self(PyValueError::new_err(message)) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs new file mode 100644 index 0000000000..3bc0aa1cf1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs @@ -0,0 +1,69 @@ +//! Migration identity binding with unchanged metadata and hash format. + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_migrations::main::{migration_fingerprint, migration_ref_names}; +use std::collections::BTreeMap; + +/// Hash normalized SQL and non-storage metadata through rename identities. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_migration_fingerprint( + py: Python<'_>, + options: Option>, +) -> PyResult> { + let arguments = KeywordArguments::read( + py, + "build_migration_fingerprint", + options, + ( + &["query_sql", "metadata_json", "ref_identities", "dialect"], + &[], + ), + )?; + let mut identities: BTreeMap = BTreeMap::new(); + for item in arguments + .value("ref_identities")? + .call_method0("items")? + .try_iter()? + { + let (key, value): (String, String) = item?.extract()?; + identities.insert(key, value); + } + let sql: String = arguments.value("query_sql")?.extract()?; + let metadata: String = arguments.value("metadata_json")?.extract()?; + let dialect: Option = arguments.value("dialect")?.extract()?; + migration_fingerprint::migration_fingerprint( + &sql, + &metadata, + &identities, + dialect + .as_deref() + .filter(|value| !value.is_empty()) + .unwrap_or("generic"), + ) + .map_err(PyValueError::new_err) +} + +/// Every name whose rename identity can change the migration fingerprint. +#[pyfunction] +#[pyo3(signature = (*, query_sql, metadata_json))] +fn migration_fingerprint_ref_names<'py>( + py: Python<'py>, + query_sql: &str, + metadata_json: &str, +) -> PyResult> { + PyFrozenSet::new( + py, + migration_ref_names::migration_ref_names(query_sql, metadata_json) + .map_err(PyValueError::new_err)?, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_migration_fingerprint, module)?)?; + module.add_function(wrap_pyfunction!(migration_fingerprint_ref_names, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs new file mode 100644 index 0000000000..72ecfd1d90 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod columns; +mod errors; +mod fingerprint; +mod projections; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs new file mode 100644 index 0000000000..b872bedc9a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs @@ -0,0 +1,148 @@ +//! Adapt existing Python query shape models to pure migration proofs. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_migrations::errors::ProofError; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_migrations::main::{query_shape, renames_explain_change as proof}; +use sqlbuild_analysis::planner_migrations::models::{QueryProjection, QueryShape}; + +const MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; + +fn read_shape(shape: &Bound<'_, PyAny>) -> PyResult { + let mut projections: Vec = Vec::new(); + for item in shape.getattr("projections")?.try_iter()? { + let item = item?; + projections.push(QueryProjection { + name: item.getattr("name")?.extract()?, + expression: item.getattr("expression")?.extract()?, + references: item.getattr("references")?.extract()?, + passthrough: item.getattr("passthrough")?.extract()?, + }); + } + Ok(QueryShape { + projections, + body: shape.getattr("body")?.extract()?, + alias_clauses: shape.getattr("alias_clauses")?.extract()?, + positional: shape.getattr("positional")?.extract()?, + input_relations: shape.getattr("input_relations")?.extract()?, + local_input_columns: shape.getattr("local_input_columns")?.extract()?, + clause_references: shape.getattr("clause_references")?.extract()?, + }) +} + +/// Named projections of one plain SELECT, or None when unknowable. +#[pyfunction] +#[pyo3(signature = (*, query_sql, dialect))] +fn parse_query_shape<'py>( + py: Python<'py>, + query_sql: &str, + dialect: Option<&str>, +) -> PyResult> { + let Some(shape) = query_shape::query_shape( + query_sql, + dialect + .filter(|value| !value.is_empty()) + .unwrap_or("generic"), + ) + .map_err(PyValueError::new_err)? + else { + return Ok(py.None().into_bound(py)); + }; + let mut projections: Vec> = Vec::new(); + for item in shape.projections { + projections.push(call_with( + &imported(py, MODELS_MODULE, "QueryProjection")?, + &[ + ("name", item.name.into_pyobject(py)?.into_any()), + ("expression", item.expression.into_pyobject(py)?.into_any()), + ( + "references", + PyFrozenSet::new(py, item.references)?.into_any(), + ), + ( + "passthrough", + item.passthrough.into_pyobject(py)?.to_owned().into_any(), + ), + ], + )?); + } + let relations = match shape.input_relations { + Some(relations) => PyTuple::new(py, relations)?.into_any(), + None => py.None().into_bound(py), + }; + call_with( + &imported(py, MODELS_MODULE, "QueryShape")?, + &[ + ("projections", PyTuple::new(py, projections)?.into_any()), + ("body", shape.body.into_pyobject(py)?.into_any()), + ( + "alias_clauses", + shape.alias_clauses.into_pyobject(py)?.into_any(), + ), + ( + "positional", + shape.positional.into_pyobject(py)?.to_owned().into_any(), + ), + ("input_relations", relations), + ( + "local_input_columns", + PyFrozenSet::new(py, shape.local_input_columns)?.into_any(), + ), + ( + "clause_references", + PyFrozenSet::new(py, shape.clause_references)?.into_any(), + ), + ], + ) +} + +pub(crate) fn explain( + previous: &Bound<'_, PyAny>, + current: &Bound<'_, PyAny>, + renames: &Bound<'_, PyAny>, + input_columns: &Bound<'_, PyAny>, +) -> PyResult { + let mut pairs: Vec<(String, String)> = Vec::new(); + for item in renames.call_method0("items")?.try_iter()? { + pairs.push(item?.extract::<(String, String)>()?); + } + proof::renames_explain_change( + (&read_shape(previous)?, &read_shape(current)?), + &pairs, + |kind, name| { + input_columns + .call1((kind, name)) + .and_then(|columns| columns.extract()) + .map_err(ProofError) + }, + ) + .map_err(|error| error.0) +} + +/// Whether output renaming explains the whole query change. +#[pyfunction] +#[pyo3(signature = (**options))] +fn renames_explain_change(py: Python<'_>, options: Option>) -> PyResult { + let arguments = KeywordArguments::read( + py, + "renames_explain_change", + options, + (&["previous", "current", "renames", "input_columns"], &[]), + )?; + explain( + &arguments.value("previous")?, + &arguments.value("current")?, + &arguments.value("renames")?, + &arguments.value("input_columns")?, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(parse_query_shape, module)?)?; + module.add_function(wrap_pyfunction!(renames_explain_change, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs new file mode 100644 index 0000000000..0bcd6568d4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs @@ -0,0 +1,9 @@ +//! Register pure migration helpers. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + super::columns::register(module)?; + super::projections::register(module)?; + super::fingerprint::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_facts.rs new file mode 100644 index 0000000000..8fe0ca9604 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_facts.rs @@ -0,0 +1,101 @@ +//! Index Python compiler fact views for native Rule-fix verification. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const LINT_MODELS: &str = "sqlbuild.lint.models"; + +/// Index a compilation by model key and diagnostic owner without changing its fact views. +#[pyfunction] +#[pyo3(signature = (*, project, project_dir))] +pub(crate) fn compile_facts<'py>( + py: Python<'py>, + project: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let models = PyDict::new(py); + let model_paths = PyDict::new(py); + let paths_by_name = PyDict::new(py); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + let key = model.getattr("key")?; + let path = project_dir + .call_method1("__truediv__", (model.getattr("relative_path")?,))? + .call_method0("resolve")?; + models.set_item(&key, &model)?; + model_paths.set_item(&key, &path)?; + paths_by_name.set_item(model.getattr("name")?, &path)?; + } + let diagnostics = PyDict::new(py); + let counter = py.import("collections")?.getattr("Counter")?; + for diagnostic in project.getattr("diagnostics")?.try_iter()? { + let diagnostic = diagnostic?; + let owner = diagnostic_owner(&diagnostic, &paths_by_name, project_dir)?; + let counts = if let Some(counts) = diagnostics.get_item(&owner)? { + counts + } else { + let counts = counter.call0()?; + diagnostics.set_item(&owner, &counts)?; + counts + }; + let identity = PyTuple::new( + py, + [ + diagnostic.getattr("code")?, + text_or_empty(&diagnostic.getattr("resource_type")?)?, + text_or_empty(&diagnostic.getattr("resource_name")?)?, + diagnostic.getattr("message")?, + ], + )?; + let count: i64 = counts.get_item(&identity)?.extract()?; + counts.set_item(identity, count + 1)?; + } + call_with( + &imported(py, LINT_MODELS, "CompileFacts")?, + &[ + ("models", models.into_any()), + ("model_paths", model_paths.into_any()), + ("diagnostics", diagnostics.into_any()), + ], + ) +} + +fn diagnostic_owner<'py>( + diagnostic: &Bound<'py, PyAny>, + paths_by_name: &Bound<'py, PyDict>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let resource_type = diagnostic.getattr("resource_type")?; + if resource_type.eq("model")? + && let Some(path) = paths_by_name.get_item(diagnostic.getattr("resource_name")?)? + { + return Ok(path); + } + let mut path = diagnostic.getattr("path")?; + if !path.is_truthy()? { + let location = diagnostic.getattr("location")?; + path = if location.is_none() { + location + } else { + location.getattr("path")? + }; + } + if path.is_none() { + project_dir.call_method0("resolve") + } else { + project_dir + .call_method1("__truediv__", (path,))? + .call_method0("resolve") + } +} + +fn text_or_empty<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if value.is_truthy()? { + Ok(value.str()?.into_any()) + } else { + Ok(pyo3::types::PyString::new(value.py(), "").into_any()) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs new file mode 100644 index 0000000000..d92218d242 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -0,0 +1,672 @@ +//! Compiler-verified fixed-point repair planning, entirely in memory. + +use std::collections::BTreeSet; +use std::path::PathBuf; + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PyTuple}; +use sqlbuild_rules::fix_verification::main::refusal::refusal; +use sqlbuild_rules::fix_verification::main::with_edits::with_edits; +use sqlbuild_rules::fix_verification::models::{FixEdit, FixEligibility}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::rules::fix_facts::compile_facts; +use crate::bindings::_helpers::rules::fix_verdict::fix_verdict; + +const NATIVE: &str = "sqlbuild._native"; +const MODELS: &str = "sqlbuild.lint.models"; +const CONSTANTS: &str = "sqlbuild.lint.constants"; +const ADAPTER: &str = "sqlbuild.adapter.contract.classes.base_adapter"; +const COMPILE_ERRORS: &str = "sqlbuild.compiler.compile.exceptions"; +const LINT_ERRORS: &str = "sqlbuild.lint.exceptions"; +const VERIFICATION_FAILURE: &str = + "Compilation could not prove unchanged output columns, types, dependencies and lineage"; +const APPLIED_REASON: &str = "Meaning-preserving rewrite; compiler invariants verified"; +const MAX_PASSES: usize = 128; + +struct FixSession<'py> { + project_dir: Bound<'py, PyAny>, + original: Bound<'py, PyDict>, + config: Bound<'py, PyAny>, + adapter: Bound<'py, PyAny>, + inputs: Bound<'py, PyAny>, + reports: Vec>, + blocked: Bound<'py, PyDict>, + skipped: Bound<'py, PyDict>, + baseline: Option>, + baseline_facts: Option>, +} + +type PlannedPass<'py> = ( + Bound<'py, PyDict>, + Vec>, + Vec>, +); +type FixState = (Vec<(PathBuf, String)>, Vec); + +/// Plan fixes without exposing any unverified intermediate source on disk. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn plan_rule_fixes<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "plan_rule_fixes", + options, + ( + &[ + "project_dir", + "files", + "config", + "adapter", + "discovered_inputs", + "enabled", + ], + &[], + ), + )?; + let files = args.value("files")?.cast_into::()?; + if !args.flag("enabled")? { + return result(&files, &[], &[]); + } + let adapter = args.value("adapter")?; + let inputs = args.value("discovered_inputs")?; + if !adapter.is_instance(&imported(py, ADAPTER, "BaseAdapter")?)? || inputs.is_none() { + return Err(PyErr::from_value( + imported(py, LINT_ERRORS, "ProjectCompileError")? + .call1(("format --fix requires discovered compiler inputs and an adapter",))?, + )); + } + FixSession { + project_dir: args.value("project_dir")?, + original: files, + config: args.value("config")?, + adapter, + inputs, + reports: Vec::new(), + blocked: PyDict::new(py), + skipped: PyDict::new(py), + baseline: None, + baseline_facts: None, + } + .run() +} + +impl<'py> FixSession<'py> { + fn run(&mut self) -> PyResult> { + let py = self.original.py(); + let mut current = self.original.copy()?; + let paths: Vec> = self + .inputs + .getattr("model_files")? + .try_iter()? + .map(|model| model?.getattr("file_path")) + .collect::>()?; + let model_paths = PyFrozenSet::new(py, paths)?; + let mut seen: BTreeSet = BTreeSet::new(); + for _ in 0..MAX_PASSES { + let mut files: Vec<(PathBuf, String)> = current + .iter() + .map(|(path, contents)| Ok((path.extract()?, contents.extract()?))) + .collect::>()?; + files.sort(); + let mut blocked: Vec = self.blocked.keys().extract()?; + blocked.sort(); + if !seen.insert((files, blocked)) { + return self.decline("Rule fixes did not converge"); + } + let (edits, pending, remaining) = self.plan_pass(¤t, model_paths.as_any())?; + if edits.is_empty() { + let mut reports = self.reports.clone(); + reports.extend(not_yet_reported(py, &self.reports, &remaining)?); + let mut faults = self.blocked_faults()?; + faults.extend(self.skipped_warnings()?); + return result(¤t, &reports, &faults); + } + let Some(verified) = self.verify_pass(¤t, &edits, &pending)? else { + self.reports.extend(pending); + return self.decline(VERIFICATION_FAILURE); + }; + current = verified; + } + self.decline("Rule fix pass limit exceeded") + } + + fn plan_pass( + &mut self, + files: &Bound<'py, PyDict>, + model_paths: &Bound<'py, PyAny>, + ) -> PyResult> { + let py = files.py(); + let relation_keys = self.relation_keys()?; + let config = replace( + &self.config, + &[ + ( + "enabled_native_rules", + PyTuple::new(py, ["SQBRSQL"])?.into_any(), + ), + ("relation_keys", relation_keys), + ], + )?; + let lint = call_with( + &imported(py, NATIVE, "run_lint")?, + &[ + ("project_dir", self.project_dir.clone()), + ("config", config), + ("value_renderer", self.adapter.clone()), + ("discovered_inputs", updated_inputs(&self.inputs, files)?), + ("source_files", files.clone().into_any()), + ( + "selected_paths", + PyFrozenSet::new(py, files.keys())?.into_any(), + ), + ( + "skip_unexpandable", + true.into_pyobject(py)?.to_owned().into_any(), + ), + ], + )?; + self.skipped + .call_method1("update", (lint.getattr("unexpandable")?,))?; + let edits = PyDict::new(py); + let mut pending: Vec> = Vec::new(); + let mut remaining: Vec> = Vec::new(); + for violation in lint.getattr("violations")?.try_iter()? { + let violation = violation?; + if !violation + .getattr("code")? + .extract::()? + .starts_with("SQBRSQL") + { + continue; + } + let reason = self.refusal(&violation, model_paths)?; + let edit = violation.getattr("fix")?; + if reason.is_some() || edit.is_none() { + remaining.push(report( + &violation, + if reason.is_some() { + "refused" + } else { + "unavailable" + }, + reason + .as_deref() + .unwrap_or("No meaning-preserving automatic fix is available"), + )?); + continue; + } + let path = edit.getattr("file_path")?; + let chosen = if let Some(chosen) = edits.get_item(&path)? { + chosen.cast_into::()? + } else { + let chosen = PyList::empty(py); + edits.set_item(&path, &chosen)?; + chosen + }; + let start: usize = edit.getattr("start")?.extract()?; + let end: usize = edit.getattr("end")?.extract()?; + let mut overlaps = false; + for other in chosen.iter() { + if start < other.getattr("end")?.extract::()? + && other.getattr("start")?.extract::()? < end + { + overlaps = true; + break; + } + } + if !overlaps { + chosen.append(edit)?; + pending.push(report(&violation, "applied", APPLIED_REASON)?); + } + } + Ok((edits, pending, remaining)) + } + + fn refusal( + &self, + violation: &Bound<'py, PyAny>, + model_paths: &Bound<'py, PyAny>, + ) -> PyResult> { + let py = violation.py(); + let code = violation.getattr("code")?; + let path = violation.getattr("file_path")?; + let has_fix = !violation.getattr("fix")?.is_none(); + let facts = FixEligibility { + model_body: model_paths.contains(&path)?, + has_fix, + safe_code: imported(py, CONSTANTS, "SAFE_RULE_FIX_CODES")?.contains(&code)?, + conditionless_join: code.eq("SQBRSQL003")?, + snowflake: self.config.getattr("dialect")?.eq("snowflake")?, + }; + if let Some(reason) = refusal(&facts) { + return Ok(Some(reason.to_owned())); + } + let unavailable = violation.getattr("fix_unavailable_reason")?; + if !unavailable.is_none() || !has_fix { + return unavailable.extract(); + } + self.blocked + .get_item(path)? + .map_or(Ok(None), |reason| reason.extract().map(Some)) + } + + fn verify_pass( + &mut self, + current: &Bound<'py, PyDict>, + edits: &Bound<'py, PyDict>, + pending: &[Bound<'py, PyAny>], + ) -> PyResult>> { + let mut accepted = edits.copy()?; + let mut confirmed = false; + while !accepted.is_empty() { + let verdict = self.verdict(&edited_files(&self.unblocked(current)?, &accepted)?)?; + if let Some(verdict) = &verdict + && verdict.getattr("verified")?.is_truthy()? + { + confirmed = true; + break; + } + let hits = failing_edits(verdict.as_ref(), &accepted)?; + if hits.is_empty() { + accepted = self.isolate(current, &accepted)?; + break; + } + for (path, reason) in hits.iter() { + self.blocked.set_item(&path, reason)?; + accepted.del_item(path)?; + } + } + let candidate = edited_files(&self.unblocked(current)?, &accepted)?; + if !accepted.is_empty() && !confirmed { + let verdict = self.verdict(&candidate)?; + if !verdict + .as_ref() + .map_or(Ok(false), |value| value.getattr("verified")?.is_truthy())? + { + return Ok(None); + } + } + let mut reports: Vec> = Vec::new(); + for report in self.reports.iter().chain(pending) { + reports.push( + if let Some(reason) = self.blocked.get_item(report.getattr("file_path")?)? { + refused(report, &reason.extract::()?)? + } else { + report.clone() + }, + ); + } + self.reports = reports; + Ok(Some(candidate)) + } + + fn isolate( + &mut self, + current: &Bound<'py, PyDict>, + edits: &Bound<'py, PyDict>, + ) -> PyResult> { + let base = self.unblocked(current)?; + let accepted = PyDict::new(current.py()); + for (path, path_edits) in edits.iter() { + let single = PyDict::new(current.py()); + single.set_item(&path, &path_edits)?; + let verdict = self.verdict(&edited_files(&base, &single)?)?; + if let Some(verdict) = &verdict + && verdict.getattr("verified")?.is_truthy()? + { + accepted.set_item(path, path_edits)?; + } else { + let hits = failing_edits(verdict.as_ref(), &single)?; + self.blocked.set_item( + &path, + hits.get_item(&path)? + .unwrap_or(VERIFICATION_FAILURE.into_pyobject(current.py())?.into_any()), + )?; + } + } + Ok(accepted) + } + + fn unblocked(&self, files: &Bound<'py, PyDict>) -> PyResult> { + let result = files.copy()?; + for (path, _) in self.blocked.iter() { + if result.contains(&path)? { + result.set_item(&path, self.original.get_item(&path)?)?; + } + } + Ok(result) + } + + fn baseline(&mut self) -> PyResult> { + if let Some(baseline) = &self.baseline { + return Ok(baseline.clone()); + } + let baseline = call_with( + &imported(self.inputs.py(), NATIVE, "build_compiled_project")?, + &[ + ("discovered_inputs", self.inputs.clone()), + ("adapter", self.adapter.clone()), + ], + )?; + self.baseline = Some(baseline.clone()); + Ok(baseline) + } + + fn relation_keys(&mut self) -> PyResult> { + let py = self.inputs.py(); + let baseline = match self.baseline() { + Ok(value) => value, + Err(error) + if error.value(py).is_instance(&imported( + py, + COMPILE_ERRORS, + "CompileInputError", + )?)? => + { + return Ok(PyDict::new(py).into_any()); + } + Err(error) => return Err(error), + }; + imported(py, NATIVE, "compiled_relation_keys")?.call1((baseline,)) + } + + fn verdict(&mut self, files: &Bound<'py, PyDict>) -> PyResult>> { + let py = files.py(); + match self.compile_verdict(files) { + Ok(verdict) => Ok(Some(verdict)), + Err(error) if error.is_instance_of::(py) => Ok(None), + Err(error) => Err(error), + } + } + + fn compile_verdict(&mut self, files: &Bound<'py, PyDict>) -> PyResult> { + let py = files.py(); + let baseline = self.baseline()?; + if self.baseline_facts.is_none() { + self.baseline_facts = Some(compile_facts(py, &baseline, &self.project_dir)?); + } + let compiled = call_with( + &imported(py, NATIVE, "build_compiled_project")?, + &[ + ("discovered_inputs", updated_inputs(&self.inputs, files)?), + ("adapter", self.adapter.clone()), + ], + )?; + let mut edited: Vec> = Vec::new(); + for (path, text) in files.iter() { + if !text.eq(self.original.get_item(&path)?)? { + edited.push(path.call_method0("resolve")?); + } + } + let before = self + .baseline_facts + .as_ref() + .ok_or_else(|| pyo3::exceptions::PyRuntimeError::new_err("missing compile facts"))?; + fix_verdict( + py, + before, + &compile_facts(py, &compiled, &self.project_dir)?, + PyFrozenSet::new(py, edited)?.as_any(), + ) + } + + fn blocked_faults(&self) -> PyResult>> { + sorted_items(&self.blocked)? + .into_iter() + .map(|(path, reason)| verification_fault(&path, &reason.extract::()?)) + .collect() + } + + fn skipped_warnings(&self) -> PyResult>> { + sorted_items(&self.skipped)? + .into_iter() + .map(|(path, reason)| { + violation( + &path, + ( + "format-fix-skipped", + &format!("Rule fixes skipped this file: {}", reason.str()?), + "warning", + "Fix the reported problem, then rerun `sqb format --fix`.", + ), + ) + }) + .collect() + } + + fn decline(&self, reason: &str) -> PyResult> { + let py = self.original.py(); + let affected = self.blocked.copy()?; + let mut reports: Vec> = Vec::new(); + for report in &self.reports { + if report.getattr("status")?.eq("applied")? { + affected.set_item(report.getattr("file_path")?, py.None())?; + } + reports.push(refused(report, reason)?); + } + let mut faults: Vec> = sorted_items(&affected)? + .into_iter() + .map(|(path, _)| verification_fault(&path, reason)) + .collect::>()?; + faults.extend(self.skipped_warnings()?); + result(&self.original, &reports, &faults) + } +} + +fn replace<'py>( + object: &Bound<'py, PyAny>, + changes: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let kwargs = + crate::bindings::_helpers::compile_driver::python_calls::keywords(object.py(), changes)?; + object + .py() + .import("dataclasses")? + .getattr("replace")? + .call((object,), Some(&kwargs)) +} + +fn updated_inputs<'py>( + inputs: &Bound<'py, PyAny>, + files: &Bound<'py, PyDict>, +) -> PyResult> { + let py = inputs.py(); + let mut models: Vec> = Vec::new(); + for model in inputs.getattr("model_files")?.try_iter()? { + let model = model?; + let current = model.getattr("contents")?; + if let Some(text) = files.get_item(model.getattr("file_path")?)? + && !text.eq(¤t)? + { + let start = current.call_method1("index", (model.getattr("query_sql")?,))?; + let query = text + .get_item(pyo3::types::PySlice::new( + py, + start.extract()?, + isize::MAX, + 1, + ))? + .call_method0("strip")?; + models.push(model.call_method1("with_text", (text, query))?); + } else { + models.push(model); + } + } + replace( + inputs, + &[ + ("model_files", PyTuple::new(py, models)?.into_any()), + ("native_session", py.None().into_bound(py)), + ], + ) +} + +fn edited_files<'py>( + files: &Bound<'py, PyDict>, + edits: &Bound<'py, PyDict>, +) -> PyResult> { + let candidate = files.copy()?; + for (path, path_edits) in edits.iter() { + let edits: Vec = path_edits + .try_iter()? + .map(|edit| { + let edit = edit?; + Ok(FixEdit { + start: edit.getattr("start")?.extract()?, + end: edit.getattr("end")?.extract()?, + replacement: edit.getattr("replacement")?.extract()?, + }) + }) + .collect::>()?; + let text: String = candidate + .get_item(&path)? + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err("edit path"))? + .extract()?; + candidate.set_item(path, with_edits(&text, edits))?; + } + Ok(candidate) +} + +fn failing_edits<'py>( + verdict: Option<&Bound<'py, PyAny>>, + edits: &Bound<'py, PyDict>, +) -> PyResult> { + let result = PyDict::new(edits.py()); + let Some(verdict) = verdict else { + return Ok(result); + }; + let resolved = PyDict::new(edits.py()); + for (path, _) in edits.iter() { + resolved.set_item(path.call_method0("resolve")?, path)?; + } + for (path, reason) in verdict.getattr("failing")?.cast_into::()?.iter() { + if let Some(original) = resolved.get_item(path)? { + result.set_item(original, reason)?; + } + } + Ok(result) +} + +fn not_yet_reported<'py>( + py: Python<'py>, + reports: &[Bound<'py, PyAny>], + remaining: &[Bound<'py, PyAny>], +) -> PyResult>> { + let recorded = PyDict::new(py); + for report in reports { + let count = recorded + .get_item(report)? + .map_or(Ok(0), |count| count.extract::())?; + recorded.set_item(report, count + 1)?; + } + let mut fresh: Vec> = Vec::new(); + for report in remaining { + let count = recorded + .get_item(report)? + .map_or(Ok(0), |count| count.extract::())?; + if count > 0 { + recorded.set_item(report, count - 1)?; + } else { + fresh.push(report.clone()); + } + } + Ok(fresh) +} + +fn report<'py>( + source: &Bound<'py, PyAny>, + status: &str, + reason: &str, +) -> PyResult> { + let py = source.py(); + call_with( + &imported(py, MODELS, "RuleFixResult")?, + &[ + ("file_path", source.getattr("file_path")?), + ("code", source.getattr("code")?), + ("line", source.getattr("line")?), + ("status", status.into_pyobject(py)?.into_any()), + ("reason", reason.into_pyobject(py)?.into_any()), + ], + ) +} + +fn refused<'py>(report: &Bound<'py, PyAny>, reason: &str) -> PyResult> { + if !report.getattr("status")?.eq("applied")? { + return Ok(report.clone()); + } + replace( + report, + &[ + ("status", "refused".into_pyobject(report.py())?.into_any()), + ("reason", reason.into_pyobject(report.py())?.into_any()), + ], + ) +} + +fn violation<'py>( + path: &Bound<'py, PyAny>, + details: (&str, &str, &str, &str), +) -> PyResult> { + let py = path.py(); + call_with( + &imported(py, MODELS, "LintViolation")?, + &[ + ("file_path", path.clone()), + ("line", 1_i64.into_pyobject(py)?.into_any()), + ("column", 1_i64.into_pyobject(py)?.into_any()), + ("code", details.0.into_pyobject(py)?.into_any()), + ("message", details.1.into_pyobject(py)?.into_any()), + ("severity", details.2.into_pyobject(py)?.into_any()), + ("engine", "native".into_pyobject(py)?.into_any()), + ("remediation", details.3.into_pyobject(py)?.into_any()), + ], + ) +} + +fn verification_fault<'py>(path: &Bound<'py, PyAny>, reason: &str) -> PyResult> { + violation( + path, + ( + "format-fix-verification-failed", + reason, + "fault", + "Restructure the SQL manually and recompile; the original file was retained.", + ), + ) +} + +fn sorted_items<'py>( + items: &Bound<'py, PyDict>, +) -> PyResult, Bound<'py, PyAny>)>> { + let sorted = items + .py() + .import("builtins")? + .getattr("sorted")? + .call1((items.items(),))?; + sorted.extract() +} + +fn result<'py>( + files: &Bound<'py, PyDict>, + reports: &[Bound<'py, PyAny>], + faults: &[Bound<'py, PyAny>], +) -> PyResult> { + Ok(PyTuple::new( + files.py(), + [ + files.clone().into_any(), + PyTuple::new(files.py(), reports)?.into_any(), + PyList::new(files.py(), faults)?.into_any(), + ], + )? + .into_any()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs new file mode 100644 index 0000000000..e6ebf6a74b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs @@ -0,0 +1,112 @@ +//! Convert compiler fact views to native equality and diagnostic facts. + +use std::collections::BTreeMap; + +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use sqlbuild_rules::fix_verification::main::compare::compare; +use sqlbuild_rules::fix_verification::models::{DiagnosticComparison, ModelComparison}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const LINT_MODELS: &str = "sqlbuild.lint.models"; + +/// Compare compile facts using native decisions and the existing Python boundary models. +#[pyfunction] +#[pyo3(signature = (*, before, after, edited))] +pub(crate) fn fix_verdict<'py>( + py: Python<'py>, + before: &Bound<'py, PyAny>, + after: &Bound<'py, PyAny>, + edited: &Bound<'py, PyAny>, +) -> PyResult> { + let before_models = before.getattr("models")?.cast_into::()?; + let after_models = after.getattr("models")?.cast_into::()?; + let same_models = before_models + .call_method0("keys")? + .eq(after_models.call_method0("keys")?)?; + let owners = PyList::empty(py); + let indices = PyDict::new(py); + let mut models: Vec = Vec::new(); + let mut diagnostics: Vec = Vec::new(); + if same_models { + let paths = before.getattr("model_paths")?; + for (key, model) in before_models.iter() { + let other = after_models + .get_item(&key)? + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err("model key"))?; + let path = paths.get_item(&key)?; + let owner = owner_index((&owners, &indices), &path)?; + let columns = model.getattr("inferred_columns")?; + let other_columns = other.getattr("inferred_columns")?; + models.push(ModelComparison { + owner, + edited: edited.contains(&path)?, + columns_known: !columns.is_none() && !other_columns.is_none(), + columns_equal: columns.eq(&other_columns)?, + dependencies_equal: model.getattr("deps")?.eq(other.getattr("deps")?)?, + lineage_equal: lineage(&model)?.eq(lineage(&other)?)?, + }); + } + let previous = before.getattr("diagnostics")?.cast_into::()?; + let current = after.getattr("diagnostics")?.cast_into::()?; + let diagnostic_owners = PyDict::new(py); + for (owner, _) in previous.iter().chain(current.iter()) { + diagnostic_owners.set_item(owner, py.None())?; + } + for (path, _) in diagnostic_owners.iter() { + diagnostics.push(DiagnosticComparison { + owner: owner_index((&owners, &indices), &path)?, + edited: edited.contains(&path)?, + before: counts(previous.get_item(&path)?)?, + after: counts(current.get_item(&path)?)?, + }); + } + } + let result = compare(same_models, &models, &diagnostics); + let failing = PyDict::new(py); + for (owner, reason) in result.failing { + failing.set_item(owners.get_item(owner)?, reason)?; + } + call_with( + &imported(py, LINT_MODELS, "FixVerdict")?, + &[ + ("failing", failing.into_any()), + ( + "unattributed", + result.unattributed.into_pyobject(py)?.to_owned().into_any(), + ), + ], + ) +} + +fn owner_index( + lookup: (&Bound<'_, PyList>, &Bound<'_, PyDict>), + owner: &Bound<'_, PyAny>, +) -> PyResult { + let (owners, indices) = lookup; + if let Some(index) = indices.get_item(owner)? { + return index.extract(); + } + let index = owners.len(); + owners.append(owner)?; + indices.set_item(owner, index)?; + Ok(index) +} + +fn lineage<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let value = model.getattr("fast_lineage_columns")?; + if value.is_none() { + Ok(value) + } else { + let items = value.try_iter()?.collect::>>()?; + Ok(PyTuple::new(model.py(), items)?.into_any()) + } +} + +fn counts( + value: Option>, +) -> PyResult> { + value.map_or_else(|| Ok(BTreeMap::new()), |value| value.extract()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs new file mode 100644 index 0000000000..930aeaec07 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs @@ -0,0 +1,92 @@ +//! Route lint bodies through the existing native declaration and header scanners. + +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const NATIVE: &str = "sqlbuild._native"; +const MODELS: &str = "sqlbuild.lint.models"; + +/// Classify a lint path, retaining the caller's supplied relative path when available. +#[pyfunction] +#[pyo3(signature = (*, file_path, project_dir, relative_path=None))] +pub(crate) fn lint_file_role<'py>( + file_path: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, + relative_path: Option>, +) -> PyResult> { + let py = file_path.py(); + let relative = if let Some(relative) = relative_path { + relative + } else { + if !file_path + .call_method1("is_relative_to", (project_dir,))? + .is_truthy()? + { + return imported(py, MODELS, "LintFileRole")?.call0(); + } + file_path.call_method1("relative_to", (project_dir,))? + }; + call_with( + &imported(py, MODELS, "LintFileRole")?, + &[ + ( + "declaration_kind", + call_with( + &imported(py, NATIVE, "named_declaration_role_kind")?, + &[("relative_path", relative.clone())], + )?, + ), + ("in_project", true.into_pyobject(py)?.to_owned().into_any()), + ( + "in_hook_directory", + relative.call_method1("is_relative_to", ("hooks",))?, + ), + ], + ) +} + +/// Use audit measurement ranges when present, otherwise the native SQL header boundaries. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn lint_body_ranges<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "lint_body_ranges", + options, + ( + &["contents", "headers", "file_path", "project_dir"], + &["role"], + ), + )?; + let mut role = args.value("role")?; + if role.is_none() { + role = lint_file_role(&args.value("file_path")?, &args.value("project_dir")?, None)?; + } + let kind = role.getattr("declaration_kind")?; + if role.getattr("in_project")?.is_truthy()? + && (kind.eq("audit")? || kind.eq("singular_audit")?) + { + let ranges = call_with( + &imported(py, NATIVE, "measurement_body_ranges")?, + &[("contents", args.value("contents")?)], + )?; + if ranges.is_truthy()? { + return Ok(ranges); + } + } + call_with( + &imported(py, NATIVE, "sql_body_ranges")?, + &[ + ("contents", args.value("contents")?), + ("headers", args.value("headers")?), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs new file mode 100644 index 0000000000..ce6ed95913 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs @@ -0,0 +1,501 @@ +//! Native lint expansion orchestration around the existing user-macro boundary. + +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use sqlbuild_rules::lint_text::main::dependency_sites::dependency_sites; +use sqlbuild_rules::lint_text::models::InterpolationSite; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, +}; +use crate::bindings::_helpers::rules::lint_headers::{lint_body_ranges, lint_file_role}; + +const NATIVE: &str = "sqlbuild._native"; +const MODELS: &str = "sqlbuild.lint.models"; +const EXPANSION_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_ERRORS: &str = "sqlbuild.compiler.compile.exceptions"; +const LINT_ERRORS: &str = "sqlbuild.lint.exceptions"; +const STOP: &str = "sqlbuild.lint.classes.stop_context"; + +/// Expand a single body and finish its native SQL preparation. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn prepare_lint_body<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let pending = expand_lint_body(py, options)?; + let prepared = + imported(py, NATIVE, "prepare_lint_sql")?.call1((pending.getattr("request")?,))?; + finish_body(&pending, &prepared) +} + +/// Expand authored SQL while retaining source offsets and pre-expansion substitutions. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn expand_lint_body<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "expand_lint_body", + options, + ( + &[ + "role", + "file_path", + "contents", + "body_range", + "context", + "dialect", + ], + &[ + "external_identifiers", + "allows_ceremonial_select", + "allows_dynamic_output_star", + "compiled_expansion", + ], + ), + )?; + let (mut start, mut end): (usize, usize) = args.value("body_range")?.extract()?; + let body = args.value("contents")?.get_item(pyo3::types::PySlice::new( + py, + start as isize, + end as isize, + 1, + ))?; + let mut input = body.clone(); + let mut sites = PyTuple::empty(py).into_any(); + let role = args.value("role")?; + let kind = role.getattr("declaration_kind")?; + let in_project = role.getattr("in_project")?.is_truthy()?; + let neutralizer = if in_project && kind.eq("audit")? { + Some("neutralize_lint_audit_parameters") + } else if role.getattr("in_hook_directory")?.is_truthy()? + || (in_project && kind.eq("sql_hook")?) + { + Some("neutralize_lint_context_interpolation") + } else { + None + }; + if let Some(neutralizer) = neutralizer { + let neutralized = imported(py, NATIVE, neutralizer)?.call1((&body,))?; + input = neutralized.get_item(0)?; + sites = site_objects(&neutralized.get_item(1)?)?; + } + let compiled = args.value("compiled_expansion")?; + let expanded; + let passes; + if !compiled.is_none() + && body + .call_method0("strip")? + .eq(compiled.getattr("authored_sql")?)? + { + start += body.len()? - body.call_method0("lstrip")?.len()?; + input = compiled.getattr("authored_sql")?; + end = start + input.len()?; + expanded = compiled.getattr("expanded_sql")?; + passes = compiled.getattr("passes")?; + } else if !input.extract::()?.contains('@') + && !input.extract::()?.contains("${") + { + expanded = input.clone(); + passes = PyTuple::empty(py).into_any(); + } else { + let value = call_with( + &imported(py, NATIVE, "expand_sql_with_spans")?, + &[ + ("sql", input.clone()), + ("file_path", args.value("file_path")?), + ("context", args.value("context")?), + ], + ) + .map_err(|error| expansion_error(py, &args, error))?; + expanded = value.get_item(0)?; + passes = value.get_item(1)?; + } + let mut prior_sites: Vec> = Vec::new(); + for site in sites.try_iter()? { + prior_sites.push(site?.getattr("neutralized_start")?); + } + let request = keywords( + py, + &[ + ("expanded", expanded.clone()), + ("before_expansion", input.clone()), + ("prior_sites", PyList::new(py, prior_sites)?.into_any()), + ("dialect", args.value("dialect")?), + ], + )?; + let external = if args.passed("external_identifiers")? { + args.value("external_identifiers")? + } else { + PyTuple::empty(py).into_any() + }; + call_with( + &imported(py, MODELS, "PendingLintBody")?, + &[ + ("file_path", args.value("file_path")?), + ("body_start", start.into_pyobject(py)?.into_any()), + ("body_end", end.into_pyobject(py)?.into_any()), + ("expansion_input", input), + ("expanded", expanded), + ("expansion_passes", passes), + ("pre_expansion_sites", sites), + ("external_identifiers", external), + ( + "allows_ceremonial_select", + args.value_or_false("allows_ceremonial_select")?, + ), + ( + "allows_dynamic_output_star", + args.value_or_false("allows_dynamic_output_star")?, + ), + ( + "allows_empty_fixture_star", + false.into_pyobject(py)?.to_owned().into_any(), + ), + ("request", request.into_any()), + ], + ) +} + +fn expansion_error(py: Python<'_>, args: &KeywordArguments<'_>, error: PyErr) -> PyErr { + let wrapped = (|| -> PyResult> { + if !error + .value(py) + .is_instance(&imported(py, COMPILE_ERRORS, "CompileInputError")?)? + { + return Ok(None); + } + let message = format!( + "{} could not be expanded, so its SQL cannot be linted: {}", + args.value("file_path")?.str()?, + error.value(py).str()? + ); + Ok(Some(PyErr::from_value( + imported(py, LINT_ERRORS, "ProjectCompileError")?.call1((message,))?, + ))) + })(); + match wrapped { + Ok(Some(wrapped)) => { + wrapped.set_cause(py, Some(error)); + wrapped + } + Ok(None) => error, + Err(wrapping_error) => wrapping_error, + } +} + +fn site_objects<'py>(sites: &Bound<'py, PyAny>) -> PyResult> { + let class = imported(sites.py(), MODELS, "InterpolationSite")?; + let values: Vec> = sites + .try_iter()? + .map(|site| class.call1(site?.cast_into::()?)) + .collect::>()?; + Ok(PyTuple::new(sites.py(), values)?.into_any()) +} + +fn sentinel_spans<'py>(sites: &Bound<'py, PyAny>) -> PyResult> { + let py = sites.py(); + let mut spans: Vec> = Vec::new(); + for site in sites.try_iter()? { + let site = site?; + spans.push(call_with( + &imported(py, EXPANSION_MODELS, "ExpansionSpan")?, + &[ + ("source_start", site.getattr("original_start")?), + ("source_end", site.getattr("original_end")?), + ("output_start", site.getattr("neutralized_start")?), + ("output_end", site.getattr("neutralized_end")?), + ], + )?); + } + Ok(PyTuple::new(py, spans)?.into_any()) +} + +fn finish_body<'py>( + pending: &Bound<'py, PyAny>, + prepared: &Bound<'py, PyAny>, +) -> PyResult> { + let py = pending.py(); + if prepared.is_none() { + return Err(PyErr::from_value( + imported(py, LINT_ERRORS, "NativeLintError")?.call1((format!( + "{} could not be prepared for native SQL lint", + pending.getattr("file_path")?.str()? + ),))?, + )); + } + let sites = site_objects(&prepared.get_item(1)?)?; + let native_sites: Vec = sites + .try_iter()? + .map(|site| { + let site = site?; + Ok(InterpolationSite { + sentinel: site.getattr("sentinel")?.extract()?, + original_text: site.getattr("original_text")?.extract()?, + original_start: site.getattr("original_start")?.extract()?, + original_end: site.getattr("original_end")?.extract()?, + neutralized_start: site.getattr("neutralized_start")?.extract()?, + neutralized_end: site.getattr("neutralized_end")?.extract()?, + }) + }) + .collect::>()?; + let dependencies = dependency_sites(&native_sites) + .map_err(crate::bindings::_helpers::boundary::panics::value_error)?; + let mut passes: Vec> = + vec![sentinel_spans(&pending.getattr("pre_expansion_sites")?)?]; + passes.extend( + pending + .getattr("expansion_passes")? + .try_iter()? + .collect::>>()?, + ); + passes.push(sentinel_spans(&sites)?); + let values = keywords( + py, + &[ + ("lint_text", prepared.get_item(0)?), + ("passes", PyTuple::new(py, passes)?.into_any()), + ( + "dependency_identifiers", + PyTuple::new(py, dependencies.identifiers)?.into_any(), + ), + ( + "externally_referenced_ctes", + PyTuple::new(py, prepared.get_item(2)?.extract::>()?)?.into_any(), + ), + ( + "dependency_relations", + PyTuple::new(py, dependencies.relations)?.into_any(), + ), + ], + )?; + for name in [ + "file_path", + "body_start", + "body_end", + "external_identifiers", + "allows_ceremonial_select", + "allows_dynamic_output_star", + "allows_empty_fixture_star", + ] { + values.set_item(name, pending.getattr(name)?)?; + } + imported(py, MODELS, "LintBody")?.call((), Some(&values)) +} + +fn prepare_batch<'py>( + py: Python<'py>, + pending: &[Bound<'py, PyAny>], +) -> PyResult>> { + if pending.is_empty() { + return Ok(Vec::new()); + } + let requests: Vec> = pending + .iter() + .map(|body| body.getattr("request")) + .collect::>()?; + let results = + imported(py, NATIVE, "prepare_lint_sql_batch")?.call1((PyList::new(py, requests)?,))?; + let mut prepared: Vec> = Vec::new(); + for (body, result) in pending.iter().zip(results.try_iter()?) { + let result = result?; + prepared.push(if result.get_item(0)?.is_truthy()? { + result.get_item(1)? + } else { + imported(py, NATIVE, "prepare_lint_sql")?.call1((body.getattr("request")?,))? + }); + } + if prepared.len() != pending.len() || results.len()? != pending.len() { + return Err(pyo3::exceptions::PyValueError::new_err( + "native preparation batch length mismatch", + )); + } + Ok(prepared) +} + +/// Expand each file body in order, preserving the first expansion failure with its partial work. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn expand_file_bodies<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "expand_file_bodies", + options, + ( + &[ + "file_path", + "contents", + "headers", + "context", + "dialect", + "project_dir", + "relative_path", + "dynamic_output_paths", + "compiled_expansions", + ], + &[], + ), + )?; + imported(py, STOP, "LintStopContext")?.call_method0("check")?; + let path = args.value("file_path")?; + let role = lint_file_role( + &path, + &args.value("project_dir")?, + Some(args.value("relative_path")?), + )?; + let dynamic = args.value("dynamic_output_paths")?; + let dynamic_star = dynamic.is_truthy()? && dynamic.contains(path.call_method0("resolve")?)?; + let expansions = args.value("compiled_expansions")?; + let compiled = if expansions.is_truthy()? { + expansions.call_method1("get", (&path,))? + } else { + py.None().into_bound(py) + }; + let external = call_with( + &imported(py, NATIVE, "external_identifiers_for_headers")?, + &[ + ("contents", args.value("contents")?), + ("headers", args.value("headers")?), + ], + )?; + let mut ceremonial = false; + for header in args.value("headers")?.try_iter()? { + let kind = header?.getattr("kind")?; + if kind.eq("TEST")? || kind.eq("SCENARIO")? { + ceremonial = true; + break; + } + } + let mut bodies: Vec> = Vec::new(); + let mut failure = py.None().into_bound(py); + let expansion = (|| -> PyResult<()> { + let ranges = lint_body_ranges( + py, + Some(keywords( + py, + &[ + ("contents", args.value("contents")?), + ("headers", args.value("headers")?), + ("file_path", path.clone()), + ("project_dir", args.value("project_dir")?), + ("role", role.clone()), + ], + )?), + )?; + for range in ranges.try_iter()? { + let pending = expand_lint_body( + py, + Some(keywords( + py, + &[ + ("role", role.clone()), + ("file_path", path.clone()), + ("contents", args.value("contents")?), + ("body_range", range?), + ("context", args.value("context")?), + ("dialect", args.value("dialect")?), + ("external_identifiers", external.clone()), + ( + "allows_ceremonial_select", + ceremonial.into_pyobject(py)?.to_owned().into_any(), + ), + ( + "allows_dynamic_output_star", + dynamic_star.into_pyobject(py)?.to_owned().into_any(), + ), + ("compiled_expansion", compiled.clone()), + ], + )?), + )?; + let changes = keywords( + py, + &[( + "allows_empty_fixture_star", + ceremonial.into_pyobject(py)?.to_owned().into_any(), + )], + )?; + bodies.push( + py.import("dataclasses")? + .getattr("replace")? + .call((pending,), Some(&changes))?, + ); + } + Ok(()) + })(); + if let Err(error) = expansion { + if !error + .value(py) + .is_instance(&imported(py, LINT_ERRORS, "ProjectCompileError")?)? + { + return Err(error); + } + failure = error.value(py).clone().into_any(); + } + call_with( + &imported(py, MODELS, "ExpandedLintFile")?, + &[ + ("file_path", path), + ("bodies", PyTuple::new(py, bodies)?.into_any()), + ("failure", failure), + ], + ) +} + +/// Batch SQL preparation, then publish bodies or failures in the original file order. +#[pyfunction] +#[pyo3(signature = (*, expanded_files, dialect, skip_unexpandable))] +pub(crate) fn prepare_expanded_files<'py>( + expanded_files: &Bound<'py, PyAny>, + dialect: &str, + skip_unexpandable: bool, +) -> PyResult> { + let _ = dialect; + let py = expanded_files.py(); + imported(py, STOP, "LintStopContext")?.call_method0("check")?; + let files: Vec> = expanded_files.try_iter()?.collect::>()?; + let mut pending: Vec> = Vec::new(); + for file in &files { + pending.extend( + file.getattr("bodies")? + .try_iter()? + .collect::>>()?, + ); + } + let prepared = prepare_batch(py, &pending)?; + let mut bodies: Vec> = Vec::new(); + let unexpandable = PyDict::new(py); + let mut position = 0; + for file in files { + let mut file_bodies: Vec> = Vec::new(); + for body in file.getattr("bodies")?.try_iter()? { + file_bodies.push(finish_body(&body?, &prepared[position])?); + position += 1; + } + let failure = file.getattr("failure")?; + if failure.is_none() { + bodies.extend(file_bodies); + } else if skip_unexpandable { + let path = file.getattr("file_path")?; + let message: String = failure.str()?.extract()?; + let prefix = format!("{} ", path.str()?); + unexpandable.set_item(path, message.strip_prefix(&prefix).unwrap_or(&message))?; + } else { + return Err(PyErr::from_value(failure)); + } + } + call_with( + &imported(py, MODELS, "PreparedLintFiles")?, + &[ + ("bodies", PyTuple::new(py, bodies)?.into_any()), + ("unexpandable", unexpandable.into_any()), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index b5ef88405a..d2234ad0e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -6,9 +6,14 @@ pub(crate) mod dsl_headers; pub(crate) mod early_sql_lint; pub(crate) mod evaluation; pub(crate) mod findings; +pub(crate) mod fix_facts; +pub(crate) mod fix_session; +pub(crate) mod fix_verdict; pub(crate) mod guidance; pub(crate) mod lint; pub(crate) mod lint_expansion; +pub(crate) mod lint_headers; +pub(crate) mod lint_preparation; pub(crate) mod lint_sql_bodies; pub(crate) mod lint_text; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index 7d9f87a4b2..657cdb351b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -1,6 +1,8 @@ //! Register the native rules, lint and DSL header functions. +use crate::bindings::_helpers::rules::{fix_facts::compile_facts, fix_verdict::fix_verdict}; use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::{types::PyModuleMethods, wrap_pyfunction}; use crate::bindings::_helpers::rules::{ builtin_catalogue, catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, @@ -9,6 +11,36 @@ use crate::bindings::_helpers::rules::{ }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + crate::bindings::_helpers::rules::lint_headers::lint_file_role, + module + )?)?; + module.add_function(wrap_pyfunction!( + crate::bindings::_helpers::rules::lint_headers::lint_body_ranges, + module + )?)?; + module.add_function(wrap_pyfunction!( + crate::bindings::_helpers::rules::lint_preparation::prepare_lint_body, + module + )?)?; + module.add_function(wrap_pyfunction!( + crate::bindings::_helpers::rules::lint_preparation::expand_lint_body, + module + )?)?; + module.add_function(wrap_pyfunction!( + crate::bindings::_helpers::rules::lint_preparation::expand_file_bodies, + module + )?)?; + module.add_function(wrap_pyfunction!( + crate::bindings::_helpers::rules::lint_preparation::prepare_expanded_files, + module + )?)?; + module.add_function(wrap_pyfunction!( + crate::bindings::_helpers::rules::fix_session::plan_rule_fixes, + module + )?)?; + module.add_function(wrap_pyfunction!(compile_facts, module)?)?; + module.add_function(wrap_pyfunction!(fix_verdict, module)?)?; evaluation::register(module)?; lint::register(module)?; lint_text::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs index f2a30c19e2..2e32ce2c90 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::rules::lint_expansion::build_lint_expansion_context; const NATIVE_MODULE: &str = "sqlbuild._native"; -const EXPANSION_MODULE: &str = "sqlbuild.lint._helpers.expansion"; +const EXPANSION_MODULE: &str = "sqlbuild._native"; const STOP_CONTEXT_MODULE: &str = "sqlbuild.lint.classes.stop_context"; const LINT_MODELS_MODULE: &str = "sqlbuild.lint.models"; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 316a1427d3..23cb732f73 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -47,6 +47,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { refactor::registration::register(module)?; plan_output::registration::register(module)?; planner_changes::registration::register(module)?; + crate::bindings::_helpers::planner_migrations::registration::register(module)?; planner_identity::registration::register(module)?; planner_selection::registration::register(module)?; planner_scope::registration::register(module)?; diff --git a/crates/sqlbuild-rules/src/fix_verification/constants.rs b/crates/sqlbuild-rules/src/fix_verification/constants.rs new file mode 100644 index 0000000000..5a268f70d6 --- /dev/null +++ b/crates/sqlbuild-rules/src/fix_verification/constants.rs @@ -0,0 +1,6 @@ +pub const COLUMNS_UNKNOWN: &str = + "Output columns could not be inferred, so the fix cannot be verified"; +pub const COLUMNS_CHANGED: &str = "The fix would change output columns, types or nullability"; +pub const DEPENDENCIES_CHANGED: &str = "The fix would change model dependencies"; +pub const LINEAGE_CHANGED: &str = "The fix would change column lineage"; +pub const NEW_DIAGNOSTICS: &str = "The fix would add compile diagnostics: "; diff --git a/crates/sqlbuild-rules/src/fix_verification/main/compare.rs b/crates/sqlbuild-rules/src/fix_verification/main/compare.rs new file mode 100644 index 0000000000..5e6fa94418 --- /dev/null +++ b/crates/sqlbuild-rules/src/fix_verification/main/compare.rs @@ -0,0 +1,67 @@ +use std::collections::BTreeSet; + +use crate::fix_verification::constants::{ + COLUMNS_CHANGED, COLUMNS_UNKNOWN, DEPENDENCIES_CHANGED, LINEAGE_CHANGED, NEW_DIAGNOSTICS, +}; +use crate::fix_verification::models::{DiagnosticComparison, FixComparisonResult, ModelComparison}; + +/// Attribute changed compile invariants while allowing edited owners to lose diagnostics. +pub fn compare( + same_models: bool, + models: &[ModelComparison], + diagnostics: &[DiagnosticComparison], +) -> FixComparisonResult { + let mut result = FixComparisonResult::default(); + if !same_models { + result.unattributed = true; + return result; + } + for model in models { + let reason = if model.edited && !model.columns_known { + Some(COLUMNS_UNKNOWN) + } else if !model.columns_equal { + Some(COLUMNS_CHANGED) + } else if !model.dependencies_equal { + Some(DEPENDENCIES_CHANGED) + } else if !model.lineage_equal { + Some(LINEAGE_CHANGED) + } else { + None + }; + if let Some(reason) = reason { + if model.edited { + result + .failing + .entry(model.owner) + .or_insert_with(|| reason.to_owned()); + } else { + result.unattributed = true; + } + } + } + for diagnostic in diagnostics { + if !diagnostic.edited { + if diagnostic.before != diagnostic.after { + result.unattributed = true; + } + continue; + } + let codes: BTreeSet<&str> = diagnostic + .after + .iter() + .filter_map(|(identity, count)| { + (*count > *diagnostic.before.get(identity).unwrap_or(&0)) + .then_some(identity.0.as_str()) + }) + .collect(); + if !codes.is_empty() { + result.failing.entry(diagnostic.owner).or_insert_with(|| { + format!( + "{NEW_DIAGNOSTICS}{}", + codes.into_iter().collect::>().join(", ") + ) + }); + } + } + result +} diff --git a/crates/sqlbuild-rules/src/fix_verification/main/mod.rs b/crates/sqlbuild-rules/src/fix_verification/main/mod.rs new file mode 100644 index 0000000000..58596d6c92 --- /dev/null +++ b/crates/sqlbuild-rules/src/fix_verification/main/mod.rs @@ -0,0 +1,3 @@ +pub mod compare; +pub mod refusal; +pub mod with_edits; diff --git a/crates/sqlbuild-rules/src/fix_verification/main/refusal.rs b/crates/sqlbuild-rules/src/fix_verification/main/refusal.rs new file mode 100644 index 0000000000..0a1de29978 --- /dev/null +++ b/crates/sqlbuild-rules/src/fix_verification/main/refusal.rs @@ -0,0 +1,14 @@ +use crate::fix_verification::models::FixEligibility; + +/// Refuse edits whose model ownership or dialect lacks a semantic proof. +pub fn refusal(facts: &FixEligibility) -> Option<&'static str> { + if !facts.model_body { + Some("Compiler-verified Rule fixes currently require a model SQL body") + } else if facts.has_fix && !facts.safe_code { + Some("This proposed rewrite is not proven meaning-preserving in this dialect") + } else if facts.conditionless_join && !facts.snowflake { + Some("Conditionless JOIN equivalence is only established for Snowflake") + } else { + None + } +} diff --git a/crates/sqlbuild-rules/src/fix_verification/main/with_edits.rs b/crates/sqlbuild-rules/src/fix_verification/main/with_edits.rs new file mode 100644 index 0000000000..1128ad0d96 --- /dev/null +++ b/crates/sqlbuild-rules/src/fix_verification/main/with_edits.rs @@ -0,0 +1,19 @@ +use crate::fix_verification::models::FixEdit; + +/// Apply nonoverlapping authored edits in reverse character-offset order. +pub fn with_edits(contents: &str, mut edits: Vec) -> String { + edits.sort_by_key(|edit| std::cmp::Reverse(edit.start)); + let mut result = contents.to_owned(); + for edit in edits { + let start = result + .char_indices() + .nth(edit.start) + .map_or(result.len(), |(index, _)| index); + let end = result + .char_indices() + .nth(edit.end) + .map_or(result.len(), |(index, _)| index); + result.replace_range(start..end, &edit.replacement); + } + result +} diff --git a/crates/sqlbuild-rules/src/fix_verification/mod.rs b/crates/sqlbuild-rules/src/fix_verification/mod.rs new file mode 100644 index 0000000000..2c7939a562 --- /dev/null +++ b/crates/sqlbuild-rules/src/fix_verification/mod.rs @@ -0,0 +1,5 @@ +//! Compiler invariant decisions for conservative SQL fixes. + +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-rules/src/fix_verification/models.rs b/crates/sqlbuild-rules/src/fix_verification/models.rs new file mode 100644 index 0000000000..3d5cc6dc99 --- /dev/null +++ b/crates/sqlbuild-rules/src/fix_verification/models.rs @@ -0,0 +1,46 @@ +use std::collections::BTreeMap; + +/// Proof requirements for a proposed automatic rule fix. +#[derive(Debug)] +pub struct FixEligibility { + pub model_body: bool, + pub has_fix: bool, + pub safe_code: bool, + pub conditionless_join: bool, + pub snowflake: bool, +} + +/// One authored edit measured in Unicode character offsets. +#[derive(Debug)] +pub struct FixEdit { + pub start: usize, + pub end: usize, + pub replacement: String, +} + +/// Equality facts extracted from compiler-owned Python views. +#[derive(Debug)] +pub struct ModelComparison { + pub owner: usize, + pub edited: bool, + pub columns_known: bool, + pub columns_equal: bool, + pub dependencies_equal: bool, + pub lineage_equal: bool, +} + +/// Location-insensitive diagnostic multiplicities for one owner. +#[derive(Debug)] +pub struct DiagnosticComparison { + pub owner: usize, + pub edited: bool, + pub before: BTreeMap<(String, String, String, String), i64>, + pub after: BTreeMap<(String, String, String, String), i64>, +} + +/// Failures attributed to edited owners and changes outside those owners. +#[derive(Debug, Default)] +pub struct FixComparisonResult { + pub failing: BTreeMap, + pub unattributed: bool, +} diff --git a/crates/sqlbuild-rules/src/lib.rs b/crates/sqlbuild-rules/src/lib.rs index 3b9babee52..d6eb3c2b9b 100644 --- a/crates/sqlbuild-rules/src/lib.rs +++ b/crates/sqlbuild-rules/src/lib.rs @@ -8,6 +8,7 @@ pub mod custom_rule_cache; pub mod dsl_headers; pub mod engine; pub mod errors; +pub mod fix_verification; pub mod lint_text; pub mod models; pub mod rule_cache; diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sites.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sites.rs index 3911e70e3e..9b9cc86bea 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sites.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sites.rs @@ -15,6 +15,11 @@ use crate::lint_text::constants::{ use crate::lint_text::errors::LintTextError; use crate::lint_text::models::{InterpolationSite, Neutralized}; +pub(crate) static DEPENDENCY_SITE_PATTERN: LazyLock> = + LazyLock::new(|| Regex::new(r"(?i)^__(?:ref|source)\s*\(")); +pub(crate) static KEYED_RELATION_SITE_PATTERN: LazyLock> = + LazyLock::new(|| Regex::new(r#"(?i)^__(ref|source|seed)\s*\(\s*['"]([^'"]+)['"]\s*\)$"#)); + static LINT_SITE: LazyLock> = LazyLock::new(|| Regex::new(LINT_SITE_PATTERN)); static FORMAT_SITE: LazyLock> = diff --git a/crates/sqlbuild-rules/src/lint_text/main/dependency_sites.rs b/crates/sqlbuild-rules/src/lint_text/main/dependency_sites.rs new file mode 100644 index 0000000000..a96a5ed93e --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/dependency_sites.rs @@ -0,0 +1,21 @@ +use crate::lint_text::_helpers::sites::{DEPENDENCY_SITE_PATTERN, KEYED_RELATION_SITE_PATTERN}; +use crate::lint_text::models::{DependencySites, InterpolationSite}; + +/// Derive dependency sentinels and stable relation identities from interpolation sites. +pub fn dependency_sites(sites: &[InterpolationSite]) -> Result { + let dependency = DEPENDENCY_SITE_PATTERN.as_ref().map_err(Clone::clone)?; + let keyed_relation = KEYED_RELATION_SITE_PATTERN.as_ref().map_err(Clone::clone)?; + let mut result = DependencySites::default(); + for site in sites { + if dependency.is_match(&site.original_text) { + result.identifiers.push(site.sentinel.clone()); + } + if let Some(captures) = keyed_relation.captures(&site.original_text) { + result.relations.push(( + site.sentinel.clone(), + format!("{}:{}", captures[1].to_lowercase(), &captures[2]), + )); + } + } + Ok(result) +} diff --git a/crates/sqlbuild-rules/src/lint_text/main/mod.rs b/crates/sqlbuild-rules/src/lint_text/main/mod.rs index 9b6cd55683..b599ea6761 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/mod.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/mod.rs @@ -1,4 +1,5 @@ pub mod apply_suppressions; +pub mod dependency_sites; pub mod first_header_only; pub mod format_sql_bodies; pub mod interpolation_text_at; diff --git a/crates/sqlbuild-rules/src/lint_text/models.rs b/crates/sqlbuild-rules/src/lint_text/models.rs index dafaf23b63..bef8d138d3 100644 --- a/crates/sqlbuild-rules/src/lint_text/models.rs +++ b/crates/sqlbuild-rules/src/lint_text/models.rs @@ -1,5 +1,12 @@ //! Interpolation sites, suppression outcomes and fixture-null candidates. +/// Dependency sentinels and keyed relations derived from authored interpolation sites. +#[derive(Debug, Default)] +pub struct DependencySites { + pub identifiers: Vec, + pub relations: Vec<(String, String)>, +} + /// One interpolation occurrence replaced by a sentinel; offsets are character positions. #[derive(Clone, Debug, PartialEq, Eq)] pub struct InterpolationSite { diff --git a/fensu.toml b/fensu.toml index f0f89d3c32..1e559ae0a3 100644 --- a/fensu.toml +++ b/fensu.toml @@ -28,6 +28,11 @@ rule = "FFR001" path = "src/sqlbuild/cli/commands/models.py" reason = "The TYPE_CHECKING block contains annotation-only imports, avoiding executor and integration imports for read-only commands; runtime declarations remain structured models." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/lint/__init__.py" +reason = "Native bindings own lint entry points, expansion preparation and compiler-verified fix planning; this package retains boundary models, custom-rule and macro callbacks." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/sql_values/constants.py" diff --git a/python-boundary.toml b/python-boundary.toml index 852366b9f6..1a50270759 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 42632 +remaining_lines = 39461 [[stays]] path = "executor" @@ -624,11 +624,3 @@ note = "A test file that no longer parses is left unformatted, as before; no Pyt [[handled_native_errors]] location = "cli/commands/_helpers/refactor/compile_facts.py::compile_for_refactor" note = "Reports the Python stage errors the native compile driver propagates as refactor diagnostics; a native compiler failure is a RuntimeError and is not caught." - -[[handled_native_errors]] -location = "lint/_helpers/fixes.py::_FixSession._relation_keys" -note = "A baseline that does not compile yields no relation keys, as before the compile entry became native; no Python implementation runs instead." - -[[handled_native_errors]] -location = "lint/_helpers/fixes.py::_FixSession._verdict" -note = "A candidate fix whose project fails to compile is unverified and not applied, as before the compile entry became native; no Python implementation runs instead." diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 9ef86c388e..172ebbb4f1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -31,6 +31,7 @@ from sqlbuild.cli.commands.models import ( ContractCommandRequest, CostCommandRequest, DbtInitCommandRequest, + DbtSqlbuildWorkContext, DiffCommandRequest, FreshnessCommandRequest, JanitorCommandRequest, @@ -201,6 +202,7 @@ from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( AuditPlanEntry, ChangeDetectionResult, + ColumnRenameHint, CursorBounds, DeferralInputs, DirectModelVersionIdentities, @@ -216,6 +218,7 @@ from sqlbuild.compiler.planner.models import ( PlannerScope, PlanOutput, PlanWarning, + QueryShape, ReferenceRename, SchemaAction, SchemaFinding, @@ -231,6 +234,7 @@ from sqlbuild.compiler.planner.models import ( from sqlbuild.compiler.planner.types import ( CursorGrain, CursorType, + InputColumns, MaterializationType, OnSchemaChange, PlanAction, @@ -321,21 +325,42 @@ from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.models import ( DbtCliConfigOverrides, DbtCliOptions, + DbtCommandExecutionResult, + DbtCommandResult, DbtInteropCommandArgs, + DbtInteropCompiledProject, + DbtInteropExecutionRequest, + DbtInteropInvocation, DbtInteropParsedArgs, DbtInteropPlan, + DbtInteropPlanResolution, DbtInteropRoutedArgs, DbtInteropSelectionResult, DbtLsNode, DbtManifestIndex, DbtManifestModel, DbtNodeExecutionResult, + DbtPlanEnvironment, + DbtSqlbuildPlanRequest, NormalizedDbtProfileConnection, ResolvedDbtConfig, ResolvedDbtProfileOutput, ) from sqlbuild.integrations.dbt.types import DbtInteropCommand -from sqlbuild.lint.models import HeaderSpan, LintBody, LintConfig, LintRunResult, LintViolation +from sqlbuild.lint.models import ( + CompileFacts, + ExpandedLintFile, + FixVerdict, + HeaderSpan, + LintBody, + LintConfig, + LintFileRole, + LintRunResult, + LintViolation, + PendingLintBody, + PreparedLintFiles, + RuleFixResult, +) from sqlbuild.lint.types import RelationKeyRow, RelationKeys from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.models import DisplayOptions @@ -1183,6 +1208,71 @@ def declared_relation_keys( sources: Iterable[SourceEntry], ) -> RelationKeys: ... def compiled_relation_keys(project: CompiledProjectView) -> RelationKeys: ... +def compile_facts(*, project: CompiledProjectView, project_dir: Path) -> CompileFacts: ... +def lint_file_role( + *, file_path: Path, project_dir: Path, relative_path: PurePath | None = None +) -> LintFileRole: ... +def lint_body_ranges( + *, + contents: str, + headers: tuple[HeaderSpan, ...], + file_path: Path, + project_dir: Path, + role: LintFileRole | None = None, +) -> tuple[tuple[int, int], ...]: ... +def prepare_lint_body( + *, + role: LintFileRole, + file_path: Path, + contents: str, + body_range: tuple[int, int], + context: SqlExpansionContext, + dialect: str, + external_identifiers: tuple[str, ...] = (), + allows_ceremonial_select: bool = False, + allows_dynamic_output_star: bool = False, + compiled_expansion: CompiledSqlExpansion | None = None, +) -> LintBody: ... +def expand_lint_body( + *, + role: LintFileRole, + file_path: Path, + contents: str, + body_range: tuple[int, int], + context: SqlExpansionContext, + dialect: str, + external_identifiers: tuple[str, ...] = (), + allows_ceremonial_select: bool = False, + allows_dynamic_output_star: bool = False, + compiled_expansion: CompiledSqlExpansion | None = None, +) -> PendingLintBody: ... +def expand_file_bodies( + *, + file_path: Path, + contents: str, + headers: tuple[HeaderSpan, ...], + context: SqlExpansionContext, + dialect: str, + project_dir: Path, + relative_path: Path, + dynamic_output_paths: frozenset[Path], + compiled_expansions: dict[Path, CompiledSqlExpansion] | None, +) -> ExpandedLintFile: ... +def prepare_expanded_files( + *, expanded_files: tuple[ExpandedLintFile, ...], dialect: str, skip_unexpandable: bool +) -> PreparedLintFiles: ... +def plan_rule_fixes( + *, + project_dir: Path, + files: dict[Path, str], + config: LintConfig, + adapter: TypedSqlValueRenderer | None, + discovered_inputs: DiscoveredProjectInputs | None, + enabled: bool, +) -> tuple[dict[Path, str], tuple[RuleFixResult, ...], list[LintViolation]]: ... +def fix_verdict( + *, before: CompileFacts, after: CompileFacts, edited: frozenset[Path] +) -> FixVerdict: ... def build_lint_expansion_context( *, project_dir: Path, @@ -1683,6 +1773,144 @@ def build_dbt_ls_argv( exclude: Sequence[str] | None = None, resource_types: Sequence[str] | None = None, ) -> tuple[str, ...]: ... +def resolve_dbt_execution_invocation( + request: DbtInteropExecutionRequest, +) -> DbtInteropInvocation: ... +def dbt_interop_plan_safety_gates() -> PlanSafetyGates: ... +def execute_dbt_interop_from_project(request: DbtInteropExecutionRequest) -> int: ... +def plan_dbt_interop_from_project( + *, + project_dir: Path, + args: tuple[str, ...], + dbt_runner: DbtRunner | None = None, + dbt_executable: str | None = None, + sqlbuild_executable: str = "sqb", + no_sql_validation: bool = False, + on_progress: Callable[[str], None] | None = None, + progress_stream: TextIO | None = None, + use_color: bool = False, +) -> DbtInteropPlan: ... +def debug_dbt_from_project( + *, + project_dir: Path, + args: tuple[str, ...], + dbt_runner: DbtRunner | None = None, + dbt_executable: str | None = None, + stdout_stream: TextIO, + stderr_stream: TextIO, +) -> int: ... +def resolve_dbt_connection_config( + *, + request: DbtInteropExecutionRequest, + invocation: DbtInteropInvocation, + compiled: DbtInteropCompiledProject, +) -> dict[str, object]: ... +def write_dbt_execution_plan_text( + *, + request: DbtInteropExecutionRequest, + invocation: DbtInteropInvocation, + plan: DbtInteropPlan, + merged_dbt_argv: tuple[str, ...] | None, +) -> None: ... +def execute_dbt_without_state_tracking( + *, + request: DbtInteropExecutionRequest, + invocation: DbtInteropInvocation, + merged_dbt_argv: tuple[str, ...] | None, +) -> DbtCommandExecutionResult: ... +def write_dbt_execution_summary( + *, + request: DbtInteropExecutionRequest, + invocation: DbtInteropInvocation, + execution: DbtCommandExecutionResult, +) -> None: ... +def resolve_sqlbuild_execution_plan_output( + *, + request: DbtInteropExecutionRequest, + invocation: DbtInteropInvocation, + compiled: DbtInteropCompiledProject, + manifest: DbtManifestIndex, + plan: DbtInteropPlan, + failed_sqlbuild_model_names: tuple[str, ...], +) -> PlanOutput | None: ... +def run_dbt_sqlbuild_work( + *, + request: DbtInteropExecutionRequest, + invocation: DbtInteropInvocation, + compiled: DbtInteropCompiledProject, + plan_output: PlanOutput, + connection_config: dict[str, object], +) -> int: ... +def write_sqlbuild_skip_notice( + *, + request: DbtInteropExecutionRequest, + invocation: DbtInteropInvocation, + message: str, +) -> None: ... +def execute_dbt_sqlbuild_work( + *, + context: DbtSqlbuildWorkContext, + command: DbtInteropCommand, + project: CompiledProjectView, + project_dir: Path, + fail_fast: bool, + verbose: bool, +) -> int: ... +def build_sqlbuild_plan_output( + *, + environment: DbtPlanEnvironment, + request: DbtSqlbuildPlanRequest, + hooks: ConnectionHooks, +) -> PlanOutput | None: ... +def attach_sqlbuild_plan_output( + *, + plan: DbtInteropPlan, + project_dir: Path, + discovered_inputs: DiscoveredProjectInputs, + compiled: DbtInteropCompiledProject, + sqlbuild_args: tuple[str, ...], + connection_progress: Any | None, + on_progress: Callable[[str], None] | None, +) -> DbtInteropPlan: ... +def dbt_failure_detail(result: DbtCommandResult) -> str | None: ... +def load_compiled_dbt_manifest( + *, + runner: DbtRunner, + dbt_options: DbtCliOptions, + full_refresh: bool, + on_progress: Callable[[str], None] | None, +) -> DbtManifestIndex: ... +def compile_dbt_interop_project( + *, + project_dir: Path, + discovered_inputs: DiscoveredProjectInputs, + manifest: DbtManifestIndex, + dbt_vars: dict[str, object], + no_sql_validation: bool, + on_progress: Callable[[str], None] | None, +) -> DbtInteropCompiledProject: ... +def resolve_dbt_interop_plan( + *, + command: DbtInteropCommand, + invocation: DbtInteropInvocation, + compiled: DbtInteropCompiledProject, + manifest: DbtManifestIndex, + sqlbuild_executable: str, + on_progress: Callable[[str], None] | None, +) -> DbtInteropPlanResolution: ... +def execute_dbt_commands( + *, + runner: DbtRunner, + options: DbtCliOptions, + merged_argv: tuple[str, ...] | None, + progress_stream: TextIO, + stdout_stream: TextIO, + use_color: bool, + skip_message: str = "Skipping dbt: no dbt work selected.", + expected_nodes: tuple[DbtLsNode, ...] | None = None, + on_node_result: Callable[[DbtNodeExecutionResult], None] | None = None, + on_progress: Callable[[str], None] | None = None, +) -> DbtCommandExecutionResult: ... def parse_dbt_ls_json_lines(*, stdout: str) -> tuple[DbtLsNode, ...]: ... def route_dbt_interop_args( *, command: DbtInteropCommand | str, parsed: DbtInteropParsedArgs @@ -4060,6 +4288,34 @@ def detect_model_changes_in_scope( expected_version_hashes: dict[str, str] | None = None, expected_metadata_jsons: dict[str, str] | None = None, ) -> dict[str, ChangeDetectionResult]: ... +def parse_query_shape(*, query_sql: str, dialect: str | None) -> QueryShape | None: ... +def renames_explain_change( + *, + previous: QueryShape, + current: QueryShape, + renames: Mapping[str, str], + input_columns: InputColumns, +) -> bool: ... +def build_migration_fingerprint( + *, query_sql: str, metadata_json: str, ref_identities: Mapping[str, str], dialect: str | None +) -> str | None: ... +def migration_fingerprint_ref_names(*, query_sql: str, metadata_json: str) -> frozenset[str]: ... +def identical_renames( + *, + previous: QueryShape, + current: QueryShape, + excluded: frozenset[str], + declared: Mapping[str, str], + input_columns: InputColumns, +) -> tuple[tuple[str, str], ...]: ... +def rename_hints( + *, + model_name: str, + previous: QueryShape, + current: QueryShape, + excluded: frozenset[str], + live_columns: frozenset[str], +) -> tuple[ColumnRenameHint, ...]: ... def detect_model_changes( *, model: CompiledModelView, diff --git a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py deleted file mode 100644 index caff1a6b4d..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py +++ /dev/null @@ -1,141 +0,0 @@ -"""SQLBuild work helpers for dbt interop commands.""" - -from __future__ import annotations - -import sys -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import ( - BuildProgressCallbacks, - ConnectionProgressReporter, - enforce_plan_safety_policies, - format_build_footer, - format_execution_header, - write_runtime_target, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands.constants import ( - EXECUTION_LIMIT_DBT_NOTE, - MISSING_ORIGIN_DBT_HELP, - RETENTION_DECREASE_DBT_HELP, - SNAPSHOT_FULL_REFRESH_DBT_HELP, - TABLE_TYPE_DOWNGRADE_DBT_HELP, -) -from sqlbuild.cli.commands.models import DbtSqlbuildWorkContext, PlanSafetyGates -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import ( - PlanOutput, -) -from sqlbuild.executor.build.models import ( - BuildCallbacks, - BuildExecutionResult, - BuildRuntimeParams, -) -from sqlbuild.executor.build.types import BuildStatus -from sqlbuild.executor.pipeline.main.run import ( - run_build_pipeline, -) -from sqlbuild.integrations.dbt.types import DbtInteropCommand -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def dbt_interop_plan_safety_gates() -> PlanSafetyGates: - """Gates for sqb dbt, which has no allow flags and confirms only on a terminal.""" - - return PlanSafetyGates( - allow_missing_migration_origin=False, - allow_snapshot_full_refresh=False, - allow_table_type_downgrade=False, - allow_retention_decrease=False, - missing_origin_help=MISSING_ORIGIN_DBT_HELP, - snapshot_full_refresh_help=SNAPSHOT_FULL_REFRESH_DBT_HELP, - table_type_downgrade_help=TABLE_TYPE_DOWNGRADE_DBT_HELP, - retention_decrease_help=RETENTION_DECREASE_DBT_HELP, - execution_limit_note=EXECUTION_LIMIT_DBT_NOTE, - ) - - -def execute_sqlbuild_build_work( - *, - context: DbtSqlbuildWorkContext, - command: DbtInteropCommand, - project: CompiledProjectView, - project_dir: Path, - fail_fast: bool, - verbose: bool, -) -> int: - plan_output: PlanOutput = context.plan_output - connection_config: dict[str, object] = context.connection_config - adapter: BaseAdapter = context.adapter - adapter_name: str = context.adapter_name - output_stream: TextIO = context.output_stream - use_color: bool = context.use_color - enforce_plan_safety_policies( - plan=plan_output, - gates=dbt_interop_plan_safety_gates(), - snapshots_config=context.snapshots_config, - target_name=project.effective_target_name, - execution_limits=context.execution_limits, - streams=(sys.stdin, output_stream), - ) - callbacks: BuildProgressCallbacks = BuildProgressCallbacks( - plan=plan_output, use_color=use_color, verbose=verbose, debug=False - ) - effective_concurrency: int = project.settings.concurrency - display_command: str = ( - "sqb build --no-tests --no-audits" - if command == DbtInteropCommand.RUN - else f"sqb {command.value}" - ) - header: str = format_execution_header( - command=display_command, target=None, concurrency=effective_concurrency - ) - style: CliStyle = CliStyle(use_color=use_color) - execution_label: str = style.object_name("SQLBuild execution") - header_detail: str = style.muted(header) - output_stream.write(f"{execution_label} {header_detail}\n\n") - output_stream.flush() - execution_connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=adapter_name, - stream=output_stream, - blank_line_after_complete=True, - use_color=use_color, - ) - result: BuildExecutionResult = run_build_pipeline( - plan=plan_output, - connection_config=connection_config, - adapter=adapter, - settings=project.settings, - runtime=BuildRuntimeParams( - run_id=project.run_id, - run_tests=command == DbtInteropCommand.BUILD, - run_audits=command == DbtInteropCommand.BUILD, - fail_fast=fail_fast, - max_concurrency=effective_concurrency, - use_color=use_color, - ), - callbacks=BuildCallbacks( - on_node_start=lambda name, resource_kind: callbacks.on_node_start( - name=name, resource_kind=resource_kind - ), - on_node_complete=callbacks.on_node_complete, - on_sub_progress=callbacks.on_sub_progress, - on_connection_start=execution_connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - execution_connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - execution_connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ), - ) - write_runtime_target(target_dir=project_dir / "target", plan_output=plan_output, result=result) - footer: str = format_build_footer(result=result, elapsed=callbacks.elapsed, use_color=use_color) - output_stream.write("\n" + footer + "\n") - output_stream.flush() - return 0 if result.status == BuildStatus.SUCCESS else 1 diff --git a/src/sqlbuild/cli/commands/main/dbt/dbt_sqlbuild_work.py b/src/sqlbuild/cli/commands/main/dbt/dbt_sqlbuild_work.py deleted file mode 100644 index 79a4263063..0000000000 --- a/src/sqlbuild/cli/commands/main/dbt/dbt_sqlbuild_work.py +++ /dev/null @@ -1,31 +0,0 @@ -"""SQLBuild work public entry for dbt interop orchestration.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.commands._helpers.dbt.sqlbuild_work import execute_sqlbuild_build_work -from sqlbuild.cli.commands.models import DbtSqlbuildWorkContext -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.integrations.dbt.types import DbtInteropCommand - - -def execute_dbt_sqlbuild_work( - *, - context: DbtSqlbuildWorkContext, - command: DbtInteropCommand, - project: CompiledProjectView, - project_dir: Path, - fail_fast: bool, - verbose: bool, -) -> int: - """Execute SQLBuild build work selected by a dbt interop command.""" - - return execute_sqlbuild_build_work( - context=context, - command=command, - project=project, - project_dir=project_dir, - fail_fast=fail_fast, - verbose=verbose, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/column_renames.py b/src/sqlbuild/compiler/planner/_helpers/migrations/column_renames.py deleted file mode 100644 index 6b38fba99f..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/column_renames.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Recognize renamed output columns only when the rest of the query is provably unchanged.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.compiler.planner._helpers.migrations.projections import renames_explain_change -from sqlbuild.compiler.planner.models import ColumnRenameHint, QueryProjection, QueryShape -from sqlbuild.compiler.planner.types import InputColumns - - -def identical_renames( - *, - previous: QueryShape, - current: QueryShape, - excluded: frozenset[str], - declared: Mapping[str, str], - input_columns: InputColumns, -) -> tuple[tuple[str, str], ...]: - """Return same-expression (old, new) pairs only when renaming is the whole change.""" - - removed: tuple[QueryProjection, ...] - added: tuple[QueryProjection, ...] - removed, added = _removed_and_added(previous=previous, current=current, excluded=excluded) - pairs: list[tuple[str, str]] = [] - column: QueryProjection - for column in added: - matches: tuple[QueryProjection, ...] = tuple( - origin for origin in removed if origin.expression == column.expression - ) - if len(matches) != 1: - continue - claims: int = sum(1 for other in added if other.expression == matches[0].expression) - if claims == 1: - pairs.append((matches[0].name, column.name)) - if not pairs or not renames_explain_change( - previous=previous, - current=current, - renames={**declared, **dict(pairs)}, - input_columns=input_columns, - ): - return () - return tuple(pairs) - - -def rename_hints( - *, - model_name: str, - previous: QueryShape, - current: QueryShape, - excluded: frozenset[str], - live_columns: frozenset[str], -) -> tuple[ColumnRenameHint, ...]: - """Point added columns that resemble removed live columns at an explicit migrate_from.""" - - removed: tuple[QueryProjection, ...] - added: tuple[QueryProjection, ...] - removed, added = _removed_and_added(previous=previous, current=current, excluded=excluded) - live_removed: tuple[QueryProjection, ...] = tuple( - origin for origin in removed if origin.key in live_columns - ) - same_binding: bool = previous.body == current.body - hints: list[ColumnRenameHint] = [] - column: QueryProjection - for column in added: - if column.key in live_columns: - continue - identical: tuple[str, ...] = tuple( - origin.name - for origin in live_removed - if same_binding and origin.expression == column.expression - ) - similar: tuple[str, ...] = tuple( - origin.name - for origin in live_removed - if origin.references and origin.references == column.references - ) - if identical or similar: - hints.append( - ColumnRenameHint( - model_name=model_name, - added_column=column.name, - candidate_columns=identical or similar, - identical=bool(identical), - ) - ) - return tuple(hints) - - -def _removed_and_added( - *, previous: QueryShape, current: QueryShape, excluded: frozenset[str] -) -> tuple[tuple[QueryProjection, ...], tuple[QueryProjection, ...]]: - """Return previous-only and current-only output columns by case-insensitive name.""" - - previous_names: frozenset[str] = frozenset(item.key for item in previous.projections) - current_names: frozenset[str] = frozenset(item.key for item in current.projections) - return ( - tuple( - item - for item in previous.projections - if item.key not in current_names and item.key not in excluded - ), - tuple( - item - for item in current.projections - if item.key not in previous_names and item.key not in excluded - ), - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py index c4b55e4ef3..2c5d8723e4 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py @@ -5,7 +5,14 @@ from collections.abc import Mapping from dataclasses import dataclass, replace -from sqlbuild._native import get_config_str, model_definition_hash +from sqlbuild._native import ( + get_config_str, + identical_renames, + model_definition_hash, + parse_query_shape, + rename_hints, + renames_explain_change, +) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType @@ -16,19 +23,11 @@ from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location from sqlbuild.compiler.migrations.models import ColumnMigrationEvent from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDiscovery -from sqlbuild.compiler.planner._helpers.migrations.column_renames import ( - identical_renames, - rename_hints, -) from sqlbuild.compiler.planner._helpers.migrations.planning import ( missing_origin_policy, planning_database, redirected_cursor_snapshots, ) -from sqlbuild.compiler.planner._helpers.migrations.projections import ( - parse_query_shape, - renames_explain_change, -) from sqlbuild.compiler.planner.classes.known_input_columns import KnownInputColumns from sqlbuild.compiler.planner.classes.migration_state_inspection import ( MigrationStateInspection, diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py b/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py deleted file mode 100644 index 31e74658f5..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py +++ /dev/null @@ -1,335 +0,0 @@ -"""Local migration fingerprints that identify the same table across renames.""" - -from __future__ import annotations - -import hashlib -import json -import re -from collections.abc import Mapping -from typing import Any - -from sqlbuild._native import version_identity_metadata_payload -from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY -from sqlbuild.compiler.planner.constants import ( - MIGRATION_FINGERPRINT_EXCLUDED_CONFIG_KEYS, - MIGRATION_LOCAL_NAME_PREFIX, - MIGRATION_MODEL_NAME_METADATA_KEY, - MIGRATION_REF_PLACEHOLDER_PREFIX, -) -from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql - -_REF_MARKER: re.Pattern[str] = re.compile(r"""__ref\(\s*(?:"([^"]+)"|'([^']+)')\s*\)""") -_IDENTIFIER_NAME_KEY: str = "name" -_IDENTIFIER_QUOTED_KEY: str = "quoted" -_ALIAS_KEY: str = "alias" -_WITH_KEY: str = "with" -_CTES_KEY: str = "ctes" -_SPAN_KEY: str = "span" -_COMMENTS_SUFFIX: str = "comments" -_CONFIG_KEY: str = "config" -_GENERIC_DIALECT: str = "generic" -_TABLE_KEY: str = "table" -_RECURSIVE_KEY: str = "recursive" -_CTE_BODY_KEY: str = "this" -_ALIASED_RELATION_KEYS: frozenset[str] = frozenset({_TABLE_KEY, "subquery"}) -_QUALIFIED_REFERENCE_KEYS: frozenset[str] = frozenset({"column", "star"}) -_RELATION_QUALIFIER_KEYS: tuple[str, ...] = ("schema", "catalog") - - -def build_migration_fingerprint( - *, - query_sql: str, - metadata_json: str, - ref_identities: Mapping[str, str], - dialect: str | None, -) -> str | None: - """Hash normalized SQL and non-storage metadata with refs resolved through renames.""" - - normalized_sql: str | None = _normalized_query_sql( - query_sql=query_sql, ref_identities=ref_identities, dialect=dialect - ) - if normalized_sql is None: - return None - payload: str = json.dumps( - { - "metadata": _identity_metadata( - metadata_json=metadata_json, ref_identities=ref_identities - ), - "sql": normalized_sql, - }, - sort_keys=True, - separators=(",", ":"), - default=str, - ) - return hashlib.sha256(payload.encode("utf-8")).hexdigest() - - -def migration_fingerprint_ref_names(*, query_sql: str, metadata_json: str) -> frozenset[str]: - """Return every model name whose rename identity can change the migration fingerprint.""" - - names: set[str] = { - match.group(1) or match.group(2) for match in _REF_MARKER.finditer(query_sql) - } - payload: Any = version_identity_metadata_payload(metadata_json) - config: Any = ( - {str(key): value for key, value in payload.items()}.get(_CONFIG_KEY) - if isinstance(payload, dict) - else None - ) - cursor_inputs: Any = ( - {str(key): value for key, value in config.items()}.get(CURSOR_INPUTS_CONFIG_KEY) - if isinstance(config, dict) - else None - ) - if isinstance(cursor_inputs, dict): - names.update(str(name) for name in cursor_inputs) - return frozenset(names) - - -def _identity_metadata(*, metadata_json: str, ref_identities: Mapping[str, str]) -> Any: - payload: Any = version_identity_metadata_payload(metadata_json) - if not isinstance(payload, dict): - return payload - identity: dict[str, Any] = {str(key): value for key, value in payload.items()} - _ = identity.pop(MIGRATION_MODEL_NAME_METADATA_KEY, None) - config: Any = identity.get(_CONFIG_KEY) - if isinstance(config, dict): - identity[_CONFIG_KEY] = { - key: _resolved_cursor_inputs(value=value, ref_identities=ref_identities) - if key == CURSOR_INPUTS_CONFIG_KEY - else value - for key, value in config.items() - if key not in MIGRATION_FINGERPRINT_EXCLUDED_CONFIG_KEYS - } - return identity - - -def _resolved_cursor_inputs(*, value: Any, ref_identities: Mapping[str, str]) -> Any: - if not isinstance(value, dict): - return value - return {ref_identities.get(str(name), str(name)): column for name, column in value.items()} - - -def _normalized_query_sql( - *, query_sql: str, ref_identities: Mapping[str, str], dialect: str | None -) -> str | None: - placeholder_sql: str = _REF_MARKER.sub( - lambda match: _ref_placeholder( - identity=ref_identities.get( - match.group(1) or match.group(2), match.group(1) or match.group(2) - ) - ), - query_sql, - ) - polyglot: Any = import_polyglot_sql() - effective_dialect: str = dialect or _GENERIC_DIALECT - try: - parsed: dict[str, Any] = polyglot.parse_one( - placeholder_sql, dialect=effective_dialect - ).to_dict() - except polyglot.PolyglotError: - return None - stripped: Any = strip_ast_formatting(parsed) - definitions: list[tuple[str, bool]] = _local_names(stripped) - if not _has_unambiguous_local_scopes(root=stripped, definitions=definitions): - return None - ordered_names: list[str] = list(dict.fromkeys(name for name, _ in definitions)) - local_names: dict[str, str] = { - name: f"{MIGRATION_LOCAL_NAME_PREFIX}{index}" - for index, name in enumerate(ordered_names, start=1) - } - cte_names: dict[str, str] = {name: local_names[name] for name, is_cte in definitions if is_cte} - canonical: Any = _canonical_local_names(node=stripped, names=local_names, cte_names=cte_names) - generated: list[str] = polyglot.generate(canonical, dialect=effective_dialect) - return "\n".join(generated) - - -def _ref_placeholder(*, identity: str) -> str: - digest: str = hashlib.sha256(identity.encode("utf-8")).hexdigest()[:16] - return f"{MIGRATION_REF_PLACEHOLDER_PREFIX}{digest}" - - -def strip_ast_formatting(node: Any) -> Any: - """Drop source spans and comments from a parsed SQL tree.""" - - if isinstance(node, dict): - return { - key: [] if key.endswith(_COMMENTS_SUFFIX) else strip_ast_formatting(value) - for key, value in node.items() - if key != _SPAN_KEY - } - if isinstance(node, list): - return [strip_ast_formatting(value) for value in node] - return node - - -def _has_unambiguous_local_scopes(*, root: Any, definitions: list[tuple[str, bool]]) -> bool: - """Accept one top-level WITH whose names never shadow physical relations.""" - - statements: list[Any] = list(root.values()) if isinstance(root, dict) else [] - top_level: Any = statements[0].get(_WITH_KEY) if len(statements) == 1 else None - clauses: list[Any] = _with_clauses(root) - if clauses and (not isinstance(top_level, dict) or len(clauses) > 1): - return False - if isinstance(top_level, dict) and top_level.get(_RECURSIVE_KEY): - return False - ctes: list[Any] = [ - cte for cte in (top_level or {}).get(_CTES_KEY) or () if isinstance(cte, dict) - ] - cte_names: list[str] = _identifier_names([cte.get(_ALIAS_KEY) for cte in ctes]) - if len(cte_names) != len(ctes) or len(set(cte_names)) != len(cte_names): - return False - index: int - cte: Any - for index, cte in enumerate(ctes): - if set(_relation_names(cte.get(_CTE_BODY_KEY))) & set(cte_names[index:]): - return False - physical: set[str] = set(_relation_names(root)) - set(cte_names) - aliases: set[str] = {name for name, is_cte in definitions if not is_cte} - return not physical & aliases - - -def _with_clauses(node: Any) -> list[Any]: - found: list[Any] = [] - if isinstance(node, list): - item: Any - for item in node: - found.extend(_with_clauses(item)) - return found - if not isinstance(node, dict): - return found - key: str - value: Any - for key, value in node.items(): - if key == _WITH_KEY and isinstance(value, dict): - found.append(value) - found.extend(_with_clauses(value)) - return found - - -def _relation_names(node: Any) -> list[str]: - """Return lower-cased unqualified relation names referenced under a node.""" - - found: list[str] = [] - if isinstance(node, list): - item: Any - for item in node: - found.extend(_relation_names(item)) - return found - if not isinstance(node, dict): - return found - key: str - value: Any - for key, value in node.items(): - if key == _TABLE_KEY and _is_relation(value) and _unqualified(value): - found.extend(_identifier_names([value.get(_IDENTIFIER_NAME_KEY)])) - found.extend(_relation_names(value)) - return found - - -def _unqualified(relation: dict[str, Any]) -> bool: - return all(relation.get(key) is None for key in _RELATION_QUALIFIER_KEYS) - - -def _local_names(node: Any) -> list[tuple[str, bool]]: - """Return (name, is_cte) for CTE and relation alias definitions in first-seen order.""" - - found: list[tuple[str, bool]] = [] - if isinstance(node, list): - item: Any - for item in node: - found.extend(_local_names(item)) - return found - if not isinstance(node, dict): - return found - with_clause: Any = node.get(_WITH_KEY) - if isinstance(with_clause, dict): - cte_aliases: list[Any] = [ - cte.get(_ALIAS_KEY) for cte in with_clause.get(_CTES_KEY) or () if isinstance(cte, dict) - ] - found.extend((name, True) for name in _identifier_names(cte_aliases)) - key: str - value: Any - for key, value in node.items(): - if key in _ALIASED_RELATION_KEYS and _is_relation(value): - found.extend((name, False) for name in _identifier_names([value.get(_ALIAS_KEY)])) - found.extend(_local_names(value)) - return found - - -def _identifier_names(identifiers: list[Any]) -> list[str]: - return [ - str(identifier[_IDENTIFIER_NAME_KEY]).lower() - for identifier in identifiers - if _is_identifier(identifier) - ] - - -def _canonical_local_names( - *, node: Any, names: Mapping[str, str], cte_names: Mapping[str, str] -) -> Any: - """Rename CTE and alias names only where they define or qualify a relation.""" - - if isinstance(node, list): - return [ - _canonical_local_names(node=value, names=names, cte_names=cte_names) for value in node - ] - if not isinstance(node, dict): - return node - rewritten: dict[str, Any] = { - key: _canonical_local_names(node=value, names=names, cte_names=cte_names) - for key, value in node.items() - } - key: str - value: Any - for key, value in node.items(): - if key in _QUALIFIED_REFERENCE_KEYS and isinstance(value, dict): - rewritten[key] = { - **rewritten[key], - _TABLE_KEY: _renamed(identifier=value.get(_TABLE_KEY), names=names), - } - elif key in _ALIASED_RELATION_KEYS and _is_relation(value): - rewritten[key] = { - **rewritten[key], - _ALIAS_KEY: _renamed(identifier=value.get(_ALIAS_KEY), names=names), - **_cte_reference(relation=value, cte_names=cte_names), - } - elif key == _WITH_KEY and isinstance(value, dict): - rewritten[key] = { - **rewritten[key], - _CTES_KEY: [ - {**cte, _ALIAS_KEY: _renamed(identifier=cte.get(_ALIAS_KEY), names=names)} - for cte in rewritten[key].get(_CTES_KEY) or () - ], - } - return rewritten - - -def _cte_reference(*, relation: dict[str, Any], cte_names: Mapping[str, str]) -> dict[str, Any]: - """Rename an unqualified relation name that refers to a CTE.""" - - name: Any = relation.get(_IDENTIFIER_NAME_KEY) - if not _unqualified(relation) or not _is_identifier(name): - return {} - return {_IDENTIFIER_NAME_KEY: _renamed(identifier=name, names=cte_names)} - - -def _renamed(*, identifier: Any, names: Mapping[str, str]) -> Any: - if not _is_identifier(identifier): - return identifier - canonical: str | None = names.get(str(identifier[_IDENTIFIER_NAME_KEY]).lower()) - if canonical is None: - return identifier - return {**identifier, _IDENTIFIER_NAME_KEY: canonical, _IDENTIFIER_QUOTED_KEY: False} - - -def _is_relation(node: Any) -> bool: - return isinstance(node, dict) and not _is_identifier(node) - - -def _is_identifier(node: Any) -> bool: - return ( - isinstance(node, dict) - and isinstance(node.get(_IDENTIFIER_NAME_KEY), str) - and _IDENTIFIER_QUOTED_KEY in node - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/projections.py b/src/sqlbuild/compiler/planner/_helpers/migrations/projections.py deleted file mode 100644 index ac50991eb0..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/projections.py +++ /dev/null @@ -1,445 +0,0 @@ -"""Formatting-insensitive query shapes used to prove that a change is only a column rename.""" - -from __future__ import annotations - -import json -from collections.abc import Mapping -from typing import Any - -from sqlbuild.compiler.planner._helpers.migrations.fingerprint import strip_ast_formatting -from sqlbuild.compiler.planner.constants import REF_INPUT_FUNCTION, SOURCE_INPUT_FUNCTION -from sqlbuild.compiler.planner.models import QueryProjection, QueryShape -from sqlbuild.compiler.planner.types import InputColumns -from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql - -_GENERIC_DIALECT: str = "generic" -_SELECT_KEY: str = "select" -_EXPRESSIONS_KEY: str = "expressions" -_ALIAS_KEY: str = "alias" -_THIS_KEY: str = "this" -_COLUMN_KEY: str = "column" -_NAME_KEY: str = "name" -_QUOTED_KEY: str = "quoted" -_TABLE_KEY: str = "table" -_LITERAL_KEY: str = "literal" -_LITERAL_TYPE_KEY: str = "literal_type" -_NUMBER_LITERAL: str = "number" -_GROUP_BY_KEY: str = "group_by" -_ORDER_BY_KEY: str = "order_by" -_DISTINCT_ON_KEY: str = "distinct_on" -_ALIAS_SCOPED_KEYS: tuple[str, ...] = (_ORDER_BY_KEY, "qualify") -_FROM_KEY: str = "from" -_JOINS_KEY: str = "joins" -_WITH_KEY: str = "with" -_ON_KEY: str = "on" -_LATERAL_VIEWS_KEY: str = "lateral_views" -_CTES_KEY: str = "ctes" -_COLUMNS_KEY: str = "columns" -_COLUMN_ALIASES_KEY: str = "column_aliases" -_FUNCTION_KEY: str = "function" -_ARGS_KEY: str = "args" -_SUBQUERY_KEY: str = "subquery" -_RELATION_FUNCTIONS: frozenset[str] = frozenset({SOURCE_INPUT_FUNCTION, REF_INPUT_FUNCTION}) -_UNSCANNED_CLAUSE_KEYS: frozenset[str] = frozenset( - {_EXPRESSIONS_KEY, _WITH_KEY, _FROM_KEY, _JOINS_KEY} -) - - -def parse_query_shape(*, query_sql: str, dialect: str | None) -> QueryShape | None: - """Return the named top-level projections of one plain SELECT, or None when unknowable.""" - - polyglot: Any = import_polyglot_sql() - try: - parsed: Any = polyglot.parse_one(query_sql, dialect=dialect or _GENERIC_DIALECT).to_dict() - except polyglot.PolyglotError: - return None - root: Any = strip_ast_formatting(parsed) - if not isinstance(root, dict) or set(root) != {_SELECT_KEY}: - return None - select: Any = root[_SELECT_KEY] - if not isinstance(select, dict) or not isinstance(select.get(_EXPRESSIONS_KEY), list): - return None - projections: list[QueryProjection] = [] - expression: Any - for expression in select[_EXPRESSIONS_KEY]: - projection: QueryProjection | None = _projection(expression) - if projection is None: - return None - projections.append(projection) - if len({projection.key for projection in projections}) != len(projections): - return None - canonical: dict[str, Any] = _canonical(select) - body: dict[str, Any] = { - key: value - for key, value in canonical.items() - if key != _EXPRESSIONS_KEY and key not in _ALIAS_SCOPED_KEYS - } - inputs: tuple[tuple[tuple[str, str], ...], frozenset[str]] | None = _select_inputs(canonical) - return QueryShape( - projections=tuple(projections), - body=_encode(body), - alias_clauses=_encode({key: canonical.get(key) for key in _ALIAS_SCOPED_KEYS}), - positional=_has_positional_reference(canonical), - input_relations=inputs[0] if inputs is not None else None, - local_input_columns=inputs[1] if inputs is not None else frozenset(), - clause_references=_unqualified_references( - [value for key, value in canonical.items() if key not in _UNSCANNED_CLAUSE_KEYS] - + [join.get(_ON_KEY) for join in _dicts(canonical.get(_JOINS_KEY))] - ), - ) - - -def renames_explain_change( - *, - previous: QueryShape, - current: QueryShape, - renames: Mapping[str, str], - input_columns: InputColumns, -) -> bool: - """Return whether renaming output columns, matched by name, is the whole query change.""" - - if not renames or previous.body != current.body: - return False - lowered: dict[str, str] = {old.lower(): new.lower() for old, new in renames.items()} - renamed_previous: dict[str, str] = { - lowered.get(item.key, item.key): item.expression for item in previous.projections - } - current_projections: dict[str, str] = { - item.key: item.expression for item in current.projections - } - if ( - len(renamed_previous) != len(previous.projections) - or renamed_previous != current_projections - ): - return False - reordered: bool = tuple(lowered.get(item.key, item.key) for item in previous.projections) != ( - tuple(item.key for item in current.projections) - ) - if reordered and (previous.positional or current.positional): - return False - if not all( - _binding_proven( - previous=previous, - current=current, - origin=origin, - destination=destination, - input_columns=input_columns, - ) - for origin, destination in lowered.items() - ): - return False - restored: Any = _substitute_alias_references( - node=json.loads(current.alias_clauses), - names={new: old for old, new in lowered.items()}, - ) - return _encode(restored) == previous.alias_clauses - - -def _binding_proven( - *, - previous: QueryShape, - current: QueryShape, - origin: str, - destination: str, - input_columns: InputColumns, -) -> bool: - """Return whether references to a renamed column provably bind the same in both queries.""" - - referenced: frozenset[str] = ( - previous.clause_references - | current.clause_references - | _other_projection_references(shape=previous, excluded=origin) - | _other_projection_references(shape=current, excluded=destination) - ) - if origin not in referenced and destination not in referenced: - return True - inputs: frozenset[str] | None = _known_input_columns(shape=current, input_columns=input_columns) - if inputs is None or destination in inputs: - return False - if origin in inputs: - return any(item.key == origin and item.passthrough for item in previous.projections) - return origin not in ( - current.clause_references | _other_projection_references(shape=current, excluded="") - ) - - -def _other_projection_references(*, shape: QueryShape, excluded: str) -> frozenset[str]: - return frozenset().union( - *(item.references for item in shape.projections if item.key != excluded) - ) - - -def _known_input_columns( - *, shape: QueryShape, input_columns: InputColumns -) -> frozenset[str] | None: - if shape.input_relations is None: - return None - known: set[str] = set(shape.local_input_columns) - kind: str - name: str - for kind, name in shape.input_relations: - columns: frozenset[str] | None = input_columns(kind, name) - if columns is None: - return None - known.update(columns) - return frozenset(known) - - -def _select_inputs( - select: Mapping[str, Any], -) -> tuple[tuple[tuple[str, str], ...], frozenset[str]] | None: - """Return a SELECT's referenced relations and derived-table columns, or None when unknown.""" - - if _dicts(select.get(_LATERAL_VIEWS_KEY)): - return None - ctes: dict[str, frozenset[str] | None] = _cte_columns(select.get(_WITH_KEY)) - items: list[Any] = list(_clause_items(select.get(_FROM_KEY))) - items.extend(join.get(_THIS_KEY) for join in _dicts(select.get(_JOINS_KEY))) - relations: list[tuple[str, str]] = [] - local: set[str] = set() - item: Any - for item in items: - resolved: tuple[tuple[str, str] | None, frozenset[str] | None] = _input_item( - item=item, ctes=ctes - ) - if resolved[0] is not None: - relations.append(resolved[0]) - elif resolved[1] is not None: - local.update(resolved[1]) - else: - return None - return tuple(relations), frozenset(local) - - -def _input_item( - *, item: Any, ctes: Mapping[str, frozenset[str] | None] -) -> tuple[tuple[str, str] | None, frozenset[str] | None]: - """Resolve one FROM or JOIN item to a relation name or to its known columns.""" - - if not isinstance(item, dict) or len(item) != 1: - return None, None - if _ALIAS_KEY in item: - alias: Any = item[_ALIAS_KEY] - if not isinstance(alias, dict): - return None, None - renamed: tuple[str, ...] = _identifier_names(alias.get(_COLUMN_ALIASES_KEY)) - inner: tuple[tuple[str, str] | None, frozenset[str] | None] = _input_item( - item=alias.get(_THIS_KEY), ctes=ctes - ) - if renamed and (inner[0] is not None or inner[1] is not None): - return None, frozenset(renamed) - return inner - function: Any = item.get(_FUNCTION_KEY) - if isinstance(function, dict): - args: Any = function.get(_ARGS_KEY) - kind: str = str(function.get(_NAME_KEY, "")).lower() - if ( - kind not in _RELATION_FUNCTIONS - or not isinstance(args, list) - or len(args) != 1 - or not isinstance(args[0], dict) - or not isinstance(args[0].get(_COLUMN_KEY), dict) - ): - return None, None - relation: str | None = _identifier_name(args[0][_COLUMN_KEY].get(_NAME_KEY)) - return (None if relation is None else (kind, relation)), None - table: Any = item.get(_TABLE_KEY) - if isinstance(table, dict): - name: str | None = _identifier_name(table.get(_NAME_KEY)) - if name is None or table.get("schema") is not None or table.get("catalog") is not None: - return None, None - return None, ctes.get(name) - subquery: Any = item.get(_SUBQUERY_KEY) - if isinstance(subquery, dict): - return None, _output_names(subquery.get(_THIS_KEY)) - return None, None - - -def _cte_columns(with_clause: Any) -> dict[str, frozenset[str] | None]: - if not isinstance(with_clause, dict): - return {} - columns: dict[str, frozenset[str] | None] = {} - cte: dict[str, Any] - for cte in _dicts(with_clause.get(_CTES_KEY)): - name: str | None = _identifier_name(cte.get(_ALIAS_KEY)) - if name is None: - continue - declared: tuple[str, ...] = _identifier_names(cte.get(_COLUMNS_KEY)) - columns[name] = frozenset(declared) if declared else _output_names(cte.get(_THIS_KEY)) - return columns - - -def _output_names(query: Any) -> frozenset[str] | None: - """Return the lower-cased output names of a plain SELECT, or None when any is unknown.""" - - if not isinstance(query, dict) or set(query) != {_SELECT_KEY}: - return None - expressions: Any = query[_SELECT_KEY].get(_EXPRESSIONS_KEY) - if not isinstance(expressions, list): - return None - projections: list[QueryProjection | None] = [_projection(value) for value in expressions] - if any(projection is None for projection in projections): - return None - return frozenset(projection.key for projection in projections if projection is not None) - - -def _identifier_names(identifiers: Any) -> tuple[str, ...]: - return tuple( - name.lower() - for identifier in (identifiers if isinstance(identifiers, list) else []) - if (name := _identifier_name(identifier)) is not None - ) - - -def _dicts(value: Any) -> list[dict[str, Any]]: - return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else [] - - -def _unqualified_references(node: Any) -> frozenset[str]: - """Return lower-cased names of unqualified column references anywhere under node.""" - - found: set[str] = set() - pending: list[Any] = [node] - while pending: - current: Any = pending.pop() - if isinstance(current, list): - pending.extend(current) - continue - if not isinstance(current, dict): - continue - column: Any = current.get(_COLUMN_KEY) - if isinstance(column, dict) and column.get(_TABLE_KEY) is None: - name: str | None = _identifier_name(column.get(_NAME_KEY)) - if name is not None: - found.add(name.lower()) - pending.extend(current.values()) - return frozenset(found) - - -def _substitute_alias_references(*, node: Any, names: Mapping[str, str]) -> Any: - """Rename unqualified references to output aliases, leaving nested queries untouched.""" - - if isinstance(node, list): - return [_substitute_alias_references(node=value, names=names) for value in node] - if not isinstance(node, dict) or _SELECT_KEY in node: - return node - column: Any = node.get(_COLUMN_KEY) - if isinstance(column, dict) and column.get(_TABLE_KEY) is None: - identifier: Any = column.get(_NAME_KEY) - name: str | None = _identifier_name(identifier) - if name is not None and identifier.get(_QUOTED_KEY) is False and name in names: - return { - **node, - _COLUMN_KEY: {**column, _NAME_KEY: {**identifier, _NAME_KEY: names[name]}}, - } - return { - key: _substitute_alias_references(node=value, names=names) for key, value in node.items() - } - - -def _has_positional_reference(select: Mapping[str, Any]) -> bool: - """Return whether grouping, ordering, or DISTINCT ON refer to output columns by position.""" - - items: list[Any] = [*_clause_items(select.get(_GROUP_BY_KEY))] - items.extend( - item.get(_THIS_KEY) if isinstance(item, dict) else item - for item in _clause_items(select.get(_ORDER_BY_KEY)) - ) - items.extend(_clause_items(select.get(_DISTINCT_ON_KEY))) - return any( - isinstance(item, dict) - and isinstance(item.get(_LITERAL_KEY), dict) - and item[_LITERAL_KEY].get(_LITERAL_TYPE_KEY) == _NUMBER_LITERAL - for item in items - ) - - -def _clause_items(clause: Any) -> list[Any]: - if isinstance(clause, list): - return clause - if isinstance(clause, dict) and isinstance(clause.get(_EXPRESSIONS_KEY), list): - return list(clause[_EXPRESSIONS_KEY]) - return [] - - -def _projection(expression: Any) -> QueryProjection | None: - if not isinstance(expression, dict) or len(expression) != 1: - return None - if _ALIAS_KEY in expression: - alias: Any = expression[_ALIAS_KEY] - name: str | None = ( - _identifier_name(alias.get(_ALIAS_KEY)) if isinstance(alias, dict) else None - ) - body: Any = alias.get(_THIS_KEY) if isinstance(alias, dict) else None - if name is None or body is None: - return None - return QueryProjection( - name=name, - expression=_encode(_canonical(body)), - references=_references(body), - passthrough=_is_bare_column(node=body, name=name), - ) - if _COLUMN_KEY in expression: - column: Any = expression[_COLUMN_KEY] - name = _identifier_name(column.get(_NAME_KEY)) if isinstance(column, dict) else None - if name is None: - return None - return QueryProjection( - name=name, - expression=_encode(_canonical(expression)), - references=_references(expression), - passthrough=_is_bare_column(node=expression, name=name), - ) - return None - - -def _is_bare_column(*, node: Any, name: str) -> bool: - """Return whether node is an unqualified reference to a column with the given name.""" - - column: Any = node.get(_COLUMN_KEY) if isinstance(node, dict) else None - if not isinstance(column, dict) or column.get(_TABLE_KEY) is not None: - return False - referenced: str | None = _identifier_name(column.get(_NAME_KEY)) - return referenced is not None and referenced.lower() == name.lower() - - -def _identifier_name(identifier: Any) -> str | None: - if not isinstance(identifier, dict) or not isinstance(identifier.get(_NAME_KEY), str): - return None - return str(identifier[_NAME_KEY]) - - -def _references(node: Any) -> frozenset[str]: - found: set[str] = set() - pending: list[Any] = [node] - while pending: - current: Any = pending.pop() - if isinstance(current, list): - pending.extend(current) - continue - if not isinstance(current, dict): - continue - column: Any = current.get(_COLUMN_KEY) - if isinstance(column, dict): - name: str | None = _identifier_name(column.get(_NAME_KEY)) - if name is not None: - found.add(name.lower()) - pending.extend(current.values()) - return frozenset(found) - - -def _canonical(node: Any) -> Any: - """Lower-case unquoted identifiers so letter case alone never looks like a change.""" - - if isinstance(node, list): - return [_canonical(value) for value in node] - if not isinstance(node, dict): - return node - canonical: dict[str, Any] = {key: _canonical(value) for key, value in node.items()} - if isinstance(canonical.get(_NAME_KEY), str) and canonical.get(_QUOTED_KEY) is False: - canonical[_NAME_KEY] = str(canonical[_NAME_KEY]).lower() - return canonical - - -def _encode(node: Any) -> str: - return json.dumps(node, sort_keys=True, separators=(",", ":"), default=str) diff --git a/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py b/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py index d31d5f9e7e..57fd160d88 100644 --- a/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py +++ b/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py @@ -11,10 +11,10 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import without_declared_columns_hash -from sqlbuild.compiler.planner._helpers.migrations.fingerprint import ( +from sqlbuild._native import ( build_migration_fingerprint, migration_fingerprint_ref_names, + without_declared_columns_hash, ) from sqlbuild.compiler.planner.constants import MIGRATION_FINGERPRINT_ALGORITHM diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py deleted file mode 100644 index bd8cf49d2d..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py +++ /dev/null @@ -1,226 +0,0 @@ -"""Execution helpers for ordinary dbt interop pipelines.""" - -from __future__ import annotations - -import json -import time -from collections.abc import Callable -from pathlib import Path -from typing import TextIO, cast - -from sqlbuild._native import ( - TransientStatusReporter, - dbt_execution_ls_argv, - parse_dbt_ls_json_lines, - render_dbt_node_result, -) -from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner -from sqlbuild.integrations.dbt.constants import DBT_UNIQUE_ID_SEPARATOR -from sqlbuild.integrations.dbt.main.runtime._execute_json_event_stream import ( - execute_dbt_json_event_stream, -) -from sqlbuild.integrations.dbt.models import ( - DbtCliOptions, - DbtCommandExecutionResult, - DbtCommandResult, - DbtLsNode, - DbtNodeExecutionResult, - DbtNodeMessage, -) -from sqlbuild.presentation.classes.cli_style import CliStyle - -_COMMAND_AND_SUBCOMMAND_ARGUMENT_COUNT: int = 2 - - -def execute_dbt_commands( - *, - runner: DbtRunner, - options: DbtCliOptions, - merged_argv: tuple[str, ...] | None, - progress_stream: TextIO, - stdout_stream: TextIO, - use_color: bool, - skip_message: str = "Skipping dbt: no dbt work selected.", - expected_nodes: tuple[DbtLsNode, ...] | None = None, - on_node_result: Callable[[DbtNodeExecutionResult], None] | None = None, - on_progress: Callable[[str], None] | None = None, -) -> DbtCommandExecutionResult: - """Execute the merged dbt command, or skip when no dbt work exists.""" - - if merged_argv is None: - style: CliStyle = CliStyle(use_color=use_color) - progress_stream.write(style.muted(skip_message) + "\n") - progress_stream.flush() - return DbtCommandExecutionResult(returncode=0) - argv: tuple[str, ...] = merged_argv - style = CliStyle(use_color=use_color) - dbt_command_name: str = Path(argv[0]).name - command_detail: str = ( - f"{dbt_command_name} {argv[1]}" - if len(argv) >= _COMMAND_AND_SUBCOMMAND_ARGUMENT_COUNT - else dbt_command_name - ) - progress_stream.write( - f"{style.dbt_execution_label('dbt execution')} {style.muted(command_detail)}\n\n" - ) - progress_stream.flush() - expected_unique_ids: frozenset[str] | None = ( - frozenset(node.unique_id for node in expected_nodes) if expected_nodes is not None else None - ) - expected_total: int | None = ( - len(expected_unique_ids) - if expected_unique_ids is not None - else _dbt_execution_expected_total( - runner=runner, - options=options, - argv=argv, - stream=progress_stream, - use_color=use_color, - ) - ) - target_path: Path | None = _effective_dbt_target_path(options=options) - returncode: int - results: tuple[DbtNodeExecutionResult, ...] - returncode, results = execute_dbt_json_event_stream( - argv=argv, - cwd=options.project_dir, - stream=stdout_stream, - use_color=use_color, - target_path=target_path, - display_total=expected_total, - on_node_result=on_node_result, - ) - if on_progress is not None: - on_progress("Finalizing dbt run...") - streamed_unique_ids: frozenset[str] = frozenset(result.unique_id for result in results) - if expected_unique_ids is not None: - results = tuple(result for result in results if result.unique_id in expected_unique_ids) - results = _append_missing_run_results( - results=results, - target_path=target_path, - expected_unique_ids=expected_unique_ids, - ) - appended_results: tuple[DbtNodeExecutionResult, ...] = tuple( - result for result in results if result.unique_id not in streamed_unique_ids - ) - streamed_result_count: int = len(results) - len(appended_results) - for offset, result in enumerate(appended_results, start=1): - stdout_stream.write( - render_dbt_node_result( - result=result, - use_color=style.use_color, - display_index=streamed_result_count + offset, - display_total=expected_total if expected_total is not None else len(results), - ) - ) - stdout_stream.flush() - if on_node_result is not None: - on_node_result(result) - if on_progress is not None: - on_progress("Finalized dbt run.") - return DbtCommandExecutionResult(returncode=returncode, node_results=results) - - -def _dbt_execution_expected_total( - *, - runner: DbtRunner, - options: DbtCliOptions, - argv: tuple[str, ...], - stream: TextIO, - use_color: bool, -) -> int | None: - ls_argv: list[str] | None = dbt_execution_ls_argv(argv=argv) - if ls_argv is None: - return None - status: TransientStatusReporter = TransientStatusReporter(stream=stream, use_color=use_color) - status.start("Resolving dbt execution selection...") - start: float = time.monotonic() - try: - result: DbtCommandResult = runner.invoke(argv=tuple(ls_argv), cwd=options.project_dir) - finally: - status.complete( - message=f"Resolved dbt execution selection. ({time.monotonic() - start:.2f}s)" - ) - if result.returncode != 0: - return None - return len(parse_dbt_ls_json_lines(stdout=result.stdout)) - - -def _append_missing_run_results( - *, - results: tuple[DbtNodeExecutionResult, ...], - target_path: Path | None, - expected_unique_ids: frozenset[str] | None = None, -) -> tuple[DbtNodeExecutionResult, ...]: - if target_path is None: - return results - run_results_path: Path = target_path / "run_results.json" - if not run_results_path.exists(): - return results - existing_unique_ids: frozenset[str] = frozenset(result.unique_id for result in results) - try: - payload: object = json.loads(run_results_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return results - if not isinstance(payload, dict) or not isinstance(payload.get("results"), list): - return results - raw_results: list[object] = cast(list[object], payload["results"]) - appended: list[DbtNodeExecutionResult] = list(results) - for index, raw_result in enumerate(raw_results, start=1): - if not isinstance(raw_result, dict): - continue - result_payload: dict[str, object] = cast(dict[str, object], raw_result) - unique_id: object = result_payload.get("unique_id") - status: object = result_payload.get("status") - if not isinstance(unique_id, str) or unique_id in existing_unique_ids: - continue - if expected_unique_ids is not None and unique_id not in expected_unique_ids: - continue - appended.append( - DbtNodeExecutionResult( - unique_id=unique_id, - resource_type=_resource_type_from_unique_id(unique_id), - node_name=_node_name_from_unique_id(unique_id), - status=status if isinstance(status, str) else "unknown", - index=index, - total=len(raw_results), - execution_time=_execution_time_from_run_result(result_payload), - messages=_messages_from_run_result(result_payload), - ) - ) - return tuple(appended) - - -def _effective_dbt_target_path(*, options: DbtCliOptions) -> Path | None: - if options.target_path is not None: - return options.target_path - if options.project_dir is not None: - return options.project_dir / "target" - return None - - -def _resource_type_from_unique_id(unique_id: str) -> str: - return ( - unique_id.split(DBT_UNIQUE_ID_SEPARATOR, 1)[0] - if DBT_UNIQUE_ID_SEPARATOR in unique_id - else "node" - ) - - -def _node_name_from_unique_id(unique_id: str) -> str: - return unique_id.rsplit(DBT_UNIQUE_ID_SEPARATOR, 1)[-1] - - -def _execution_time_from_run_result(raw_result: dict[str, object]) -> float | None: - value: object = raw_result.get("execution_time") - return float(value) if isinstance(value, int | float) else None - - -def _messages_from_run_result(raw_result: dict[str, object]) -> tuple[DbtNodeMessage, ...]: - message: object = raw_result.get("message") - if isinstance(message, str) and message.strip(): - return (DbtNodeMessage(level="error", message=message.strip()),) - failures: object = raw_result.get("failures") - if isinstance(failures, int) and failures > 0: - return (DbtNodeMessage(level="error", message=f"{failures} failure(s)"),) - return () diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py deleted file mode 100644 index b2c8f288f5..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py +++ /dev/null @@ -1,233 +0,0 @@ -"""Ordinary execution phases for dbt interop.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild._native import ( - ConnectionProgressReporter, - apply_command_warehouse, - build_effective_connection_config, - render_dbt_execution_summary_footer, - render_dbt_interop_plan, - resolve_target_config, -) -from sqlbuild.cli.commands.main.dbt.dbt_sqlbuild_work import execute_dbt_sqlbuild_work -from sqlbuild.cli.commands.models import DbtSqlbuildWorkContext -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.integrations.dbt._helpers.pipeline.execute import execute_dbt_commands -from sqlbuild.integrations.dbt._helpers.pipeline.plan_output import build_sqlbuild_plan_output -from sqlbuild.integrations.dbt.main.profile._resolve_connection_config import ( - resolve_connection_config, -) -from sqlbuild.integrations.dbt.main.runtime._report_progress import report_progress -from sqlbuild.integrations.dbt.models import ( - DbtCommandExecutionResult, - DbtInteropCompiledProject, - DbtInteropExecutionRequest, - DbtInteropInvocation, - DbtInteropPlan, - DbtManifestIndex, - DbtPlanEnvironment, - DbtSqlbuildPlanRequest, -) -from sqlbuild.presentation.models import DisplayOptions -from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.spec.contracts.models import ExecutionLimitsConfig - - -def resolve_dbt_connection_config( - *, - request: DbtInteropExecutionRequest, - invocation: DbtInteropInvocation, - compiled: DbtInteropCompiledProject, -) -> dict[str, object]: - """Resolve the connection used only by selected SQLBuild work.""" - - return apply_command_warehouse( - config=resolve_connection_config( - raw_config=build_effective_connection_config( - discovered_inputs=invocation.discovered_inputs - ), - project_dir=request.project_dir, - adapter_name=compiled.adapter_name, - discovered_inputs=invocation.discovered_inputs, - ), - adapter_name=compiled.adapter_name, - discovered_inputs=invocation.discovered_inputs, - project_dir=request.project_dir, - selected_target=None, - ) - - -def write_dbt_execution_plan_text( - *, - request: DbtInteropExecutionRequest, - invocation: DbtInteropInvocation, - plan: DbtInteropPlan, - merged_dbt_argv: tuple[str, ...] | None, -) -> None: - """Render the selected work without model-state classifications.""" - - if request.json_output: - return - display_plan: DbtInteropPlan = ( - replace(plan, dbt_command_argv=merged_dbt_argv, supplemental_dbt_command_argvs=()) - if merged_dbt_argv is not None - else plan - ) - rendered_plan: str = render_dbt_interop_plan( - plan=display_plan, - json_output=False, - use_color=request.use_color, - display_options=DisplayOptions(max_entries_per_section=None if request.verbose else 10), - ) - invocation.output_stream.write(rendered_plan + "\n\n") - invocation.output_stream.flush() - - -def execute_dbt_without_state_tracking( - *, - request: DbtInteropExecutionRequest, - invocation: DbtInteropInvocation, - merged_dbt_argv: tuple[str, ...] | None, -) -> DbtCommandExecutionResult: - """Execute ordinary dbt events without SQLBuild state reads or writes.""" - - return execute_dbt_commands( - runner=invocation.runner, - options=invocation.dbt_options, - merged_argv=merged_dbt_argv, - progress_stream=invocation.output_stream, - stdout_stream=invocation.dbt_output_stream, - use_color=request.use_color, - skip_message="Skipping dbt: no dbt work selected.", - on_progress=request.on_progress, - ) - - -def write_dbt_execution_summary( - *, - request: DbtInteropExecutionRequest, - invocation: DbtInteropInvocation, - execution: DbtCommandExecutionResult, -) -> None: - """Write the ordinary dbt event summary footer.""" - - summary_footer: str | None = render_dbt_execution_summary_footer( - node_results=execution.node_results, - use_color=request.use_color, - ) - if summary_footer is not None: - invocation.output_stream.write("\n" + summary_footer + "\n") - invocation.output_stream.flush() - - -def resolve_sqlbuild_execution_plan_output( - *, - request: DbtInteropExecutionRequest, - invocation: DbtInteropInvocation, - compiled: DbtInteropCompiledProject, - manifest: DbtManifestIndex, - plan: DbtInteropPlan, - failed_sqlbuild_model_names: tuple[str, ...], -) -> PlanOutput | None: - """Plan selected SQLBuild work after dbt using actual dbt failures as blockers.""" - - if plan.sqlbuild_skip_reason is not None: - return None - selected_model_names: tuple[str, ...] = plan.selection.sqlbuild_model_names - if not selected_model_names: - return None - invocation.output_stream.write("\n") - invocation.output_stream.flush() - connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=compiled.adapter_name, - stream=invocation.output_stream, - blank_line_after_complete=True, - use_color=request.use_color, - ) - return build_sqlbuild_plan_output( - environment=DbtPlanEnvironment( - project_dir=request.project_dir, - discovered_inputs=invocation.discovered_inputs, - project=compiled.project, - adapter=compiled.adapter, - adapter_name=compiled.adapter_name, - ), - request=DbtSqlbuildPlanRequest( - selected_model_names=selected_model_names, - sqlbuild_args=invocation.effective_sqlbuild_args, - external_blocked_model_names=failed_sqlbuild_model_names, - ), - hooks=ConnectionHooks( - on_connection_start=connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, *, elapsed_seconds: ( - connection_progress.on_connection_complete( - connection_count=connection_count, - elapsed_seconds=elapsed_seconds, - ) - ), - on_connection_error=lambda connection_count, *, elapsed_seconds: ( - connection_progress.on_connection_error( - connection_count=connection_count, - elapsed_seconds=elapsed_seconds, - ) - ), - ), - ) - - -def run_dbt_sqlbuild_work( - *, - request: DbtInteropExecutionRequest, - invocation: DbtInteropInvocation, - compiled: DbtInteropCompiledProject, - plan_output: PlanOutput, - connection_config: dict[str, object], -) -> int: - """Execute the SQLBuild portion of the interop plan and return its exit code.""" - - return execute_dbt_sqlbuild_work( - context=DbtSqlbuildWorkContext( - plan_output=plan_output, - connection_config=connection_config, - adapter=compiled.adapter, - adapter_name=compiled.adapter_name, - output_stream=invocation.output_stream, - use_color=request.use_color, - snapshots_config=invocation.discovered_inputs.project_config.snapshots, - execution_limits=_execution_limits( - discovered_inputs=invocation.discovered_inputs, - target_name=compiled.project.effective_target_name, - ), - ), - command=request.command, - project=compiled.project, - project_dir=request.project_dir, - fail_fast=request.fail_fast, - verbose=request.verbose, - ) - - -def write_sqlbuild_skip_notice( - *, request: DbtInteropExecutionRequest, invocation: DbtInteropInvocation, message: str -) -> None: - """Write the SQLBuild skip transition.""" - - invocation.output_stream.write("\n") - invocation.output_stream.flush() - report_progress(on_progress=request.on_progress, message=message) - - -def _execution_limits( - *, discovered_inputs: DiscoveredProjectInputs, target_name: str | None -) -> ExecutionLimitsConfig: - if target_name is None: - return ExecutionLimitsConfig() - return resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ).execution_limits diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py deleted file mode 100644 index 3532cc35e7..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Shared prologue phases for dbt interop pipelines.""" - -from __future__ import annotations - -import sys -import time -from collections.abc import Callable -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import ( - DbtCombinedGraph, - build_compiled_project, - build_dbt_combined_graph, - discover_project_inputs, - load_dbt_manifest_index, - plan_dbt_interop_command, - resolve_dbt_executable, - resolve_effective_adapter_name, - route_dbt_interop_args, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt._helpers.pipeline.plan_output import dbt_failure_detail -from sqlbuild.integrations.dbt.classes.dbt_compile_reference_resolver import ( - DbtCompileReferenceResolver, -) -from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner -from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError -from sqlbuild.integrations.dbt.main.cli._parse_execution_args import parse_dbt_execution_args -from sqlbuild.integrations.dbt.main.config._resolve_plan_options import resolve_dbt_plan_options -from sqlbuild.integrations.dbt.main.config._resolve_vars_mapping import resolve_dbt_vars_mapping -from sqlbuild.integrations.dbt.main.manifest._resolve_manifest_path import ( - resolve_dbt_manifest_path, -) -from sqlbuild.integrations.dbt.main.runtime._report_progress import report_progress -from sqlbuild.integrations.dbt.main.runtime._resolve_interop_adapter import ( - resolve_dbt_interop_adapter, -) -from sqlbuild.integrations.dbt.models import ( - DbtCliOptions, - DbtCommandResult, - DbtInteropCommandArgs, - DbtInteropCompiledProject, - DbtInteropExecutionRequest, - DbtInteropInvocation, - DbtInteropPlan, - DbtInteropPlanResolution, - DbtInteropRoutedArgs, - DbtManifestIndex, -) -from sqlbuild.integrations.dbt.types import DbtInteropCommand - - -def resolve_dbt_execution_invocation( - request: DbtInteropExecutionRequest, -) -> DbtInteropInvocation: - """Route args, discover the project, and resolve dbt options for execution.""" - - dbt_executable: str = request.dbt_executable or resolve_dbt_executable() - output_stream: TextIO = request.progress_stream or sys.stdout - routed: DbtInteropRoutedArgs = route_dbt_interop_args( - command=request.command, - parsed=parse_dbt_execution_args(command=request.command, args=request.args), - ) - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=request.project_dir - ) - dbt_options: DbtCliOptions = resolve_dbt_plan_options( - project_dir=request.project_dir, - discovered_inputs=discovered_inputs, - dbt_args=routed.dbt_args, - ) - dbt_vars: dict[str, object] = resolve_dbt_vars_mapping( - project_config=discovered_inputs.project_config.dbt, - local_config=discovered_inputs.local_config.dbt, - dbt_args=routed.dbt_args, - ) - return DbtInteropInvocation( - dbt_executable=dbt_executable, - output_stream=output_stream, - dbt_output_stream=request.dbt_stdout_stream or output_stream, - routed=routed, - discovered_inputs=discovered_inputs, - effective_sqlbuild_args=routed.sqlbuild_args, - dbt_options=dbt_options, - dbt_vars=dbt_vars, - runner=request.dbt_runner or DbtRunner(dbt_executable=dbt_executable), - ) - - -def load_compiled_dbt_manifest( - *, - runner: DbtRunner, - dbt_options: DbtCliOptions, - full_refresh: bool, - on_progress: Callable[[str], None] | None, -) -> DbtManifestIndex: - """Compile the dbt project and load its manifest index.""" - - dbt_compile_start: float = time.monotonic() - report_progress(on_progress=on_progress, message="Compiling dbt project...") - compile_result: DbtCommandResult = runner.compile( - options=dbt_options, - full_refresh=full_refresh, - ) - if compile_result.returncode != 0: - raise DbtInteropRuntimeError("dbt compile failed", help=dbt_failure_detail(compile_result)) - report_progress( - on_progress=on_progress, - message=f"Compiled dbt project. ({time.monotonic() - dbt_compile_start:.2f}s)", - ) - manifest_start: float = time.monotonic() - report_progress(on_progress=on_progress, message="Loading dbt manifest...") - manifest_path: Path = resolve_dbt_manifest_path(options=dbt_options) - manifest: DbtManifestIndex = load_dbt_manifest_index(manifest_path=manifest_path) - report_progress( - on_progress=on_progress, - message=f"Loaded dbt manifest. ({time.monotonic() - manifest_start:.2f}s)", - ) - return manifest - - -def compile_dbt_interop_project( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - manifest: DbtManifestIndex, - dbt_vars: dict[str, object], - no_sql_validation: bool, - on_progress: Callable[[str], None] | None, -) -> DbtInteropCompiledProject: - """Resolve the adapter and compile the SQLBuild project against the dbt manifest.""" - - sqlbuild_compile_start: float = time.monotonic() - report_progress(on_progress=on_progress, message="Compiling SQLBuild project...") - adapter_name: str = resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_dbt_interop_adapter( - adapter_name=adapter_name, project_dir=project_dir - ) - project: CompiledProjectView = build_compiled_project( - discovered_inputs=discovered_inputs, - adapter=adapter, - no_sql_validation=no_sql_validation, - cli_vars=dbt_vars, - external_sql_reference_resolver=DbtCompileReferenceResolver(dbt_manifest=manifest), - ) - report_progress( - on_progress=on_progress, - message=f"Compiled SQLBuild project. ({time.monotonic() - sqlbuild_compile_start:.2f}s)", - ) - return DbtInteropCompiledProject(adapter_name=adapter_name, adapter=adapter, project=project) - - -def resolve_dbt_interop_plan( - *, - command: DbtInteropCommand, - invocation: DbtInteropInvocation, - compiled: DbtInteropCompiledProject, - manifest: DbtManifestIndex, - sqlbuild_executable: str, - on_progress: Callable[[str], None] | None, -) -> DbtInteropPlanResolution: - """Build the combined dbt graph and resolve dbt and SQLBuild selection.""" - - graph_start: float = time.monotonic() - report_progress(on_progress=on_progress, message="Building dbt interop graph...") - graph: DbtCombinedGraph = build_dbt_combined_graph(manifest=manifest, project=compiled.project) - report_progress( - on_progress=on_progress, - message=f"Built dbt interop graph. ({time.monotonic() - graph_start:.2f}s)", - ) - selection_start: float = time.monotonic() - report_progress(on_progress=on_progress, message="Resolving dbt and SQLBuild selection...") - plan: DbtInteropPlan = plan_dbt_interop_command( - command=command, - project=compiled.project, - manifest=manifest, - graph=graph, - dbt_runner=invocation.runner, - dbt_options=invocation.dbt_options, - command_args=DbtInteropCommandArgs( - select=tuple(invocation.routed.select), - exclude=tuple(invocation.routed.exclude), - dbt_command_args=tuple(invocation.routed.dbt_args), - sqlbuild_command_args=tuple(invocation.effective_sqlbuild_args), - dbt_executable=invocation.dbt_executable, - sqlbuild_executable=sqlbuild_executable, - ), - ) - report_progress( - on_progress=on_progress, - message=f"Resolved dbt and SQLBuild selection. ({time.monotonic() - selection_start:.2f}s)", - ) - return DbtInteropPlanResolution(graph=graph, plan=plan) diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py deleted file mode 100644 index 6486e5cc97..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py +++ /dev/null @@ -1,215 +0,0 @@ -"""Native SQLBuild plan output helpers for dbt interop pipelines.""" - -from __future__ import annotations - -import time -from collections.abc import Callable -from dataclasses import replace -from pathlib import Path -from typing import Any - -from sqlbuild._native import ( - apply_command_warehouse, - build_effective_connection_config, - resolve_model_deferral_inputs, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.execution.display_plan import build_display_only_sqlbuild_plan -from sqlbuild.compiler.planner.main.execution.execution import build_execution_plan -from sqlbuild.compiler.planner.models import ( - CursorOverrides, - DeferralInputs, - PlannerOverrides, - PlannerPolicies, - PlannerSelection, - PlanOutput, -) -from sqlbuild.integrations.dbt.constants import DBT_FULL_REFRESH_FLAG -from sqlbuild.integrations.dbt.main.profile._resolve_connection_config import ( - resolve_connection_config, -) -from sqlbuild.integrations.dbt.models import ( - DbtCommandResult, - DbtInteropCompiledProject, - DbtInteropPlan, - DbtPlanEnvironment, - DbtSqlbuildPlanRequest, -) -from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback - - -def dbt_failure_detail(result: DbtCommandResult) -> str | None: - detail: str = (result.stderr or result.stdout).strip() - return detail or None - - -def build_sqlbuild_plan_output( - *, - environment: DbtPlanEnvironment, - request: DbtSqlbuildPlanRequest, - hooks: ConnectionHooks, -) -> PlanOutput | None: - """Build a native SQLBuild plan for selected SQLBuild models.""" - - if not request.selected_model_names: - return None - planning_project: CompiledProjectView = environment.project - connection_config: dict[str, object] = apply_command_warehouse( - config=resolve_connection_config( - raw_config=build_effective_connection_config( - discovered_inputs=environment.discovered_inputs - ), - project_dir=environment.project_dir, - adapter_name=environment.adapter_name, - discovered_inputs=environment.discovered_inputs, - ), - adapter_name=environment.adapter_name, - discovered_inputs=environment.discovered_inputs, - project_dir=environment.project_dir, - selected_target=None, - ) - connection: Any = _connect_for_plan( - adapter=environment.adapter, - connection_config=connection_config, - hooks=hooks, - ) - try: - deferral: DeferralInputs = _build_deferral_inputs( - environment=environment, - request=request, - connection=connection, - ) - try: - return build_execution_plan( - project=planning_project, - adapter=environment.adapter, - connection=connection, - selection=PlannerSelection(select=request.selected_model_names), - overrides=PlannerOverrides( - cursor_overrides=_parse_cursor_overrides(request.sqlbuild_args), - full_refresh=DBT_FULL_REFRESH_FLAG in request.sqlbuild_args, - external_blocked_model_names=request.external_blocked_model_names, - ), - deferral=deferral, - policies=PlannerPolicies(), - on_progress=hooks.on_progress, - project_config=environment.discovered_inputs.project_config, - local_config=environment.discovered_inputs.local_config, - ) - except PlannerInputError: - return build_display_only_sqlbuild_plan( - project=planning_project, - selected_model_names=request.selected_model_names, - full_refresh=DBT_FULL_REFRESH_FLAG in request.sqlbuild_args, - ) - finally: - environment.adapter.close(connection) - - -def attach_sqlbuild_plan_output( - *, - plan: DbtInteropPlan, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - compiled: DbtInteropCompiledProject, - sqlbuild_args: tuple[str, ...], - connection_progress: Any | None, - on_progress: Callable[[str], None] | None, -) -> DbtInteropPlan: - """Attach native SQLBuild planning only when SQLBuild work is selected.""" - - sqlbuild_plan_output: PlanOutput | None = build_sqlbuild_plan_output( - environment=DbtPlanEnvironment( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - project=compiled.project, - adapter=compiled.adapter, - adapter_name=compiled.adapter_name, - ), - request=DbtSqlbuildPlanRequest( - selected_model_names=plan.selection.sqlbuild_model_names, - sqlbuild_args=sqlbuild_args, - ), - hooks=_connection_progress_hooks( - connection_progress=connection_progress, - on_progress=on_progress, - ), - ) - if sqlbuild_plan_output is None: - return plan - return replace(plan, sqlbuild_plan_output=sqlbuild_plan_output) - - -def _connect_for_plan( - *, - adapter: BaseAdapter, - connection_config: dict[str, object], - hooks: ConnectionHooks, -) -> Any: - if hooks.on_connection_start is not None: - hooks.on_connection_start(1) - start: float = time.monotonic() - try: - connection: Any = adapter.connect(connection_config) - except Exception: - if hooks.on_connection_error is not None: - hooks.on_connection_error(connection_count=1, elapsed_seconds=time.monotonic() - start) - raise - if hooks.on_connection_complete is not None: - hooks.on_connection_complete(connection_count=1, elapsed_seconds=time.monotonic() - start) - return connection - - -def _build_deferral_inputs( - *, - environment: DbtPlanEnvironment, - request: DbtSqlbuildPlanRequest, - connection: Any, -) -> DeferralInputs: - defer_to: str | None = _parse_value(args=request.sqlbuild_args, flag="--defer-to") - return resolve_model_deferral_inputs( - project=environment.project, - discovered_inputs=environment.discovered_inputs, - adapter=environment.adapter, - connection=connection, - defer_to=defer_to, - ) - - -def _parse_cursor_overrides(args: tuple[str, ...]) -> CursorOverrides: - return CursorOverrides( - start_ts=_parse_value(args=args, flag="--start-cursor-ts"), - end_ts=_parse_value(args=args, flag="--end-cursor-ts"), - start_int=_parse_value(args=args, flag="--start-cursor-int"), - end_int=_parse_value(args=args, flag="--end-cursor-int"), - ) - - -def _parse_value(*, args: tuple[str, ...], flag: str) -> str | None: - if flag not in args: - return None - index: int = args.index(flag) - if index + 1 >= len(args): - return None - return args[index + 1] - - -def _connection_progress_hooks( - *, - connection_progress: Any | None, - on_progress: Callable[[str], None] | None, -) -> ConnectionHooks: - if connection_progress is None: - return ConnectionHooks(on_progress=on_progress) - on_connection_complete: ConnectionElapsedCallback = connection_progress.on_connection_complete - on_connection_error: ConnectionElapsedCallback = connection_progress.on_connection_error - return ConnectionHooks( - on_progress=on_progress, - on_connection_start=connection_progress.on_connection_start, - on_connection_complete=on_connection_complete, - on_connection_error=on_connection_error, - ) diff --git a/src/sqlbuild/integrations/dbt/main/pipeline/debug.py b/src/sqlbuild/integrations/dbt/main/pipeline/debug.py deleted file mode 100644 index 6cb34b3b6b..0000000000 --- a/src/sqlbuild/integrations/dbt/main/pipeline/debug.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Runtime pipeline for `sqb dbt debug`.""" - -from __future__ import annotations - -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import discover_project_inputs, resolve_dbt_config -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner -from sqlbuild.integrations.dbt.main.config._parse_config_overrides import ( - parse_dbt_config_overrides, -) -from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtCommandResult, ResolvedDbtConfig - - -def debug_dbt_from_project( - *, - project_dir: Path, - args: tuple[str, ...], - dbt_runner: DbtRunner | None = None, - dbt_executable: str | None = None, - stdout_stream: TextIO, - stderr_stream: TextIO, -) -> int: - """Run dbt debug from SQLBuild project configuration.""" - - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) - resolved: ResolvedDbtConfig = resolve_dbt_config( - project_root=project_dir, - config=discovered_inputs.project_config.dbt, - local_config=discovered_inputs.local_config.dbt, - overrides=parse_dbt_config_overrides(args), - require_project_dir=True, - ) - options: DbtCliOptions = DbtCliOptions( - project_dir=resolved.project_dir, - profiles_dir=resolved.profiles_dir, - target=resolved.target, - ) - result: DbtCommandResult = (dbt_runner or DbtRunner(dbt_executable=dbt_executable)).debug( - options=options, - args=_strip_sqlbuild_and_config_args(args), - ) - stdout_stream.write(result.stdout) - stderr_stream.write(result.stderr) - stdout_stream.flush() - stderr_stream.flush() - return result.returncode - - -def _strip_sqlbuild_and_config_args(args: tuple[str, ...]) -> tuple[str, ...]: - stripped: list[str] = [] - skip_next: bool = False - value_flags: set[str] = {"--project-dir", "--profiles-dir", "--target", "--target-path"} - local_flags: set[str] = {"--no-connection"} - for index, arg in enumerate(args): - if skip_next: - skip_next = False - continue - if arg in local_flags: - continue - if arg in value_flags: - skip_next = index + 1 < len(args) - continue - stripped.append(arg) - return tuple(stripped) diff --git a/src/sqlbuild/integrations/dbt/main/pipeline/execute.py b/src/sqlbuild/integrations/dbt/main/pipeline/execute.py deleted file mode 100644 index cb4375e9ef..0000000000 --- a/src/sqlbuild/integrations/dbt/main/pipeline/execute.py +++ /dev/null @@ -1,135 +0,0 @@ -"""Runtime execution pipeline for ordinary dbt interop commands.""" - -from __future__ import annotations - -from sqlbuild._native import ( - build_failed_sqlbuild_model_names, - build_merged_dbt_execution_argv, - plan_has_executable_work, -) -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.integrations.dbt._helpers.pipeline.execution_phases import ( - execute_dbt_without_state_tracking, - resolve_dbt_connection_config, - resolve_sqlbuild_execution_plan_output, - run_dbt_sqlbuild_work, - write_dbt_execution_plan_text, - write_dbt_execution_summary, - write_sqlbuild_skip_notice, -) -from sqlbuild.integrations.dbt._helpers.pipeline.interop_prologue import ( - compile_dbt_interop_project, - load_compiled_dbt_manifest, - resolve_dbt_execution_invocation, - resolve_dbt_interop_plan, -) -from sqlbuild.integrations.dbt.exceptions import DbtInteropArgumentError -from sqlbuild.integrations.dbt.models import ( - DbtCommandExecutionResult, - DbtInteropCompiledProject, - DbtInteropExecutionRequest, - DbtInteropInvocation, - DbtInteropPlanResolution, - DbtManifestIndex, -) -from sqlbuild.integrations.dbt.types import DbtInteropCommand - - -def execute_dbt_interop_from_project(request: DbtInteropExecutionRequest) -> int: - """Execute dbt first, then native SQLBuild work selected by the combined graph.""" - - if request.command not in ( - DbtInteropCommand.RUN, - DbtInteropCommand.BUILD, - ): - raise DbtInteropArgumentError( - f"unsupported dbt interop execution command: {request.command}" - ) - invocation: DbtInteropInvocation = resolve_dbt_execution_invocation(request) - manifest: DbtManifestIndex = load_compiled_dbt_manifest( - runner=invocation.runner, - dbt_options=invocation.dbt_options, - full_refresh=False, - on_progress=request.on_progress, - ) - compiled: DbtInteropCompiledProject = compile_dbt_interop_project( - project_dir=request.project_dir, - discovered_inputs=invocation.discovered_inputs, - manifest=manifest, - dbt_vars=invocation.dbt_vars, - no_sql_validation=request.no_sql_validation, - on_progress=request.on_progress, - ) - resolution: DbtInteropPlanResolution = resolve_dbt_interop_plan( - command=request.command, - invocation=invocation, - compiled=compiled, - manifest=manifest, - sqlbuild_executable=request.sqlbuild_executable, - on_progress=request.on_progress, - ) - merged_dbt_argv: tuple[str, ...] | None = build_merged_dbt_execution_argv( - command=request.command, - options=invocation.dbt_options, - routed_args=invocation.routed.dbt_args, - plan=resolution.plan, - ) - write_dbt_execution_plan_text( - request=request, - invocation=invocation, - plan=resolution.plan, - merged_dbt_argv=merged_dbt_argv, - ) - execution: DbtCommandExecutionResult = execute_dbt_without_state_tracking( - request=request, - invocation=invocation, - merged_dbt_argv=merged_dbt_argv, - ) - write_dbt_execution_summary( - request=request, - invocation=invocation, - execution=execution, - ) - failed_sqlbuild_model_names: tuple[str, ...] = build_failed_sqlbuild_model_names( - graph=resolution.graph, - manifest=manifest, - node_results=execution.node_results, - ) - if execution.returncode != 0 and not execution.node_results: - return execution.returncode - plan_output: PlanOutput | None = resolve_sqlbuild_execution_plan_output( - request=request, - invocation=invocation, - compiled=compiled, - manifest=manifest, - plan=resolution.plan, - failed_sqlbuild_model_names=failed_sqlbuild_model_names, - ) - if plan_output is None: - if resolution.plan.sqlbuild_skip_reason is not None: - write_sqlbuild_skip_notice( - request=request, - invocation=invocation, - message="No SQLBuild work selected.", - ) - return execution.returncode - if not plan_has_executable_work(plan=plan_output): - write_sqlbuild_skip_notice( - request=request, - invocation=invocation, - message="Skipping SQLBuild: no executable work selected.", - ) - return execution.returncode - connection_config: dict[str, object] = resolve_dbt_connection_config( - request=request, - invocation=invocation, - compiled=compiled, - ) - sqlbuild_exit_code: int = run_dbt_sqlbuild_work( - request=request, - invocation=invocation, - compiled=compiled, - plan_output=plan_output, - connection_config=connection_config, - ) - return max(execution.returncode, sqlbuild_exit_code) diff --git a/src/sqlbuild/integrations/dbt/main/pipeline/plan.py b/src/sqlbuild/integrations/dbt/main/pipeline/plan.py deleted file mode 100644 index 47b74d6a66..0000000000 --- a/src/sqlbuild/integrations/dbt/main/pipeline/plan.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Runtime planning pipeline for `sqb dbt plan`.""" - -from __future__ import annotations - -import time -from collections.abc import Callable -from pathlib import Path -from typing import Any, TextIO - -from sqlbuild._native import ConnectionProgressReporter -from sqlbuild.integrations.dbt._helpers.pipeline.interop_prologue import ( - compile_dbt_interop_project, - load_compiled_dbt_manifest, - resolve_dbt_execution_invocation, - resolve_dbt_interop_plan, -) -from sqlbuild.integrations.dbt._helpers.pipeline.plan_output import ( - attach_sqlbuild_plan_output, -) -from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner -from sqlbuild.integrations.dbt.main.runtime._report_progress import report_progress -from sqlbuild.integrations.dbt.models import ( - DbtInteropCompiledProject, - DbtInteropExecutionRequest, - DbtInteropInvocation, - DbtInteropPlan, - DbtInteropPlanResolution, - DbtManifestIndex, -) -from sqlbuild.integrations.dbt.types import DbtInteropCommand - - -def plan_dbt_interop_from_project( - *, - project_dir: Path, - args: tuple[str, ...], - dbt_runner: DbtRunner | None = None, - dbt_executable: str | None = None, - sqlbuild_executable: str = "sqb", - no_sql_validation: bool = False, - on_progress: Callable[[str], None] | None = None, - progress_stream: TextIO | None = None, - use_color: bool = False, -) -> DbtInteropPlan: - """Build a dbt interop plan from real project files and dbt artifacts.""" - - invocation: DbtInteropInvocation = resolve_dbt_execution_invocation( - DbtInteropExecutionRequest( - command=DbtInteropCommand.PLAN, - project_dir=project_dir, - args=args, - dbt_runner=dbt_runner, - dbt_executable=dbt_executable, - sqlbuild_executable=sqlbuild_executable, - no_sql_validation=no_sql_validation, - on_progress=on_progress, - progress_stream=progress_stream, - use_color=use_color, - ) - ) - manifest: DbtManifestIndex = load_compiled_dbt_manifest( - runner=invocation.runner, - dbt_options=invocation.dbt_options, - full_refresh=False, - on_progress=on_progress, - ) - compiled: DbtInteropCompiledProject = compile_dbt_interop_project( - project_dir=project_dir, - discovered_inputs=invocation.discovered_inputs, - manifest=manifest, - dbt_vars=invocation.dbt_vars, - no_sql_validation=no_sql_validation, - on_progress=on_progress, - ) - connection_progress: Any | None = ( - ConnectionProgressReporter( - adapter_name=compiled.adapter_name, - stream=progress_stream, - use_color=use_color, - ) - if progress_stream is not None - else None - ) - generation_start: float = time.monotonic() - resolution: DbtInteropPlanResolution = resolve_dbt_interop_plan( - command=DbtInteropCommand.PLAN, - invocation=invocation, - compiled=compiled, - manifest=manifest, - sqlbuild_executable=sqlbuild_executable, - on_progress=on_progress, - ) - plan: DbtInteropPlan = attach_sqlbuild_plan_output( - plan=resolution.plan, - project_dir=project_dir, - discovered_inputs=invocation.discovered_inputs, - compiled=compiled, - sqlbuild_args=invocation.effective_sqlbuild_args, - connection_progress=connection_progress, - on_progress=on_progress, - ) - report_progress( - on_progress=on_progress, - message=f"Generated dbt interop plan. ({time.monotonic() - generation_start:.2f}s)", - ) - return plan diff --git a/src/sqlbuild/lint/_helpers/expansion.py b/src/sqlbuild/lint/_helpers/expansion.py deleted file mode 100644 index b06a8229e5..0000000000 --- a/src/sqlbuild/lint/_helpers/expansion.py +++ /dev/null @@ -1,339 +0,0 @@ -"""Prepare authored SQL bodies for linting by expanding them like compile does.""" - -from __future__ import annotations - -import re -from dataclasses import replace -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild._native import ( - expand_sql_with_spans, - external_identifiers_for_headers, -) -from sqlbuild.compiler.compile.constants import MACRO_TOKEN -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompiledSqlExpansion, - ExpansionSpan, - SqlExpansionContext, -) -from sqlbuild.compiler.scopes.types import DeclarationKind -from sqlbuild.lint._helpers.headers import lint_body_ranges, lint_file_role -from sqlbuild.lint.classes.stop_context import LintStopContext -from sqlbuild.lint.constants import ( - HEADER_KIND_SCENARIO, - HEADER_KIND_TEST, - RELATION_IDENTITY_TEMPLATE, - TEMPLATE_INTERPOLATION_START, -) -from sqlbuild.lint.exceptions import NativeLintError, ProjectCompileError -from sqlbuild.lint.models import ( - ExpandedLintFile, - HeaderSpan, - InterpolationSite, - LintBody, - LintFileRole, - PendingLintBody, - PreparedLintFiles, -) -from sqlbuild.lint.types import NativePreparedSql - -_DEPENDENCY_INTRINSIC_PATTERN: re.Pattern[str] = re.compile( - r"^__(?:ref|source)\s*\(", re.IGNORECASE -) -_KEYED_RELATION_PATTERN: re.Pattern[str] = re.compile( - r"^__(ref|source|seed)\s*\(\s*['\"]([^'\"]+)['\"]\s*\)$", re.IGNORECASE -) - - -def prepare_lint_body( - *, - role: LintFileRole, - file_path: Path, - contents: str, - body_range: tuple[int, int], - context: SqlExpansionContext, - dialect: str, - external_identifiers: tuple[str, ...] = (), - allows_ceremonial_select: bool = False, - allows_dynamic_output_star: bool = False, - compiled_expansion: CompiledSqlExpansion | None = None, -) -> LintBody: - """Expand one authored body and neutralize whatever interpolation remains.""" - - pending: PendingLintBody = expand_lint_body( - role=role, - file_path=file_path, - contents=contents, - body_range=body_range, - context=context, - dialect=dialect, - external_identifiers=external_identifiers, - allows_ceremonial_select=allows_ceremonial_select, - allows_dynamic_output_star=allows_dynamic_output_star, - compiled_expansion=compiled_expansion, - ) - return finish_lint_body( - pending=pending, prepared=_native.prepare_lint_sql(pending.request), dialect=dialect - ) - - -def expand_lint_body( - *, - role: LintFileRole, - file_path: Path, - contents: str, - body_range: tuple[int, int], - context: SqlExpansionContext, - dialect: str, - external_identifiers: tuple[str, ...] = (), - allows_ceremonial_select: bool = False, - allows_dynamic_output_star: bool = False, - compiled_expansion: CompiledSqlExpansion | None = None, -) -> PendingLintBody: - """Expand one authored body like compile does, leaving its native preparation pending.""" - - body_start: int - body_end: int - body_start, body_end = body_range - authored_body: str = contents[body_start:body_end] - pre_expansion_sites: tuple[InterpolationSite, ...] = () - expansion_input: str = authored_body - if _is_generic_audit_path(role=role): - expansion_input, pre_expansion_sites = _sites( - _native.neutralize_lint_audit_parameters(authored_body) - ) - elif _is_sql_hook_path(role=role): - expansion_input, pre_expansion_sites = _sites( - _native.neutralize_lint_context_interpolation(authored_body) - ) - expanded: str - expansion_passes: tuple[tuple[ExpansionSpan, ...], ...] - try: - if ( - compiled_expansion is not None - and authored_body.strip() == compiled_expansion.authored_sql - ): - body_start += len(authored_body) - len(authored_body.lstrip()) - body_end = body_start + len(compiled_expansion.authored_sql) - expansion_input = compiled_expansion.authored_sql - expanded = compiled_expansion.expanded_sql - expansion_passes = compiled_expansion.passes - elif ( - MACRO_TOKEN not in expansion_input - and TEMPLATE_INTERPOLATION_START not in expansion_input - ): - expanded = expansion_input - expansion_passes = () - else: - expanded, expansion_passes = expand_sql_with_spans( - sql=expansion_input, file_path=file_path, context=context - ) - except CompileInputError as error: - raise ProjectCompileError( - f"{file_path} could not be expanded, so its SQL cannot be linted: {error}" - ) from error - return PendingLintBody( - file_path=file_path, - body_start=body_start, - body_end=body_end, - expansion_input=expansion_input, - expanded=expanded, - expansion_passes=expansion_passes, - pre_expansion_sites=pre_expansion_sites, - external_identifiers=external_identifiers, - allows_ceremonial_select=allows_ceremonial_select, - allows_dynamic_output_star=allows_dynamic_output_star, - allows_empty_fixture_star=False, - request={ - "expanded": expanded, - "before_expansion": expansion_input, - "prior_sites": [site.neutralized_start for site in pre_expansion_sites], - "dialect": dialect, - }, - ) - - -def prepare_lint_sql_batch( - pending: tuple[PendingLintBody, ...], -) -> tuple[NativePreparedSql | None, ...]: - """Prepare pending bodies in one GIL-free native call; failures re-raise per body, in order.""" - - if not pending: - return () - results: list[tuple[bool, NativePreparedSql | None]] = _native.prepare_lint_sql_batch( - [body.request for body in pending] - ) - return tuple( - prepared if succeeded else _native.prepare_lint_sql(body.request) - for body, (succeeded, prepared) in zip(pending, results, strict=True) - ) - - -def finish_lint_body( - *, pending: PendingLintBody, prepared: NativePreparedSql | None, dialect: str -) -> LintBody: - """Build the lint body from one body's native preparation.""" - - del dialect - if prepared is None: - raise NativeLintError(f"{pending.file_path} could not be prepared for native SQL lint") - neutralized: str = prepared[0] - sites: tuple[InterpolationSite, ...] = tuple(InterpolationSite(*site) for site in prepared[1]) - externally_referenced_ctes: tuple[str, ...] = tuple(prepared[2]) - dependency_identifiers: tuple[str, ...] = tuple( - site.sentinel - for site in sites - if _DEPENDENCY_INTRINSIC_PATTERN.match(site.original_text) is not None - ) - return LintBody( - file_path=pending.file_path, - body_start=pending.body_start, - body_end=pending.body_end, - lint_text=neutralized, - passes=( - _sentinel_spans(sites=pending.pre_expansion_sites), - *pending.expansion_passes, - _sentinel_spans(sites=sites), - ), - external_identifiers=pending.external_identifiers, - dependency_identifiers=dependency_identifiers, - externally_referenced_ctes=externally_referenced_ctes, - allows_ceremonial_select=pending.allows_ceremonial_select, - allows_dynamic_output_star=pending.allows_dynamic_output_star, - allows_empty_fixture_star=pending.allows_empty_fixture_star, - dependency_relations=tuple( - ( - site.sentinel, - RELATION_IDENTITY_TEMPLATE.format(kind=match.group(1).lower(), name=match.group(2)), - ) - for site in sites - if (match := _KEYED_RELATION_PATTERN.match(site.original_text)) is not None - ), - ) - - -def _sites( - neutralized: tuple[str, list[tuple[str, int, int, int, int, str]]], -) -> tuple[str, tuple[InterpolationSite, ...]]: - return neutralized[0], tuple(InterpolationSite(*site) for site in neutralized[1]) - - -def _sentinel_spans(*, sites: tuple[InterpolationSite, ...]) -> tuple[ExpansionSpan, ...]: - """Express sentinel substitutions as expansion spans for offset mapping.""" - - return tuple( - ExpansionSpan( - source_start=site.original_start, - source_end=site.original_end, - output_start=site.neutralized_start, - output_end=site.neutralized_end, - ) - for site in sites - ) - - -def _is_generic_audit_path(*, role: LintFileRole) -> bool: - """Return whether a lint input is an authored generic-audit definition.""" - - return role.in_project and role.declaration_kind is DeclarationKind.AUDIT - - -def _is_sql_hook_path(*, role: LintFileRole) -> bool: - return role.in_hook_directory or ( - role.in_project and role.declaration_kind is DeclarationKind.SQL_HOOK - ) - - -def expand_file_bodies( - *, - file_path: Path, - contents: str, - headers: tuple[HeaderSpan, ...], - context: SqlExpansionContext, - dialect: str, - project_dir: Path, - relative_path: Path, - dynamic_output_paths: frozenset[Path], - compiled_expansions: dict[Path, CompiledSqlExpansion] | None, -) -> ExpandedLintFile: - """Expand one file's bodies up to the first body that cannot be expanded.""" - - LintStopContext.check() - bodies: list[PendingLintBody] = [] - role: LintFileRole = lint_file_role( - file_path=file_path, project_dir=project_dir, relative_path=relative_path - ) - allows_dynamic_output_star: bool = ( - bool(dynamic_output_paths) and file_path.resolve() in dynamic_output_paths - ) - compiled_expansion: CompiledSqlExpansion | None = (compiled_expansions or {}).get(file_path) - external_identifiers: tuple[str, ...] = external_identifiers_for_headers( - contents=contents, headers=headers - ) - allows_ceremonial_select: bool = any( - header.kind in {HEADER_KIND_TEST, HEADER_KIND_SCENARIO} for header in headers - ) - allows_empty_fixture_star: bool = allows_ceremonial_select - body_start: int - body_end: int - try: - for body_start, body_end in lint_body_ranges( - contents=contents, - headers=headers, - file_path=file_path, - project_dir=project_dir, - role=role, - ): - bodies.append( - replace( - expand_lint_body( - role=role, - file_path=file_path, - contents=contents, - body_range=(body_start, body_end), - context=context, - dialect=dialect, - external_identifiers=external_identifiers, - allows_ceremonial_select=allows_ceremonial_select, - allows_dynamic_output_star=allows_dynamic_output_star, - compiled_expansion=compiled_expansion, - ), - allows_empty_fixture_star=allows_empty_fixture_star, - ) - ) - except ProjectCompileError as error: - return ExpandedLintFile(file_path=file_path, bodies=tuple(bodies), failure=error) - return ExpandedLintFile(file_path=file_path, bodies=tuple(bodies), failure=None) - - -def prepare_expanded_files( - *, expanded_files: tuple[ExpandedLintFile, ...], dialect: str, skip_unexpandable: bool -) -> PreparedLintFiles: - """Prepare every expanded body natively at once, then report failures in file order.""" - - LintStopContext.check() - pending: list[PendingLintBody] = [] - for expanded in expanded_files: - pending.extend(expanded.bodies) - prepared: tuple[NativePreparedSql | None, ...] = prepare_lint_sql_batch(tuple(pending)) - bodies: list[LintBody] = [] - unexpandable: dict[Path, str] = {} - position: int = 0 - for expanded in expanded_files: - file_bodies: list[LintBody] = [] - for body in expanded.bodies: - file_bodies.append( - finish_lint_body(pending=body, prepared=prepared[position], dialect=dialect) - ) - position += 1 - if expanded.failure is None: - bodies.extend(file_bodies) - elif skip_unexpandable: - unexpandable[expanded.file_path] = str(expanded.failure).removeprefix( - f"{expanded.file_path} " - ) - else: - raise expanded.failure - return PreparedLintFiles(bodies=tuple(bodies), unexpandable=unexpandable) diff --git a/src/sqlbuild/lint/_helpers/fixes.py b/src/sqlbuild/lint/_helpers/fixes.py deleted file mode 100644 index 0dbc1b6233..0000000000 --- a/src/sqlbuild/lint/_helpers/fixes.py +++ /dev/null @@ -1,395 +0,0 @@ -"""Conservative, compiler-verified fixed-point Rule repair planning.""" - -from __future__ import annotations - -from collections import Counter -from dataclasses import dataclass, field, replace -from pathlib import Path - -from sqlbuild._native import build_compiled_project, compiled_relation_keys, run_lint -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import CompiledProjectView, TypedSqlValueRenderer -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.lint.constants import ( - LINT_ENGINE_NATIVE, - SAFE_RULE_FIX_CODES, - VIOLATION_SEVERITY_FAULT, - VIOLATION_SEVERITY_WARNING, -) -from sqlbuild.lint.exceptions import ProjectCompileError -from sqlbuild.lint.main.compile_facts import compile_facts -from sqlbuild.lint.main.fix_verdict import fix_verdict -from sqlbuild.lint.models import ( - CompileFacts, - FixVerdict, - LintConfig, - LintEdit, - LintRunResult, - LintViolation, - RuleFixResult, -) -from sqlbuild.lint.types import RelationKeys, RuleFixStatus - -_MAX_PASSES: int = 128 -_APPLIED: str = "applied" -_CONDITIONLESS_JOIN_CODE: str = "SQBRSQL003" -_SNOWFLAKE_DIALECT: str = "snowflake" -_APPLIED_REASON: str = "Meaning-preserving rewrite; compiler invariants verified" -_VERIFICATION_FAILURE: str = ( - "Compilation could not prove unchanged output columns, types, dependencies and lineage" -) -_SKIPPED_CODE: str = "format-fix-skipped" - - -@dataclass -class _FixSession: - """One fixed-point run that verifies each file's edits and reverts only failing files.""" - - project_dir: Path - original: dict[Path, str] - config: LintConfig - adapter: BaseAdapter - inputs: DiscoveredProjectInputs - reports: list[RuleFixResult] = field(default_factory=list) - blocked: dict[Path, str] = field(default_factory=dict) - skipped: dict[Path, str] = field(default_factory=dict) - baseline: CompiledProjectView | None = None - baseline_facts: CompileFacts | None = None - - def run(self) -> tuple[dict[Path, str], tuple[RuleFixResult, ...], list[LintViolation]]: - current: dict[Path, str] = dict(self.original) - model_paths: frozenset[Path] = frozenset( - model.file_path for model in self.inputs.model_files - ) - seen: set[tuple[tuple[tuple[Path, str], ...], tuple[Path, ...]]] = set() - for _ in range(_MAX_PASSES): - state: tuple[tuple[tuple[Path, str], ...], tuple[Path, ...]] = ( - tuple(sorted(current.items())), - tuple(sorted(self.blocked)), - ) - if state in seen: - return self._decline(reason="Rule fixes did not converge") - seen.add(state) - edits, pending, remaining = self._plan_pass(files=current, model_paths=model_paths) - if not edits: - return ( - current, - (*self.reports, *_not_yet_reported(reports=self.reports, remaining=remaining)), - [*self._blocked_faults(), *self._skipped_warnings()], - ) - verified: dict[Path, str] | None = self._verify_pass( - current=current, edits=edits, pending=pending - ) - if verified is None: - self.reports.extend(pending) - return self._decline(reason=_VERIFICATION_FAILURE) - current = verified - return self._decline(reason="Rule fix pass limit exceeded") - - def _plan_pass( - self, *, files: dict[Path, str], model_paths: frozenset[Path] - ) -> tuple[dict[Path, list[LintEdit]], list[RuleFixResult], list[RuleFixResult]]: - result: LintRunResult = run_lint( - project_dir=self.project_dir, - config=replace( - self.config, - enabled_native_rules=("SQBRSQL",), - relation_keys=self._relation_keys(), - ), - value_renderer=self.adapter, - discovered_inputs=_updated_inputs(inputs=self.inputs, files=files), - source_files=files, - selected_paths=frozenset(files), - skip_unexpandable=True, - ) - self.skipped.update(result.unexpandable) - edits: dict[Path, list[LintEdit]] = {} - pending: list[RuleFixResult] = [] - remaining: list[RuleFixResult] = [] - for violation in result.violations: - if not violation.code.startswith("SQBRSQL"): - continue - reason: str | None = self._refusal(violation=violation, model_paths=model_paths) - if reason is not None or violation.fix is None: - remaining.append( - _report( - violation=violation, - status="refused" if reason is not None else "unavailable", - reason=reason or "No meaning-preserving automatic fix is available", - ) - ) - continue - edit: LintEdit = violation.fix - chosen: list[LintEdit] = edits.setdefault(edit.file_path, []) - if any(edit.start < other.end and other.start < edit.end for other in chosen): - continue - chosen.append(edit) - pending.append(_report(violation=violation, status="applied", reason=_APPLIED_REASON)) - return edits, pending, remaining - - def _refusal(self, *, violation: LintViolation, model_paths: frozenset[Path]) -> str | None: - if violation.file_path not in model_paths: - return "Compiler-verified Rule fixes currently require a model SQL body" - if violation.fix is not None and violation.code not in SAFE_RULE_FIX_CODES: - return "This proposed rewrite is not proven meaning-preserving in this dialect" - if violation.code == _CONDITIONLESS_JOIN_CODE and self.config.dialect != _SNOWFLAKE_DIALECT: - return "Conditionless JOIN equivalence is only established for Snowflake" - if violation.fix_unavailable_reason is not None or violation.fix is None: - return violation.fix_unavailable_reason - return self.blocked.get(violation.file_path) - - def _verify_pass( - self, - *, - current: dict[Path, str], - edits: dict[Path, list[LintEdit]], - pending: list[RuleFixResult], - ) -> dict[Path, str] | None: - """Attribute failures from one compile of all edits; isolate files when none is to blame.""" - - accepted: dict[Path, list[LintEdit]] = dict(edits) - confirmed: bool = False - while accepted: - verdict: FixVerdict | None = self._verdict( - files=_with_edits(files=self._unblocked(current), edits=accepted) - ) - if verdict is not None and verdict.verified: - confirmed = True - break - hits: dict[Path, str] = ( - {} if verdict is None else _failing_edits(verdict=verdict, edits=accepted) - ) - if not hits: - accepted = self._isolate(current=current, edits=accepted) - break - for path, reason in hits.items(): - self.blocked[path] = reason - del accepted[path] - candidate: dict[Path, str] = _with_edits(files=self._unblocked(current), edits=accepted) - if accepted and not confirmed and not self._verified(files=candidate): - return None - self.reports[:] = [ - _refused(report=report, reason=self.blocked[report.file_path]) - if report.file_path in self.blocked - else report - for report in (*self.reports, *pending) - ] - return candidate - - def _isolate( - self, *, current: dict[Path, str], edits: dict[Path, list[LintEdit]] - ) -> dict[Path, list[LintEdit]]: - """Check each file's edits alone and keep only the files that verify.""" - - base: dict[Path, str] = self._unblocked(current) - accepted: dict[Path, list[LintEdit]] = {} - for path, path_edits in edits.items(): - verdict: FixVerdict | None = self._verdict( - files=_with_edits(files=base, edits={path: path_edits}) - ) - if verdict is not None and verdict.verified: - accepted[path] = path_edits - else: - self.blocked[path] = ( - _failing_edits(verdict=verdict, edits={path: path_edits}).get( - path, _VERIFICATION_FAILURE - ) - if verdict is not None - else _VERIFICATION_FAILURE - ) - return accepted - - def _unblocked(self, files: dict[Path, str]) -> dict[Path, str]: - return { - path: self.original[path] if path in self.blocked else contents - for path, contents in files.items() - } - - def _relation_keys(self) -> RelationKeys: - """Declared keys from the baseline compile, so determinism proofs match compile.""" - - if self.baseline is None: - try: - self.baseline = build_compiled_project( - discovered_inputs=self.inputs, adapter=self.adapter - ) - except CompileInputError: - return {} - return compiled_relation_keys(self.baseline) - - def _verdict(self, *, files: dict[Path, str]) -> FixVerdict | None: - """Compare one candidate compilation with the baseline; None when it cannot compile.""" - - verdict: FixVerdict | None - try: - if self.baseline is None: - self.baseline = build_compiled_project( - discovered_inputs=self.inputs, adapter=self.adapter - ) - if self.baseline_facts is None: - self.baseline_facts = compile_facts( - project=self.baseline, project_dir=self.project_dir - ) - compiled: CompiledProjectView = build_compiled_project( - discovered_inputs=_updated_inputs(inputs=self.inputs, files=files), - adapter=self.adapter, - ) - verdict = fix_verdict( - before=self.baseline_facts, - after=compile_facts(project=compiled, project_dir=self.project_dir), - edited=frozenset( - path.resolve() - for path, contents in files.items() - if contents != self.original[path] - ), - ) - except Exception: - verdict = None - return verdict - - def _verified(self, *, files: dict[Path, str]) -> bool: - verdict: FixVerdict | None = self._verdict(files=files) - return verdict is not None and verdict.verified - - def _blocked_faults(self) -> list[LintViolation]: - return [ - _verification_fault(path=path, reason=reason) - for path, reason in sorted(self.blocked.items()) - ] - - def _skipped_warnings(self) -> list[LintViolation]: - return [ - LintViolation( - file_path=path, - line=1, - column=1, - code=_SKIPPED_CODE, - message=f"Rule fixes skipped this file: {reason}", - severity=VIOLATION_SEVERITY_WARNING, - engine=LINT_ENGINE_NATIVE, - remediation="Fix the reported problem, then rerun `sqb format --fix`.", - ) - for path, reason in sorted(self.skipped.items()) - ] - - def _decline( - self, *, reason: str - ) -> tuple[dict[Path, str], tuple[RuleFixResult, ...], list[LintViolation]]: - affected: set[Path] = { - report.file_path for report in self.reports if report.status == _APPLIED - } | set(self.blocked) - return ( - self.original, - tuple(_refused(report=report, reason=reason) for report in self.reports), - [ - *(_verification_fault(path=path, reason=reason) for path in sorted(affected)), - *self._skipped_warnings(), - ], - ) - - -def plan_rule_fixes( - *, - project_dir: Path, - files: dict[Path, str], - config: LintConfig, - adapter: TypedSqlValueRenderer | None, - discovered_inputs: DiscoveredProjectInputs | None, - enabled: bool, -) -> tuple[dict[Path, str], tuple[RuleFixResult, ...], list[LintViolation]]: - """Plan in memory; never expose an unverified intermediate version on disk.""" - - if not enabled: - return files, (), [] - if not isinstance(adapter, BaseAdapter) or discovered_inputs is None: - raise ProjectCompileError("format --fix requires discovered compiler inputs and an adapter") - session: _FixSession = _FixSession( - project_dir=project_dir, - original=files, - config=config, - adapter=adapter, - inputs=discovered_inputs, - ) - return session.run() - - -def _failing_edits(*, verdict: FixVerdict, edits: dict[Path, list[LintEdit]]) -> dict[Path, str]: - """Failing files of a verdict among the edited files, keyed as the edits are.""" - - by_resolved: dict[Path, Path] = {path.resolve(): path for path in edits} - return { - by_resolved[path]: reason for path, reason in verdict.failing.items() if path in by_resolved - } - - -def _with_edits(*, files: dict[Path, str], edits: dict[Path, list[LintEdit]]) -> dict[Path, str]: - candidate: dict[Path, str] = dict(files) - for path, path_edits in edits.items(): - contents: str = candidate[path] - for edit in sorted(path_edits, key=lambda item: item.start, reverse=True): - contents = contents[: edit.start] + edit.replacement + contents[edit.end :] - candidate[path] = contents - return candidate - - -def _not_yet_reported( - *, reports: list[RuleFixResult], remaining: list[RuleFixResult] -) -> list[RuleFixResult]: - """Final-pass outcomes, minus those an earlier pass already recorded identically.""" - - recorded: Counter[RuleFixResult] = Counter(reports) - fresh: list[RuleFixResult] = [] - for report in remaining: - if recorded[report] > 0: - recorded[report] -= 1 - else: - fresh.append(report) - return fresh - - -def _report(*, violation: LintViolation, status: RuleFixStatus, reason: str) -> RuleFixResult: - return RuleFixResult( - file_path=violation.file_path, - code=violation.code, - line=violation.line, - status=status, - reason=reason, - ) - - -def _refused(*, report: RuleFixResult, reason: str = _VERIFICATION_FAILURE) -> RuleFixResult: - if report.status != _APPLIED: - return report - return replace(report, status="refused", reason=reason) - - -def _verification_fault(*, path: Path, reason: str = _VERIFICATION_FAILURE) -> LintViolation: - return LintViolation( - file_path=path, - line=1, - column=1, - code="format-fix-verification-failed", - message=reason, - severity=VIOLATION_SEVERITY_FAULT, - engine=LINT_ENGINE_NATIVE, - remediation="Restructure the SQL manually and recompile; the original file was retained.", - ) - - -def _updated_inputs( - *, inputs: DiscoveredProjectInputs, files: dict[Path, str] -) -> DiscoveredProjectInputs: - return replace( - inputs, - model_files=tuple( - model.with_text( - files[model.file_path], - files[model.file_path][model.contents.index(model.query_sql) :].strip(), - ) - if model.file_path in files and files[model.file_path] != model.contents - else model - for model in inputs.model_files - ), - native_session=None, - ) diff --git a/src/sqlbuild/lint/_helpers/headers.py b/src/sqlbuild/lint/_helpers/headers.py deleted file mode 100644 index b9eeb3232b..0000000000 --- a/src/sqlbuild/lint/_helpers/headers.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Lintable body ranges of SQL files by project role.""" - -from __future__ import annotations - -from pathlib import Path, PurePath - -from sqlbuild._native import ( - measurement_body_ranges, - named_declaration_role_kind, - sql_body_ranges, -) -from sqlbuild.compiler.compile.constants import HOOK_DIRECTORY_NAME -from sqlbuild.compiler.scopes.types import DeclarationKind -from sqlbuild.lint.models import HeaderSpan, LintFileRole - - -def lint_file_role( - *, file_path: Path, project_dir: Path, relative_path: PurePath | None = None -) -> LintFileRole: - """Return the path facts lint needs, reusing a caller's project-relative path.""" - - if relative_path is None: - if not file_path.is_relative_to(project_dir): - return LintFileRole() - relative_path = file_path.relative_to(project_dir) - return LintFileRole( - declaration_kind=named_declaration_role_kind(relative_path=relative_path), - in_project=True, - in_hook_directory=relative_path.is_relative_to(HOOK_DIRECTORY_NAME), - ) - - -def lint_body_ranges( - *, - contents: str, - headers: tuple[HeaderSpan, ...], - file_path: Path, - project_dir: Path, - role: LintFileRole | None = None, -) -> tuple[tuple[int, int], ...]: - """Return lintable authored bodies using resource-specific DSL boundaries.""" - - file_role: LintFileRole = ( - role if role is not None else lint_file_role(file_path=file_path, project_dir=project_dir) - ) - if file_role.in_project and file_role.declaration_kind in { - DeclarationKind.AUDIT, - DeclarationKind.SINGULAR_AUDIT, - }: - measurement_ranges: tuple[tuple[int, int], ...] = measurement_body_ranges(contents=contents) - if measurement_ranges: - return measurement_ranges - return sql_body_ranges(contents=contents, headers=headers) diff --git a/src/sqlbuild/lint/constants.py b/src/sqlbuild/lint/constants.py index a5a45b6d9f..951a4b5ae4 100644 --- a/src/sqlbuild/lint/constants.py +++ b/src/sqlbuild/lint/constants.py @@ -10,9 +10,6 @@ VIOLATION_SEVERITY_FAULT: LintSeverity = LintSeverity.FAULT VIOLATION_SEVERITY_WARNING: LintSeverity = LintSeverity.WARNING -HEADER_KIND_SCENARIO: str = "SCENARIO" -HEADER_KIND_TEST: str = "TEST" - DEFAULT_MAX_LITERAL_LENGTH: int = 100 DEFAULT_MAX_RANKING_ORDER_BY: int = 5 MAX_LITERAL_LENGTH_THRESHOLD: str = "max_literal_length" @@ -30,13 +27,5 @@ } ) -TEMPLATE_INTERPOLATION_START: str = "${" -RELATION_IDENTITY_TEMPLATE: str = "{kind}:{name}" RELATION_KIND_SOURCE: str = "source" RELATION_KIND_SEED: str = "seed" - -FIX_COLUMNS_UNKNOWN: str = "Output columns could not be inferred, so the fix cannot be verified" -FIX_COLUMNS_CHANGED: str = "The fix would change output columns, types or nullability" -FIX_DEPENDENCIES_CHANGED: str = "The fix would change model dependencies" -FIX_LINEAGE_CHANGED: str = "The fix would change column lineage" -FIX_NEW_DIAGNOSTICS: str = "The fix would add compile diagnostics: " diff --git a/src/sqlbuild/lint/main/compile_facts.py b/src/sqlbuild/lint/main/compile_facts.py deleted file mode 100644 index 43e5939eb3..0000000000 --- a/src/sqlbuild/lint/main/compile_facts.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Index a compilation so two compilations can be compared file by file.""" - -from __future__ import annotations - -from collections import Counter -from pathlib import Path - -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompilerDiagnostic -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.lint.models import CompileFacts -from sqlbuild.lint.types import DiagnosticIdentity - - -def compile_facts(*, project: CompiledProjectView, project_dir: Path) -> CompileFacts: - """Index a compilation so two compilations can be compared file by file.""" - - model_paths: dict[CompiledObjectKey, Path] = { - model.key: (project_dir / model.relative_path).resolve() for model in project.models - } - paths_by_name: dict[str, Path] = { - model.name: model_paths[model.key] for model in project.models - } - diagnostics: dict[Path, Counter[DiagnosticIdentity]] = {} - for diagnostic in project.diagnostics: - owner: Path = _owner( - diagnostic=diagnostic, paths_by_name=paths_by_name, project_dir=project_dir - ) - diagnostics.setdefault(owner, Counter())[_identity(diagnostic)] += 1 - return CompileFacts( - models={model.key: model for model in project.models}, - model_paths=model_paths, - diagnostics=diagnostics, - ) - - -def _owner( - *, diagnostic: CompilerDiagnostic, paths_by_name: dict[str, Path], project_dir: Path -) -> Path: - if ( - diagnostic.resource_type == CompiledResourceType.MODEL - and diagnostic.resource_name in paths_by_name - ): - return paths_by_name[diagnostic.resource_name] - path: Path | None = diagnostic.path or ( - diagnostic.location.path if diagnostic.location is not None else None - ) - return (project_dir / path).resolve() if path is not None else project_dir.resolve() - - -def _identity(diagnostic: CompilerDiagnostic) -> DiagnosticIdentity: - """Location-insensitive identity, so moved lines do not count as new diagnostics.""" - - return ( - diagnostic.code, - str(diagnostic.resource_type or ""), - diagnostic.resource_name or "", - diagnostic.message, - ) diff --git a/src/sqlbuild/lint/main/fix_verdict.py b/src/sqlbuild/lint/main/fix_verdict.py deleted file mode 100644 index a68ccfc252..0000000000 --- a/src/sqlbuild/lint/main/fix_verdict.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Attribute compile differences between a baseline and a fixed project to edited files.""" - -from __future__ import annotations - -from collections import Counter -from pathlib import Path - -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.lint.constants import ( - FIX_COLUMNS_CHANGED, - FIX_COLUMNS_UNKNOWN, - FIX_DEPENDENCIES_CHANGED, - FIX_LINEAGE_CHANGED, - FIX_NEW_DIAGNOSTICS, -) -from sqlbuild.lint.models import CompileFacts, FixVerdict -from sqlbuild.lint.types import DiagnosticIdentity - - -def fix_verdict( - *, before: CompileFacts, after: CompileFacts, edited: frozenset[Path] -) -> FixVerdict: - """Edited files may lose diagnostics but not gain any; everything else must not change.""" - - if set(before.models) != set(after.models): - return FixVerdict(failing={}, unattributed=True) - failing: dict[Path, str] = {} - unattributed: bool = False - for key, model in before.models.items(): - path: Path = before.model_paths[key] - reason: str | None = _model_change( - before=model, after=after.models[key], require_columns=path in edited - ) - if reason is None: - continue - if path in edited: - failing.setdefault(path, reason) - else: - unattributed = True - empty: Counter[DiagnosticIdentity] = Counter() - for owner in set(before.diagnostics) | set(after.diagnostics): - previous: Counter[DiagnosticIdentity] = before.diagnostics.get(owner, empty) - current: Counter[DiagnosticIdentity] = after.diagnostics.get(owner, empty) - if owner in edited: - added: Counter[DiagnosticIdentity] = current - previous - if added: - codes: str = ", ".join(sorted({identity[0] for identity in added})) - failing.setdefault(owner, FIX_NEW_DIAGNOSTICS + codes) - elif previous != current: - unattributed = True - return FixVerdict(failing=failing, unattributed=unattributed) - - -def _model_change( - *, before: CompiledModelView, after: CompiledModelView, require_columns: bool -) -> str | None: - """Edited models need known, equal columns; others must not change; absent lineage matches.""" - - if require_columns and (before.inferred_columns is None or after.inferred_columns is None): - return FIX_COLUMNS_UNKNOWN - if before.inferred_columns != after.inferred_columns: - return FIX_COLUMNS_CHANGED - if before.deps != after.deps: - return FIX_DEPENDENCIES_CHANGED - lineage_before: object = ( - None if before.fast_lineage_columns is None else tuple(before.fast_lineage_columns) - ) - lineage_after: object = ( - None if after.fast_lineage_columns is None else tuple(after.fast_lineage_columns) - ) - if lineage_before != lineage_after: - return FIX_LINEAGE_CHANGED - return None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_build_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_build_performance.py index 999f5e3af7..ff1b53d553 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_build_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_build_performance.py @@ -37,7 +37,8 @@ BuildPerformanceGuardTestCase( description="models_3000_job_build_1000_model_benchmark_from_empty_warehouse", expected_max_wall_seconds=90.0, - expected_max_rss_bytes=1_600 * _MIB, + # Temporary native-migration headroom pending memory optimisation. + expected_max_rss_bytes=1_728 * _MIB, ) ], ids=lambda case: case.description, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py index ca097d770a..e585f087af 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py @@ -424,5 +424,46 @@ def test_given_unused_cte_output_when_fixing_then_model_rows_are_unchanged( ) +@pytest.mark.parametrize( + "test_case", + [FormatCompileIntegrationTestCase("macro and CRLF source", "UNION DISTINCT")], + ids=lambda case: case.description, +) +def test_given_macro_and_crlf_source_when_fixing_then_previews_preserve_bytes_and_write_keeps_mode( + test_case: FormatCompileIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + macros: Path = tmp_path / "macros" + macros.mkdir() + (macros / "values.py").write_text('def order_value():\n return "1"\n') + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + original: bytes = ( + b'MODEL (description "Orders");\r\n' + b"SELECT @order_value() AS order_id UNION SELECT 2 AS order_id\r\n" + ) + model.write_bytes(original) + model.chmod(0o640) + original_mode: int = model.stat().st_mode + + assert main(["--project-dir", str(tmp_path), "format", "--fix", "--check", "--json"]) == 1 + capsys.readouterr() + assert model.read_bytes() == original + assert main(["--project-dir", str(tmp_path), "format", "--fix", "--diff"]) == 0 + capsys.readouterr() + assert model.read_bytes() == original + + assert main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) == 0 + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + assert any( + fix["code"] == "SQBRSQL008" and fix["status"] == "applied" for fix in payload["rule_fixes"] + ) + written: bytes = model.read_bytes() + assert b"@order_value()" in written + assert test_case.expected_literal.encode() in written + assert written.count(b"\n") == written.count(b"\r\n") + assert model.stat().st_mode == original_mode + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-vv"])) diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py b/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py index f161626d30..7ec1102362 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.integrations.dbt.main.pipeline.plan import plan_dbt_interop_from_project +from sqlbuild._native import plan_dbt_interop_from_project from sqlbuild.integrations.dbt.models import DbtInteropPlan from sqlbuild.integrations.dbt.types import DbtInteropSkipReason from tests.integration.src.sqlbuild.integrations.dbt._test_types import ( diff --git a/tests/integration/src/sqlbuild/lint/main/test_fix_verdict.py b/tests/integration/src/sqlbuild/lint/main/test_fix_verdict.py index d4a6746ecc..9fe90a5ec7 100644 --- a/tests/integration/src/sqlbuild/lint/main/test_fix_verdict.py +++ b/tests/integration/src/sqlbuild/lint/main/test_fix_verdict.py @@ -6,9 +6,8 @@ import pytest +from sqlbuild._native import compile_facts, fix_verdict from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.lint.main.compile_facts import compile_facts -from sqlbuild.lint.main.fix_verdict import fix_verdict from sqlbuild.lint.models import FixVerdict from tests.integration.src.sqlbuild.lint.main._test_types import FixVerdictTestCase from tests.integration.src.sqlbuild.lint.main.helpers import compile_models, without_lineage @@ -27,6 +26,37 @@ @pytest.mark.parametrize( "test_case", ( + FixVerdictTestCase( + description="a changed model inventory cannot be attributed to one edit", + before_models=(("orders.sql", _ORDERS),), + after_models=(("orders.sql", _ORDERS), ("payments.sql", _PAYMENTS)), + edited=("orders.sql",), + expected_failing=(), + expected_unattributed=True, + ), + FixVerdictTestCase( + description="an edited model without inferred columns cannot verify", + before_models=( + ( + "orders.sql", + 'MODEL (description "Orders", sql_analysis false);\nSELECT 1 AS order_id\n', + ), + ), + after_models=( + ( + "orders.sql", + 'MODEL (description "Orders", sql_analysis false);\nSELECT 1 AS order_id\n', + ), + ), + edited=("orders.sql",), + expected_failing=( + ( + "orders.sql", + "Output columns could not be inferred, so the fix cannot be verified", + ), + ), + expected_unattributed=False, + ), FixVerdictTestCase( description="an edit that adds a diagnostic fails its own file", before_models=(("orders.sql", _ORDERS), ("payments.sql", _PAYMENTS)), diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py index 6580f51a52..0a1e83037c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import enforce_plan_safety_policies -from sqlbuild.cli.commands._helpers.dbt.sqlbuild_work import dbt_interop_plan_safety_gates +from sqlbuild._native import dbt_interop_plan_safety_gates, enforce_plan_safety_policies from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py index cb91d85e2b..3b34366c44 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py @@ -74,7 +74,7 @@ def plan_dbt_interop_from_project( return build_empty_dbt_plan() monkeypatch.setattr( - "sqlbuild.integrations.dbt.main.pipeline.plan.plan_dbt_interop_from_project", + "sqlbuild._native.plan_dbt_interop_from_project", plan_dbt_interop_from_project, ) write_minimal_sqlbuild_project(tmp_path) @@ -125,7 +125,7 @@ def execute_dbt_interop_from_project(request: DbtInteropExecutionRequest) -> int return 0 monkeypatch.setattr( - "sqlbuild.integrations.dbt.main.pipeline.execute.execute_dbt_interop_from_project", + "sqlbuild._native.execute_dbt_interop_from_project", execute_dbt_interop_from_project, ) write_minimal_sqlbuild_project(tmp_path) @@ -200,7 +200,7 @@ def run_sqlbuild_debug( return 0 monkeypatch.setattr( - "sqlbuild.integrations.dbt.main.pipeline.debug.debug_dbt_from_project", + "sqlbuild._native.debug_dbt_from_project", debug_dbt_from_project, ) monkeypatch.setattr("sqlbuild._native.run_debug_command", run_sqlbuild_debug) @@ -293,7 +293,7 @@ def plan_dbt_interop_from_project( run_dbt_profile_init, ) monkeypatch.setattr( - "sqlbuild.integrations.dbt.main.pipeline.plan.plan_dbt_interop_from_project", + "sqlbuild._native.plan_dbt_interop_from_project", plan_dbt_interop_from_project, ) @@ -382,7 +382,7 @@ def plan_dbt_interop_from_project( run_dbt_profile_init, ) monkeypatch.setattr( - "sqlbuild.integrations.dbt.main.pipeline.plan.plan_dbt_interop_from_project", + "sqlbuild._native.plan_dbt_interop_from_project", plan_dbt_interop_from_project, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py index b343ca4b77..f393bb7136 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py @@ -10,6 +10,28 @@ ) +@dataclass(frozen=True) +class RenameLookupFailureTestCase: + description: str + expected_error: str + + +@dataclass(frozen=True) +class StoredMigrationHashTestCase: + description: str + metadata_json: str + expected_fingerprint: str + + +@dataclass(frozen=True) +class StoredProjectionTestCase: + description: str + query_sql: str + expected_references: frozenset[str] + expected_expression_hash: str + expected_fingerprint: str + + @dataclass(frozen=True) class MigrationFingerprintTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py index 2a04d45b9f..516ac3c56d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py @@ -5,10 +5,9 @@ import json from datetime import UTC, datetime +from sqlbuild._native import build_migration_fingerprint, parse_query_shape from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.migrations.types import OldNameViewAction -from sqlbuild.compiler.planner._helpers.migrations.fingerprint import build_migration_fingerprint -from sqlbuild.compiler.planner._helpers.migrations.projections import parse_query_shape from sqlbuild.compiler.planner.models import OldNameViewPlanEntry, QueryShape from sqlbuild.compiler.planner.types import InputColumns @@ -51,6 +50,11 @@ def input_lookup(relations: dict[tuple[str, str], frozenset[str]]) -> InputColum return lambda kind, name: relations.get((kind, name)) +def unavailable_input_columns(kind: str, name: str) -> frozenset[str] | None: + """Represent an input inspection failure encountered during a rename proof.""" + raise RuntimeError(f"cannot inspect {kind} {name}") + + def old_name_entry( *, action: str, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py index 8d90f66d5b..e5682b411a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py @@ -4,12 +4,10 @@ import pytest -from sqlbuild.compiler.planner._helpers.migrations.column_renames import ( +from sqlbuild._native import ( identical_renames, - rename_hints, -) -from sqlbuild.compiler.planner._helpers.migrations.projections import ( parse_query_shape, + rename_hints, renames_explain_change, ) from sqlbuild.compiler.planner.models import ColumnRenameHint @@ -17,11 +15,13 @@ IdenticalRenameTestCase, RenameExplainsChangeTestCase, RenameHintTestCase, + RenameLookupFailureTestCase, UnreadableQueryShapeTestCase, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations.helpers import ( input_lookup, query_shape, + unavailable_input_columns, ) _ORDERS: str = 'FROM __source("raw_orders")' @@ -36,6 +36,12 @@ current_sql=f"SELECT order_id, amount AS revenue {_ORDERS}", expected_renames=(("amount", "revenue"),), ), + IdenticalRenameTestCase( + description="quoted Unicode names and numeric expressions retain projection order", + previous_sql='SELECT 12345678901234567890 AS "Öld", NULL AS missing', + current_sql='SELECT NULL AS absent, 12345678901234567890 AS "Néw"', + expected_renames=(("missing", "absent"), ("Öld", "Néw")), + ), IdenticalRenameTestCase( description="formatting, comments, and keyword case are ignored", previous_sql=f"SELECT order_id, ROUND(amount, 2) AS revenue {_ORDERS}", @@ -383,5 +389,28 @@ def test_given_query_without_readable_output_names_when_parsing_then_returns_non ) +@pytest.mark.parametrize( + "test_case", + [ + RenameLookupFailureTestCase( + description="input inspection fails", + expected_error="cannot inspect __source raw_orders", + ) + ], + ids=lambda case: case.description, +) +def test_given_input_lookup_failure_when_proving_rename_then_propagates_error( + test_case: RenameLookupFailureTestCase, +) -> None: + with pytest.raises(RuntimeError, match=test_case.expected_error): + identical_renames( + previous=query_shape(f"SELECT amount {_ORDERS} ORDER BY amount"), + current=query_shape(f"SELECT amount AS revenue {_ORDERS} ORDER BY revenue"), + excluded=frozenset(), + declared={}, + input_columns=unavailable_input_columns, + ) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py index 211a7bb3f6..86148da004 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py @@ -4,9 +4,11 @@ import pytest +from sqlbuild._native import build_migration_fingerprint from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( IneligibleFingerprintTestCase, MigrationFingerprintTestCase, + StoredMigrationHashTestCase, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations.helpers import ( BASE_CONFIG, @@ -15,6 +17,41 @@ ) +@pytest.mark.parametrize( + "test_case", + [ + StoredMigrationHashTestCase( + description="empty config", + metadata_json='{"config": {}}', + expected_fingerprint="7734ff9fad2e1b98899e6f9ade8f30aee082ece92540d972b7f54816b28d6b70", + ), + StoredMigrationHashTestCase( + description="ordered keys and large integer", + metadata_json='{"config": {"unique_key": ["b", "a"], "scale": 12345678901234567890}}', + expected_fingerprint="24c0e337e8fdabcffca3ca478fc59883dd2187e849d036a547638909e8f22f09", + ), + StoredMigrationHashTestCase( + description="Unicode numeric and null metadata", + metadata_json='{"config": {"cursor_inputs": {"orders": "id"}, "amount": 1.0, "note": "café", "optional": null}}', + expected_fingerprint="e9ca28719a39368e4e00095c8361457354240f71f4f552f2e44f871709de1226", + ), + ], + ids=lambda case: case.description, +) +def test_given_recorded_metadata_when_fingerprinting_then_preserves_stored_hash( + test_case: StoredMigrationHashTestCase, +) -> None: + assert ( + build_migration_fingerprint( + query_sql='SELECT o.id FROM __ref("orders") o', + metadata_json=test_case.metadata_json, + ref_identities={"orders": "previous_orders"}, + dialect="duckdb", + ) + == test_case.expected_fingerprint + ) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_query_shape.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_query_shape.py new file mode 100644 index 0000000000..678171429d --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_query_shape.py @@ -0,0 +1,65 @@ +"""Pinned query-shape and fingerprint evidence for structured expressions.""" + +from __future__ import annotations + +import hashlib + +import pytest + +from sqlbuild._native import build_migration_fingerprint, parse_query_shape +from sqlbuild.compiler.planner.models import QueryShape +from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( + StoredProjectionTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + StoredProjectionTestCase( + description="CASE tuple pairs preserve recorded proof boundaries", + query_sql='SELECT CASE WHEN amount > 1 THEN amount ELSE 0 END AS total FROM __ref("orders")', + expected_references=frozenset(), + expected_expression_hash="3b459b87711a9f5500b8f13f9c7eadae4da1d8245d67cdf926f57179a0f339a1", + expected_fingerprint="734b3eafe12854d5451634531a66bc6971a229549ceabd25e23e72637f1346f9", + ), + StoredProjectionTestCase( + description="struct references retain their expression identity", + query_sql="SELECT {'total': amount} AS details FROM __ref(\"orders\")", + expected_references=frozenset({"amount"}), + expected_expression_hash="a160e71b93536688672e65f73ca5ff09ac0020689aad1b4282c64b8c03a664f5", + expected_fingerprint="ed1d753dd09893556d7fa08c91c8ce431600419dafd80939e843013ab705edb5", + ), + StoredProjectionTestCase( + description="decode references retain their expression identity", + query_sql='SELECT DECODE(amount, 1, 2, 0) AS total FROM __ref("orders")', + expected_references=frozenset({"amount"}), + expected_expression_hash="210d8215f7e3655cf5482d4e7c5026fad6f3c2a2b63f762c3bdcde05d7929ba7", + expected_fingerprint="6e71b7abdf66bbb2f7c97b64deaf3d63a25d3d7c0e04b7d904e9b67a5af30619", + ), + ], + ids=lambda case: case.description, +) +def test_given_structured_expression_when_proving_migration_then_preserves_recorded_identity( + test_case: StoredProjectionTestCase, +) -> None: + shape: QueryShape | None = parse_query_shape(query_sql=test_case.query_sql, dialect="duckdb") + assert shape is not None + assert shape.projections[0].references == test_case.expected_references + assert ( + hashlib.sha256(shape.projections[0].expression.encode()).hexdigest() + == test_case.expected_expression_hash + ) + assert ( + build_migration_fingerprint( + query_sql=test_case.query_sql, + metadata_json="{}", + ref_identities={}, + dialect="duckdb", + ) + == test_case.expected_fingerprint + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py index 6703d65bae..71a660d9a8 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py @@ -11,10 +11,9 @@ import sqlbuild.compiler.planner.classes.fixture_column_inferences as fixture_inferences import sqlbuild.compiler.planner.classes.migration_fingerprint_cache as fingerprint_cache -from sqlbuild._native import infer_fixture_column_facts +from sqlbuild._native import build_migration_fingerprint, infer_fixture_column_facts from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.models import FixtureColumnInference -from sqlbuild.compiler.planner._helpers.migrations.fingerprint import build_migration_fingerprint ORDERS_SQL: str = 'SELECT o.order_id, o.amount_cents FROM __ref("stg_orders") AS o' CURSOR_METADATA: str = json.dumps( diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_migration_fingerprint_cache.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_migration_fingerprint_cache.py index 7c38f4c20a..0c6851289e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_migration_fingerprint_cache.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_migration_fingerprint_cache.py @@ -9,7 +9,7 @@ import pytest import sqlbuild.compiler.planner.classes.migration_fingerprint_cache as fingerprint_cache -from sqlbuild.compiler.planner._helpers.migrations.fingerprint import build_migration_fingerprint +from sqlbuild._native import build_migration_fingerprint from sqlbuild.compiler.planner.classes.migration_fingerprint_cache import ( MigrationFingerprintCache, ) diff --git a/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py b/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py index 7fa1804900..b26afb0d0e 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py @@ -11,6 +11,14 @@ from sqlbuild.spec.contracts.models import DbtConfig, LocalDbtConfig +@dataclass(frozen=True) +class DbtResultBackfillTestCase: + description: str + message: str + failures: int + expected_message: str + + @dataclass(frozen=True) class DbtConfigResolutionTestCase: description: str diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_debug.py b/tests/unit/src/sqlbuild/integrations/dbt/test_debug.py index b002631d1e..b1b3b2e14d 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_debug.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_debug.py @@ -5,8 +5,8 @@ import pytest +from sqlbuild._native import debug_dbt_from_project from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner -from sqlbuild.integrations.dbt.main.pipeline.debug import debug_dbt_from_project from sqlbuild.integrations.dbt.models import DbtCommandResult from tests.unit.src.sqlbuild.integrations.dbt._test_types import DbtDebugPipelineTestCase from tests.unit.src.sqlbuild.integrations.dbt.helpers import RecordingDbtInvoker diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py index 61109771df..1f94a81d74 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from io import StringIO from pathlib import Path from types import SimpleNamespace @@ -7,20 +8,21 @@ import pytest +from sqlbuild import _native as execute_module +from sqlbuild import _native as phases_module +from sqlbuild import _native as plan_module from sqlbuild._native import ( DbtCombinedGraph, build_dbt_interop_plan, build_failed_sqlbuild_model_names, build_merged_dbt_execution_argv, + execute_dbt_commands, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt._helpers.pipeline import execute as dbt_execute_module -from sqlbuild.integrations.dbt._helpers.pipeline import execution_phases as phases_module from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner -from sqlbuild.integrations.dbt.main.pipeline import execute as execute_module -from sqlbuild.integrations.dbt.main.pipeline import plan as plan_module +from sqlbuild.integrations.dbt.main.runtime import _execute_json_event_stream as event_module from sqlbuild.integrations.dbt.models import ( DbtCliOptions, DbtCommandExecutionResult, @@ -40,6 +42,7 @@ DbtCompileFullRefreshPipelineTestCase, DbtExecutionSelectionStatusTestCase, DbtExecutionSummaryFooterTestCase, + DbtResultBackfillTestCase, ) from tests.unit.src.sqlbuild.integrations.dbt.helpers import ( CompileOnlyDbtRunner, @@ -437,13 +440,13 @@ def invoke(self, **kwargs: object) -> object: ), ) monkeypatch.setattr( - dbt_execute_module, + event_module, "execute_dbt_json_event_stream", lambda **kwargs: (0, streamed_results), ) progress_stream: StringIO = StringIO() - result: DbtCommandExecutionResult = dbt_execute_module.execute_dbt_commands( + result: DbtCommandExecutionResult = execute_dbt_commands( runner=cast(DbtRunner, NoSelectionRunner()), options=DbtCliOptions(project_dir=Path("/dbt")), merged_argv=("dbt", "test", "--select", planned_node.unique_id), @@ -457,3 +460,86 @@ def invoke(self, **kwargs: object) -> object: assert result.node_results[0].unique_id == planned_node.unique_id assert test_case.expected_output_fragment in progress_stream.getvalue() assert test_case.expected_completion_fragment not in progress_stream.getvalue() + + +@pytest.mark.parametrize( + "test_case", + [ + DbtResultBackfillTestCase( + description="unicode message takes precedence over failures", + message=" café\u2003", + failures=3, + expected_message="café", + ), + DbtResultBackfillTestCase( + description="blank message falls back to failure count", + message="\u2003", + failures=3, + expected_message="3 failure(s)", + ), + ], + ids=lambda case: case.description, +) +def test_given_missing_streamed_nodes_when_finalizing_then_backfills_selected_artifact_results( + test_case: DbtResultBackfillTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + streamed: DbtNodeExecutionResult = DbtNodeExecutionResult( + unique_id="model.analytics.orders", + resource_type="model", + node_name="orders", + status="error", + index=1, + total=2, + execution_time=0.25, + ) + (tmp_path / "run_results.json").write_text( + json.dumps( + { + "results": [ + {"unique_id": streamed.unique_id, "status": "error"}, + { + "unique_id": "model.analytics.customers", + "status": "skipped", + "message": test_case.message, + "failures": test_case.failures, + }, + {"unique_id": "model.analytics.inventory", "status": "success"}, + ] + } + ), + encoding="utf-8", + ) + monkeypatch.setattr( + event_module, "execute_dbt_json_event_stream", lambda **kwargs: (1, (streamed,)) + ) + output: StringIO = StringIO() + callbacks: list[DbtNodeExecutionResult] = [] + progress: list[str] = [] + + result: DbtCommandExecutionResult = execute_dbt_commands( + runner=DbtRunner(), + options=DbtCliOptions(target_path=tmp_path), + merged_argv=("dbt", "build"), + progress_stream=StringIO(), + stdout_stream=output, + use_color=False, + expected_nodes=( + DbtLsNode(unique_id=streamed.unique_id), + DbtLsNode(unique_id="model.analytics.customers"), + ), + on_node_result=callbacks.append, + on_progress=progress.append, + ) + + assert result.returncode == 1 + assert result.node_results[0] == streamed + assert len(result.node_results) == 2 + appended: DbtNodeExecutionResult = result.node_results[1] + assert appended.unique_id == "model.analytics.customers" + assert appended.status == "skipped" + assert (appended.index, appended.total) == (2, 3) + assert appended.messages[0].message == test_case.expected_message + assert callbacks == [appended] + assert progress == ["Finalizing dbt run...", "Finalized dbt run."] + assert "customers" in output.getvalue() + assert "inventory" not in output.getvalue() diff --git a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py index 6b68156b5c..b7d5eea802 100644 --- a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py @@ -11,13 +11,13 @@ from sqlbuild._native import ( build_lint_expansion_context, expand_sql_with_spans, + lint_file_role, map_expanded_offset, + prepare_lint_body, run_lint, ) from sqlbuild.compiler.compile.models import MappedOffset, SqlExpansionContext from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.lint._helpers.expansion import prepare_lint_body -from sqlbuild.lint._helpers.headers import lint_file_role from sqlbuild.lint.exceptions import ProjectCompileError from sqlbuild.lint.models import LintBody, LintConfig, LintRunResult from tests.unit.src.sqlbuild.lint._test_types import (