diff --git a/crates/sqlbuild-analysis/src/compiler/constants.rs b/crates/sqlbuild-analysis/src/compiler/constants.rs index 179f2ae1b7..6ad99688fa 100644 --- a/crates/sqlbuild-analysis/src/compiler/constants.rs +++ b/crates/sqlbuild-analysis/src/compiler/constants.rs @@ -13,3 +13,7 @@ pub(crate) const FUNCTION_TYPE_DIALECTS: [(&str, &str, DialectType); 4] = [ pub(crate) const CTES_FIELD: &str = "ctes"; pub(crate) const ALIAS_FIELD: &str = "alias"; pub(crate) const NAME_FIELD: &str = "name"; + +/// Opening and closing quotes of identifiers SQL tests list as expected columns. +pub(crate) const SQL_TEST_IDENTIFIER_QUOTES: [(char, char); 3] = + [('"', '"'), ('`', '`'), ('[', ']')]; diff --git a/crates/sqlbuild-analysis/src/compiler/main/missing_expected_columns.rs b/crates/sqlbuild-analysis/src/compiler/main/missing_expected_columns.rs new file mode 100644 index 0000000000..b3a0829cb2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/main/missing_expected_columns.rs @@ -0,0 +1,43 @@ +//! Explain expected-output columns that a model does not produce. + +use sqlbuild_core::text::main::python_casefold::python_casefold; + +use crate::compiler::constants::SQL_TEST_IDENTIFIER_QUOTES; + +/// The message for listed expected columns absent from casefolded model output names, if any. +#[must_use] +pub fn missing_expected_columns( + model_name: &str, + expected_columns: &[String], + available_columns: &[String], +) -> Option { + let missing: Vec<&str> = expected_columns + .iter() + .filter(|column| !available_columns.contains(&normalized_identifier(column))) + .map(String::as_str) + .collect(); + if missing.is_empty() { + return None; + } + Some(format!( + "expected output '{model_name}' lists columns that model '{model_name}' does not output: {}. Expected CTEs are compared only on the columns they list, matched by name", + missing.join(", ") + )) +} + +fn normalized_identifier(identifier: &str) -> String { + let closing: Option = identifier.chars().next().and_then(|first| { + SQL_TEST_IDENTIFIER_QUOTES + .iter() + .find(|(open, _)| *open == first) + .map(|(_, close)| *close) + }); + match closing { + Some(close) if identifier.ends_with(close) => { + let inner: &str = identifier.get(1..identifier.len() - 1).unwrap_or_default(); + let doubled: String = [close, close].iter().collect(); + python_casefold(&inner.replace(&doubled, &close.to_string())) + } + _ => python_casefold(identifier), + } +} diff --git a/crates/sqlbuild-analysis/src/compiler/main/mod.rs b/crates/sqlbuild-analysis/src/compiler/main/mod.rs index 0975b294d5..e33178c38d 100644 --- a/crates/sqlbuild-analysis/src/compiler/main/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/main/mod.rs @@ -1,4 +1,6 @@ pub mod function_type_validation; +pub mod missing_expected_columns; +pub mod scenario_expected_comparison_sql; pub mod sql_scenario_classification; pub mod sql_test_assembly; pub mod sql_test_chains; diff --git a/crates/sqlbuild-analysis/src/compiler/main/scenario_expected_comparison_sql.rs b/crates/sqlbuild-analysis/src/compiler/main/scenario_expected_comparison_sql.rs new file mode 100644 index 0000000000..3ccdd0b378 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/main/scenario_expected_comparison_sql.rs @@ -0,0 +1,17 @@ +//! The query comparing a scenario's actual output with its expected SQL. + +/// Actual, expected and mismatched row counts of one scenario expectation. +#[must_use] +pub fn scenario_expected_comparison_sql( + actual_sql: &str, + expected_sql: &str, + set_difference_operator: &str, +) -> String { + format!( + "WITH __actual AS ({actual_sql}), __expected AS ({expected_sql}) SELECT \ + (SELECT COUNT(*) FROM __actual) AS actual_count, \ + (SELECT COUNT(*) FROM __expected) AS expected_count, \ + (SELECT COUNT(*) FROM (SELECT * FROM __actual {set_difference_operator} SELECT * FROM __expected\ + ) AS __sqlbuild_mismatch) AS mismatched_count" + ) +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs index 7dec7aa968..8d9c11ec79 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod checks; pub(crate) mod edges; pub(crate) mod nodes; +pub(crate) mod selection; pub(crate) mod values; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/selection.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/selection.rs new file mode 100644 index 0000000000..8148ad5255 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/selection.rs @@ -0,0 +1,219 @@ +//! Breadth-first closures and directed path nodes over string-keyed DAG edges. + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +use sqlbuild_core::text::main::python_isspace::python_isspace; +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +use crate::dag_artifact::constants::{ + SELECTOR_KIND_CHECK, SELECTOR_KIND_NAME, SELECTOR_KIND_PATH, SELECTOR_KIND_SEPARATOR, + SELECTOR_KIND_TAG, SELECTOR_KIND_TEST, SELECTOR_MODELS_PREFIX, +}; +use crate::dag_artifact::errors::DagSelectionError; +use crate::dag_artifact::models::{DagSelectionIndex, DagSelectionNode}; +use crate::graph::main::parse_selector::parse_selector; +use crate::graph::models::ParsedSelector; + +/// Upstream and downstream neighbours of each node id. +pub(crate) type Adjacency = BTreeMap>; + +/// Every id reachable from `starts`, plus the starts when `include_starts`. +pub(crate) fn closure( + starts: &BTreeSet, + edges: &Adjacency, + include_starts: bool, +) -> BTreeSet { + let mut visited: BTreeSet = if include_starts { + starts.clone() + } else { + BTreeSet::new() + }; + let mut frontier: VecDeque<&String> = starts.iter().collect(); + while let Some(current) = frontier.pop_front() { + for neighbor in edges.get(current).into_iter().flatten() { + if visited.insert(neighbor.clone()) { + frontier.push_back(neighbor); + } + } + } + visited +} + +/// Ids on directed paths from `start` to `end`, or `None` when `end` is not downstream. +pub(crate) fn path_nodes( + (start, end): (&str, &str), + (upstream, downstream): (&Adjacency, &Adjacency), +) -> Option> { + let reachable: BTreeSet = + closure(&BTreeSet::from([start.to_owned()]), downstream, false); + if !reachable.contains(end) { + return None; + } + let mut upstream_of_end: BTreeSet = BTreeSet::new(); + let mut stack: Vec = vec![end.to_owned()]; + while let Some(current) = stack.pop() { + if !upstream_of_end.insert(current.clone()) { + continue; + } + for parent in upstream.get(¤t).into_iter().flatten() { + if reachable.contains(parent) || parent == start { + stack.push(parent.clone()); + } + } + } + let mut nodes: BTreeSet = reachable.intersection(&upstream_of_end).cloned().collect(); + nodes.insert(start.to_owned()); + nodes.insert(end.to_owned()); + Some(nodes) +} + +pub(crate) fn resolve_expression( + index: &DagSelectionIndex<'_>, + expression: &str, +) -> Result, DagSelectionError> { + let mut selected: BTreeSet = BTreeSet::new(); + for token in expression + .split(python_isspace) + .filter(|token| !token.is_empty()) + { + let mut token_selection: Option> = None; + for part in token.split(',') { + let matched: BTreeSet = resolve_atomic(index, part)?; + token_selection = Some(match token_selection { + None => matched, + Some(previous) => previous.intersection(&matched).cloned().collect(), + }); + } + selected.extend(token_selection.unwrap_or_default()); + } + Ok(selected) +} + +fn resolve_atomic( + index: &DagSelectionIndex<'_>, + raw: &str, +) -> Result, DagSelectionError> { + match parse_selector(raw).map_err(DagSelectionError::Selector)? { + ParsedSelector::Path { + start, + end, + upstream, + downstream, + } => { + let start_id: String = node_id_for_name(index, &start)?; + let end_id: String = node_id_for_name(index, &end)?; + let mut matched: BTreeSet = + path_nodes((&start_id, &end_id), (&index.upstream, &index.downstream)).ok_or_else( + || { + DagSelectionError::Input(format!( + "no SQLBuild DAG path exists between {} and {}", + python_str_repr(&start_id), + python_str_repr(&end_id) + )) + }, + )?; + if upstream { + matched.extend(closure(&BTreeSet::from([start_id]), &index.upstream, true)); + } + if downstream { + matched.extend(closure(&BTreeSet::from([end_id]), &index.downstream, true)); + } + Ok(matched) + } + ParsedSelector::Kind { + kind, + value, + upstream, + downstream, + } => { + let mut matched: BTreeSet = match_kind(index, &kind, &value)?; + let anchors: BTreeSet = matched.clone(); + if upstream { + matched.extend(closure(&anchors, &index.upstream, true)); + } + if downstream { + matched.extend(closure(&anchors, &index.downstream, true)); + } + Ok(matched) + } + } +} + +fn match_kind( + index: &DagSelectionIndex<'_>, + kind: &str, + value: &str, +) -> Result, DagSelectionError> { + let ids = |matches: &dyn Fn(&DagSelectionNode) -> bool| -> BTreeSet { + index + .nodes + .iter() + .filter(|(_, node)| matches(node)) + .map(|(id, _)| (*id).to_owned()) + .collect() + }; + Ok(match kind { + SELECTOR_KIND_TEST => { + return Err(DagSelectionError::UnitTest(format!( + "{SELECTOR_KIND_TEST}{SELECTOR_KIND_SEPARATOR}{value}" + ))); + } + SELECTOR_KIND_TAG => ids(&|node| node.tags.iter().any(|tag| tag == value)), + SELECTOR_KIND_PATH => { + let folder: String = value.replace('\\', "/"); + let folder: &str = folder.trim_matches('/'); + ids(&|node| path_is_under(node.path.as_deref(), folder)) + } + SELECTOR_KIND_CHECK => index + .graph + .checks + .iter() + .filter(|check| check.name == value || check.id == value) + .flat_map(|check| check.checked_asset_ids.iter().cloned()) + .collect(), + SELECTOR_KIND_NAME => ids(&|node| node.name == value), + _ => ids(&|node| node.name == value && node.kind == kind), + }) +} + +fn path_is_under(path: Option<&str>, folder: &str) -> bool { + let Some(path) = path else { + return false; + }; + let normalized: String = path.replace('\\', "/"); + let normalized: &str = normalized.trim_matches('/'); + let prefixed: String = if !folder.is_empty() && !folder.starts_with(SELECTOR_MODELS_PREFIX) { + format!("{SELECTOR_MODELS_PREFIX}{folder}") + } else { + folder.to_owned() + }; + [folder, prefixed.as_str()] + .into_iter() + .filter(|candidate| !candidate.is_empty()) + .any(|candidate| { + normalized == candidate + || normalized + .strip_prefix(candidate) + .is_some_and(|rest| rest.starts_with('/')) + }) +} + +fn node_id_for_name( + index: &DagSelectionIndex<'_>, + name: &str, +) -> Result { + let matching: Vec<&str> = index + .nodes + .iter() + .filter(|(_, node)| node.name == name) + .map(|(id, _)| *id) + .collect(); + match matching.as_slice() { + [only] => Ok((*only).to_owned()), + _ => Err(DagSelectionError::Input(format!( + "SQLBuild selector name {} matched {} DAG nodes", + python_str_repr(name), + matching.len() + ))), + } +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/constants.rs b/crates/sqlbuild-analysis/src/dag_artifact/constants.rs index 8b877efc20..77a41ef817 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/constants.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/constants.rs @@ -27,3 +27,10 @@ pub(crate) const LOADER_NODE_META_KEY: &str = "sqlbuild_loader_node"; pub(crate) const LOADER_DESTINATION_PREFIX: &str = "__loader__"; /// The most parts a loader destination may have. pub(crate) const LOADER_MAX_PARTS: usize = 3; +pub(crate) const SELECTOR_KIND_NAME: &str = "name"; +pub(crate) const SELECTOR_KIND_TAG: &str = "tag"; +pub(crate) const SELECTOR_KIND_PATH: &str = "path"; +pub(crate) const SELECTOR_KIND_CHECK: &str = "check"; +pub(crate) const SELECTOR_KIND_TEST: &str = "test"; +pub(crate) const SELECTOR_KIND_SEPARATOR: &str = ":"; +pub(crate) const SELECTOR_MODELS_PREFIX: &str = "models/"; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/errors.rs b/crates/sqlbuild-analysis/src/dag_artifact/errors.rs index b37aa7a6eb..461e6c7aa0 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/errors.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/errors.rs @@ -14,3 +14,14 @@ pub enum DagArtifactError { /// The emitter refused the encoded artifact. Emit(JsonEmitError), } + +/// Why a Dagster selection over the DAG artifact failed, in the order Python raised them. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DagSelectionError { + /// The selector token did not parse. + Selector(crate::graph::errors::SelectorError), + /// A `test:` selector, which only `sqb test` and `sqb build` run. + UnitTest(String), + /// `DagsterDagInputError` with this message. + Input(String), +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs index 60780629b8..80514c0a8a 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs @@ -1 +1,2 @@ pub mod build_dag_json; +pub mod resolve_dag_selection; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/resolve_dag_selection.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/resolve_dag_selection.rs new file mode 100644 index 0000000000..2995d66569 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/resolve_dag_selection.rs @@ -0,0 +1,50 @@ +//! Resolve SQLBuild selectors over a DAG artifact into the selected node ids. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::dag_artifact::_helpers::selection::{Adjacency, resolve_expression}; +use crate::dag_artifact::errors::DagSelectionError; +use crate::dag_artifact::models::{DagSelectionGraph, DagSelectionIndex, DagSelectionNode}; + +/// Node ids `select` matches minus those `exclude` matches. +pub fn resolve_dag_selection( + graph: &DagSelectionGraph, + (select, exclude): (&str, Option<&str>), +) -> Result, DagSelectionError> { + let mut nodes: BTreeMap<&str, &DagSelectionNode> = BTreeMap::new(); + for node in &graph.nodes { + nodes.insert(node.id.as_str(), node); + } + let mut upstream: Adjacency = Adjacency::new(); + let mut downstream: Adjacency = Adjacency::new(); + for id in nodes.keys() { + upstream.insert((*id).to_owned(), BTreeSet::new()); + downstream.insert((*id).to_owned(), BTreeSet::new()); + } + for (from_id, to_id) in &graph.edges { + if !nodes.contains_key(from_id.as_str()) || !nodes.contains_key(to_id.as_str()) { + continue; + } + upstream + .entry(to_id.clone()) + .or_default() + .insert(from_id.clone()); + downstream + .entry(from_id.clone()) + .or_default() + .insert(to_id.clone()); + } + let index: DagSelectionIndex<'_> = DagSelectionIndex { + graph, + nodes, + upstream, + downstream, + }; + let mut selected: BTreeSet = resolve_expression(&index, select)?; + if let Some(exclude) = exclude.filter(|exclude| !exclude.is_empty()) { + for excluded in resolve_expression(&index, exclude)? { + selected.remove(&excluded); + } + } + Ok(selected) +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/models.rs b/crates/sqlbuild-analysis/src/dag_artifact/models.rs index 660a118be2..7ec5cf89a3 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/models.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/models.rs @@ -216,3 +216,38 @@ pub struct DagProjectInput { pub target_database: Option, pub target_schema: Option, } + +/// One DAG artifact node as the Dagster selection reads it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DagSelectionNode { + pub id: String, + pub name: String, + pub kind: String, + pub path: Option, + pub tags: Vec, +} + +/// One DAG artifact check: its name, id and checked asset ids. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DagSelectionCheck { + pub name: String, + pub id: String, + pub checked_asset_ids: Vec, +} + +/// The DAG artifact's nodes, `(from, to)` edges and checks. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DagSelectionGraph { + pub nodes: Vec, + pub edges: Vec<(String, String)>, + pub checks: Vec, +} + +/// The node lookup and edge indexes every selector token resolves against. +#[derive(Debug)] +pub(crate) struct DagSelectionIndex<'a> { + pub(crate) graph: &'a DagSelectionGraph, + pub(crate) nodes: std::collections::BTreeMap<&'a str, &'a DagSelectionNode>, + pub(crate) upstream: crate::dag_artifact::_helpers::selection::Adjacency, + pub(crate) downstream: crate::dag_artifact::_helpers::selection::Adjacency, +} diff --git a/crates/sqlbuild-analysis/src/diff_selection/constants.rs b/crates/sqlbuild-analysis/src/diff_selection/constants.rs index e85095e908..13daec5375 100644 --- a/crates/sqlbuild-analysis/src/diff_selection/constants.rs +++ b/crates/sqlbuild-analysis/src/diff_selection/constants.rs @@ -11,3 +11,6 @@ pub(crate) const EMPTY_SELECTOR_CODE: &str = "S001"; pub(crate) const UNKNOWN_NAME_CODE: &str = "S007"; pub(crate) const UNKNOWN_TAG_CODE: &str = "S008"; pub(crate) const UNKNOWN_PATH_CODE: &str = "S009"; +pub(crate) const RAW_QUERY_READ_ONLY_ROOTS: [&str; 5] = + ["except", "intersect", "select", "union", "values"]; +pub(crate) const RAW_QUERY_GUARD: &str = r#"{"maxFunctionCallDepth": 512}"#; diff --git a/crates/sqlbuild-analysis/src/diff_selection/main/mod.rs b/crates/sqlbuild-analysis/src/diff_selection/main/mod.rs index 5f8a0fa05f..82abb2ca13 100644 --- a/crates/sqlbuild-analysis/src/diff_selection/main/mod.rs +++ b/crates/sqlbuild-analysis/src/diff_selection/main/mod.rs @@ -1 +1,2 @@ +pub mod raw_query_shape; pub mod resolve_diff_selection; diff --git a/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs b/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs new file mode 100644 index 0000000000..c553e213e1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs @@ -0,0 +1,45 @@ +//! Parse one raw diff query as the Polyglot wheel's guarded `parse_one` does and read its shape. + +use polyglot_sql::dialects::Dialect; +use polyglot_sql::traversal::ExpressionWalk; +use polyglot_sql::{ComplexityGuardOptions, Expression, ParseOptions}; + +use crate::diff_selection::constants::{RAW_QUERY_GUARD, RAW_QUERY_READ_ONLY_ROOTS}; +use crate::diff_selection::models::RawQueryShape; + +/// The query's shape, or the text of the exception the wheel raises for it. +pub fn raw_query_shape(sql: &str, dialect: &str) -> Result { + let parser: Dialect = + Dialect::get_by_name(dialect).ok_or_else(|| format!("Unknown dialect: {dialect}"))?; + let guard: ComplexityGuardOptions = + serde_json::from_str(RAW_QUERY_GUARD).map_err(|error| error.to_string())?; + let mut statements: Vec = parser + .parse_with_options( + sql, + &ParseOptions { + complexity_guard: Some(guard), + }, + ) + .map_err(|error| error.to_string())?; + if statements.len() != 1 { + return Err(format!("Expected 1 statement, found {}", statements.len())); + } + let root: Expression = statements.remove(0); + let root_key: String = root.variant_name().to_lowercase(); + let node_keys: Vec = if RAW_QUERY_READ_ONLY_ROOTS.contains(&root_key.as_str()) { + root.dfs() + .map(|node| node.variant_name().to_lowercase()) + .collect() + } else { + Vec::new() + }; + let writes_or_locks: bool = match &root { + Expression::Select(select) => select.into.is_some() || !select.locks.is_empty(), + _ => false, + }; + Ok(RawQueryShape { + root_key, + node_keys, + writes_or_locks, + }) +} diff --git a/crates/sqlbuild-analysis/src/diff_selection/models.rs b/crates/sqlbuild-analysis/src/diff_selection/models.rs index d95cbdf0dd..f20f588539 100644 --- a/crates/sqlbuild-analysis/src/diff_selection/models.rs +++ b/crates/sqlbuild-analysis/src/diff_selection/models.rs @@ -14,3 +14,13 @@ pub struct DiffGraph { /// `(resource, model folder)`. pub paths: Vec<(usize, String)>, } + +/// The parsed shape of one raw diff query: lowercased expression kinds and write clauses. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RawQueryShape { + pub root_key: String, + /// Every node kind in depth-first order, collected only for read-only roots. + pub node_keys: Vec, + /// The root `SELECT` has an `INTO` target or row-locking clauses. + pub writes_or_locks: bool, +} diff --git a/crates/sqlbuild-analysis/src/identifier_fitting/_helpers/bytes.rs b/crates/sqlbuild-analysis/src/identifier_fitting/_helpers/bytes.rs new file mode 100644 index 0000000000..03373d0092 --- /dev/null +++ b/crates/sqlbuild-analysis/src/identifier_fitting/_helpers/bytes.rs @@ -0,0 +1,20 @@ +//! UTF-8 byte truncation and short digests. + +use sha2::{Digest, Sha256}; + +use crate::identifier_fitting::constants::SHORTENED_NAME_HASH_LENGTH; + +/// The longest prefix whose UTF-8 encoding fits in `max_bytes`, never splitting a character. +pub(crate) fn truncate_bytes(value: &str, max_bytes: usize) -> &str { + let mut end: usize = max_bytes.min(value.len()); + while !value.is_char_boundary(end) { + end -= 1; + } + &value[..end] +} + +/// The first hex digits of the SHA-256 digest of `value`. +pub(crate) fn short_hash(value: &str) -> String { + let digest: String = format!("{:x}", Sha256::digest(value.as_bytes())); + digest[..SHORTENED_NAME_HASH_LENGTH].to_owned() +} diff --git a/crates/sqlbuild-analysis/src/identifier_fitting/_helpers/mod.rs b/crates/sqlbuild-analysis/src/identifier_fitting/_helpers/mod.rs new file mode 100644 index 0000000000..14e49498ab --- /dev/null +++ b/crates/sqlbuild-analysis/src/identifier_fitting/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod bytes; diff --git a/crates/sqlbuild-analysis/src/identifier_fitting/constants.rs b/crates/sqlbuild-analysis/src/identifier_fitting/constants.rs new file mode 100644 index 0000000000..8d0306b3f6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/identifier_fitting/constants.rs @@ -0,0 +1,4 @@ +//! Identifier fitting constants. + +/// Hex digits of the SHA-256 digest a shortened name keeps. +pub(crate) const SHORTENED_NAME_HASH_LENGTH: usize = 8; diff --git a/crates/sqlbuild-analysis/src/identifier_fitting/main/fit_artifact_logical_name.rs b/crates/sqlbuild-analysis/src/identifier_fitting/main/fit_artifact_logical_name.rs new file mode 100644 index 0000000000..9aa50e4a15 --- /dev/null +++ b/crates/sqlbuild-analysis/src/identifier_fitting/main/fit_artifact_logical_name.rs @@ -0,0 +1,41 @@ +//! Fit a readable logical name after a fixed prefix. + +use crate::identifier_fitting::_helpers::bytes::{short_hash, truncate_bytes}; +use crate::identifier_fitting::constants::SHORTENED_NAME_HASH_LENGTH; + +/// The logical name, or its truncation with a hash suffix; `Err` carries the input error message. +pub fn fit_artifact_logical_name( + (logical_name, fixed_prefix): (&str, &str), + identifier_limit: i64, + artifact_label: &str, +) -> Result { + let max_logical_length: i64 = identifier_limit - byte_length(fixed_prefix); + if max_logical_length < 1 { + return Err(format!( + "{artifact_label} prefix '{fixed_prefix}' does not fit within identifier limit {identifier_limit}" + )); + } + if byte_length(logical_name) <= max_logical_length { + return Ok(logical_name.to_owned()); + } + let suffix_length: i64 = byte_length_of(SHORTENED_NAME_HASH_LENGTH) + 1; + if max_logical_length <= suffix_length { + return Err(format!( + "{artifact_label} name for '{logical_name}' cannot fit within identifier limit {identifier_limit}" + )); + } + let kept: usize = usize::try_from(max_logical_length - suffix_length).unwrap_or(0); + Ok(format!( + "{}_{}", + truncate_bytes(logical_name, kept), + short_hash(logical_name) + )) +} + +fn byte_length(value: &str) -> i64 { + byte_length_of(value.len()) +} + +fn byte_length_of(length: usize) -> i64 { + i64::try_from(length).unwrap_or(i64::MAX) +} diff --git a/crates/sqlbuild-analysis/src/identifier_fitting/main/fit_auxiliary_relation_name.rs b/crates/sqlbuild-analysis/src/identifier_fitting/main/fit_auxiliary_relation_name.rs new file mode 100644 index 0000000000..36389b76d4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/identifier_fitting/main/fit_auxiliary_relation_name.rs @@ -0,0 +1,31 @@ +//! Append a suffix to a relation name within an identifier limit. + +use crate::identifier_fitting::_helpers::bytes::{short_hash, truncate_bytes}; + +/// `base_name + suffix`, shortening the base with a stable hash when too long; `Err` carries the message. +pub fn fit_auxiliary_relation_name( + base_name: &str, + suffix: &str, + identifier_limit: i64, +) -> Result { + let whole: i64 = length(base_name) + length(suffix); + if whole <= identifier_limit { + return Ok(format!("{base_name}{suffix}")); + } + let hash_part: String = format!("_{}", short_hash(base_name)); + let kept_length: i64 = identifier_limit - length(suffix) - length(&hash_part); + if kept_length < 1 { + return Err(format!( + "auxiliary relation suffix '{suffix}' for '{base_name}' cannot fit within identifier limit {identifier_limit}" + )); + } + let kept: usize = usize::try_from(kept_length).unwrap_or(0); + Ok(format!( + "{}{hash_part}{suffix}", + truncate_bytes(base_name, kept) + )) +} + +fn length(value: &str) -> i64 { + i64::try_from(value.len()).unwrap_or(i64::MAX) +} diff --git a/crates/sqlbuild-analysis/src/identifier_fitting/main/mod.rs b/crates/sqlbuild-analysis/src/identifier_fitting/main/mod.rs new file mode 100644 index 0000000000..21551a11b4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/identifier_fitting/main/mod.rs @@ -0,0 +1,2 @@ +pub mod fit_artifact_logical_name; +pub mod fit_auxiliary_relation_name; diff --git a/crates/sqlbuild-analysis/src/identifier_fitting/mod.rs b/crates/sqlbuild-analysis/src/identifier_fitting/mod.rs new file mode 100644 index 0000000000..526e5bf21f --- /dev/null +++ b/crates/sqlbuild-analysis/src/identifier_fitting/mod.rs @@ -0,0 +1,5 @@ +//! Fit generated relation names within identifier limits measured in UTF-8 bytes. + +pub(crate) mod _helpers; +mod constants; +pub mod main; diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 1b29ef8804..7e636a4508 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -15,6 +15,7 @@ pub mod diff_selection; pub mod explicit_references; pub mod fingerprint_storage; pub mod graph; +pub mod identifier_fitting; pub mod lineage; pub mod lineage_command; pub mod lineage_graph; diff --git a/crates/sqlbuild-core/src/build_aggregate/constants.rs b/crates/sqlbuild-core/src/build_aggregate/constants.rs new file mode 100644 index 0000000000..0ac789c3c3 --- /dev/null +++ b/crates/sqlbuild-core/src/build_aggregate/constants.rs @@ -0,0 +1,10 @@ +//! Result status and outcome values the aggregation reads. + +pub(crate) const SUCCESS: &str = "success"; +pub(crate) const FAILED: &str = "failed"; +pub(crate) const SKIPPED: &str = "skipped"; +pub(crate) const TEST_PASS: &str = "pass"; +pub(crate) const AUDIT_WARN: &str = "warn"; +pub(crate) const AUDIT_ERROR: &str = "error"; +pub(crate) const AUDIT_INSUFFICIENT: &str = "insufficient"; +pub(crate) const SOURCE_FRESHNESS_BLOCKED_CODE: &str = "B005"; diff --git a/crates/sqlbuild-core/src/build_aggregate/main/aggregate_build_counts.rs b/crates/sqlbuild-core/src/build_aggregate/main/aggregate_build_counts.rs new file mode 100644 index 0000000000..f749904e5b --- /dev/null +++ b/crates/sqlbuild-core/src/build_aggregate/main/aggregate_build_counts.rs @@ -0,0 +1,73 @@ +//! Aggregate counts and overall status of one build. + +use crate::build_aggregate::constants::{ + AUDIT_ERROR, AUDIT_INSUFFICIENT, AUDIT_WARN, FAILED, SKIPPED, SOURCE_FRESHNESS_BLOCKED_CODE, + SUCCESS, TEST_PASS, +}; +use crate::build_aggregate::models::{BuildCounts, BuildResultFacts}; + +/// Success, failure, skipped, warning and insufficient counts, and whether the build failed. +#[must_use] +pub fn aggregate_build_counts(facts: &BuildResultFacts) -> BuildCounts { + let mut counts: BuildCounts = BuildCounts::default(); + for model in &facts.models { + match model.status.as_str() { + SUCCESS => counts.success += 1, + FAILED => counts.failure += 1, + SKIPPED => { + counts.skipped += 1; + if model.error_code.as_deref() == Some(SOURCE_FRESHNESS_BLOCKED_CODE) { + counts.failure += 1; + } + } + _ => {} + } + counts.warning += model.warning_count + matching(&model.audit_outcomes, AUDIT_WARN); + } + let statuses = facts + .seed_statuses + .iter() + .chain(facts.functions.iter().map(|(status, _)| status)) + .chain(&facts.load_statuses); + for status in statuses { + counts.success += usize::from(status == SUCCESS); + counts.failure += usize::from(status == FAILED); + counts.skipped += usize::from(status == SKIPPED); + } + counts.warning += facts + .functions + .iter() + .map(|(_, warnings)| warnings) + .sum::(); + for outcome in &facts.test_outcomes { + if outcome == TEST_PASS { + counts.success += 1; + } else { + counts.failure += 1; + } + } + let model_audits = facts.models.iter().flat_map(|model| &model.audit_outcomes); + counts.insufficient = facts + .source_audit_outcomes + .iter() + .chain(&facts.end_audit_outcomes) + .chain(model_audits) + .filter(|outcome| *outcome == AUDIT_INSUFFICIENT) + .count(); + counts.failure += matching(&facts.source_audit_outcomes, AUDIT_ERROR); + counts.warning += matching(&facts.source_audit_outcomes, AUDIT_WARN) + + matching(&facts.end_audit_outcomes, AUDIT_WARN); + counts.failed = counts.failure > 0 + || facts + .end_audit_outcomes + .iter() + .any(|outcome| outcome == AUDIT_ERROR); + counts +} + +fn matching(outcomes: &[String], expected: &str) -> usize { + outcomes + .iter() + .filter(|outcome| *outcome == expected) + .count() +} diff --git a/crates/sqlbuild-core/src/build_aggregate/main/mod.rs b/crates/sqlbuild-core/src/build_aggregate/main/mod.rs new file mode 100644 index 0000000000..e94806f546 --- /dev/null +++ b/crates/sqlbuild-core/src/build_aggregate/main/mod.rs @@ -0,0 +1 @@ +pub mod aggregate_build_counts; diff --git a/crates/sqlbuild-core/src/build_aggregate/mod.rs b/crates/sqlbuild-core/src/build_aggregate/mod.rs new file mode 100644 index 0000000000..4bd1773ec1 --- /dev/null +++ b/crates/sqlbuild-core/src/build_aggregate/mod.rs @@ -0,0 +1,5 @@ +//! Aggregate counts and overall status of one build's execution results. + +mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-core/src/build_aggregate/models.rs b/crates/sqlbuild-core/src/build_aggregate/models.rs new file mode 100644 index 0000000000..2ceed85105 --- /dev/null +++ b/crates/sqlbuild-core/src/build_aggregate/models.rs @@ -0,0 +1,34 @@ +//! Build result facts and their aggregate counts. + +/// The status, error code, warning count and audit outcomes of one model result. +#[derive(Debug, Clone, Default)] +pub struct ModelResultFacts { + pub status: String, + pub error_code: Option, + pub warning_count: usize, + pub audit_outcomes: Vec, +} + +/// The statuses and outcomes of every result one build produced. +#[derive(Debug, Clone, Default)] +pub struct BuildResultFacts { + pub models: Vec, + pub seed_statuses: Vec, + /// `(status, warning count)` of each function result. + pub functions: Vec<(String, usize)>, + pub load_statuses: Vec, + pub test_outcomes: Vec, + pub source_audit_outcomes: Vec, + pub end_audit_outcomes: Vec, +} + +/// Aggregate counts and whether the build failed. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct BuildCounts { + pub success: usize, + pub failure: usize, + pub skipped: usize, + pub warning: usize, + pub insufficient: usize, + pub failed: bool, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 2e9cf4e654..f2911bf659 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -3,6 +3,7 @@ #![forbid(unsafe_code)] pub mod audit_command; +pub mod build_aggregate; pub mod build_command; pub mod build_progress; pub mod check_command; diff --git a/crates/sqlbuild-core/src/nested_progress/main/mod.rs b/crates/sqlbuild-core/src/nested_progress/main/mod.rs index 1f0a78ad20..b42103fe36 100644 --- a/crates/sqlbuild-core/src/nested_progress/main/mod.rs +++ b/crates/sqlbuild-core/src/nested_progress/main/mod.rs @@ -5,3 +5,4 @@ pub mod render_audit_row; pub mod render_group_header; pub mod render_nested_item; pub mod render_nested_spinner_line; +pub mod sql_test_resource_id; diff --git a/crates/sqlbuild-core/src/nested_progress/main/sql_test_resource_id.rs b/crates/sqlbuild-core/src/nested_progress/main/sql_test_resource_id.rs new file mode 100644 index 0000000000..8d68a7cc68 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/main/sql_test_resource_id.rs @@ -0,0 +1,12 @@ +//! The execution-protocol identity of one SQL test case. + +/// `sql_test:`, or `sql_test:::` for one parameterised case. +#[must_use] +pub fn sql_test_resource_id(test_name: &str, case: Option<(&str, i64, &str)>) -> String { + match case { + Some((source_path, block_index, case_name)) => { + format!("sql_test:{source_path}:{block_index}:{case_name}") + } + None => format!("sql_test:{test_name}"), + } +} diff --git a/crates/sqlbuild-pysource/src/macro_modules/_helpers/load_plan.rs b/crates/sqlbuild-pysource/src/macro_modules/_helpers/load_plan.rs new file mode 100644 index 0000000000..b9f3be44fb --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/_helpers/load_plan.rs @@ -0,0 +1,71 @@ +//! The synthetic package name, parent packages and dependency-first order of macro modules. + +use std::cmp::Reverse; +use std::collections::{BTreeSet, BinaryHeap, HashMap}; + +use sha2::{Digest, Sha256}; + +use crate::macro_modules::constants::{SYNTHETIC_DIGEST_LENGTH, SYNTHETIC_PACKAGE_PREFIX}; +use crate::macro_modules::models::MacroModuleSpec; + +/// The root package named by a digest of every module's name, path and contents. +pub(crate) fn synthetic_root(modules: &[MacroModuleSpec]) -> String { + let mut by_name: Vec<&MacroModuleSpec> = modules.iter().collect(); + by_name.sort_by(|left, right| left.name.cmp(&right.name)); + let joined: String = by_name + .iter() + .map(|module| format!("{}\0{}\0{}", module.name, module.file_path, module.contents)) + .collect::>() + .join("\0"); + let digest: String = format!("{:x}", Sha256::digest(joined.as_bytes())); + format!( + "{SYNTHETIC_PACKAGE_PREFIX}{}", + &digest[..SYNTHETIC_DIGEST_LENGTH] + ) +} + +/// The root and every parent package of a module, shallowest first. +pub(crate) fn parent_packages(root: &str, modules: &[MacroModuleSpec]) -> Vec { + let mut packages: BTreeSet<(usize, String)> = BTreeSet::from([(0, root.to_owned())]); + for module in modules { + let parts: Vec<&str> = module.name.split('.').collect(); + for index in 1..parts.len() { + let package: String = format!("{root}.{}", parts[..index].join(".")); + packages.insert((package.matches('.').count(), package)); + } + } + packages.into_iter().map(|(_, package)| package).collect() +} + +/// Kahn order over module dependencies, taking the smallest ready name first. +pub(crate) fn load_order(modules: &[MacroModuleSpec]) -> Vec { + let mut remaining: HashMap<&str, usize> = HashMap::new(); + let mut dependents: HashMap<&str, Vec<&str>> = HashMap::new(); + for module in modules { + remaining.insert(&module.name, module.dependencies.len()); + for dependency in &module.dependencies { + dependents + .entry(dependency.as_str()) + .or_default() + .push(&module.name); + } + } + let mut ready: BinaryHeap> = remaining + .iter() + .filter(|(_, count)| **count == 0) + .map(|(name, _)| Reverse(*name)) + .collect(); + let mut order: Vec = Vec::new(); + while let Some(Reverse(name)) = ready.pop() { + order.push(name.to_owned()); + for dependent in dependents.get(name).into_iter().flatten() { + if let Some(count) = remaining.get_mut(dependent) { + *count -= 1; + if *count == 0 { + ready.push(Reverse(dependent)); + } + } + } + } + order +} diff --git a/crates/sqlbuild-pysource/src/macro_modules/_helpers/mod.rs b/crates/sqlbuild-pysource/src/macro_modules/_helpers/mod.rs index 18d81a3751..157f515dc0 100644 --- a/crates/sqlbuild-pysource/src/macro_modules/_helpers/mod.rs +++ b/crates/sqlbuild-pysource/src/macro_modules/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod declarations; pub(crate) mod function_calls; +pub(crate) mod load_plan; pub(crate) mod module_analysis; pub(crate) mod module_names; pub(crate) mod project_analysis; diff --git a/crates/sqlbuild-pysource/src/macro_modules/constants.rs b/crates/sqlbuild-pysource/src/macro_modules/constants.rs index 2fdb0bd3ab..d3ceef59cc 100644 --- a/crates/sqlbuild-pysource/src/macro_modules/constants.rs +++ b/crates/sqlbuild-pysource/src/macro_modules/constants.rs @@ -12,3 +12,7 @@ pub(crate) const PRIVATE_PREFIX: &str = "_"; pub(crate) const PATH_ROOTS: [char; 2] = ['/', '\\']; /// Separator of the names in a reported call or import cycle. pub(crate) const CYCLE_SEPARATOR: &str = " -> "; +/// Prefix of the synthetic package macro modules are executed under. +pub(crate) const SYNTHETIC_PACKAGE_PREFIX: &str = "_sqlbuild_project_macros_"; +/// Hex digits of the content digest naming the synthetic package. +pub(crate) const SYNTHETIC_DIGEST_LENGTH: usize = 16; diff --git a/crates/sqlbuild-pysource/src/macro_modules/main/macro_load_plan.rs b/crates/sqlbuild-pysource/src/macro_modules/main/macro_load_plan.rs new file mode 100644 index 0000000000..343befc1cc --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/main/macro_load_plan.rs @@ -0,0 +1,15 @@ +//! The synthetic package and dependency-first execution order of project macro modules. + +use crate::macro_modules::_helpers::load_plan::{load_order, parent_packages, synthetic_root}; +use crate::macro_modules::models::{MacroLoadPlan, MacroModuleSpec}; + +/// Root package named by a content digest, parent packages, and Kahn order with smallest name first. +#[must_use] +pub fn macro_load_plan(modules: &[MacroModuleSpec]) -> MacroLoadPlan { + let root: String = synthetic_root(modules); + MacroLoadPlan { + packages: parent_packages(&root, modules), + order: load_order(modules), + root, + } +} diff --git a/crates/sqlbuild-pysource/src/macro_modules/main/mod.rs b/crates/sqlbuild-pysource/src/macro_modules/main/mod.rs index bf4f23865d..fdfda80311 100644 --- a/crates/sqlbuild-pysource/src/macro_modules/main/mod.rs +++ b/crates/sqlbuild-pysource/src/macro_modules/main/mod.rs @@ -1 +1,2 @@ pub mod inventory_macro_modules; +pub mod macro_load_plan; diff --git a/crates/sqlbuild-pysource/src/macro_modules/models.rs b/crates/sqlbuild-pysource/src/macro_modules/models.rs index 64f5eea25f..151af0b39f 100644 --- a/crates/sqlbuild-pysource/src/macro_modules/models.rs +++ b/crates/sqlbuild-pysource/src/macro_modules/models.rs @@ -64,3 +64,20 @@ pub struct MacroModuleInventory { /// Every module in discovery order, present only when the strict analysis passed. pub modules: Option>, } + +/// One macro module to load: its module name, file path text, contents and module dependencies. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MacroModuleSpec { + pub name: String, + pub file_path: String, + pub contents: String, + pub dependencies: Vec, +} + +/// The synthetic package root, its packages shallowest first, and the module execution order. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MacroLoadPlan { + pub root: String, + pub packages: Vec, + pub order: Vec, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs index 1e4d77872b..d13e5033ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs @@ -11,4 +11,5 @@ pub(crate) mod reference_facts; pub(crate) mod relation_names; mod required_columns; pub(crate) mod schema_request; +pub(crate) mod sql_test_comparisons; pub(crate) mod sql_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs new file mode 100644 index 0000000000..c45df16436 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs @@ -0,0 +1,242 @@ +//! Render executable SQL test and scenario comparison queries from planned entries. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use serde_json::{Value, json}; +use sqlbuild_analysis::compiler::main::missing_expected_columns::missing_expected_columns; +use sqlbuild_analysis::compiler::main::scenario_expected_comparison_sql::scenario_expected_comparison_sql; +use sqlbuild_analysis::compiler::main::sql_test_difference_sampling::render_difference_sample_json; +use sqlbuild_analysis::compiler::main::sql_test_rendering::render_json; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::types::CompilerDetach; + +const RENDER_WORKERS: usize = 4; +const DEFAULT_SET_DIFFERENCE: &str = "EXCEPT"; +const TOP_CLAUSE_DIALECT: &str = "tsql"; + +/// Operator, dialect and optional probed chain step of one comparison render. +pub(crate) struct RenderSettings { + set_difference_operator: String, + sql_analysis_dialect: Option, + probe_step_index: Option, +} + +impl RenderSettings { + fn read(options: Option<&Bound<'_, PyDict>>) -> PyResult { + Ok(Self { + set_difference_operator: optional_value(options, "set_difference_operator")? + .unwrap_or_else(|| DEFAULT_SET_DIFFERENCE.to_owned()), + sql_analysis_dialect: optional_value(options, "sql_analysis_dialect")?, + probe_step_index: optional_value(options, "probe_step_index")?, + }) + } +} + +fn optional_value FromPyObject<'py>>( + options: Option<&Bound<'_, PyDict>>, + name: &str, +) -> PyResult> { + let value: Option> = match options { + Some(options) => options.get_item(name)?, + None => None, + }; + match value { + Some(value) if !value.is_none() => Ok(Some(value.extract()?)), + _ => Ok(None), + } +} + +/// Comparison SQL of each planned SQL test, or of one chain step's actual-output probe. +#[pyfunction] +#[pyo3(signature = (test_entries, **options))] +fn render_sql_test_comparison_sql( + py: Python<'_>, + test_entries: Vec>, + options: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + rendered_comparisons(py, &test_entries, &RenderSettings::read(options)?) +} + +fn rendered_comparisons( + py: Python<'_>, + test_entries: &[Bound<'_, PyAny>], + settings: &RenderSettings, +) -> PyResult> { + if test_entries.is_empty() { + return Ok(Vec::new()); + } + let mut requests: Vec = Vec::with_capacity(test_entries.len()); + for entry in test_entries { + let mut request: Value = comparison_request( + entry, + &settings.set_difference_operator, + settings.sql_analysis_dialect.as_deref(), + )?; + if let (Some(index), Some(fields)) = (settings.probe_step_index, request.as_object_mut()) { + fields.insert("probeStepIndex".to_owned(), json!(index)); + } + requests.push(request); + } + let request_json: String = + serde_json::to_string(&json!({"requests": requests, "workers": RENDER_WORKERS})) + .map_err(value_error)?; + let response: String = py + .compiler_detach(|| render_json(&request_json)) + .map_err(value_error)?; + let rendered: Vec = serde_json::from_str(&response).map_err(value_error)?; + rendered + .iter() + .map(|item| { + item.get("sql") + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| value_error("native SQL-test rendering returned an invalid result")) + }) + .collect() +} + +/// The comparison SQL of one planned SQL test under `adapter`'s operator and dialect. +pub(crate) fn single_comparison_sql( + entry: &Bound<'_, PyAny>, + adapter: &Bound<'_, PyAny>, +) -> PyResult { + let settings = RenderSettings { + set_difference_operator: adapter + .call_method0("render_set_difference_operator")? + .extract()?, + sql_analysis_dialect: adapter.call_method0("sql_analysis_dialect")?.extract()?, + probe_step_index: None, + }; + rendered_comparisons(entry.py(), std::slice::from_ref(entry), &settings)? + .into_iter() + .next() + .ok_or_else(|| value_error("native SQL-test rendering returned an invalid batch response")) +} + +/// The native comparison render request of one planned SQL test. +pub(crate) fn comparison_request( + entry: &Bound<'_, PyAny>, + set_difference_operator: &str, + sql_analysis_dialect: Option<&str>, +) -> PyResult { + let chain: Vec = entry + .getattr("chain")? + .try_iter()? + .map(|step| chain_step_request(&step?)) + .collect::>()?; + let assertions: Vec = entry + .getattr("assertions")? + .try_iter()? + .map(|assertion| { + let assertion: Bound<'_, PyAny> = assertion?; + Ok(json!({ + "name": assertion.getattr("name")?.extract::()?, + "resolvedSql": assertion.getattr("resolved_sql")?.extract::()?, + "liftedCtes": assertion.getattr("lifted_ctes")?.extract::>()?, + "comparisonBodySql": assertion.getattr("comparison_body_sql")?.extract::>()?, + })) + }) + .collect::>()?; + Ok(json!({ + "chain": chain, + "assertions": assertions, + "sqlAnalysisEnabled": entry.getattr("sql_analysis_enabled")?.extract::()?, + "setDifferenceOperator": set_difference_operator, + "sqlAnalysisDialect": sql_analysis_dialect, + })) +} + +/// The native request of one planned chain step. +pub(crate) fn chain_step_request(step: &Bound<'_, PyAny>) -> PyResult { + Ok(json!({ + "modelName": step.getattr("model_name")?.extract::()?, + "resolvedSql": step.getattr("resolved_sql")?.extract::()?, + "expectedCteSql": step.getattr("expected_cte_sql")?.extract::>()?, + "liftedCtes": step.getattr("lifted_ctes")?.extract::>()?, + "comparisonBodySql": step.getattr("comparison_body_sql")?.extract::>()?, + "expectedColumns": step.getattr("expected_columns")?.extract::>>()?, + "expectedLiftedCtes": step.getattr("expected_lifted_ctes")?.extract::>()?, + })) +} + +/// A bounded query for one direction of an expected-output difference of one chain step. +#[pyfunction] +#[pyo3(signature = (**options))] +fn render_sql_test_difference_sample_sql( + py: Python<'_>, + options: Option<&Bound<'_, PyDict>>, +) -> PyResult { + let request = Request::new("render_sql_test_difference_sample_sql", options)?; + let test_entry: Bound<'_, PyAny> = request.get("test_entry")?; + let step: Bound<'_, PyAny> = request.get("step")?; + if step.getattr("expected_cte_sql")?.is_none() { + return Ok(String::new()); + } + let settings: RenderSettings = RenderSettings::read(options)?; + let request_json: String = serde_json::to_string(&json!({ + "step": chain_step_request(&step)?, + "sqlAnalysisEnabled": test_entry.getattr("sql_analysis_enabled")?.extract::()?, + "setDifferenceOperator": settings.set_difference_operator, + "sqlAnalysisDialect": settings.sql_analysis_dialect, + "direction": request.get("direction")?.extract::()?, + "sampleLimit": request.get("sample_limit")?.extract::()?, + "useTopClause": settings.sql_analysis_dialect.as_deref() == Some(TOP_CLAUSE_DIALECT), + })) + .map_err(value_error)?; + let response: String = py + .compiler_detach(|| render_difference_sample_json(&request_json)) + .map_err(value_error)?; + let rendered: Value = serde_json::from_str(&response).map_err(value_error)?; + rendered + .get("sql") + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| value_error("native SQL-test difference rendering returned invalid SQL")) +} + +/// Describe listed expected columns absent from casefolded model output names. +#[pyfunction] +#[pyo3(signature = (*, model_name, expected_columns, available_columns))] +pub(crate) fn describe_missing_expected_columns( + model_name: &str, + expected_columns: Vec, + available_columns: Bound<'_, PyAny>, +) -> PyResult> { + let available: Vec = available_columns + .try_iter()? + .map(|column| column?.extract::()) + .collect::>()?; + Ok(missing_expected_columns( + model_name, + &expected_columns, + &available, + )) +} + +/// SQL comparing a scenario's actual output with its expected SQL. +#[pyfunction] +#[pyo3(signature = (*, actual_sql, expected_sql, set_difference_operator))] +pub(crate) fn build_scenario_expected_comparison_sql( + actual_sql: &str, + expected_sql: &str, + set_difference_operator: &str, +) -> String { + scenario_expected_comparison_sql(actual_sql, expected_sql, set_difference_operator) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_sql_test_comparison_sql, module)?)?; + module.add_function(wrap_pyfunction!( + render_sql_test_difference_sample_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(describe_missing_expected_columns, module)?)?; + module.add_function(wrap_pyfunction!( + build_scenario_expected_comparison_sql, + module + )?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs index 46f16013cc..63025f70be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs @@ -6,24 +6,6 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::types::CompilerDetach; -#[pyfunction] -fn render_sql_test_comparisons_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::compiler::main::sql_test_rendering::render_json(request_json) - }) - .map_err(value_error) -} - -#[pyfunction] -fn render_sql_test_difference_sample_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::compiler::main::sql_test_difference_sampling::render_difference_sample_json( - request_json, - ) - }) - .map_err(value_error) -} - #[pyfunction] fn extract_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult { py.compiler_detach(|| { @@ -33,11 +15,6 @@ fn extract_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult) -> PyResult<()> { - module.add_function(wrap_pyfunction!(render_sql_test_comparisons_json, module)?)?; - module.add_function(wrap_pyfunction!( - render_sql_test_difference_sample_json, - module - )?)?; module.add_function(wrap_pyfunction!(extract_sql_tests_json, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index a35915276a..7b59a30828 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -20,6 +20,7 @@ use crate::bindings::_helpers::check::check_core::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; const NATIVE_MODULE: &str = "sqlbuild._native"; const CURSOR_OVERRIDES_MODULE: &str = "sqlbuild._native"; @@ -28,7 +29,6 @@ const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const PIPELINE_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const FRESHNESS_MODULE: &str = "sqlbuild.cli.commands._helpers.freshness.source_freshness"; const COST_CONTEXT_MODULE: &str = "sqlbuild._native"; -const GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const CHECKS_MODULE: &str = "sqlbuild.executor.python_nodes.main._checks"; const EXECUTOR_PYTHON_MODELS: &str = "sqlbuild.executor.python_nodes.models"; const EXECUTION_COMMAND: &str = "sqb build"; @@ -493,10 +493,7 @@ pub(crate) fn run_post_build_python_checks<'py>( let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?; let plan: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; let load_results: Bound<'py, PyAny> = result.getattr("load_results")?; - let graph: Bound<'py, PyAny> = call_with( - &imported(py, GRAPH_MODULE, "build_discovered_python_node_graph")?, - &[("discovered_inputs", discovered.clone())], - )?; + let graph: Bound<'py, PyAny> = python_node_graph(&discovered)?; let by_loader = || -> PyResult> { Ok(load_results_by_loader_name(&plan.getattr("source_map")?, &load_results)?.into_any()) }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index 122d228eeb..bae571a778 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -15,9 +15,9 @@ use crate::bindings::_helpers::build_command::support::{ boolean, emit, frozenset, or_none, text, tuple, }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; const NATIVE_MODULE: &str = "sqlbuild._native"; -const GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const PYTHON_NODE_MODELS: &str = "sqlbuild.compiler.python_nodes.models"; const LOADER_DAG_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; @@ -299,10 +299,7 @@ fn lifecycle_plan<'py>( ) -> PyResult> { let py: Python<'py> = discovered.py(); let plan_output: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; - let graph: Bound<'py, PyAny> = call_with( - &imported(py, GRAPH_MODULE, "build_discovered_python_node_graph")?, - &[("discovered_inputs", discovered.clone())], - )?; + let graph: Bound<'py, PyAny> = python_node_graph(discovered)?; let selection: Bound<'py, PyAny> = call_with( &imported(py, PYTHON_NODE_MODELS, "PythonSqlRunSelection")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index eb7034d6d9..59b75fb7af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -22,6 +22,7 @@ use crate::bindings::_helpers::check::check_results::{ use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::compile_outputs::runtime_target::write_python_check_runtime_target; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::graph::python_selection::python_node_names; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -35,7 +36,6 @@ const SELECTOR_PARSE_MODULE: &str = "sqlbuild._native"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; -const PYTHON_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const PYTHON_NODE_MODELS: &str = "sqlbuild.compiler.python_nodes.models"; const PYTHON_NODE_TYPES: &str = "sqlbuild.compiler.python_nodes.types"; const EXECUTOR_MODELS: &str = "sqlbuild.executor.python_nodes.models"; @@ -291,14 +291,7 @@ fn python_selection<'py>( discovered: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyFrozenSet>)> { let py: Python<'py> = request.py(); - let python_graph: Bound<'py, PyAny> = call_with( - &imported( - py, - PYTHON_GRAPH_MODULE, - "build_discovered_python_node_graph", - )?, - &[("discovered_inputs", discovered.clone())], - )?; + let python_graph: Bound<'py, PyAny> = python_node_graph(discovered)?; call_with( &imported(py, REJECT_UNIT_TESTS_MODULE, "reject_unit_test_selectors")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 3e6257e344..cd267b1829 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -13,6 +13,7 @@ use sqlbuild_cache::artifacts::main::new_sql_test_record::new_sql_test_record; use sqlbuild_cache::artifacts::main::reusable_sql_test_record::reusable_sql_test_record; use sqlbuild_cache::artifacts::models::SqlTestArtifactRecord; +use crate::bindings::_helpers::analysis::sql_test_comparisons::single_comparison_sql; use crate::bindings::_helpers::compile_command::artifact_batch::{ArtifactBatch, timed}; use crate::bindings::_helpers::compile_command::reuse_recording::{ ArtifactRecorder, CompileArtifactWrites, artifact_kept, artifact_moved, artifact_tree_moved, @@ -50,7 +51,6 @@ const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; const MODEL_CHAIN_MODULE: &str = "sqlbuild._native"; const PLAN_ARTIFACTS_MODULE: &str = "sqlbuild._native"; -const COMPARISON_SQL_MODULE: &str = "sqlbuild.executor.testing.main.comparison_sql"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const TARGET_OPTIONS: (&[&str], &[&str]) = (&["target_dir", "adapter", "project"], &["manifest"]); const STAGING_CHANGED_HELP: &str = @@ -271,21 +271,7 @@ fn write_planned_target<'py>( .div(TESTS_DIR)? .div(planned_sql_test_path(&entry)?)?; let comparison_sql: String = timed(py, "comparison_render_ms", || { - call_with( - &imported(py, COMPARISON_SQL_MODULE, "build_sql_test_comparison_sql")?, - &[ - ("test_entry", entry.clone()), - ( - "set_difference_operator", - adapter.call_method0("render_set_difference_operator")?, - ), - ( - "sql_analysis_dialect", - adapter.call_method0("sql_analysis_dialect")?, - ), - ], - )? - .extract() + single_comparison_sql(&entry, adapter) })?; batch.queue_sql(&path, &comparison_sql)?; managed.push(path); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index c407dc3e15..2a3cf543f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -11,10 +11,10 @@ use crate::bindings::_helpers::compile_command::target_writer::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{Status, call_with, imported}; use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const PYTHON_NODES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; @@ -113,14 +113,7 @@ pub(crate) fn write_dag_artifact<'py>( let started: f64 = monotonic(py)?; status.start("Writing DAG artifact...")?; let discovered: Bound<'py, PyAny> = analysis.getattr("discovered_inputs")?; - let python_graph: Bound<'py, PyAny> = call_with( - &imported( - py, - PYTHON_NODES_MODULE, - "build_discovered_python_node_graph", - )?, - &[("discovered_inputs", discovered.clone())], - )?; + let python_graph: Bound<'py, PyAny> = python_node_graph(&discovered)?; let path: Bound<'py, PyAny> = resolved_dag_path(project_dir, dag_path)?; let project_name: String = discovered .getattr("project_config")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 2b188c660f..f023e6ef3d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -9,12 +9,12 @@ use crate::bindings::_helpers::compile_driver::phases::monotonic; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::graph::python_node_graph::run_selection_from_inputs; const NATIVE_MODULE: &str = "sqlbuild._native"; const EXECUTION_PLAN_MODULE: &str = "sqlbuild._native"; const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const MATERIALIZATIONS_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.materializations"; -const RUN_SELECTION_MODULE: &str = "sqlbuild.compiler.python_nodes.main._run_selection"; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const SINGLE_CONNECTION: i64 = 1; @@ -58,19 +58,8 @@ pub(crate) fn pipeline_result<'py>( &native.getattr("build_project_graph_from_compiled_project")?, &[("project", project.clone())], )?; - run_selection = call_with( - &imported( - py, - RUN_SELECTION_MODULE, - "resolve_python_sql_run_selection_from_inputs", - )?, - &[ - ("select", select.clone()), - ("exclude", exclude.clone()), - ("project_graph", graph), - ("discovered_inputs", discovered.clone()), - ], - )?; + run_selection = + run_selection_from_inputs((select.extract()?, exclude.extract()?), &graph, discovered)?; selected_sql_keys = run_selection.getattr("sql_keys")?; selected_python_node_names = run_selection.getattr("python_node_names")?; python_read_source_names = call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs index 403eda0b72..8491a94905 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs @@ -12,9 +12,9 @@ use sqlbuild_analysis::python_plan::main::upstream_closure::upstream_closure; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; -const NODE_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const NATIVE_MODULE: &str = "sqlbuild._native"; const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.compiler.python_nodes.models"; const PYTHON_NODE_TYPES_MODULE: &str = "sqlbuild.compiler.python_nodes.types"; @@ -63,14 +63,7 @@ fn upstream_names(graph: &Bound<'_, PyAny>) -> PyResult(discovered: &Bound<'py, PyAny>) -> PyResult> { - call_with( - &imported( - discovered.py(), - NODE_GRAPH_MODULE, - "build_discovered_python_node_graph", - )?, - &[("discovered_inputs", discovered.clone())], - ) + python_node_graph(discovered) } /// Return sources declared by the selected Python nodes and their Python upstreams. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs index 8e8784396f..8a6a95628b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs @@ -14,10 +14,10 @@ use sqlbuild_analysis::target_paths::main::sql_test_output_path::sql_test_output use sqlbuild_analysis::target_paths::models::SqlTestPathFacts; use sqlbuild_core::text::main::python_isspace::python_isspace; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::analysis::sql_test_comparisons::single_comparison_sql; +use crate::bindings::_helpers::compile_driver::python_calls::imported; const LIFECYCLE_TYPES: &str = "sqlbuild.adapter.contract.types"; -const COMPARISON_SQL_MODULE: &str = "sqlbuild.executor.testing.main.comparison_sql"; const RUN_DIR: &str = "run"; const TESTS_DIR: &str = "tests"; const CHECKS_DIR: &str = "checks"; @@ -172,7 +172,6 @@ pub(crate) fn write_test_runtime_target( plan: &Bound<'_, PyAny>, results: &Bound<'_, PyAny>, ) -> PyResult<()> { - let py: Python<'_> = target_dir.py(); let mut executed: HashSet = HashSet::new(); for result in results.try_iter()? { executed.insert(test_key(&result?, "test_name")?); @@ -183,21 +182,7 @@ pub(crate) fn write_test_runtime_target( if !executed.contains(&test_key(&entry, "name")?) { continue; } - let sql: String = call_with( - &imported(py, COMPARISON_SQL_MODULE, "build_sql_test_comparison_sql")?, - &[ - ("test_entry", entry.clone()), - ( - "set_difference_operator", - adapter.call_method0("render_set_difference_operator")?, - ), - ( - "sql_analysis_dialect", - adapter.call_method0("sql_analysis_dialect")?, - ), - ], - )? - .extract()?; + let sql: String = single_comparison_sql(&entry, adapter)?; write_text( &tests_dir.div(planned_sql_test_path(&entry)?)?, &format!("{}\n", sql.trim_end_matches(python_isspace)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs index 3831b2b702..12aa7365ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs @@ -1,18 +1,36 @@ //! Building, intersecting, merging, splitting and capping half-open cursor intervals. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use std::cmp::Ordering; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; use pyo3::types::{PySlice, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cursor::algebra::main::cap_from_end::cap_from_end; use sqlbuild_cursor::algebra::main::cap_from_start::cap_from_start; use sqlbuild_cursor::algebra::main::clamp::clamp; +use sqlbuild_cursor::algebra::main::compare::compare; use sqlbuild_cursor::algebra::main::merge::merge; use sqlbuild_cursor::algebra::main::observed_partition::observed_partition; use sqlbuild_cursor::algebra::main::split_into_batches::split_into_batches; use sqlbuild_cursor::algebra::main::validate_interval::validate_interval; use sqlbuild_cursor::algebra::models::{Interval, Scalar}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::cursor::conversion::CursorSession; +use crate::bindings::_helpers::cursor::scalars::as_scalar; + +const MICROBATCH_MODELS_MODULE: &str = "sqlbuild.microbatches.models"; +const MICROBATCH_EXCEPTIONS_MODULE: &str = "sqlbuild.microbatches.exceptions"; +const RAW_INTERVAL_CURSOR_TYPES: [&str; 2] = ["timestamp", "integer"]; + +/// One stored interval with its parsed bounds. +struct RawInterval<'py> { + start: Scalar, + end: Scalar, + interval: Bound<'py, PyAny>, +} fn python_intervals<'py>( session: &CursorSession<'py>, @@ -84,6 +102,69 @@ fn merge_intervals<'py>(intervals: &Bound<'py, PyAny>) -> PyResult( + intervals: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = intervals.py(); + let kind: String = cursor_type.str()?.extract()?; + if !RAW_INTERVAL_CURSOR_TYPES.contains(&kind.as_str()) { + return Err( + imported(py, MICROBATCH_EXCEPTIONS_MODULE, "MicrobatchStateError") + .and_then(|class| class.call1((format!("unsupported cursor type: {kind}"),))) + .map_or_else(|error| error, PyErr::from_value), + ); + } + let session: CursorSession<'py> = CursorSession::new(py)?; + let mut parsed: Vec> = Vec::new(); + for interval in intervals.try_iter()? { + let interval: Bound<'py, PyAny> = interval?; + parsed.push(RawInterval { + start: as_scalar(&session, &interval.getattr("start")?, cursor_type)?, + end: as_scalar(&session, &interval.getattr("end")?, cursor_type)?, + interval, + }); + } + let mut failure: Option = None; + parsed.sort_by(|left, right| { + session + .checked(compare(&left.start, &right.start, &session)) + .unwrap_or_else(|error| { + failure.get_or_insert(error); + Ordering::Equal + }) + }); + if let Some(error) = failure { + return Err(error); + } + let class: Bound<'py, PyAny> = imported(py, MICROBATCH_MODELS_MODULE, "MicrobatchInterval")?; + let mut merged: Vec> = Vec::new(); + for next in parsed { + let Some(last) = merged.last_mut() else { + merged.push(next); + continue; + }; + if session.checked(compare(&last.end, &next.start, &session))? == Ordering::Less { + merged.push(next); + continue; + } + if session.checked(compare(&last.end, &next.end, &session))? == Ordering::Less { + last.interval = call_with( + &class, + &[ + ("start", last.interval.getattr("start")?), + ("end", next.interval.getattr("end")?), + ], + )?; + last.end = next.end; + } + } + PyTuple::new(py, merged.into_iter().map(|raw| raw.interval)) +} + /// An interval split into batches of a positive number of grain units. #[pyfunction(name = "cursor_split_into_batches")] #[pyo3(signature = (*, interval, step))] @@ -124,6 +205,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(partition, module)?)?; module.add_function(wrap_pyfunction!(clamp_intervals, module)?)?; module.add_function(wrap_pyfunction!(merge_intervals, module)?)?; + module.add_function(wrap_pyfunction!(merge_raw_intervals, module)?)?; module.add_function(wrap_pyfunction!(split, module)?)?; module.add_function(wrap_pyfunction!(cap_start, module)?)?; module.add_function(wrap_pyfunction!(cap_end, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs index c6582585cb..f551d426da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs @@ -170,7 +170,7 @@ pub(crate) fn parse_value<'py>( } /// `raw` unchanged when it is already a typed value, else parsed. -fn as_scalar<'py>( +pub(crate) fn as_scalar<'py>( session: &CursorSession<'py>, raw: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs index 96eb544955..b90720c229 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod artifact; pub(crate) mod registration; pub(crate) mod rows; +pub(crate) mod selection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs index 29c2baeab9..70ba39f2c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs @@ -2,8 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::dag_artifact::artifact; +use crate::bindings::_helpers::dag_artifact::{artifact, selection}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - artifact::register(module) + artifact::register(module)?; + selection::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs new file mode 100644 index 0000000000..70099991bc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs @@ -0,0 +1,218 @@ +//! Dagster asset selection: SQLBuild selectors resolved over a DAG artifact mapping. + +use pyo3::exceptions::PyOSError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyFrozenSet, PyList, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::dag_artifact::errors::DagSelectionError; +use sqlbuild_analysis::dag_artifact::main::resolve_dag_selection::resolve_dag_selection as resolve; +use sqlbuild_analysis::dag_artifact::models::{ + DagSelectionCheck, DagSelectionGraph, DagSelectionNode, +}; +use sqlbuild_analysis::planner_selection::main::unit_test_selector_rejection::unit_test_selector_rejection; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::planner_selection::carrier::Carrier; + +const DAGSTER_EXCEPTIONS_MODULE: &str = "sqlbuild.integrations.dagster.exceptions"; +const PATHLIB_MODULE: &str = "pathlib"; +const JSON_MODULE: &str = "json"; +const ABC_MODULE: &str = "collections.abc"; +const DAG_VERSION: i64 = 1; +const DAG_LIST_KEYS: [&str; 3] = ["nodes", "edges", "checks"]; + +fn text(value: &Bound<'_, PyAny>) -> PyResult { + value.str()?.extract() +} + +fn field<'py>(mapping: &Bound<'py, PyAny>, key: &str) -> PyResult> { + mapping.call_method1("get", (key,)) +} + +fn items<'py>(mapping: &Bound<'py, PyAny>, key: &str) -> PyResult>> { + let empty: Bound<'py, PyTuple> = PyTuple::empty(mapping.py()); + mapping + .call_method1("get", (key, empty))? + .try_iter()? + .collect() +} + +fn texts(mapping: &Bound<'_, PyAny>, key: &str) -> PyResult> { + items(mapping, key)?.iter().map(text).collect() +} + +fn selection_graph(dag: &Bound<'_, PyAny>) -> PyResult { + let nodes: Vec = items(dag, "nodes")? + .iter() + .map(|node| { + let path: Bound<'_, PyAny> = field(node, "path")?; + Ok(DagSelectionNode { + id: text(&node.get_item("id")?)?, + name: text(&field(node, "name")?)?, + kind: text(&field(node, "kind")?)?, + path: if path.is_instance_of::() { + Some(path.extract()?) + } else { + None + }, + tags: texts(node, "tags")?, + }) + }) + .collect::>()?; + let edges: Vec<(String, String)> = items(dag, "edges")? + .iter() + .map(|edge| { + Ok(( + text(&field(edge, "from_id")?)?, + text(&field(edge, "to_id")?)?, + )) + }) + .collect::>()?; + let checks: Vec = items(dag, "checks")? + .iter() + .map(|check| { + Ok(DagSelectionCheck { + name: text(&field(check, "name")?)?, + id: text(&field(check, "id")?)?, + checked_asset_ids: texts(check, "checked_asset_ids")?, + }) + }) + .collect::>()?; + Ok(DagSelectionGraph { + nodes, + edges, + checks, + }) +} + +fn selection_error(py: Python<'_>, error: DagSelectionError, raw: &str) -> PyErr { + let carrier = match Carrier::single(&PyString::new(py, raw)) { + Ok(carrier) => carrier, + Err(failure) => return failure, + }; + match error { + DagSelectionError::Selector(error) => carrier.error(error), + DagSelectionError::UnitTest(selector) => { + carrier.error(unit_test_selector_rejection(&selector)) + } + DagSelectionError::Input(message) => { + imported(py, DAGSTER_EXCEPTIONS_MODULE, "DagsterDagInputError") + .and_then(|class| class.call1((message,))) + .map_or_else(|failure| failure, PyErr::from_value) + } + } +} + +/// Ids of DAG artifact nodes `select` matches minus those `exclude` matches. +#[pyfunction] +#[pyo3(signature = (*, dag, select, exclude = None))] +fn resolve_dag_selection<'py>( + dag: &Bound<'py, PyAny>, + select: &str, + exclude: Option<&str>, +) -> PyResult> { + let py: Python<'py> = dag.py(); + let graph: DagSelectionGraph = selection_graph(dag)?; + let selected = + resolve(&graph, (select, exclude)).map_err(|error| selection_error(py, error, select))?; + PyFrozenSet::new(py, selected) +} + +fn raised(error_class: &Bound<'_, PyAny>, message: String, cause: Option) -> PyErr { + let py: Python<'_> = error_class.py(); + match error_class.call1((message,)) { + Ok(instance) => { + let error: PyErr = PyErr::from_value(instance); + error.set_cause(py, cause); + error + } + Err(failure) => failure, + } +} + +/// A SQLBuild DAG artifact from a mapping or JSON file path, validated, raising `error_class`. +#[pyfunction] +#[pyo3(signature = (dag, *, error_class))] +fn load_sqlbuild_dag<'py>( + dag: Bound<'py, PyAny>, + error_class: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = dag.py(); + let path_class: Bound<'py, PyAny> = imported(py, PATHLIB_MODULE, "Path")?; + let payload: Bound<'py, PyAny> = if dag.is_instance_of::() + || dag.is_instance(&path_class)? + { + let path: Bound<'py, PyAny> = path_class.call1((&dag,))?; + let encoding: Bound<'py, PyDict> = PyDict::new(py); + encoding.set_item("encoding", "utf-8")?; + let content: Bound<'py, PyAny> = match path.call_method("read_text", (), Some(&encoding)) { + Ok(content) => content, + Err(error) if error.is_instance_of::(py) => { + return Err(raised( + error_class, + format!("could not read SQLBuild DAG artifact: {}", path.str()?), + Some(error), + )); + } + Err(error) => return Err(error), + }; + let json: Bound<'py, PyModule> = py.import(JSON_MODULE)?; + let loaded: Bound<'py, PyAny> = match json.call_method1("loads", (content,)) { + Ok(loaded) => loaded, + Err(error) if error.is_instance(py, &json.getattr("JSONDecodeError")?) => { + let message: String = error.value(py).getattr("msg")?.extract()?; + return Err(raised( + error_class, + format!( + "SQLBuild DAG artifact is not valid JSON: {}: {message}", + path.str()? + ), + Some(error), + )); + } + Err(error) => return Err(error), + }; + if !loaded.is_instance(&imported(py, ABC_MODULE, "Mapping")?)? { + return Err(raised( + error_class, + "SQLBuild DAG artifact must be a JSON object".to_owned(), + None, + )); + } + loaded + } else { + dag + }; + let version: Bound<'py, PyAny> = payload.call_method1("get", ("version",))?; + if version.ne(DAG_VERSION)? { + return Err(raised( + error_class, + format!( + "unsupported SQLBuild DAG artifact version: {}", + version.repr()? + ), + None, + )); + } + for key in DAG_LIST_KEYS { + if !payload + .call_method1("get", (key,))? + .is_instance_of::() + { + return Err(raised( + error_class, + format!("SQLBuild DAG artifact field '{key}' must be a list"), + None, + )); + } + } + Ok(payload) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_dag_selection, module)?)?; + module.add_function(wrap_pyfunction!(load_sqlbuild_dag, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs index a2dea05e2a..d0a3f88ba1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs @@ -3,3 +3,4 @@ pub(crate) mod facts; pub(crate) mod query; pub(crate) mod registration; pub(crate) mod support; +pub(crate) mod tolerances; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index 420f59afef..df0ee54d16 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -12,24 +12,28 @@ use sqlbuild_cli::diff_report::main::read_only_query_problem::read_only_query_pr use sqlbuild_cli::diff_report::models::QueryShapeFacts; use sqlbuild_cli::diff_report::types::PhaseSeconds; +use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::diff_report::support::{ call, cli_error, error_text, none, object, progress_line, }; +use crate::bindings::_helpers::diff_report::tolerances::cli_tolerance_overrides; +use crate::bindings::types::CompilerDetach; +use sqlbuild_analysis::diff_selection::main::raw_query_shape::raw_query_shape; +use sqlbuild_analysis::diff_selection::models::RawQueryShape; const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const NAMESPACE_MODULE: &str = "sqlbuild._native"; const RUNTIME_MODULE: &str = "sqlbuild._native"; -const POLYGLOT_MODULE: &str = "sqlbuild.compiler.sql_analysis.main.import_polyglot_sql"; +const GENERIC_DIALECT: &str = "generic"; const COMMAND_MODELS_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.executor.diff.classes.query_artifact_lifecycle"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; const CURSOR_MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; const EXECUTE_QUERY_MODULE: &str = "sqlbuild.executor.diff.main._execute_query"; const DIFF_MODELS_MODULE: &str = "sqlbuild.executor.diff.models"; -const DIFF_CONFIG_MODULE: &str = "sqlbuild.executor.diff.main._config"; const NAMESPACELESS_ADAPTERS: [&str; 2] = ["duckdb", "motherduck"]; fn failure(py: Python<'_>, message: String, code: &str, help: Option<&str>) -> PyErr { @@ -114,13 +118,6 @@ fn query_input( } } -fn node_key(node: &Bound<'_, PyAny>) -> PyResult { - match node.getattr_opt("key")? { - Some(key) => Ok(key.str()?.to_string().to_lowercase()), - None => Ok(String::new()), - } -} - fn read_only_query( py: Python<'_>, sql: &str, @@ -139,69 +136,29 @@ fn read_only_query( None, )); } - let polyglot: Bound<'_, PyAny> = - imported(py, POLYGLOT_MODULE, "import_polyglot_sql")?.call0()?; - if polyglot.is_none() { - return Err(failure( - py, - "raw-query diff requires the bundled SQL parser".to_owned(), - "C233", - None, - )); - } let dialect: Bound<'_, PyAny> = adapter.call_method0("sql_analysis_dialect")?; - let dialect: Bound<'_, PyAny> = if dialect.is_truthy()? { - dialect + let dialect: String = if dialect.is_truthy()? { + dialect.extract()? } else { - object(py, "generic")? - }; - let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("dialect", dialect)?; - let expression: Bound<'_, PyAny> = - match polyglot.call_method("parse_one", (normalized,), Some(&keywords)) { - Ok(expression) => expression, - Err(error) if error.is_instance_of::(py) => { - let replacement: PyErr = failure( - py, - format!( - "{label} query is not one valid SQL statement: {}", - error_text(py, &error)? - ), - "C234", - None, - ); - replacement.set_cause(py, Some(error)); - return Err(replacement); - } - Err(error) => return Err(error), - }; - let mut node_keys: Vec = Vec::new(); - let root_key: String = node_key(&expression)?; - let read_only_root: bool = - ["except", "intersect", "select", "union", "values"].contains(&root_key.as_str()); - if read_only_root { - for node in expression.call_method0("walk")?.try_iter()? { - node_keys.push(node_key(&node?)?); - } - } - let arguments: Bound<'_, PyAny> = expression.getattr_opt("args")?.unwrap_or_else(|| none(py)); - let writes_or_locks: bool = match arguments.downcast::() { - Ok(arguments) => { - arguments - .get_item("into")? - .is_some_and(|into| !into.is_none()) - || arguments - .get_item("locks")? - .map_or(Ok(false), |locks| locks.is_truthy())? - } - Err(_) => false, + GENERIC_DIALECT.to_owned() }; + let shape: RawQueryShape = py + .compiler_detach(|| Ok::<_, String>(raw_query_shape(normalized, &dialect))) + .map_err(value_error)? + .map_err(|message| { + failure( + py, + format!("{label} query is not one valid SQL statement: {message}"), + "C234", + None, + ) + })?; read_only_query_problem( label, &QueryShapeFacts { - root_key, - node_keys, - writes_or_locks, + root_key: shape.root_key, + node_keys: shape.node_keys, + writes_or_locks: shape.writes_or_locks, }, ) .map_err(|error| user_failure(py, error))?; @@ -358,9 +315,9 @@ pub(crate) fn execution_options<'py>( ("exhaustive", &attribute("exhaustive")?), ], )?; - let tolerances: Bound<'py, PyAny> = call( - &imported(py, DIFF_CONFIG_MODULE, "parse_cli_tolerance_overrides")?, - &[("values", &attribute("tolerance_overrides")?)], + let tolerances: Bound<'py, PyAny> = cli_tolerance_overrides( + py, + &attribute("tolerance_overrides")?.extract::>()?, )?; let schema_only: Bound<'py, PyAny> = attribute("schema_only")?; let collect: Bound<'py, PyAny> = object(py, !schema_only.is_truthy()?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs index 78fa537525..6f861c78b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs @@ -17,6 +17,7 @@ use crate::bindings::_helpers::diff_report::command::{ execute_prepared_query_diff, run_diff_command, }; use crate::bindings::_helpers::diff_report::facts::{diff_facts, json_of}; +use crate::bindings::_helpers::diff_report::tolerances; use crate::bindings::_helpers::frontier::encoder::emit_error; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use sqlbuild_cli::diff_report::types::PhaseSeconds; @@ -216,5 +217,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(render_diff_output, module)?)?; module.add_function(wrap_pyfunction!(render_diff_error_json, module)?)?; module.add_function(wrap_pyfunction!(run_diff_command, module)?)?; - module.add_function(wrap_pyfunction!(execute_prepared_query_diff, module)?) + module.add_function(wrap_pyfunction!(execute_prepared_query_diff, module)?)?; + tolerances::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs new file mode 100644 index 0000000000..308f7960de --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs @@ -0,0 +1,349 @@ +//! Row-diff tolerance and sampling configuration from model config and CLI overrides. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyString}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; +const EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; +const DECIMAL_MODULE: &str = "decimal"; +const ABSOLUTE_KEY: &str = "absolute"; +const RELATIVE_KEY: &str = "relative"; +const TOLERANCE_KEYS: [&str; 2] = [ABSOLUTE_KEY, RELATIVE_KEY]; +const DEFAULT_TOLERANCES_LABEL: &str = "row_diff_tolerances"; +const CLI_TOLERANCE_MESSAGE: &str = + "diff --tolerance must use COLUMN:absolute=VALUE or COLUMN:relative=VALUE"; + +/// `ExecutorInputError(message, code=code)`. +fn input_error(py: Python<'_>, message: String, code: &str) -> PyErr { + imported(py, EXCEPTIONS_MODULE, "ExecutorInputError") + .and_then(|class| { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("code", code)?; + class.call((message,), Some(&keywords)) + }) + .map_or_else(|error| error, PyErr::from_value) +} + +/// `RowDiffTolerances` parsed from raw model configuration. +#[pyfunction] +#[pyo3(signature = (*, raw, label = DEFAULT_TOLERANCES_LABEL))] +fn parse_row_diff_tolerances<'py>( + raw: Bound<'py, PyAny>, + label: &str, +) -> PyResult> { + parsed_tolerances(&raw, label) +} + +fn parsed_tolerances<'py>(raw: &Bound<'py, PyAny>, label: &str) -> PyResult> { + let py: Python<'py> = raw.py(); + let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "RowDiffTolerances")?; + if raw.is_none() { + return class.call0(); + } + let Ok(mapping) = raw.downcast::() else { + return Err(input_error( + py, + format!("{label} must be a mapping"), + "X401", + )); + }; + let by_type: Bound<'py, PyDict> = tolerance_section( + py, + mapping.get_item("by_type")?, + (&format!("{label}.by_type"), true), + )?; + let by_column: Bound<'py, PyDict> = tolerance_section( + py, + mapping.get_item("by_column")?, + (&format!("{label}.by_column"), false), + )?; + call_with( + &class, + &[ + ("by_type", by_type.into_any()), + ("by_column", by_column.into_any()), + ], + ) +} + +fn tolerance_section<'py>( + py: Python<'py>, + raw: Option>, + (label, normalize_key): (&str, bool), +) -> PyResult> { + let parsed: Bound<'py, PyDict> = PyDict::new(py); + let Some(raw) = raw.filter(|raw| !raw.is_none()) else { + return Ok(parsed); + }; + let Ok(mapping) = raw.downcast::() else { + return Err(input_error( + py, + format!("{label} must be a mapping"), + "X401", + )); + }; + for (key, rule) in mapping.iter() { + let Ok(text) = key.downcast::() else { + return Err(input_error( + py, + format!("{label} keys must be non-empty strings"), + "X402", + )); + }; + let key_text: String = text.extract()?; + if key_text.is_empty() { + return Err(input_error( + py, + format!("{label} keys must be non-empty strings"), + "X402", + )); + } + let parsed_key: Bound<'py, PyAny> = if normalize_key { + key.call_method0("lower")? + } else { + key.clone() + }; + parsed.set_item( + parsed_key, + tolerance_rule(&rule, &format!("{label}.{key_text}"))?, + )?; + } + Ok(parsed) +} + +fn tolerance_rule<'py>(raw: &Bound<'py, PyAny>, label: &str) -> PyResult> { + let py: Python<'py> = raw.py(); + let Ok(mapping) = raw.downcast::() else { + return Err(input_error( + py, + format!("{label} must be a mapping"), + "X401", + )); + }; + let mut unsupported: Vec = Vec::new(); + for key in mapping.keys() { + let known: bool = key + .extract::() + .is_ok_and(|text| TOLERANCE_KEYS.contains(&text.as_str())); + if !known { + unsupported.push(key.str()?.extract()?); + } + } + if !unsupported.is_empty() { + return Err(input_error( + py, + format!( + "{label} contains unsupported keys: {}", + unsupported.join(", ") + ), + "X403", + )); + } + let absolute: Option> = optional_decimal( + mapping.get_item(ABSOLUTE_KEY)?, + &format!("{label}.absolute"), + )?; + let relative: Option> = optional_decimal( + mapping.get_item(RELATIVE_KEY)?, + &format!("{label}.relative"), + )?; + if absolute.is_none() && relative.is_none() { + return Err(input_error( + py, + format!("{label} must define absolute or relative"), + "X404", + )); + } + call_with( + &imported(py, MODELS_MODULE, "RowDiffTolerance")?, + &[ + ( + "absolute", + absolute.unwrap_or_else(|| py.None().into_bound(py)), + ), + ( + "relative", + relative.unwrap_or_else(|| py.None().into_bound(py)), + ), + ], + ) +} + +fn optional_decimal<'py>( + raw: Option>, + label: &str, +) -> PyResult>> { + let Some(raw) = raw.filter(|raw| !raw.is_none()) else { + return Ok(None); + }; + let py: Python<'py> = raw.py(); + let decimal_module: Bound<'py, PyModule> = py.import(DECIMAL_MODULE)?; + let decimal_class: Bound<'py, PyAny> = decimal_module.getattr("Decimal")?; + let numeric: bool = !raw.is_instance_of::() + && (raw.is_instance_of::() + || raw.is_instance_of::() + || raw.is_instance_of::() + || raw.is_instance(&decimal_class)?); + if !numeric { + return Err(input_error(py, format!("{label} must be numeric"), "X405")); + } + match decimal_class.call1((raw.str()?,)) { + Ok(value) => Ok(Some(value)), + Err(error) if error.is_instance(py, &decimal_module.getattr("InvalidOperation")?) => { + let replacement: PyErr = input_error(py, format!("{label} must be numeric"), "X405"); + replacement.set_cause(py, Some(error)); + Err(replacement) + } + Err(error) => Err(error), + } +} + +fn optional_integer<'py>(raw: &Bound<'py, PyAny>, label: &str) -> PyResult> { + if raw.is_none() { + return Ok(None); + } + if raw.is_instance_of::() || !raw.is_instance_of::() { + return Err(input_error( + raw.py(), + format!("{label} must be an integer"), + "X406", + )); + } + raw.extract().map(Some) +} + +/// `RowDiffTolerances` from repeated `COLUMN:KIND=VALUE` CLI specifications. +#[pyfunction] +#[pyo3(signature = (*, values))] +fn parse_cli_tolerance_overrides<'py>( + py: Python<'py>, + values: Vec, +) -> PyResult> { + cli_tolerance_overrides(py, &values) +} + +/// `parse_cli_tolerance_overrides` for native callers. +pub(crate) fn cli_tolerance_overrides<'py>( + py: Python<'py>, + values: &[String], +) -> PyResult> { + let by_column: Bound<'py, PyDict> = PyDict::new(py); + for value in values { + let (column_and_kind, amount) = value.split_once('=').unwrap_or((value, "")); + let separator: bool = value.contains('='); + let (column, kind) = column_and_kind + .split_once(':') + .map_or((column_and_kind, None), |(column, kind)| { + (column, Some(kind)) + }); + let valid_kind: Option<&str> = kind.filter(|kind| TOLERANCE_KEYS.contains(kind)); + let (Some(kind), true, false, false) = + (valid_kind, separator, column.is_empty(), amount.is_empty()) + else { + return Err(input_error(py, CLI_TOLERANCE_MESSAGE.to_owned(), "X408")); + }; + let rule: Bound<'py, PyDict> = match by_column.get_item(column)? { + Some(rule) => rule.downcast_into()?, + None => { + let rule: Bound<'py, PyDict> = PyDict::new(py); + by_column.set_item(column, &rule)?; + rule + } + }; + rule.set_item(kind, amount)?; + } + let raw: Bound<'py, PyDict> = PyDict::new(py); + raw.set_item("by_column", by_column)?; + parsed_tolerances(raw.as_any(), DEFAULT_TOLERANCES_LABEL) +} + +/// Model sampling configuration resolved with invocation overrides, or `None`. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_row_diff_sampling<'py>( + options: Option<&Bound<'py, PyDict>>, +) -> PyResult>> { + let request = Request::new("resolve_row_diff_sampling", options)?; + let py: Python<'py> = request.py(); + let override_values: Bound<'py, PyAny> = request.get("override")?; + if override_values.getattr("exhaustive")?.is_truthy()? { + return Ok(None); + } + let label: String = request.get("label")?.extract()?; + let overridden = |field: &str, raw: &str| -> PyResult> { + let value: Bound<'py, PyAny> = override_values.getattr(field)?; + if value.is_none() { + request.get(raw) + } else { + Ok(value) + } + }; + let row_limit_label: String = format!("{label}.row_diff_sample_rows"); + let row_limit: Option = + optional_integer(&overridden("row_limit", "raw_row_limit")?, &row_limit_label)?; + if row_limit.is_some_and(|value| value < 0) { + return Err(input_error( + py, + format!("{row_limit_label} must be zero or greater"), + "X407", + )); + } + let Some(row_limit) = row_limit.filter(|value| *value != 0) else { + return Ok(None); + }; + let seed: Option = optional_integer( + &overridden("seed", "raw_seed")?, + &format!("{label}.row_diff_sample_seed"), + )?; + call_with( + &imported(py, MODELS_MODULE, "RowDiffSampling")?, + &[ + ("row_limit", row_limit.into_bound_py_any(py)?), + ("seed", seed.unwrap_or(0).into_bound_py_any(py)?), + ], + ) + .map(Some) +} + +/// Invocation tolerance rules overlaid on compiled model rules. +#[pyfunction] +#[pyo3(signature = (**options))] +fn merge_row_diff_tolerances<'py>( + options: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("merge_row_diff_tolerances", options)?; + let py: Python<'py> = request.py(); + let base: Bound<'py, PyAny> = request.get("base")?; + let overlay: Bound<'py, PyAny> = request.get("override")?; + if overlay.is_none() { + return Ok(base); + } + let merged = |field: &str| -> PyResult> { + let combined: Bound<'py, PyDict> = PyDict::new(py); + combined.update(base.getattr(field)?.downcast()?.as_mapping())?; + combined.update(overlay.getattr(field)?.downcast()?.as_mapping())?; + Ok(combined.into_any()) + }; + call_with( + &imported(py, MODELS_MODULE, "RowDiffTolerances")?, + &[ + ("by_type", merged("by_type")?), + ("by_column", merged("by_column")?), + ("sampling", base.getattr("sampling")?), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(parse_row_diff_tolerances, module)?)?; + module.add_function(wrap_pyfunction!(parse_cli_tolerance_overrides, module)?)?; + module.add_function(wrap_pyfunction!(resolve_row_diff_sampling, module)?)?; + module.add_function(wrap_pyfunction!(merge_row_diff_tolerances, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs index e33df4195c..af9d80ac3c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs @@ -3,6 +3,7 @@ mod execution_gates; pub(crate) mod keys; pub(crate) mod project_graph; mod python_lifecycle; +pub(crate) mod python_node_graph; pub(crate) mod python_selection; pub(crate) mod registration; pub(crate) mod views; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs new file mode 100644 index 0000000000..c3ff893545 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs @@ -0,0 +1,354 @@ +//! The executable Python-node graph built from discovered project inputs. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; +use sqlbuild_analysis::python_selection::models::{InventoryIndexes, InventoryNode}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::graph::python_selection::resolve_python_sql_run_selection; +use crate::bindings::_helpers::graph::views::compiled_graph_keys; + +const USER_INVENTORY_MODULE: &str = "sqlbuild.compiler.python_nodes._helpers.user_inventory"; +const NODE_MODELS_MODULE: &str = "sqlbuild.compiler.python_nodes.models"; +const NODE_TYPES_MODULE: &str = "sqlbuild.compiler.python_nodes.types"; +const SQL_REF_MODULE: &str = "sqlbuild.python_nodes.models"; +const NAME_KEY: &str = "name"; + +/// One discovered function collection, its node kind and whether it carries documentation. +const KINDS: [(&str, &str, bool); 4] = [ + ("loader_functions", "LOADER", false), + ("task_functions", "TASK", true), + ("asset_functions", "ASSET", true), + ("check_functions", "CHECK", true), +]; + +/// Python classes and shims every node is built with. +struct Vocabulary<'py> { + node: Bound<'py, PyAny>, + kinds: Bound<'py, PyAny>, + sql_ref: Bound<'py, PyAny>, + user_facts: Bound<'py, PyAny>, + providers: Bound<'py, PyDict>, +} + +fn attribute_value<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult> { + value.getattr(name) +} + +fn optional_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(value.clone()); + } + value.getattr("value") +} + +/// The kind-specific decorator configuration and metadata record of one discovered function. +fn kind_parts<'py>( + py: Python<'py>, + kind: &str, + discovered: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyDict>, &'static str, Bound<'py, PyAny>)> { + let config: Bound<'py, PyDict> = PyDict::new(py); + let get = |name: &str| attribute_value(discovered, name); + let metadata = |class: &str, fields: &[&str]| -> PyResult> { + let pairs: Vec<(&str, Bound<'py, PyAny>)> = fields + .iter() + .map(|field| Ok((*field, get(field)?))) + .collect::>()?; + call_with(&imported(py, NODE_MODELS_MODULE, class)?, &pairs) + }; + Ok(match kind { + "LOADER" => { + config.set_item("columns", get("columns")?)?; + config.set_item("contract", get("contract")?)?; + config.set_item("cursor_column", get("cursor_column")?)?; + config.set_item("destination", get("destination")?)?; + config.set_item("unique_key", get("unique_key")?)?; + config.set_item("write_strategy", optional_value(&get("write_strategy")?)?)?; + let record = metadata( + "DiscoveredPythonLoaderMetadata", + &[ + "destination", + "write_strategy", + "cursor_column", + "unique_key", + "columns", + "contract", + "connection_mode", + ], + )?; + (config, "loader", record) + } + "TASK" => ( + config, + "task", + metadata("DiscoveredPythonTaskMetadata", &["retry"])?, + ), + "ASSET" => { + config.set_item("columns", get("columns")?)?; + config.set_item("column_lineage", get("column_lineage")?)?; + let record = metadata( + "DiscoveredPythonAssetMetadata", + &["columns", "column_lineage", "retry"], + )?; + (config, "asset", record) + } + _ => { + config.set_item("severity", get("severity")?.getattr("value")?)?; + let record = metadata("DiscoveredPythonCheckMetadata", &["severity"])?; + (config, "check", record) + } + }) +} + +/// One `DiscoveredPythonNode` with its identity and provider usages read by the user shim. +fn python_node<'py>( + vocabulary: &Vocabulary<'py>, + (kind, documented): (&str, bool), + discovered: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let get = |name: &str| attribute_value(discovered, name); + let (config, metadata_field, metadata) = kind_parts(py, kind, discovered)?; + if documented { + for field in ["description", "group", "meta", "tags"] { + config.set_item(field, get(field)?)?; + } + } + let kind_member: Bound<'py, PyAny> = vocabulary.kinds.getattr(kind)?; + let (usages, identity): (Bound<'py, PyAny>, Bound<'py, PyAny>) = call_with( + &vocabulary.user_facts, + &[ + ("kind", kind_member.getattr("value")?), + ("name", get("name")?), + ("function", get("function")?), + ("file_path", get("file_path")?), + ("provider_by_name", vocabulary.providers.clone().into_any()), + ("decorator_config", config.into_any()), + ], + )? + .extract()?; + let depends_on: Bound<'py, PyAny> = get("depends_on")?; + let mut sql_deps: Vec> = Vec::new(); + for dependency in depends_on.try_iter()? { + let dependency: Bound<'py, PyAny> = dependency?; + if dependency.is_instance(&vocabulary.sql_ref)? { + sql_deps.push(dependency); + } + } + let mut pairs: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("kind", kind_member), + ("file_path", get("file_path")?), + ("relative_path", get("relative_path")?), + ("name", get("name")?), + ("function", get("function")?), + ("depends_on", depends_on), + ("sql_deps", PyTuple::new(py, sql_deps)?.into_any()), + ("provider_usages", usages), + ("identity", identity), + ("description", get("description")?), + (metadata_field, metadata), + ]; + if documented { + for field in ["tags", "group", "meta"] { + pairs.push((field, get(field)?)); + } + } + call_with(&vocabulary.node, &pairs) +} + +/// Each node's upstream Python-node names, resolving decorated dependencies through the shim. +fn upstream_names( + py: Python<'_>, + nodes: &[Bound<'_, PyAny>], + sql_ref: &Bound<'_, PyAny>, +) -> PyResult>> { + let by_key: Bound<'_, PyDict> = PyDict::new(py); + for node in nodes { + by_key.set_item(node.getattr("function")?, node.getattr("name")?)?; + } + for node in nodes { + let name: Bound<'_, PyAny> = node.getattr("name")?; + by_key.set_item( + PyTuple::new(py, [PyString::new(py, NAME_KEY).into_any(), name.clone()])?, + name, + )?; + } + let dependency_key: Bound<'_, PyAny> = + imported(py, USER_INVENTORY_MODULE, "python_node_dependency_key")?; + nodes + .iter() + .map(|node| { + let mut names: Vec = Vec::new(); + for dependency in node.getattr("depends_on")?.try_iter()? { + let dependency: Bound<'_, PyAny> = dependency?; + if dependency.is_instance(sql_ref)? { + continue; + } + let key: Bound<'_, PyAny> = dependency_key.call1((dependency,))?; + if let Some(upstream) = by_key.get_item(key)? { + names.push(upstream.extract()?); + } + } + Ok(names) + }) + .collect() +} + +fn inventory_node(node: &Bound<'_, PyAny>, upstream: Vec) -> PyResult { + Ok(InventoryNode { + name: node.getattr("name")?.extract()?, + tags: node.getattr("tags")?.extract()?, + relative_path: node + .getattr("relative_path")? + .call_method0("as_posix")? + .extract()?, + upstream, + }) +} + +/// The internal executable Python-node graph of discovered project inputs. +pub(crate) fn python_node_graph<'py>( + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered_inputs.py(); + let providers: Bound<'py, PyDict> = PyDict::new(py); + for provider in discovered_inputs.getattr("providers")?.try_iter()? { + let provider: Bound<'py, PyAny> = provider?; + providers.set_item(provider.getattr("name")?, provider)?; + } + let vocabulary: Vocabulary<'py> = Vocabulary { + node: imported(py, NODE_MODELS_MODULE, "DiscoveredPythonNode")?, + kinds: imported(py, NODE_TYPES_MODULE, "PythonNodeKind")?, + sql_ref: imported(py, SQL_REF_MODULE, "SqlResourceRef")?, + user_facts: imported(py, USER_INVENTORY_MODULE, "python_node_user_facts")?, + providers, + }; + let mut nodes: Vec> = Vec::new(); + for (collection, kind, documented) in KINDS { + for discovered in discovered_inputs.getattr(collection)?.try_iter()? { + nodes.push(python_node(&vocabulary, (kind, documented), &discovered?)?); + } + } + let inventory: Vec = nodes + .iter() + .zip(upstream_names(py, &nodes, &vocabulary.sql_ref)?) + .map(|(node, upstream)| inventory_node(node, upstream)) + .collect::>()?; + graph_record(py, &nodes, graph_indexes(&inventory)) +} + +type NameRows = Vec<(String, Vec)>; + +fn graph_record<'py>( + py: Python<'py>, + nodes: &[Bound<'py, PyAny>], + indexes: InventoryIndexes, +) -> PyResult> { + let InventoryIndexes { + edges, + upstream, + downstream, + tags, + paths, + } = indexes; + let edge_class: Bound<'py, PyAny> = + imported(py, NODE_MODELS_MODULE, "PythonNodeDependencyEdge")?; + let dependency_edges: Vec> = edges + .into_iter() + .map(|(upstream_name, downstream_name)| { + call_with( + &edge_class, + &[ + ( + "upstream_name", + PyString::new(py, &upstream_name).into_any(), + ), + ( + "downstream_name", + PyString::new(py, &downstream_name).into_any(), + ), + ], + ) + }) + .collect::>()?; + let adjacency = |rows: NameRows| -> PyResult> { + let mapping: Bound<'py, PyDict> = PyDict::new(py); + for (name, names) in rows { + mapping.set_item(name, PyTuple::new(py, names)?)?; + } + Ok(mapping) + }; + let tag_index: Bound<'py, PyDict> = PyDict::new(py); + for (tag, names) in tags { + tag_index.set_item(tag, PyFrozenSet::new(py, names)?)?; + } + let path_index: Bound<'py, PyDict> = PyDict::new(py); + for (name, path) in paths { + path_index.set_item(name, path)?; + } + let by_name: Bound<'py, PyDict> = PyDict::new(py); + let by_selector: Bound<'py, PyDict> = PyDict::new(py); + for node in nodes { + let name: Bound<'py, PyAny> = node.getattr("name")?; + let kind: String = node.getattr("kind")?.getattr("value")?.extract()?; + by_name.set_item(&name, node)?; + by_selector.set_item(format!("{kind}:{}", name.str()?), node)?; + } + call_with( + &imported(py, NODE_MODELS_MODULE, "PythonNodeGraph")?, + &[ + ("nodes", PyTuple::new(py, nodes)?.into_any()), + ( + "dependency_edges", + PyTuple::new(py, dependency_edges)?.into_any(), + ), + ("upstream_deps", adjacency(upstream)?.into_any()), + ("downstream_deps", adjacency(downstream)?.into_any()), + ("tag_index", tag_index.into_any()), + ("path_index", path_index.into_any()), + ("nodes_by_name", by_name.into_any()), + ("nodes_by_typed_selector", by_selector.into_any()), + ], + ) +} + +/// `PythonSqlRunSelection` of run selectors over the project graph and discovered Python nodes. +pub(crate) fn run_selection_from_inputs<'py>( + (select, exclude): (Vec, Vec), + project_graph: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_graph.py(); + let python_graph: Bound<'py, PyAny> = python_node_graph(discovered_inputs)?; + let (sql_keys, python_node_names) = + resolve_python_sql_run_selection(project_graph, &python_graph, select, exclude)?; + call_with( + &imported(py, NODE_MODELS_MODULE, "PythonSqlRunSelection")?, + &[ + ("sql_keys", compiled_graph_keys(py, sql_keys)?.into_any()), + ( + "python_node_names", + PyFrozenSet::new(py, python_node_names)?.into_any(), + ), + ], + ) +} + +/// Build the internal executable Python-node graph from discovered project inputs. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_python_node_graph<'py>( + options: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("build_python_node_graph", options)?; + python_node_graph(&request.get("discovered_inputs")?) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_python_node_graph, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index b61ae69be3..7cb087cb02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -160,7 +160,7 @@ fn resolve_python_check_excludes<'py>( /// `(sql keys, python node names)` run selectors resolve to across SQL resources and Python nodes. #[pyfunction] #[pyo3(signature = (*, project_graph, python_graph, select, exclude))] -fn resolve_python_sql_run_selection( +pub(crate) fn resolve_python_sql_run_selection( project_graph: &Bound<'_, PyAny>, python_graph: &Bound<'_, PyAny>, select: Vec, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs index f9e662f18c..971b8975e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs @@ -6,5 +6,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::graph::execution_gates::register(module)?; crate::bindings::_helpers::graph::python_selection::register(module)?; crate::bindings::_helpers::graph::python_lifecycle::register(module)?; + crate::bindings::_helpers::graph::python_node_graph::register(module)?; crate::bindings::_helpers::graph::views::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs index 01218702a9..79a3b3be7e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs @@ -29,7 +29,7 @@ fn native_graph_keys(keys: &Bound<'_, PyAny>) -> PyResult> { /// `CompiledObjectKey`s for native graph key pairs. #[pyfunction] -fn compiled_graph_keys<'py>( +pub(crate) fn compiled_graph_keys<'py>( py: Python<'py>, keys: Vec, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index ccc4eacfbb..a63b0b867b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -66,6 +66,7 @@ pub(crate) mod query_command; pub(crate) mod records; pub(crate) mod refactor; pub(crate) mod references; +pub(crate) mod relation_naming; pub(crate) mod render; pub(crate) mod result_output; pub(crate) mod rules; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 88011119fd..7fd07624a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -35,10 +35,10 @@ use crate::bindings::_helpers::planner_identity::metadata::{ compiled_function_fingerprint_sql, function_local_hashes, model_declared_columns_hash, model_version_identity_metadata_json, }; +use crate::bindings::_helpers::type_system::equality::types_equal as native_types_equal; const NATIVE_MODULE: &str = "sqlbuild._native"; const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; -const TYPES_EQUAL_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; const DYNAMIC_FAMILIES_MODULE: &str = "sqlbuild._native"; const DEBUG_EVENT_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; const LOG_SQL_MODULE: &str = "sqlbuild.diagnostics.main.log_sql"; @@ -594,18 +594,8 @@ fn schema_findings_of<'py>( ) -> PyResult> { let py: Python<'py> = dialect.py(); let dialect_name: Option = dialect.extract()?; - let types_equal_function: Bound<'py, PyAny> = imported(py, TYPES_EQUAL_MODULE, "types_equal")?; - let mut types_equal = |left: &str, right: &str| -> PyResult { - call_with( - &types_equal_function, - &[ - ("left", PyString::new(py, left).into_any()), - ("right", PyString::new(py, right).into_any()), - ("dialect", dialect.clone()), - ], - )? - .is_truthy() - }; + let mut types_equal = + |left: &str, right: &str| -> PyResult { native_types_equal(dialect, left, right) }; let matching_function: Bound<'py, PyAny> = imported(py, DYNAMIC_FAMILIES_MODULE, "matching_dynamic_families")?; let mut single_family_type = |name: &str| -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs index ad4968c18c..62cfa9cd58 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs @@ -5,7 +5,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::{ planner_changes, planner_entries, planner_identity, planner_migrations, planner_models, planner_phases, planner_resolution, planner_scenario, planner_scope, planner_selection, - planner_strategy, planner_warehouse, + planner_strategy, planner_warehouse, relation_naming, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -21,5 +21,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { planner_strategy::registration::register(module)?; planner_resolution::registration::register(module)?; planner_entries::registration::register(module)?; + relation_naming::registration::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs index 748fa0f48c..4898e1db42 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs @@ -1,11 +1,10 @@ //! Source contract validation and adapter-controlled projection rendering. -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, -}; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_resolution::values::{cursor_wrap, method, string}; use crate::bindings::_helpers::planner_strategy::values::{PLANNER_EXCEPTIONS_MODULE, input_error}; use crate::bindings::_helpers::references::source_relations::render_source_relation; +use crate::bindings::_helpers::type_system::equality::types_equal; use pyo3::IntoPyObject; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -16,8 +15,6 @@ use sqlbuild_sqltext::resolution::main::{ reference_spans::reference_spans, source_alias::source_alias, }; -const TYPES_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; - fn source_error(py: Python<'_>, message: &str, code: &str) -> PyErr { input_error( py, @@ -86,15 +83,11 @@ fn check_columns( if actual.is_empty() { continue; } - let equal = call_with( - &imported(py, TYPES_MODULE, "types_equal")?, - &[ - ("left", string(py, actual)), - ("right", string(py, &kind)), - ("dialect", adapter.getattr("sql_analysis_dialect_name")?), - ], - )? - .is_truthy()?; + let equal: bool = types_equal( + &adapter.getattr("sql_analysis_dialect_name")?, + actual, + &kind, + )?; if !equal { return Err(source_error( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs index 4aa6985c0e..edda0220c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs @@ -8,10 +8,13 @@ use crate::bindings::_helpers::planner_scenario::constants::{ SCENARIO_PLAN_RELATION_COLLISION, }; use crate::bindings::_helpers::planner_scenario::values::{MODELS, TYPES, error, string}; +use crate::bindings::_helpers::relation_naming::fitting::fitted_logical_name; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_scenario::main::{hash_prefix, parse_artifact_name}; +const SCENARIO_ARTIFACT_LABEL: &str = "Scenario artifact"; + #[pyfunction] #[pyo3(signature = (**kwargs))] pub(crate) fn compute_scenario_hash_prefix( @@ -106,24 +109,17 @@ pub(crate) fn build_scenario_hash_index<'py>( #[pyfunction] #[pyo3(signature = (*, logical_name, fixed_prefix, identifier_limit))] -pub(crate) fn fit_scenario_artifact_logical_name<'py>( - py: Python<'py>, - logical_name: Bound<'py, PyAny>, - fixed_prefix: Bound<'py, PyAny>, - identifier_limit: Bound<'py, PyAny>, -) -> PyResult> { - call_with( - &imported( - py, - "sqlbuild.adapter.relations.main.fit_artifact_logical_name", - "fit_artifact_logical_name", - )?, - &[ - ("logical_name", logical_name), - ("fixed_prefix", fixed_prefix), - ("identifier_limit", identifier_limit), - ("artifact_label", string(py, "Scenario artifact")), - ], +pub(crate) fn fit_scenario_artifact_logical_name( + py: Python<'_>, + logical_name: &str, + fixed_prefix: &str, + identifier_limit: i64, +) -> PyResult { + fitted_logical_name( + py, + (logical_name, fixed_prefix), + identifier_limit, + SCENARIO_ARTIFACT_LABEL, ) } @@ -133,17 +129,16 @@ pub(crate) fn build_scenario_artifact_physical_name( hash_prefix: &str, kind: &str, logical_name: Bound<'_, PyAny>, - identifier_limit: Bound<'_, PyAny>, + identifier_limit: i64, ) -> PyResult { let py = logical_name.py(); let prefix = format!("__sqb_{hash_prefix}__{kind}__"); let logical: String = fit_scenario_artifact_logical_name( py, - logical_name, - string(py, &prefix), + &logical_name.extract::()?, + &prefix, identifier_limit, - )? - .extract()?; + )?; Ok(format!("{prefix}{logical}")) } @@ -160,8 +155,7 @@ pub(crate) fn build_scenario_artifact_name( .call1((kind,))? .getattr("value")? .extract()?; - let limit = pyo3::IntoPyObjectExt::into_bound_py_any(identifier_limit, py)?; - build_scenario_artifact_physical_name(hash_prefix, &kind, logical_name, limit) + build_scenario_artifact_physical_name(hash_prefix, &kind, logical_name, identifier_limit) } #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs index 9f2e10cf29..73b24b16fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs @@ -9,11 +9,11 @@ use sqlbuild_core::query_command::main::render_dag_summary::render_dag_summary; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::query_command::project::{ProjectAdapter, project_adapter}; const NATIVE_MODULE: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; -const PYTHON_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; fn array_length(document: &serde_json::Value, key: &str) -> PyResult { document @@ -69,14 +69,7 @@ fn run_dag_command<'py>(py: Python<'py>, options: Option>) -> ("external_sql_reference_resolver", resolver), ], )?; - let python_graph: Bound<'py, PyAny> = call_with( - &imported( - py, - PYTHON_GRAPH_MODULE, - "build_discovered_python_node_graph", - )?, - &[("discovered_inputs", discovered.clone())], - )?; + let python_graph: Bound<'py, PyAny> = python_node_graph(&discovered)?; let dag_json: String = call_with( &imported(py, NATIVE_MODULE, "build_dag_json")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs index 1ac9d93180..595e088c2d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs @@ -7,12 +7,12 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::text::main::python_casefold::python_casefold; use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::type_system::equality::types_equal as native_types_equal; const ADOPTION_MODELS_MODULE: &str = "sqlbuild.compiler.contract_adoption.models"; const ADOPTION_TYPES_MODULE: &str = "sqlbuild.compiler.contract_adoption.types"; const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const TYPES_EQUAL_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; const DYNAMIC_FAMILIES_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; const MODEL_RESOURCE: &str = "model"; @@ -241,16 +241,11 @@ fn types_equal( adapter: &Bound<'_, PyAny>, (left, right): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), ) -> PyResult { - let py: Python<'_> = adapter.py(); - keyword_call( - &py.import(TYPES_EQUAL_MODULE)?.getattr("types_equal")?, - vec![ - ("left", left.clone()), - ("right", right.clone()), - ("dialect", adapter.call_method0("sql_analysis_dialect")?), - ], - )? - .is_truthy() + native_types_equal( + &adapter.call_method0("sql_analysis_dialect")?, + &left.extract::()?, + &right.extract::()?, + ) } /// The finding for a physical column no declared column names, if any. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs new file mode 100644 index 0000000000..86f0264adf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs @@ -0,0 +1,63 @@ +//! Generated relation names fitted within identifier limits, raising `SharedInputError`. + +use pyo3::prelude::{Bound, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::identifier_fitting::main::fit_artifact_logical_name::fit_artifact_logical_name as fit_logical; +use sqlbuild_analysis::identifier_fitting::main::fit_auxiliary_relation_name::fit_auxiliary_relation_name as fit_auxiliary; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const SHARED_EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; + +fn shared_error(py: Python<'_>, message: String) -> PyErr { + imported(py, SHARED_EXCEPTIONS_MODULE, "SharedInputError") + .and_then(|class| class.call1((message,))) + .map_or_else(|error| error, PyErr::from_value) +} + +/// A readable logical component fitted after a fixed prefix, with a hash suffix when shortened. +#[pyfunction] +#[pyo3(signature = (**options))] +fn fit_artifact_logical_name(options: Option<&Bound<'_, PyDict>>) -> PyResult { + let request = Request::new("fit_artifact_logical_name", options)?; + fitted_logical_name( + request.py(), + ( + &request.get("logical_name")?.extract::()?, + &request.get("fixed_prefix")?.extract::()?, + ), + request.get("identifier_limit")?.extract()?, + &request.get("artifact_label")?.extract::()?, + ) +} + +/// `fit_artifact_logical_name`, raising `SharedInputError` when the name cannot fit. +pub(crate) fn fitted_logical_name( + py: Python<'_>, + names: (&str, &str), + identifier_limit: i64, + artifact_label: &str, +) -> PyResult { + fit_logical(names, identifier_limit, artifact_label) + .map_err(|message| shared_error(py, message)) +} + +/// `base_name + suffix` fitted within the limit, keeping the suffix and a base hash. +#[pyfunction] +#[pyo3(signature = (*, base_name, suffix, identifier_limit))] +fn fit_auxiliary_relation_name( + py: Python<'_>, + base_name: &str, + suffix: &str, + identifier_limit: i64, +) -> PyResult { + fit_auxiliary(base_name, suffix, identifier_limit).map_err(|message| shared_error(py, message)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(fit_artifact_logical_name, module)?)?; + module.add_function(wrap_pyfunction!(fit_auxiliary_relation_name, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/mod.rs new file mode 100644 index 0000000000..0bbf05581d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod fitting; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/registration.rs new file mode 100644 index 0000000000..7a489501cf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/registration.rs @@ -0,0 +1,9 @@ +//! Register the generated relation name fitting functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::relation_naming::fitting; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + fitting::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs index 8b561d1a36..d19447b762 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs @@ -3,10 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_pysource::macro_modules::main::inventory_macro_modules::inventory_macro_modules as inventory; +use sqlbuild_pysource::macro_modules::main::macro_load_plan::macro_load_plan; use sqlbuild_pysource::macro_modules::models::{ MacroExport, MacroFault, MacroModuleInventory, MacroModuleLoad, MacroModulePath, MacroModuleSource, }; +use sqlbuild_pysource::macro_modules::models::{MacroLoadPlan, MacroModuleSpec}; use sqlbuild_pysource::macro_modules::types::MacroScopeKind; use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_error}; @@ -108,7 +110,32 @@ pub(crate) fn inventory_macro_modules( )) } +/// `(synthetic root, packages shallowest first, module execution order)` of `(name, path, contents, dependencies)` rows. +#[pyfunction] +fn macro_module_load_plan( + modules: Vec<(String, String, String, Vec)>, +) -> (String, Vec, Vec) { + let specs: Vec = modules + .into_iter() + .map( + |(name, file_path, contents, dependencies)| MacroModuleSpec { + name, + file_path, + contents, + dependencies, + }, + ) + .collect(); + let MacroLoadPlan { + root, + packages, + order, + } = macro_load_plan(&specs); + (root, packages, order) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(inventory_macro_modules, module)?)?; + module.add_function(wrap_pyfunction!(macro_module_load_plan, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs new file mode 100644 index 0000000000..9c892671f1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs @@ -0,0 +1,120 @@ +//! Build `BuildExecutionResult` with aggregate counts and overall status from typed results. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_core::build_aggregate::main::aggregate_build_counts::aggregate_build_counts; +use sqlbuild_core::build_aggregate::models::{BuildCounts, BuildResultFacts, ModelResultFacts}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const BUILD_MODELS_MODULE: &str = "sqlbuild.executor.build.models"; +const BUILD_TYPES_MODULE: &str = "sqlbuild.executor.build.types"; +const RESULT_FIELDS: [&str; 7] = [ + "model_results", + "seed_results", + "function_results", + "load_results", + "test_results", + "source_audit_results", + "end_audit_results", +]; + +fn texts(results: &Bound<'_, PyAny>, field: &str) -> PyResult> { + results + .try_iter()? + .map(|result| result?.getattr(field)?.str()?.extract()) + .collect() +} + +fn facts(sections: &[Bound<'_, PyAny>]) -> PyResult { + let models: Vec = sections[0] + .try_iter()? + .map(|model| { + let model: Bound<'_, PyAny> = model?; + Ok(ModelResultFacts { + status: model.getattr("status")?.str()?.extract()?, + error_code: model.getattr("error_code")?.extract()?, + warning_count: model.getattr("warning_messages")?.len()?, + audit_outcomes: texts(&model.getattr("audit_results")?, "outcome")?, + }) + }) + .collect::>()?; + let functions: Vec<(String, usize)> = sections[2] + .try_iter()? + .map(|function| { + let function: Bound<'_, PyAny> = function?; + Ok(( + function.getattr("status")?.str()?.extract()?, + function.getattr("warning_messages")?.len()?, + )) + }) + .collect::>()?; + Ok(BuildResultFacts { + models, + seed_statuses: texts(§ions[1], "status")?, + functions, + load_statuses: texts(§ions[3], "status")?, + test_outcomes: texts(§ions[4], "outcome")?, + source_audit_outcomes: texts(§ions[5], "outcome")?, + end_audit_outcomes: texts(§ions[6], "outcome")?, + }) +} + +/// `BuildExecutionResult` over the seven result tuples, in `RESULT_FIELDS` order. +pub(crate) fn aggregated_build_result<'py>( + py: Python<'py>, + sections: &[Bound<'py, PyAny>], +) -> PyResult> { + let counts: BuildCounts = aggregate_build_counts(&facts(sections)?); + let status: Bound<'py, PyAny> = imported(py, BUILD_TYPES_MODULE, "BuildStatus")? + .getattr(if counts.failed { "FAILED" } else { "SUCCESS" })?; + let mut keywords: Vec<(&str, Bound<'py, PyAny>)> = vec![("status", status)]; + keywords.extend(RESULT_FIELDS.iter().copied().zip(sections.iter().cloned())); + keywords.extend([ + ("success_count", counts.success.into_bound_py_any(py)?), + ("failure_count", counts.failure.into_bound_py_any(py)?), + ("skipped_count", counts.skipped.into_bound_py_any(py)?), + ("warning_count", counts.warning.into_bound_py_any(py)?), + ( + "insufficient_count", + counts.insufficient.into_bound_py_any(py)?, + ), + ]); + call_with( + &imported(py, BUILD_MODELS_MODULE, "BuildExecutionResult")?, + &keywords, + ) +} + +/// Compute aggregate counts and overall build status. +#[pyfunction] +#[pyo3(signature = (**options))] +fn aggregate_build_result<'py>( + options: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("aggregate_build_result", options)?; + let sections: Vec> = RESULT_FIELDS + .iter() + .map(|field| request.get(field)) + .collect::>()?; + aggregated_build_result(request.py(), §ions) +} + +/// `aggregate_build_result` over retained result lists, as tuples. +pub(crate) fn aggregated_from_lists<'py>( + py: Python<'py>, + retained: &[Vec>], +) -> PyResult> { + let sections: Vec> = retained + .iter() + .map(|kept| Ok(PyTuple::new(py, kept)?.into_any())) + .collect::>()?; + aggregated_build_result(py, §ions) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(aggregate_build_result, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs index 59b16c0f77..4b5798fdde 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs @@ -4,11 +4,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; use crate::bindings::_helpers::compile_outputs::json_report::indented_json; use crate::bindings::_helpers::plan_output::values::{any, dict, items, text}; +use crate::bindings::_helpers::result_output::aggregation::aggregated_from_lists; use crate::bindings::_helpers::result_output::assets::{ clone_asset, function_assets, load_assets, model_assets, python_node_assets, seed_assets, without_none, @@ -19,7 +18,6 @@ use crate::bindings::_helpers::result_output::checks::{ use crate::bindings::_helpers::result_output::projection::{Projection, rounded}; const JSON_VERSION: i64 = 1; -const AGGREGATE_MODULE: &str = "sqlbuild.executor.build.main._aggregate_result"; const FAILED: &str = "failed"; const SUCCESS: &str = "success"; const SKIPPED: &str = "skipped"; @@ -332,15 +330,7 @@ fn build_json<'py>( assets.extend(load_assets(&projection, &retained[3])?); assets.extend(python_node_assets(&projection, &nodes)?); let counted: Bound<'py, PyAny> = if partial { - let keywords: Vec<(&str, Bound<'py, PyAny>)> = BUILD_RESULT_FIELDS - .iter() - .zip(&retained) - .map(|(field, kept)| Ok((*field, PyTuple::new(py, kept)?.into_any()))) - .collect::>()?; - call_with( - &imported(py, AGGREGATE_MODULE, "aggregate_build_result")?, - &keywords, - )? + aggregated_from_lists(py, &retained)? } else { result.clone() }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs index 1e2e9b28c9..ac7be681a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod aggregation; pub(crate) mod assets; pub(crate) mod checks; pub(crate) mod documents; @@ -7,4 +8,5 @@ pub(crate) mod integration_records; pub(crate) mod integration_values; pub(crate) mod projection; pub(crate) mod registration; +pub(crate) mod resource_ids; pub(crate) mod terminal_index; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs index e7f3325b56..1cc2df689e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs @@ -3,8 +3,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; use pyo3::types::{PyFloat, PyInt}; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::plan_output::values::text; +use crate::bindings::_helpers::result_output::resource_ids::{audit_identity, sql_test_identity}; use crate::bindings::_helpers::result_output::terminal_index::{TerminalEventIndex, current_index}; const RUN_MODELS: &str = "sqlbuild.executor.run.models"; @@ -14,8 +15,6 @@ const PYTHON_NODE_MODELS: &str = "sqlbuild.executor.python_nodes.models"; const TESTING_MODELS: &str = "sqlbuild.executor.testing.models"; const AUDITING_MODELS: &str = "sqlbuild.executor.auditing.models"; const CLONE_MODELS: &str = "sqlbuild.executor.clone.models"; -const SQL_TEST_ID_MODULE: &str = "sqlbuild.executor.testing.main._resource_id"; -const AUDIT_ID_MODULE: &str = "sqlbuild.executor.auditing.main.resource_id"; /// The executor result types that result output distinguishes. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -72,35 +71,12 @@ pub(crate) fn result_kind(result: &Bound<'_, PyAny>) -> PyResult { /// `sql_test_resource_id` of one SQL test result. pub(crate) fn sql_test_check_id(result: &Bound<'_, PyAny>) -> PyResult { - let fields: Vec<(&str, Bound<'_, PyAny>)> = - ["test_name", "source_path", "block_index", "case_name"] - .into_iter() - .map(|name| Ok((name, result.getattr(name)?))) - .collect::>()?; - call_with( - &imported(result.py(), SQL_TEST_ID_MODULE, "sql_test_resource_id")?, - &fields, - )? - .extract() + sql_test_identity(result, "test_name") } /// `audit_resource_id` of one audit result. pub(crate) fn audit_check_id(result: &Bound<'_, PyAny>) -> PyResult { - let fields: Vec<(&str, Bound<'_, PyAny>)> = [ - ("audit_name", "audit_name"), - ("attachment_kind", "attachment_kind"), - ("attached_target_kind", "attached_target_kind"), - ("attached_target_name", "attached_target_name"), - ("attached_column_name", "attached_column_name"), - ] - .into_iter() - .map(|(keyword, name)| Ok((keyword, result.getattr(name)?))) - .collect::>()?; - call_with( - &imported(result.py(), AUDIT_ID_MODULE, "audit_resource_id")?, - &fields, - )? - .extract() + audit_identity(result) } /// The canonical resource name and identifier of an executor result. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs index 890cf21372..b94c4fbf40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs @@ -3,11 +3,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::result_output::{ - documents, event_writer, integration_envelope, terminal_index, + aggregation, documents, event_writer, integration_envelope, resource_ids, terminal_index, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { terminal_index::register(module)?; + resource_ids::register(module)?; + aggregation::register(module)?; integration_envelope::register(module)?; event_writer::register(module)?; documents::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs new file mode 100644 index 0000000000..7e3c8f4812 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs @@ -0,0 +1,104 @@ +//! Canonical execution-protocol identities of SQL test cases and attached audits. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use sqlbuild_core::nested_progress::main::audit_resource_id::audit_resource_id as audit_id; +use sqlbuild_core::nested_progress::main::sql_test_resource_id::sql_test_resource_id as sql_test_id; + +/// The exact stable identity of one plain or parameterised SQL test. +#[pyfunction] +#[pyo3(signature = (*, test_name, source_path, block_index, case_name))] +pub(crate) fn sql_test_resource_id( + test_name: &str, + source_path: Bound<'_, PyAny>, + block_index: Option, + case_name: Option, +) -> PyResult { + let path: Option = if source_path.is_none() { + None + } else { + Some(source_path.call_method0("as_posix")?.extract()?) + }; + Ok(match (path, block_index, case_name) { + (Some(path), Some(block), Some(case)) => { + sql_test_id(test_name, Some((&path, block, &case))) + } + _ => sql_test_id(test_name, None), + }) +} + +/// The stable identity of one attached audit execution. +#[pyfunction] +#[pyo3(signature = (**options))] +fn audit_resource_id(options: Option<&Bound<'_, PyDict>>) -> PyResult { + let request = Request::new("audit_resource_id", options)?; + let target_kind: Option> = request + .optional("attached_target_kind")? + .filter(|kind| !kind.is_none()); + let kind: Bound<'_, PyAny> = match target_kind { + Some(kind) => kind, + None => request.get("attachment_kind")?, + }; + audit_identity_of( + &request.get("audit_name")?, + &kind, + ( + &request.get("attached_target_name")?, + &request.get("attached_column_name")?, + ), + ) +} + +fn audit_identity_of( + audit_name: &Bound<'_, PyAny>, + kind: &Bound<'_, PyAny>, + (target, column): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult { + let target: Option = target.extract()?; + let column: Option = column.extract()?; + Ok(audit_id( + &audit_name.extract::()?, + &kind.getattr("value")?.extract::()?, + (target.as_deref(), column.as_deref()), + )) +} + +/// `sql_test_resource_id` of one result or plan entry carrying `test_name_field` and case fields. +pub(crate) fn sql_test_identity( + result: &Bound<'_, PyAny>, + test_name_field: &str, +) -> PyResult { + sql_test_resource_id( + &result.getattr(test_name_field)?.extract::()?, + result.getattr("source_path")?, + result.getattr("block_index")?.extract()?, + result.getattr("case_name")?.extract()?, + ) +} + +/// `audit_resource_id` of one audit result. +pub(crate) fn audit_identity(result: &Bound<'_, PyAny>) -> PyResult { + let target_kind: Bound<'_, PyAny> = result.getattr("attached_target_kind")?; + let kind: Bound<'_, PyAny> = if target_kind.is_none() { + result.getattr("attachment_kind")? + } else { + target_kind + }; + audit_identity_of( + &result.getattr("audit_name")?, + &kind, + ( + &result.getattr("attached_target_name")?, + &result.getattr("attached_column_name")?, + ), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(sql_test_resource_id, module)?)?; + module.add_function(wrap_pyfunction!(audit_resource_id, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 8163395c66..30d5cc9aef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -8,7 +8,7 @@ use pyo3::exceptions::{PyOSError, PyTypeError, PyValueError}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; +use pyo3::types::{PyDict, PyStringMethods, PyTuple}; use pyo3::{ FromPyObject, IntoPyObject, IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction, }; @@ -34,15 +34,12 @@ use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::rules::findings::{FindingClasses, finding_fault}; use crate::bindings::_helpers::rules::selection::rules_error; use crate::bindings::_helpers::rules::sql_test_objects::{scenario_fact, sql_test_fact}; +use crate::bindings::_helpers::type_system::equality::types_equal; use crate::bindings::types::CompilerDetach; -const LEFT_ARGUMENT: &str = "left"; -const RIGHT_ARGUMENT: &str = "right"; -const DIALECT_ARGUMENT: &str = "dialect"; const NATIVE_MODULE: &str = "sqlbuild._native"; const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const CODE_IDENTITY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiled_code_identity"; -const TYPES_EQUAL_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; const DATACLASSES_MODULE: &str = "dataclasses"; const NATIVE_RULES_CACHE_FILE: &str = "native.json"; const NATIVE_RULES_MEMO_FILE: &str = "native-response.json"; @@ -67,7 +64,6 @@ struct ProjectRows<'py> { public_enums: Vec>, public_constants: Vec>, initial_findings: Vec>, - types_equal: Bound<'py, PyAny>, dialect: Bound<'py, PyAny>, } @@ -166,7 +162,6 @@ fn built_request(options: &Request<'_, '_>) -> PyResult { .get("initial_findings")? .try_iter()? .collect::>()?, - types_equal: imported(py, TYPES_EQUAL_MODULE, "types_equal")?, dialect, }; let facts: Bound<'_, NativeCompiledProject> = @@ -349,19 +344,6 @@ fn as_rules_error(py: Python<'_>, result: PyResult) -> PyResult { }) } -/// `left=`, `right=` and `dialect=`: `types_equal` takes keyword-only arguments. -fn type_arguments<'py>( - dialect: &Bound<'py, PyAny>, - left: &str, - right: &str, -) -> PyResult> { - let arguments: Bound<'py, PyDict> = PyDict::new(dialect.py()); - arguments.set_item(LEFT_ARGUMENT, left)?; - arguments.set_item(RIGHT_ARGUMENT, right)?; - arguments.set_item(DIALECT_ARGUMENT, dialect)?; - Ok(arguments) -} - fn built_models( project: &ProjectRows<'_>, request: &EvaluateRequest, @@ -369,10 +351,7 @@ fn built_models( ) -> PyResult> { let mut python_error: Option = None; let fallback = |left: &str, right: &str| -> Result { - match type_arguments(&project.dialect, left, right) - .and_then(|arguments| project.types_equal.call((), Some(&arguments))) - .and_then(|equal| equal.extract::()) - { + match types_equal(&project.dialect, left, right) { Ok(equal) => Ok(equal), Err(error) => { let message: String = error.to_string(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs index 43e3e99094..486212aaf4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs @@ -1,5 +1,6 @@ //! Scenario executor results read into plain rows and runtime artifact inputs. +use sqlbuild_analysis::compiler::main::scenario_expected_comparison_sql::scenario_expected_comparison_sql; use std::collections::HashSet; use std::path::{Path, PathBuf}; @@ -15,7 +16,6 @@ use sqlbuild_core::scenario_command::models::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::scenario_command::setup::text; const SUCCESS_STATUS: &str = "success"; const FAILED_STATUS: &str = "failed"; @@ -28,7 +28,6 @@ const LOCAL_DIR: &str = "local"; const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; const QUALIFIED_NAME_MODULE: &str = "sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name"; -const COMPARISON_SQL_MODULE: &str = "sqlbuild.executor.scenario.main.expected_comparison_sql"; fn items<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { value.try_iter()?.collect() @@ -262,25 +261,13 @@ fn expectation_sql( ], )? .extract()?; - let sql: String = call_with( - &imported( - py, - COMPARISON_SQL_MODULE, - "build_scenario_expected_comparison_sql", - )?, - &[ - ( - "actual_sql", - text(py, &format!("SELECT * FROM {actual_relation}")), - ), - ("expected_sql", expected.getattr("expected_sql")?), - ( - "set_difference_operator", - adapter.call_method0("render_set_difference_operator")?, - ), - ], - )? - .extract()?; + let sql: String = scenario_expected_comparison_sql( + &format!("SELECT * FROM {actual_relation}"), + &expected.getattr("expected_sql")?.extract::()?, + &adapter + .call_method0("render_set_difference_operator")? + .extract::()?, + ); expectations .expected .push((expected.getattr("model_name")?.extract()?, sql)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index df0faa4614..3a6644efe0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -30,6 +30,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::compile_outputs::runtime_target::write_test_runtime_target; use crate::bindings::_helpers::result_output::integration_values::json_value; +use crate::bindings::_helpers::result_output::resource_ids::sql_test_identity; const NATIVE_MODULE: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; @@ -39,7 +40,6 @@ const TEST_PLAN_MODULE: &str = "sqlbuild._native"; const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; const EXECUTOR_PIPELINE_MODELS: &str = "sqlbuild.executor.pipeline.models"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; -const RESOURCE_ID_MODULE: &str = "sqlbuild.executor.testing.main._resource_id"; const DECIMAL_KIND: &str = "decimal"; const PASS_OUTCOME: &str = "pass"; const EXPECT_LABEL: &str = "expect"; @@ -218,15 +218,7 @@ impl TestCallbackRelay { if group.is_empty() { group = UNKNOWN_GROUP.to_owned(); } - let resource_id: Bound<'_, PyAny> = call_with( - &imported(py, RESOURCE_ID_MODULE, "sql_test_resource_id")?, - &[ - ("test_name", result.getattr("test_name")?), - ("source_path", result.getattr("source_path")?), - ("block_index", result.getattr("block_index")?), - ("case_name", result.getattr("case_name")?), - ], - )?; + let resource_id: Bound<'_, PyAny> = text(py, &sql_test_identity(result, "test_name")?); call_with( &self.progress.bind(py).getattr("on_item_complete")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs index 4f39f527d9..8aa749df36 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs @@ -3,7 +3,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple}; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::analysis::sql_test_comparisons::describe_missing_expected_columns; +use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::sql_test_glue::constants::{COMPILE_TYPES, PLAN_MODELS}; use crate::bindings::_helpers::sql_test_glue::constants::{TABLE_FN, UDF}; use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; @@ -147,21 +148,14 @@ pub(super) fn expected_diagnostics( if expected.is_none() || available.is_none() { continue; } - let message = call_with( - &imported( - py, - "sqlbuild.executor.testing.main._missing_expected_columns", - "describe_missing_expected_columns", - )?, - &[ - ("model_name", name.clone()), - ("expected_columns", expected), - ("available_columns", available), - ], - )?; - if message.is_none() { + let Some(message) = describe_missing_expected_columns( + &name.extract::()?, + expected.extract()?, + available, + )? + else { continue; - } + }; let location = super::fixture_validation::fixture_location( test, &resource_types.getattr("SQL_TEST")?, @@ -171,7 +165,7 @@ pub(super) fn expected_diagnostics( "SQL test '{}': {}: {}", test.getattr("name")?.extract::()?, location, - message.extract::()? + message )); } Ok(result) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/equality.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/equality.rs new file mode 100644 index 0000000000..3af3c77ea6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/equality.rs @@ -0,0 +1,90 @@ +//! Semantic type equality for native callers, with the adapter wrapper's parse-fallback debug event. + +use std::collections::HashMap; +use std::sync::{Mutex, OnceLock, PoisonError}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_analysis::type_system::main::normalize_type::normalize_type; +use sqlbuild_analysis::type_system::models::{NormalizedType, TypeNormalization}; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::type_system::normalization::normalization_error; + +const DEBUG_EVENT_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; +const TYPE_NORMALIZATION_LOGGER: &str = "sqlbuild.adapter"; +const TYPE_PARSE_FALLBACK_MESSAGE: &str = "type normalization polyglot parse failed; falling back"; +const DEFAULT_DIALECT: &str = "generic"; +const CACHE_LIMIT: usize = 1024; + +type Cache = Mutex>; + +static NORMALIZED: OnceLock = OnceLock::new(); + +/// Whether two type strings are semantically equal under `dialect` (a `TypeDialect`, name or `None`). +pub(crate) fn types_equal(dialect: &Bound<'_, PyAny>, left: &str, right: &str) -> PyResult { + let name: String = if dialect.is_truthy()? { + dialect.str()?.extract()? + } else { + DEFAULT_DIALECT.to_owned() + }; + let left_type: NormalizedType = normalized(dialect, &name, left)?; + let right_type: NormalizedType = normalized(dialect, &name, right)?; + Ok(left_type == right_type) +} + +fn normalized(dialect: &Bound<'_, PyAny>, name: &str, type_sql: &str) -> PyResult { + let py: Python<'_> = dialect.py(); + let cache: &Cache = NORMALIZED.get_or_init(|| Mutex::new(HashMap::new())); + let key: (String, String) = (type_sql.to_owned(), name.to_owned()); + let cached: Option = cache + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(&key) + .cloned(); + if let Some(found) = cached { + return Ok(found); + } + let TypeNormalization { + normalized, + parse_error, + } = compiler_guard(|| Ok(normalize_type(type_sql, name)))? + .map_err(|error| normalization_error(py, &error))?; + if let Some(parse_error) = parse_error { + log_parse_fallback(dialect, type_sql, &parse_error)?; + } + let mut held = cache.lock().unwrap_or_else(PoisonError::into_inner); + if held.len() >= CACHE_LIMIT { + held.clear(); + } + held.insert(key, normalized.clone()); + Ok(normalized) +} + +fn log_parse_fallback( + dialect: &Bound<'_, PyAny>, + type_sql: &str, + parse_error: &str, +) -> PyResult<()> { + let py: Python<'_> = dialect.py(); + call_with( + &imported(py, DEBUG_EVENT_MODULE, "log_debug_event")?, + &[ + ( + "logger", + py.import("logging")? + .getattr("getLogger")? + .call1((TYPE_NORMALIZATION_LOGGER,))?, + ), + ( + "message", + PyString::new(py, TYPE_PARSE_FALLBACK_MESSAGE).into_any(), + ), + ("type_sql", PyString::new(py, type_sql).into_any()), + ("dialect", dialect.str()?.into_any()), + ("sqlbuild_error", PyString::new(py, parse_error).into_any()), + ], + )?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs index bcc09d3157..30a05c9b4f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod equality; pub(crate) mod normalization; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 4c229c0e09..da39e9d00a 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -55,6 +55,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { state_rows::registration::register(module)?; diagnostics::redaction::register(module)?; diagnostics::trackers::register(module)?; + analysis::sql_test_comparisons::register(module)?; Ok(()) } diff --git a/python-boundary.toml b/python-boundary.toml index 493b802784..1a723fac71 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -402,7 +402,7 @@ note = "Tracks the loaded user modules macro call results depend on, and runs a [[stays]] path = "compiler/python_nodes/_helpers/user_inventory.py" reason = "user-code" -note = "Builds Python nodes from loaded user functions: signatures, provider annotations and decorated dependencies." +note = "Reads loaded user node functions: provider parameter annotations, identities over live functions and decorated dependency names; graph construction is native." [[stays]] path = "compiler/discovery/_helpers/filesystem/core.py" @@ -464,16 +464,6 @@ path = "compiler/python_nodes/main/_hook_identities.py" reason = "user-code" note = "Builds identities of loaded user hook functions." -[[stays]] -path = "compiler/python_nodes/main/_graph.py" -reason = "user-code" -note = "Builds the Python node graph from loaded user functions." - -[[stays]] -path = "compiler/python_nodes/main/_run_selection.py" -reason = "user-code" -note = "Builds the Python node graph from loaded user functions for run selection." - [[stays]] path = "compiler/auditing/main/above.py" reason = "public-api" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 93df92569d..c41205c948 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -20,6 +20,8 @@ from sqlbuild.adapter.contract.models import ( ExpressionInferenceProfile, QueryResult, RelationInfo, + RowDiffSampling, + RowDiffTolerances, ) from sqlbuild.adapter.contract.types import ( AdapterExecute, @@ -290,17 +292,19 @@ from sqlbuild.diagnostics.models import ProcessResourceUsage from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import ( BuildExecutionResult, + FunctionExecutionResult, SchedulerState, SeedExecutionResult, ) from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult -from sqlbuild.executor.diff.models import DiffExecutionResult +from sqlbuild.executor.diff.models import DiffExecutionResult, RowDiffSamplingOverride from sqlbuild.executor.janitor.models import JanitorExecutionResult, JanitorPlan from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.python_nodes.models import ( PythonCheckExecutionResult, PythonNodeExecutionResult, ) +from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scenario.models import ScenarioRunResult, ScenarioSnapshotCaptureRunResult from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner @@ -325,9 +329,8 @@ from sqlbuild.lint.models import ( PreparedLintFiles, RuleFixResult, ) -from sqlbuild.lint.types import ( - RelationKeyRow, -) +from sqlbuild.lint.types import RelationKeyRow +from sqlbuild.microbatches.models import MicrobatchInterval from sqlbuild.presentation.models import DisplayOptions from sqlbuild.presentation.types import TransientLineOwner from sqlbuild.provider.classes.container import ProviderContainer @@ -4910,6 +4913,76 @@ def scope_test_ctes( ) -> list[tuple[str | None, list[tuple[str, str]]]]: ... def omitted_ceremonial_select(sql: str, syntax: dict[str, object]) -> int | None: ... def extract_sql_tests_json(request_json: str) -> str: ... +def parse_row_diff_tolerances( + *, raw: object, label: str = "row_diff_tolerances" +) -> RowDiffTolerances: ... +def parse_cli_tolerance_overrides(*, values: Sequence[str]) -> RowDiffTolerances: ... +def resolve_row_diff_sampling( + *, raw_row_limit: object, raw_seed: object, override: RowDiffSamplingOverride, label: str +) -> RowDiffSampling | None: ... +def merge_row_diff_tolerances( + *, base: RowDiffTolerances, override: RowDiffTolerances | None +) -> RowDiffTolerances: ... +def aggregate_build_result( + *, + model_results: tuple[ModelExecutionResult, ...], + seed_results: tuple[SeedExecutionResult, ...], + function_results: tuple[FunctionExecutionResult, ...], + load_results: tuple[LoadExecutionResult, ...], + test_results: tuple[SqlTestExecutionResult, ...], + source_audit_results: tuple[AuditExecutionResult, ...], + end_audit_results: tuple[AuditExecutionResult, ...], +) -> BuildExecutionResult: ... +def resolve_dag_selection( + *, dag: Mapping[str, Any], select: str, exclude: str | None = None +) -> frozenset[str]: ... +def load_sqlbuild_dag( + dag: Mapping[str, Any] | str | Path, *, error_class: type[Exception] +) -> Mapping[str, Any]: ... +def build_python_node_graph(*, discovered_inputs: DiscoveredProjectInputs) -> PythonNodeGraph: ... +def macro_module_load_plan( + modules: Sequence[tuple[str, str, str, Sequence[str]]], +) -> tuple[str, list[str], list[str]]: ... +def fit_artifact_logical_name( + *, logical_name: str, fixed_prefix: str, identifier_limit: int, artifact_label: str +) -> str: ... +def fit_auxiliary_relation_name(*, base_name: str, suffix: str, identifier_limit: int) -> str: ... +def merge_microbatch_intervals( + *, intervals: Sequence[MicrobatchInterval], cursor_type: str +) -> tuple[MicrobatchInterval, ...]: ... +def render_sql_test_comparison_sql( + test_entries: Sequence[SqlTestPlanEntry], + *, + set_difference_operator: str = "EXCEPT", + sql_analysis_dialect: str | None = None, + probe_step_index: int | None = None, +) -> list[str]: ... +def render_sql_test_difference_sample_sql( + *, + test_entry: SqlTestPlanEntry, + step: ChainStep, + direction: str, + sample_limit: int, + set_difference_operator: str = "EXCEPT", + sql_analysis_dialect: str | None = None, +) -> str: ... +def describe_missing_expected_columns( + *, model_name: str, expected_columns: Sequence[str], available_columns: Iterable[str] +) -> str | None: ... +def build_scenario_expected_comparison_sql( + *, actual_sql: str, expected_sql: str, set_difference_operator: str +) -> str: ... +def sql_test_resource_id( + *, test_name: str, source_path: PurePath | None, block_index: int | None, case_name: str | None +) -> str: ... +def audit_resource_id( + *, + audit_name: str, + attachment_kind: AuditAttachmentKind, + attached_target_kind: AttachedAuditTargetKind | None = None, + attached_target_name: str | None, + attached_column_name: str | None, +) -> str: ... def validated_model_cursor_intrinsics( *, sql: str, config_values: Mapping[str, object], model_name: str ) -> str: ... diff --git a/src/sqlbuild/adapter/relations/_helpers/identifier_fitting.py b/src/sqlbuild/adapter/relations/_helpers/identifier_fitting.py deleted file mode 100644 index 1b17b5bb0b..0000000000 --- a/src/sqlbuild/adapter/relations/_helpers/identifier_fitting.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Fit generated relation names within identifier limits measured in UTF-8 bytes.""" - -from __future__ import annotations - -import hashlib - -from sqlbuild.adapter.relations.constants import SHORTENED_LOGICAL_NAME_HASH_LENGTH -from sqlbuild.errors.contracts.exceptions import SharedInputError - - -def fit_artifact_logical_name_impl( - *, logical_name: str, fixed_prefix: str, identifier_limit: int, artifact_label: str -) -> str: - """Fit a readable logical component with a deterministic hash suffix.""" - - max_logical_length: int = identifier_limit - _byte_length(fixed_prefix) - if max_logical_length < 1: - raise SharedInputError( - f"{artifact_label} prefix '{fixed_prefix}' does not fit within identifier " - f"limit {identifier_limit}" - ) - if _byte_length(logical_name) <= max_logical_length: - return logical_name - - suffix_length: int = SHORTENED_LOGICAL_NAME_HASH_LENGTH + 1 - if max_logical_length <= suffix_length: - raise SharedInputError( - f"{artifact_label} name for '{logical_name}' cannot fit within identifier " - f"limit {identifier_limit}" - ) - logical_hash: str = hashlib.sha256(logical_name.encode("utf-8")).hexdigest()[ - :SHORTENED_LOGICAL_NAME_HASH_LENGTH - ] - prefix_length: int = max_logical_length - suffix_length - return f"{_truncate_bytes(value=logical_name, max_bytes=prefix_length)}_{logical_hash}" - - -def fit_auxiliary_relation_name_impl(*, base_name: str, suffix: str, identifier_limit: int) -> str: - """Append a suffix, shortening the base with a stable hash when the whole name is too long.""" - - if _byte_length(base_name) + _byte_length(suffix) <= identifier_limit: - return f"{base_name}{suffix}" - hash_part: str = ( - "_" - + hashlib.sha256(base_name.encode("utf-8")).hexdigest()[:SHORTENED_LOGICAL_NAME_HASH_LENGTH] - ) - kept_length: int = identifier_limit - _byte_length(suffix) - len(hash_part) - if kept_length < 1: - raise SharedInputError( - f"auxiliary relation suffix '{suffix}' for '{base_name}' cannot fit within " - f"identifier limit {identifier_limit}" - ) - return f"{_truncate_bytes(value=base_name, max_bytes=kept_length)}{hash_part}{suffix}" - - -def _byte_length(value: str) -> int: - return len(value.encode("utf-8")) - - -def _truncate_bytes(*, value: str, max_bytes: int) -> str: - """Keep the longest prefix whose UTF-8 encoding fits, never splitting a character.""" - - return value.encode("utf-8")[:max_bytes].decode("utf-8", errors="ignore") diff --git a/src/sqlbuild/adapter/relations/constants.py b/src/sqlbuild/adapter/relations/constants.py index 53d4ad02c5..e68a5951b2 100644 --- a/src/sqlbuild/adapter/relations/constants.py +++ b/src/sqlbuild/adapter/relations/constants.py @@ -3,7 +3,6 @@ INSPECTION_DESCRIBED_NAME_LIMIT: int = 3 INSPECTION_IN_LIST_LIMIT: int = 50 METADATA_NAME_FILTER_LIMIT: int = 250 -SHORTENED_LOGICAL_NAME_HASH_LENGTH: int = 8 STATEMENT_SEPARATOR: str = ";" QUOTED_IDENTIFIER_MIN_LENGTH: int = 2 DOUBLE_QUOTE: str = '"' diff --git a/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name.py b/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name.py deleted file mode 100644 index 2c601d9173..0000000000 --- a/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Generated relation name fitting entrypoint.""" - -from __future__ import annotations - -from sqlbuild.adapter.relations._helpers.identifier_fitting import fit_artifact_logical_name_impl - - -def fit_artifact_logical_name( - *, logical_name: str, fixed_prefix: str, identifier_limit: int, artifact_label: str -) -> str: - """Fit a readable logical component after a fixed prefix within an identifier limit.""" - - return fit_artifact_logical_name_impl( - logical_name=logical_name, - fixed_prefix=fixed_prefix, - identifier_limit=identifier_limit, - artifact_label=artifact_label, - ) diff --git a/src/sqlbuild/adapter/relations/main/fit_auxiliary_relation_name.py b/src/sqlbuild/adapter/relations/main/fit_auxiliary_relation_name.py deleted file mode 100644 index ec4f1a6dba..0000000000 --- a/src/sqlbuild/adapter/relations/main/fit_auxiliary_relation_name.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Auxiliary relation name fitting entrypoint.""" - -from __future__ import annotations - -from sqlbuild.adapter.relations._helpers.identifier_fitting import ( - fit_auxiliary_relation_name_impl, -) - - -def fit_auxiliary_relation_name(*, base_name: str, suffix: str, identifier_limit: int) -> str: - """Return ``base_name + suffix`` fitted within the limit, keeping the suffix and a base hash.""" - - return fit_auxiliary_relation_name_impl( - base_name=base_name, suffix=suffix, identifier_limit=identifier_limit - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py index 7473eecc9d..a6c0b0fcaa 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py @@ -3,8 +3,6 @@ from __future__ import annotations import ast -import hashlib -import heapq import inspect import sys import threading @@ -42,7 +40,6 @@ from sqlbuild.sql_values.models import SqlValue from sqlbuild.sql_values.types import SqlValueKind -_SYNTHETIC_PACKAGE_PREFIX: str = "_sqlbuild_project_macros_" _OBJECT_ENTRY_LENGTH: int = 2 _MACRO_MODULE_LOAD_LOCK: threading.RLock = threading.RLock() @@ -268,25 +265,22 @@ def _load_macro_modules(*, analyses: dict[str, _MacroModuleLoad]) -> dict[str, M def _load_macro_modules_locked(*, analyses: dict[str, _MacroModuleLoad]) -> dict[str, ModuleType]: - digest: str = hashlib.sha256( - "\0".join( - f"{name}\0{analyses[name].file.file_path}\0{analyses[name].file.contents}" - for name in sorted(analyses) - ).encode() - ).hexdigest()[:16] - root: str = f"{_SYNTHETIC_PACKAGE_PREFIX}{digest}" + root, package_names, load_order = _native.macro_module_load_plan( + [ + ( + name, + str(analysis.file.file_path), + analysis.file.contents, + list(analysis.module_dependencies), + ) + for name, analysis in analyses.items() + ] + ) synthetic_names: list[str] = [] previous_modules: dict[str, ModuleType] = {} modules: dict[str, ModuleType] = {} try: - package_names: set[str] = {root} - for name in analyses: - parts: list[str] = name.split(".")[:-1] - package_names.update( - f"{root}.{'/'.join(parts[:index]).replace('/', '.')}" - for index in range(1, len(parts) + 1) - ) - for package_name in sorted(package_names, key=lambda item: item.count(".")): + for package_name in package_names: package: ModuleType = ModuleType(package_name) package.__package__ = package_name package.__path__ = [] # type: ignore[attr-defined] @@ -296,21 +290,8 @@ def _load_macro_modules_locked(*, analyses: dict[str, _MacroModuleLoad]) -> dict sys.modules[package_name] = package synthetic_names.append(package_name) - remaining_dependencies: dict[str, int] = { - name: len(analysis.module_dependencies) for name, analysis in analyses.items() - } - dependents: dict[str, list[str]] = {} - for name, analysis in analyses.items(): - for dependency in analysis.module_dependencies: - dependents.setdefault(dependency, []).append(name) - ready: list[str] = [ - name - for name, dependency_count in remaining_dependencies.items() - if dependency_count == 0 - ] - heapq.heapify(ready) - while ready: - name: str = heapq.heappop(ready) + name: str + for name in load_order: analysis: _MacroModuleLoad = analyses[name] synthetic_name: str = f"{root}.{name}" module: ModuleType = ModuleType(synthetic_name) @@ -331,10 +312,6 @@ def _load_macro_modules_locked(*, analyses: dict[str, _MacroModuleLoad]) -> dict f"Failed to load macros from '{analysis.file.file_path}': {error}" ) from error modules[name] = module - for dependent in dependents.get(name, ()): - remaining_dependencies[dependent] -= 1 - if remaining_dependencies[dependent] == 0: - heapq.heappush(ready, dependent) return modules finally: for synthetic_name in reversed(synthetic_names): diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py b/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py index 9a94fa2409..c16204c3a8 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py @@ -1,34 +1,15 @@ -"""Python-node inventory over loaded user functions: signatures, providers and dependencies.""" +"""Python-node facts read from loaded user functions: identities, providers and dependencies.""" from __future__ import annotations import inspect from collections.abc import Callable -from dataclasses import replace from pathlib import Path -from typing import Protocol, get_type_hints +from typing import get_type_hints -from sqlbuild._native import python_node_graph_indexes -from sqlbuild.compiler.discovery.models import ( - DiscoveredAssetFunction, - DiscoveredCheckFunction, - DiscoveredLoaderFunction, - DiscoveredProjectInputs, - DiscoveredProvider, - DiscoveredProviderUsage, - DiscoveredTaskFunction, -) +from sqlbuild.compiler.discovery.models import DiscoveredProvider, DiscoveredProviderUsage from sqlbuild.compiler.python_nodes._helpers.identity import build_python_identity -from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonAssetMetadata, - DiscoveredPythonCheckMetadata, - DiscoveredPythonLoaderMetadata, - DiscoveredPythonNode, - DiscoveredPythonTaskMetadata, - PythonNodeDependencyEdge, - PythonNodeGraph, -) -from sqlbuild.compiler.python_nodes.types import PythonNodeKind +from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity from sqlbuild.provider.classes.provider import Provider from sqlbuild.python_nodes.main.read_asset_definition import read_asset_definition from sqlbuild.python_nodes.main.read_check_definition import read_check_definition @@ -38,265 +19,38 @@ AssetDefinition, CheckDefinition, LoaderDefinition, - SqlResourceRef, TaskDefinition, ) -def build_python_node_graph(*, discovered_inputs: DiscoveredProjectInputs) -> PythonNodeGraph: - """Build the internal executable Python-node graph from discovered project inputs.""" - - nodes: tuple[DiscoveredPythonNode, ...] = build_python_nodes( - loader_functions=discovered_inputs.loader_functions, - task_functions=discovered_inputs.task_functions, - asset_functions=discovered_inputs.asset_functions, - check_functions=discovered_inputs.check_functions, - providers=discovered_inputs.providers, - ) - edges, upstream, downstream, tags, paths = python_node_graph_indexes( - [ - (node.name, node.tags, node.relative_path.as_posix(), upstream_names) - for node, upstream_names in zip(nodes, _upstream_node_names(nodes), strict=True) - ] - ) - return PythonNodeGraph( - nodes=nodes, - dependency_edges=tuple( - PythonNodeDependencyEdge(upstream_name=upstream_name, downstream_name=downstream_name) - for upstream_name, downstream_name in edges - ), - upstream_deps={name: tuple(names) for name, names in upstream}, - downstream_deps={name: tuple(names) for name, names in downstream}, - tag_index={tag: frozenset(names) for tag, names in tags}, - path_index=dict(paths), - nodes_by_name={node.name: node for node in nodes}, - nodes_by_typed_selector={f"{node.kind.value}:{node.name}": node for node in nodes}, - ) - - -def build_python_nodes( - *, - loader_functions: tuple[DiscoveredLoaderFunction, ...] = (), - task_functions: tuple[DiscoveredTaskFunction, ...] = (), - asset_functions: tuple[DiscoveredAssetFunction, ...] = (), - check_functions: tuple[DiscoveredCheckFunction, ...] = (), - providers: tuple[DiscoveredProvider, ...] = (), -) -> tuple[DiscoveredPythonNode, ...]: - """Return the shared internal node view for all executable Python node kinds.""" - - nodes: list[DiscoveredPythonNode] = [] - provider_by_name: dict[str, DiscoveredProvider] = { - discovered_provider.name: discovered_provider for discovered_provider in providers - } - loader_function: DiscoveredLoaderFunction - for loader_function in loader_functions: - nodes.append(_build_loader_node(loader=loader_function, provider_by_name=provider_by_name)) - task_function: DiscoveredTaskFunction - for task_function in task_functions: - nodes.append(_build_task_node(task=task_function, provider_by_name=provider_by_name)) - asset_function: DiscoveredAssetFunction - for asset_function in asset_functions: - nodes.append(_build_asset_node(asset=asset_function, provider_by_name=provider_by_name)) - check_function: DiscoveredCheckFunction - for check_function in check_functions: - nodes.append(_build_check_node(check=check_function, provider_by_name=provider_by_name)) - return tuple(nodes) - - -def _upstream_node_names(nodes: tuple[DiscoveredPythonNode, ...]) -> list[list[str]]: - """Each node's Python-node dependencies, resolving live callables to node names.""" - - node_by_dependency_key: dict[object | tuple[str, str], DiscoveredPythonNode] = { - node.function: node for node in nodes - } - node: DiscoveredPythonNode - for node in nodes: - node_by_dependency_key[("name", node.name)] = node - upstream_names: list[list[str]] = [] - for node in nodes: - names: list[str] = [] - dependency: Callable[..., object] | SqlResourceRef - for dependency in node.depends_on: - if isinstance(dependency, SqlResourceRef): - continue - upstream_node: DiscoveredPythonNode | None = node_by_dependency_key.get( - _python_node_dependency_key(dependency) - ) - if upstream_node is not None: - names.append(upstream_node.name) - upstream_names.append(names) - return upstream_names - - -class _DiscoveredPythonFunction(Protocol): - @property - def file_path(self) -> Path: ... - - @property - def relative_path(self) -> Path: ... - - @property - def name(self) -> str: ... - - @property - def function(self) -> Callable[..., object]: ... - - @property - def depends_on(self) -> tuple[Callable[..., object] | SqlResourceRef, ...]: ... - - -class _DocumentedPythonFunction(_DiscoveredPythonFunction, Protocol): - @property - def tags(self) -> tuple[str, ...]: ... - - @property - def group(self) -> str | None: ... - - @property - def description(self) -> str | None: ... - - @property - def meta(self) -> dict[str, object] | None: ... - - -def _build_loader_node( - *, loader: DiscoveredLoaderFunction, provider_by_name: dict[str, DiscoveredProvider] -) -> DiscoveredPythonNode: - return replace( - _build_python_node( - kind=PythonNodeKind.LOADER, - discovered=loader, - provider_by_name=provider_by_name, - decorator_config={ - "columns": loader.columns, - "contract": loader.contract, - "cursor_column": loader.cursor_column, - "destination": loader.destination, - "unique_key": loader.unique_key, - "write_strategy": loader.write_strategy.value - if loader.write_strategy is not None - else None, - }, - ), - description=loader.description, - loader=DiscoveredPythonLoaderMetadata( - destination=loader.destination, - write_strategy=loader.write_strategy, - cursor_column=loader.cursor_column, - unique_key=loader.unique_key, - columns=loader.columns, - contract=loader.contract, - connection_mode=loader.connection_mode, - ), - ) - - -def _build_task_node( - *, task: DiscoveredTaskFunction, provider_by_name: dict[str, DiscoveredProvider] -) -> DiscoveredPythonNode: - return replace( - _build_documented_python_node( - kind=PythonNodeKind.TASK, - discovered=task, - provider_by_name=provider_by_name, - decorator_config={}, - ), - task=DiscoveredPythonTaskMetadata(retry=task.retry), - ) - - -def _build_asset_node( - *, asset: DiscoveredAssetFunction, provider_by_name: dict[str, DiscoveredProvider] -) -> DiscoveredPythonNode: - return replace( - _build_documented_python_node( - kind=PythonNodeKind.ASSET, - discovered=asset, - provider_by_name=provider_by_name, - decorator_config={"columns": asset.columns, "column_lineage": asset.column_lineage}, - ), - asset=DiscoveredPythonAssetMetadata( - columns=asset.columns, - column_lineage=asset.column_lineage, - retry=asset.retry, - ), - ) - - -def _build_check_node( - *, check: DiscoveredCheckFunction, provider_by_name: dict[str, DiscoveredProvider] -) -> DiscoveredPythonNode: - return replace( - _build_documented_python_node( - kind=PythonNodeKind.CHECK, - discovered=check, - provider_by_name=provider_by_name, - decorator_config={"severity": check.severity.value}, - ), - check=DiscoveredPythonCheckMetadata(severity=check.severity), - ) - - -def _build_documented_python_node( - *, - kind: PythonNodeKind, - discovered: _DocumentedPythonFunction, - provider_by_name: dict[str, DiscoveredProvider], - decorator_config: dict[str, object], -) -> DiscoveredPythonNode: - return replace( - _build_python_node( - kind=kind, - discovered=discovered, - provider_by_name=provider_by_name, - decorator_config={ - **decorator_config, - "description": discovered.description, - "group": discovered.group, - "meta": discovered.meta, - "tags": discovered.tags, - }, - ), - tags=discovered.tags, - group=discovered.group, - description=discovered.description, - meta=discovered.meta, - ) - - -def _build_python_node( +def python_node_user_facts( *, - kind: PythonNodeKind, - discovered: _DiscoveredPythonFunction, + kind: str, + name: str, + function: Callable[..., object], + file_path: Path, provider_by_name: dict[str, DiscoveredProvider], decorator_config: dict[str, object], -) -> DiscoveredPythonNode: - """Build the kind-independent node view; decorator_config is exactly the identity input.""" - - return DiscoveredPythonNode( - kind=kind, - file_path=discovered.file_path, - relative_path=discovered.relative_path, - name=discovered.name, - function=discovered.function, - depends_on=discovered.depends_on, - sql_deps=_sql_deps(discovered.depends_on), - provider_usages=_provider_usages( - function=discovered.function, provider_by_name=provider_by_name - ), - identity=build_python_identity( - node_type=kind.value, - node_name=discovered.name, - function=discovered.function, - project_dir=discovered.file_path.parent, +) -> tuple[tuple[DiscoveredProviderUsage, ...], PythonNodeIdentity]: + """Read provider parameters and the identity of one loaded user node function.""" + + return ( + provider_usages(function=function, provider_by_name=provider_by_name), + build_python_identity( + node_type=kind, + node_name=name, + function=function, + project_dir=file_path.parent, decorator_config=decorator_config, ), ) -def _provider_usages( +def provider_usages( *, function: Callable[..., object], provider_by_name: dict[str, DiscoveredProvider] ) -> tuple[DiscoveredProviderUsage, ...]: + """Provider parameters of one loaded user function and their annotation classes.""" + usages: list[DiscoveredProviderUsage] = [] type_hints: dict[str, object] = get_type_hints(function) parameter: inspect.Parameter @@ -321,7 +75,9 @@ def _provider_usages( return tuple(usages) -def _python_node_dependency_key(dependency: object) -> object | tuple[str, str]: +def python_node_dependency_key(dependency: object) -> object | tuple[str, str]: + """The node name a decorated dependency declares, or the dependency itself.""" + loader_definition: LoaderDefinition | None = ( read_loader_definition(dependency) if callable(dependency) else None ) @@ -343,11 +99,3 @@ def _python_node_dependency_key(dependency: object) -> object | tuple[str, str]: if check_definition is not None: return ("name", check_definition.name) return dependency - - -def _sql_deps( - dependencies: tuple[Callable[..., object] | SqlResourceRef, ...], -) -> tuple[SqlResourceRef, ...]: - return tuple( - dependency for dependency in dependencies if isinstance(dependency, SqlResourceRef) - ) diff --git a/src/sqlbuild/compiler/python_nodes/main/_graph.py b/src/sqlbuild/compiler/python_nodes/main/_graph.py deleted file mode 100644 index b968382904..0000000000 --- a/src/sqlbuild/compiler/python_nodes/main/_graph.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Public Python-node graph construction entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.python_nodes._helpers.user_inventory import build_python_node_graph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph - - -def build_discovered_python_node_graph( - *, discovered_inputs: DiscoveredProjectInputs -) -> PythonNodeGraph: - """Build the internal executable Python-node graph from discovered inputs.""" - - return build_python_node_graph(discovered_inputs=discovered_inputs) diff --git a/src/sqlbuild/compiler/python_nodes/main/_run_selection.py b/src/sqlbuild/compiler/python_nodes/main/_run_selection.py deleted file mode 100644 index c78145aebb..0000000000 --- a/src/sqlbuild/compiler/python_nodes/main/_run_selection.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Run-command Python-node selection entrypoint.""" - -from __future__ import annotations - -from sqlbuild._native import compiled_graph_keys, resolve_python_sql_run_selection -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes._helpers.user_inventory import build_python_node_graph -from sqlbuild.compiler.python_nodes.models import PythonSqlRunSelection - - -def resolve_python_sql_run_selection_from_inputs( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - project_graph: ProjectGraph, - discovered_inputs: DiscoveredProjectInputs, -) -> PythonSqlRunSelection: - """Resolve run selectors using discovered Python-node project inputs.""" - - sql_keys, python_node_names = resolve_python_sql_run_selection( - project_graph=project_graph, - python_graph=build_python_node_graph(discovered_inputs=discovered_inputs), - select=select, - exclude=exclude, - ) - return PythonSqlRunSelection( - sql_keys=compiled_graph_keys(sql_keys), python_node_names=frozenset(python_node_names) - ) diff --git a/src/sqlbuild/executor/auditing/main/resource_id.py b/src/sqlbuild/executor/auditing/main/resource_id.py deleted file mode 100644 index 52de73a259..0000000000 --- a/src/sqlbuild/executor/auditing/main/resource_id.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Canonical resource identity for an attached audit execution.""" - -from sqlbuild.compiler.auditing.types import AuditAttachmentKind -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind - - -def audit_resource_id( - *, - audit_name: str, - attachment_kind: AuditAttachmentKind, - attached_target_kind: AttachedAuditTargetKind | None = None, - attached_target_name: str | None, - attached_column_name: str | None, -) -> str: - """Format the stable execution-protocol identity of one attached audit.""" - - identity_kind: str = ( - attached_target_kind.value if attached_target_kind is not None else attachment_kind.value - ) - - parts: tuple[str | None, ...] = ( - "audit", - audit_name, - identity_kind, - attached_target_name, - attached_column_name, - ) - return ":".join(filter(None, parts)) diff --git a/src/sqlbuild/executor/build/_helpers/aggregation.py b/src/sqlbuild/executor/build/_helpers/aggregation.py deleted file mode 100644 index f76e65b1af..0000000000 --- a/src/sqlbuild/executor/build/_helpers/aggregation.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Build result aggregation helpers.""" - -from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.executor.run.models import ModelExecutionResult - - -def count_insufficient_audits( - *, - model_results: tuple[ModelExecutionResult, ...], - source_audit_results: tuple[AuditExecutionResult, ...], - end_audit_results: tuple[AuditExecutionResult, ...], -) -> int: - """Count visible non-blocking insufficient measurement outcomes.""" - - results: list[AuditExecutionResult] = [*source_audit_results, *end_audit_results] - for model_result in model_results: - results.extend(model_result.audit_results) - return sum(result.outcome == AuditOutcome.INSUFFICIENT for result in results) diff --git a/src/sqlbuild/executor/build/_helpers/end_audits.py b/src/sqlbuild/executor/build/_helpers/end_audits.py index f19dcfac46..3169b537d9 100644 --- a/src/sqlbuild/executor/build/_helpers/end_audits.py +++ b/src/sqlbuild/executor/build/_helpers/end_audits.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import audit_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope from sqlbuild.compiler.compile.models import CompiledRelationLocation @@ -14,7 +15,6 @@ from sqlbuild.executor.auditing.main.publish_completed_audit_results import ( publish_completed_audit_results, ) -from sqlbuild.executor.auditing.main.resource_id import audit_resource_id from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.runtime.observability.classes.resource_attempt_lifecycle import ( ResourceAttemptLifecycle, diff --git a/src/sqlbuild/executor/build/_helpers/source_audits.py b/src/sqlbuild/executor/build/_helpers/source_audits.py index b1c6b2fcc7..17da4ee61c 100644 --- a/src/sqlbuild/executor/build/_helpers/source_audits.py +++ b/src/sqlbuild/executor/build/_helpers/source_audits.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import audit_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope from sqlbuild.compiler.compile.models import CompiledObjectKey @@ -15,7 +16,6 @@ from sqlbuild.executor.auditing.main.publish_completed_audit_results import ( publish_completed_audit_results, ) -from sqlbuild.executor.auditing.main.resource_id import audit_resource_id from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build._helpers.blocking import downstream_blocked_keys from sqlbuild.executor.build.models import SourceAuditRunResult diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 892158d337..6284493d5c 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -15,7 +15,7 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, sql_test_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import RelationInfo @@ -108,7 +108,6 @@ from sqlbuild.executor.seed.main._execute import execute_seed from sqlbuild.executor.testing.constants import SQL_TEST_ENTRY_MISSING_CODE from sqlbuild.executor.testing.main._execute import execute_sql_test -from sqlbuild.executor.testing.main._resource_id import sql_test_resource_id from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome from sqlbuild.microbatches.classes.direct_store import ( diff --git a/src/sqlbuild/executor/build/main/_aggregate_result.py b/src/sqlbuild/executor/build/main/_aggregate_result.py deleted file mode 100644 index 861cdbd9fd..0000000000 --- a/src/sqlbuild/executor/build/main/_aggregate_result.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Public entry for typed build result aggregation.""" - -from __future__ import annotations - -from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.executor.build._helpers.aggregation import count_insufficient_audits -from sqlbuild.executor.build.constants import BUILD_SOURCE_FRESHNESS_BLOCKED_CODE -from sqlbuild.executor.build.models import ( - BuildExecutionResult, - FunctionExecutionResult, - SeedExecutionResult, -) -from sqlbuild.executor.build.types import BuildStatus -from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.run.models import ModelExecutionResult -from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.executor.testing.models import SqlTestExecutionResult -from sqlbuild.executor.testing.types import SqlTestOutcome - - -def aggregate_build_result( - *, - model_results: tuple[ModelExecutionResult, ...], - seed_results: tuple[SeedExecutionResult, ...], - function_results: tuple[FunctionExecutionResult, ...], - load_results: tuple[LoadExecutionResult, ...], - test_results: tuple[SqlTestExecutionResult, ...], - source_audit_results: tuple[AuditExecutionResult, ...], - end_audit_results: tuple[AuditExecutionResult, ...], -) -> BuildExecutionResult: - """Compute aggregate counts and overall build status.""" - - resource_counts: tuple[int, int, int, int] = _aggregate_resources( - model_results=model_results, - seed_results=seed_results, - function_results=function_results, - load_results=load_results, - test_results=test_results, - ) - success_count: int = resource_counts[0] - failure_count: int = resource_counts[1] - skipped_count: int = resource_counts[2] - warning_count: int = resource_counts[3] - insufficient_count: int = count_insufficient_audits( - model_results=model_results, - source_audit_results=source_audit_results, - end_audit_results=end_audit_results, - ) - audit_counts: tuple[int, int, bool] = _aggregate_audits( - source_audit_results=source_audit_results, - end_audit_results=end_audit_results, - ) - failure_count += audit_counts[0] - warning_count += audit_counts[1] - status: BuildStatus = ( - BuildStatus.FAILED if failure_count > 0 or audit_counts[2] else BuildStatus.SUCCESS - ) - return BuildExecutionResult( - status=status, - model_results=model_results, - seed_results=seed_results, - function_results=function_results, - load_results=load_results, - test_results=test_results, - source_audit_results=source_audit_results, - end_audit_results=end_audit_results, - success_count=success_count, - failure_count=failure_count, - skipped_count=skipped_count, - warning_count=warning_count, - insufficient_count=insufficient_count, - ) - - -def _aggregate_resources( - *, - model_results: tuple[ModelExecutionResult, ...], - seed_results: tuple[SeedExecutionResult, ...], - function_results: tuple[FunctionExecutionResult, ...], - load_results: tuple[LoadExecutionResult, ...], - test_results: tuple[SqlTestExecutionResult, ...], -) -> tuple[int, int, int, int]: - success_count: int = 0 - failure_count: int = 0 - skipped_count: int = 0 - warning_count: int = 0 - for model_result in model_results: - if model_result.status == ExecutionStatus.SUCCESS: - success_count += 1 - elif model_result.status == ExecutionStatus.FAILED: - failure_count += 1 - elif model_result.status == ExecutionStatus.SKIPPED: - skipped_count += 1 - if model_result.error_code == BUILD_SOURCE_FRESHNESS_BLOCKED_CODE: - failure_count += 1 - warning_count += len(model_result.warning_messages) - warning_count += sum( - audit_result.outcome == AuditOutcome.WARN for audit_result in model_result.audit_results - ) - for seed_result in seed_results: - success_count += seed_result.status == ExecutionStatus.SUCCESS - failure_count += seed_result.status == ExecutionStatus.FAILED - skipped_count += seed_result.status == ExecutionStatus.SKIPPED - for function_result in function_results: - success_count += function_result.status == ExecutionStatus.SUCCESS - failure_count += function_result.status == ExecutionStatus.FAILED - skipped_count += function_result.status == ExecutionStatus.SKIPPED - warning_count += len(function_result.warning_messages) - for load_result in load_results: - success_count += load_result.status == ExecutionStatus.SUCCESS - failure_count += load_result.status == ExecutionStatus.FAILED - skipped_count += load_result.status == ExecutionStatus.SKIPPED - for test_result in test_results: - success_count += test_result.outcome == SqlTestOutcome.PASS - failure_count += test_result.outcome != SqlTestOutcome.PASS - return success_count, failure_count, skipped_count, warning_count - - -def _aggregate_audits( - *, - source_audit_results: tuple[AuditExecutionResult, ...], - end_audit_results: tuple[AuditExecutionResult, ...], -) -> tuple[int, int, bool]: - source_failures: int = sum( - result.outcome == AuditOutcome.ERROR for result in source_audit_results - ) - warnings: int = sum( - result.outcome == AuditOutcome.WARN - for result in (*source_audit_results, *end_audit_results) - ) - any_end_error: bool = any(result.outcome == AuditOutcome.ERROR for result in end_audit_results) - return source_failures, warnings, any_end_error diff --git a/src/sqlbuild/executor/build/main/_execute.py b/src/sqlbuild/executor/build/main/_execute.py index ba81c28583..27b9768f1b 100644 --- a/src/sqlbuild/executor/build/main/_execute.py +++ b/src/sqlbuild/executor/build/main/_execute.py @@ -5,6 +5,7 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import aggregate_build_result from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.relation_metadata_cache_scope import ( relation_metadata_cache_scope, @@ -24,7 +25,6 @@ reconcile_retention_after_build, ) from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler -from sqlbuild.executor.build.main._aggregate_result import aggregate_build_result from sqlbuild.executor.build.models import ( BuildCallbacks, BuildCustomizations, diff --git a/src/sqlbuild/executor/diff/_helpers/config.py b/src/sqlbuild/executor/diff/_helpers/config.py deleted file mode 100644 index 0ec3179925..0000000000 --- a/src/sqlbuild/executor/diff/_helpers/config.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Configuration helpers for diff execution.""" - -from __future__ import annotations - -from decimal import Decimal, InvalidOperation -from typing import cast - -from sqlbuild.adapter.contract.models import RowDiffSampling, RowDiffTolerance, RowDiffTolerances -from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.executor.diff.constants import ( - ROW_DIFF_ABSOLUTE_TOLERANCE_KEY, - ROW_DIFF_RELATIVE_TOLERANCE_KEY, - ROW_DIFF_TOLERANCE_KEYS, -) -from sqlbuild.executor.diff.models import RowDiffSamplingOverride - - -def resolve_row_diff_sampling( - *, - raw_row_limit: object, - raw_seed: object, - override: RowDiffSamplingOverride, - label: str, -) -> RowDiffSampling | None: - """Resolve model sampling configuration with invocation overrides.""" - - if override.exhaustive: - return None - row_limit: int | None = _parse_optional_non_negative_integer( - raw=override.row_limit if override.row_limit is not None else raw_row_limit, - label=f"{label}.row_diff_sample_rows", - ) - if row_limit in (None, 0): - return None - seed: int | None = _parse_optional_integer( - raw=override.seed if override.seed is not None else raw_seed, - label=f"{label}.row_diff_sample_seed", - ) - return RowDiffSampling(row_limit=row_limit, seed=seed or 0) - - -def parse_row_diff_tolerances( - *, - raw: object, - label: str = "row_diff_tolerances", -) -> RowDiffTolerances: - """Parse raw model config into typed row diff tolerances.""" - - if raw is None: - return RowDiffTolerances() - if not isinstance(raw, dict): - raise ExecutorInputError(f"{label} must be a mapping", code="X401") - raw_mapping: dict[str, object] = cast(dict[str, object], raw) - - by_type: dict[str, RowDiffTolerance] = _parse_tolerance_section( - raw=raw_mapping.get("by_type"), - label=f"{label}.by_type", - normalize_key=True, - ) - by_column: dict[str, RowDiffTolerance] = _parse_tolerance_section( - raw=raw_mapping.get("by_column"), - label=f"{label}.by_column", - normalize_key=False, - ) - return RowDiffTolerances(by_type=by_type, by_column=by_column) - - -def parse_cli_tolerance_overrides(*, values: tuple[str, ...]) -> RowDiffTolerances: - """Parse repeated COLUMN:KIND=VALUE CLI tolerance specifications.""" - - raw_by_column: dict[str, dict[str, str]] = {} - for value in values: - column_and_kind, separator, amount = value.partition("=") - column, kind_separator, kind = column_and_kind.partition(":") - if ( - not separator - or not kind_separator - or not column - or kind not in ROW_DIFF_TOLERANCE_KEYS - or not amount - ): - raise ExecutorInputError( - "diff --tolerance must use COLUMN:absolute=VALUE or COLUMN:relative=VALUE", - code="X408", - ) - raw_by_column.setdefault(column, {})[kind] = amount - return parse_row_diff_tolerances(raw={"by_column": raw_by_column}) - - -def merge_row_diff_tolerances( - *, base: RowDiffTolerances, override: RowDiffTolerances | None -) -> RowDiffTolerances: - """Overlay invocation tolerance rules on compiled model rules.""" - - if override is None: - return base - return RowDiffTolerances( - by_type={**base.by_type, **override.by_type}, - by_column={**base.by_column, **override.by_column}, - sampling=base.sampling, - ) - - -def _parse_tolerance_section( - *, - raw: object, - label: str, - normalize_key: bool, -) -> dict[str, RowDiffTolerance]: - if raw is None: - return {} - if not isinstance(raw, dict): - raise ExecutorInputError(f"{label} must be a mapping", code="X401") - raw_mapping: dict[object, object] = cast(dict[object, object], raw) - - parsed: dict[str, RowDiffTolerance] = {} - key: object - rule: object - for key, rule in raw_mapping.items(): - if not isinstance(key, str) or not key: - raise ExecutorInputError(f"{label} keys must be non-empty strings", code="X402") - parsed_key: str = key.lower() if normalize_key else key - parsed[parsed_key] = _parse_tolerance_rule(raw=rule, label=f"{label}.{key}") - return parsed - - -def _parse_tolerance_rule(*, raw: object, label: str) -> RowDiffTolerance: - if not isinstance(raw, dict): - raise ExecutorInputError(f"{label} must be a mapping", code="X401") - raw_mapping: dict[str, object] = cast(dict[str, object], raw) - - unsupported_keys: tuple[str, ...] = tuple( - str(key) for key in raw_mapping if key not in ROW_DIFF_TOLERANCE_KEYS - ) - if unsupported_keys: - unsupported: str = ", ".join(unsupported_keys) - raise ExecutorInputError(f"{label} contains unsupported keys: {unsupported}", code="X403") - - absolute: Decimal | None = _parse_optional_decimal( - raw=raw_mapping.get(ROW_DIFF_ABSOLUTE_TOLERANCE_KEY), - label=f"{label}.absolute", - ) - relative: Decimal | None = _parse_optional_decimal( - raw=raw_mapping.get(ROW_DIFF_RELATIVE_TOLERANCE_KEY), - label=f"{label}.relative", - ) - if absolute is None and relative is None: - raise ExecutorInputError(f"{label} must define absolute or relative", code="X404") - return RowDiffTolerance(absolute=absolute, relative=relative) - - -def _parse_optional_decimal(*, raw: object, label: str) -> Decimal | None: - if raw is None: - return None - if isinstance(raw, bool): - raise ExecutorInputError(f"{label} must be numeric", code="X405") - if not isinstance(raw, str | int | float | Decimal): - raise ExecutorInputError(f"{label} must be numeric", code="X405") - try: - return Decimal(str(raw)) - except InvalidOperation as error: - raise ExecutorInputError(f"{label} must be numeric", code="X405") from error - - -def _parse_optional_integer(*, raw: object, label: str) -> int | None: - if raw is None: - return None - if isinstance(raw, bool) or not isinstance(raw, int): - raise ExecutorInputError(f"{label} must be an integer", code="X406") - return raw - - -def _parse_optional_non_negative_integer(*, raw: object, label: str) -> int | None: - value: int | None = _parse_optional_integer(raw=raw, label=label) - if value is not None and value < 0: - raise ExecutorInputError(f"{label} must be zero or greater", code="X407") - return value diff --git a/src/sqlbuild/executor/diff/_helpers/execution.py b/src/sqlbuild/executor/diff/_helpers/execution.py index 4ed4af7c97..72d630ed94 100644 --- a/src/sqlbuild/executor/diff/_helpers/execution.py +++ b/src/sqlbuild/executor/diff/_helpers/execution.py @@ -5,6 +5,11 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import ( + merge_row_diff_tolerances, + parse_row_diff_tolerances, + resolve_row_diff_sampling, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ( ColumnInfo, @@ -16,11 +21,6 @@ ) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.diff._helpers.bounds import resolve_bounded_cursors -from sqlbuild.executor.diff._helpers.config import ( - merge_row_diff_tolerances, - parse_row_diff_tolerances, - resolve_row_diff_sampling, -) from sqlbuild.executor.diff._helpers.selection import ( get_row_diff_exclude_columns, get_unique_key, diff --git a/src/sqlbuild/executor/diff/constants.py b/src/sqlbuild/executor/diff/constants.py index 5387092604..807a3ff99f 100644 --- a/src/sqlbuild/executor/diff/constants.py +++ b/src/sqlbuild/executor/diff/constants.py @@ -1,9 +1,4 @@ """Constants for diff execution.""" -ROW_DIFF_ABSOLUTE_TOLERANCE_KEY: str = "absolute" -ROW_DIFF_RELATIVE_TOLERANCE_KEY: str = "relative" DIFF_INPUT_KIND_MODEL: str = "model" DIFF_INPUT_KIND_QUERY: str = "query" -ROW_DIFF_TOLERANCE_KEYS: frozenset[str] = frozenset( - (ROW_DIFF_ABSOLUTE_TOLERANCE_KEY, ROW_DIFF_RELATIVE_TOLERANCE_KEY) -) diff --git a/src/sqlbuild/executor/diff/main/_config.py b/src/sqlbuild/executor/diff/main/_config.py deleted file mode 100644 index 6177f157ad..0000000000 --- a/src/sqlbuild/executor/diff/main/_config.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public diff invocation configuration entrypoints.""" - -from sqlbuild.adapter.contract.models import RowDiffTolerances -from sqlbuild.executor.diff._helpers.config import ( - parse_cli_tolerance_overrides as _parse_cli_tolerance_overrides, -) - - -def parse_cli_tolerance_overrides(*, values: tuple[str, ...]) -> RowDiffTolerances: - """Parse repeated invocation-level per-column tolerance rules.""" - - return _parse_cli_tolerance_overrides(values=values) diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_names.py b/src/sqlbuild/executor/janitor/_helpers/archive_names.py index af806b6cdb..31885e515f 100644 --- a/src/sqlbuild/executor/janitor/_helpers/archive_names.py +++ b/src/sqlbuild/executor/janitor/_helpers/archive_names.py @@ -5,7 +5,7 @@ import re from datetime import UTC, datetime -from sqlbuild.adapter.relations.main.fit_artifact_logical_name import fit_artifact_logical_name +from sqlbuild._native import fit_artifact_logical_name from sqlbuild.executor.janitor.constants import ( ARCHIVE_ARTIFACT_LABEL, ARCHIVE_LOOKALIKE_PREFIX, diff --git a/src/sqlbuild/executor/load/_helpers/staging.py b/src/sqlbuild/executor/load/_helpers/staging.py index 4e2308ee91..237ae5b80b 100644 --- a/src/sqlbuild/executor/load/_helpers/staging.py +++ b/src/sqlbuild/executor/load/_helpers/staging.py @@ -4,10 +4,10 @@ from typing import Any +from sqlbuild._native import fit_auxiliary_relation_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.types import LoaderLogicalType -from sqlbuild.adapter.relations.main.fit_auxiliary_relation_name import fit_auxiliary_relation_name from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) diff --git a/src/sqlbuild/executor/pipeline/_helpers/auditing.py b/src/sqlbuild/executor/pipeline/_helpers/auditing.py index aee09c8380..9cc0dbe756 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/auditing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/auditing.py @@ -15,7 +15,7 @@ from typing import Any, cast from uuid import uuid4 -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, audit_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput @@ -29,7 +29,6 @@ from sqlbuild.executor.auditing.main.publish_completed_audit_results import ( publish_completed_audit_results, ) -from sqlbuild.executor.auditing.main.resource_id import audit_resource_id from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.pipeline._helpers.connections import ( close_connections, diff --git a/src/sqlbuild/executor/pipeline/_helpers/testing.py b/src/sqlbuild/executor/pipeline/_helpers/testing.py index bf5ebd668b..81577014c0 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/testing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/testing.py @@ -12,6 +12,7 @@ from typing import Any, cast from uuid import uuid4 +from sqlbuild._native import sql_test_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import FunctionInfo @@ -31,7 +32,6 @@ from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.constants import SQL_TEST_EXECUTION_ERROR_CODE from sqlbuild.executor.testing.main._execute import execute_sql_test -from sqlbuild.executor.testing.main._resource_id import sql_test_resource_id from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.executor.testing.types import SqlTestOutcome from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/executor/run/_helpers/execution/table_targets.py b/src/sqlbuild/executor/run/_helpers/execution/table_targets.py index cb2a03f5f9..f93a559bb0 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/table_targets.py +++ b/src/sqlbuild/executor/run/_helpers/execution/table_targets.py @@ -2,8 +2,8 @@ from __future__ import annotations +from sqlbuild._native import fit_auxiliary_relation_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.relations.main.fit_auxiliary_relation_name import fit_auxiliary_relation_name from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index 9b0d288808..cf4c517f67 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -6,11 +6,14 @@ from typing import Any from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild._native import future_cursor_cap_warning, maximum_start_cap_warning +from sqlbuild._native import ( + fit_auxiliary_relation_name, + future_cursor_cap_warning, + maximum_start_cap_warning, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.adapter.relations.main.fit_auxiliary_relation_name import fit_auxiliary_relation_name from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index 2fdb534834..599fc8b238 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -16,9 +16,11 @@ compute_query_hash, cursor_bound_display, empty_cursor_window, + fit_auxiliary_relation_name, future_cursor_cap_warning, inclusive_cursor_end, maximum_start_cap_warning, + merge_microbatch_intervals, microbatch_limit_warning, resolve_effective_microbatch_batch_size, ) @@ -38,7 +40,6 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists -from sqlbuild.adapter.relations.main.fit_auxiliary_relation_name import fit_auxiliary_relation_name from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) @@ -130,7 +131,6 @@ from sqlbuild.microbatches.main.latest_replay_requirement import ( latest_active_replay_requirement, ) -from sqlbuild.microbatches.main.merge_intervals import merge_intervals from sqlbuild.microbatches.main.project_coverage import project_microbatch_coverage from sqlbuild.microbatches.main.project_replay import project_replay_requirement from sqlbuild.microbatches.models import ( @@ -415,7 +415,7 @@ def execute_microbatch_entry( if isinstance(reconciliation, ModelExecutionResult): return reconciliation state, history_context, reconciled_batches = reconciliation - resolved_intervals: tuple[MicrobatchInterval, ...] = merge_intervals( + resolved_intervals: tuple[MicrobatchInterval, ...] = merge_microbatch_intervals( intervals=tuple( MicrobatchInterval(start=render(value=batch.start), end=render(value=batch.end)) for batch in reconciled_batches @@ -1234,7 +1234,7 @@ def _microbatch_lifecycle( def _microbatch_lifecycle_plan( *, batch_plan: _MicrobatchPlan, cursor_type: str ) -> _MicrobatchLifecyclePlan: - selected_intervals: tuple[MicrobatchInterval, ...] = merge_intervals( + selected_intervals: tuple[MicrobatchInterval, ...] = merge_microbatch_intervals( intervals=tuple( MicrobatchInterval(start=render(value=batch.start), end=render(value=batch.end)) for batch in batch_plan.batches @@ -3296,7 +3296,7 @@ def _plan_microbatch_windows( batch_size=batch_size, effective_grain=effective_grain, ) - work_intervals: tuple[MicrobatchInterval, ...] = merge_intervals( + work_intervals: tuple[MicrobatchInterval, ...] = merge_microbatch_intervals( intervals=( ( MicrobatchInterval( diff --git a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py index 911926fe1f..9c15fb0b75 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import fit_auxiliary_relation_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( @@ -13,7 +14,6 @@ ) from sqlbuild.adapter.contract.types import TypeFamily from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists -from sqlbuild.adapter.relations.main.fit_auxiliary_relation_name import fit_auxiliary_relation_name from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) diff --git a/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py b/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py index bf0cf58fa7..e8863dcb82 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py +++ b/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py @@ -4,10 +4,10 @@ from typing import Any +from sqlbuild._native import fit_auxiliary_relation_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.adapter.relations.main.fit_auxiliary_relation_name import fit_auxiliary_relation_name from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) diff --git a/src/sqlbuild/executor/scenario/main/_expected_expectations.py b/src/sqlbuild/executor/scenario/main/_expected_expectations.py index 196ab590c0..d848eb1477 100644 --- a/src/sqlbuild/executor/scenario/main/_expected_expectations.py +++ b/src/sqlbuild/executor/scenario/main/_expected_expectations.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import build_scenario_expected_comparison_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, @@ -14,9 +15,6 @@ SCENARIO_EXEC_EXPECTED_FAILED, SCENARIO_EXEC_EXPECTED_INTERNAL, ) -from sqlbuild.executor.scenario.main.expected_comparison_sql import ( - build_scenario_expected_comparison_sql, -) from sqlbuild.executor.scenario.models import ScenarioExpectedExpectationExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/src/sqlbuild/executor/scenario/main/expected_comparison_sql.py b/src/sqlbuild/executor/scenario/main/expected_comparison_sql.py deleted file mode 100644 index b8ceff07f7..0000000000 --- a/src/sqlbuild/executor/scenario/main/expected_comparison_sql.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Public scenario expected-output comparison SQL entrypoint.""" - -from __future__ import annotations - - -def build_scenario_expected_comparison_sql( - *, - actual_sql: str, - expected_sql: str, - set_difference_operator: str, -) -> str: - """Build SQL that compares actual scenario output to expected SQL.""" - - return ( - "WITH " - f"__actual AS ({actual_sql}), " - f"__expected AS ({expected_sql}) " - "SELECT " - "(SELECT COUNT(*) FROM __actual) AS actual_count, " - "(SELECT COUNT(*) FROM __expected) AS expected_count, " - "(SELECT COUNT(*) FROM (" - f"SELECT * FROM __actual {set_difference_operator} SELECT * FROM __expected" - ") AS __sqlbuild_mismatch) AS mismatched_count" - ) diff --git a/src/sqlbuild/executor/testing/_helpers/difference_samples.py b/src/sqlbuild/executor/testing/_helpers/difference_samples.py index f7b3fe9141..70bb3a2911 100644 --- a/src/sqlbuild/executor/testing/_helpers/difference_samples.py +++ b/src/sqlbuild/executor/testing/_helpers/difference_samples.py @@ -4,28 +4,19 @@ import logging from dataclasses import replace -from typing import Any, Final, cast +from typing import Any, Final -import orjson - -import sqlbuild._native as _native +from sqlbuild._native import render_sql_test_difference_sample_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor from sqlbuild.diagnostics.main.log_debug_event import log_debug_event -from sqlbuild.executor.testing._helpers.native_requests import chain_step_request -from sqlbuild.executor.testing.exceptions import SqlTestRenderingError from sqlbuild.executor.testing.models import ( SqlTestColumnDifference, SqlTestDifferenceSample, StepResult, ) -from sqlbuild.executor.testing.types import ( - NativeSqlTestRenderingModule, - SqlTestDifferenceDirection, - SqlTestOutcome, -) +from sqlbuild.executor.testing.types import SqlTestDifferenceDirection, SqlTestOutcome _ROW_LIMIT: Final[int] = 3 _COLUMN_LIMIT: Final[int] = 12 @@ -110,39 +101,6 @@ def _align_single_difference( ) -def build_sql_test_difference_sample_sql( - *, - test_entry: SqlTestPlanEntry, - step: ChainStep, - direction: SqlTestDifferenceDirection, - sample_limit: int, - set_difference_operator: str = "EXCEPT", - sql_analysis_dialect: str | None = None, -) -> str: - """Build a bounded query for one direction of an expected-output difference.""" - - if step.expected_cte_sql is None: - return "" - request: dict[str, object] = { - "step": chain_step_request(step=step), - "sqlAnalysisEnabled": test_entry.sql_analysis_enabled, - "setDifferenceOperator": set_difference_operator, - "sqlAnalysisDialect": sql_analysis_dialect, - "direction": direction.value, - "sampleLimit": sample_limit, - "useTopClause": sql_analysis_dialect == TypeDialect.TSQL, - } - response: object = orjson.loads( - cast(NativeSqlTestRenderingModule, _native).render_sql_test_difference_sample_json( - orjson.dumps(request, option=orjson.OPT_SORT_KEYS).decode() - ) - ) - sql: object = response.get("sql") if isinstance(response, dict) else None - if not isinstance(sql, str): - raise SqlTestRenderingError("native SQL-test difference rendering returned invalid SQL") - return sql - - def _fetch_difference_samples( *, test_entry: SqlTestPlanEntry, @@ -151,7 +109,7 @@ def _fetch_difference_samples( adapter: BaseAdapter, connection: Any, ) -> tuple[SqlTestDifferenceSample, ...]: - sql: str = build_sql_test_difference_sample_sql( + sql: str = render_sql_test_difference_sample_sql( test_entry=test_entry, step=step, direction=direction, diff --git a/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py b/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py index 9c193493fe..c82b524ce9 100644 --- a/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py +++ b/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py @@ -3,20 +3,13 @@ from __future__ import annotations import logging -from typing import Any, cast +from typing import Any -import orjson - -import sqlbuild._native as _native +from sqlbuild._native import describe_missing_expected_columns, render_sql_test_comparison_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import SqlTestPlanEntry from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor from sqlbuild.diagnostics.main.log_debug_event import log_debug_event -from sqlbuild.executor.testing._helpers.native_requests import comparison_request -from sqlbuild.executor.testing.main._missing_expected_columns import ( - describe_missing_expected_columns, -) -from sqlbuild.executor.testing.types import NativeSqlTestRenderingModule _LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") @@ -47,20 +40,13 @@ def missing_expected_columns_message( def _probe_actual_columns( *, test_entry: SqlTestPlanEntry, step_index: int, adapter: BaseAdapter, connection: Any ) -> frozenset[str] | None: - request: dict[str, object] = comparison_request( - test_entry=test_entry, + (sql,) = render_sql_test_comparison_sql( + (test_entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), + probe_step_index=step_index, ) - request["probeStepIndex"] = step_index - response: object = orjson.loads( - cast(NativeSqlTestRenderingModule, _native).render_sql_test_comparisons_json( - orjson.dumps({"requests": [request]}, option=orjson.OPT_SORT_KEYS).decode() - ) - ) - item: object = response[0] if isinstance(response, list) and response else None - sql: object = item.get("sql") if isinstance(item, dict) else None - if not isinstance(sql, str) or not sql: + if not sql: return None try: cursor: Any = adapter.execute( diff --git a/src/sqlbuild/executor/testing/_helpers/native_requests.py b/src/sqlbuild/executor/testing/_helpers/native_requests.py deleted file mode 100644 index 37e3566c0d..0000000000 --- a/src/sqlbuild/executor/testing/_helpers/native_requests.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Serialize planned SQL-test steps for the native comparison renderer.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.models import ChainStep, SqlTestAssertionStep, SqlTestPlanEntry - - -def comparison_request( - *, - test_entry: SqlTestPlanEntry, - set_difference_operator: str, - sql_analysis_dialect: str | None, -) -> dict[str, object]: - """Serialize one planned test into a native comparison render request.""" - - return { - "chain": [chain_step_request(step=step) for step in test_entry.chain], - "assertions": [ - assertion_step_request(assertion=assertion) for assertion in test_entry.assertions - ], - "sqlAnalysisEnabled": test_entry.sql_analysis_enabled, - "setDifferenceOperator": set_difference_operator, - "sqlAnalysisDialect": sql_analysis_dialect, - } - - -def chain_step_request(*, step: ChainStep) -> dict[str, object]: - """Serialize one planned chain step for the native renderer.""" - - return { - "modelName": step.model_name, - "resolvedSql": step.resolved_sql, - "expectedCteSql": step.expected_cte_sql, - "liftedCtes": step.lifted_ctes, - "comparisonBodySql": step.comparison_body_sql, - "expectedColumns": step.expected_columns, - "expectedLiftedCtes": step.expected_lifted_ctes, - } - - -def assertion_step_request(*, assertion: SqlTestAssertionStep) -> dict[str, object]: - """Serialize one planned assertion step for the native renderer.""" - - return { - "name": assertion.name, - "resolvedSql": assertion.resolved_sql, - "liftedCtes": assertion.lifted_ctes, - "comparisonBodySql": assertion.comparison_body_sql, - } diff --git a/src/sqlbuild/executor/testing/constants.py b/src/sqlbuild/executor/testing/constants.py index 1ef960c924..5cabd16b52 100644 --- a/src/sqlbuild/executor/testing/constants.py +++ b/src/sqlbuild/executor/testing/constants.py @@ -6,6 +6,3 @@ SQL_TEST_ENTRY_MISSING_CODE: str = "T004" SQL_TEST_EXPECTED_COLUMNS_CODE: str = "T005" SQL_TEST_SIZE_BYTES: str = "bytes" - -SQL_TEST_NATIVE_RENDER_WORKERS: int = 4 -SQL_TEST_IDENTIFIER_QUOTES: dict[str, str] = {'"': '"', "`": "`", "[": "]"} diff --git a/src/sqlbuild/executor/testing/exceptions.py b/src/sqlbuild/executor/testing/exceptions.py deleted file mode 100644 index 6b18460b4c..0000000000 --- a/src/sqlbuild/executor/testing/exceptions.py +++ /dev/null @@ -1,5 +0,0 @@ -"""SQL test execution exceptions.""" - - -class SqlTestRenderingError(RuntimeError): - """Raised when native SQL-test rendering returns an invalid response.""" diff --git a/src/sqlbuild/executor/testing/main/_execute.py b/src/sqlbuild/executor/testing/main/_execute.py index 59a5d59020..f0c31495c7 100644 --- a/src/sqlbuild/executor/testing/main/_execute.py +++ b/src/sqlbuild/executor/testing/main/_execute.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import render_sql_test_comparison_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.planner.models import SqlTestPlanEntry @@ -20,7 +21,6 @@ from sqlbuild.executor.testing.main._sql_length import ( validate_unit_test_sql_length, ) -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.executor.testing.types import SqlTestOutcome from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -36,8 +36,8 @@ def execute_sql_test( ) -> SqlTestExecutionResult: """Execute one SQL unit test and collect bounded diagnostics for failed comparisons.""" - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=test_entry, + (comparison_sql,) = render_sql_test_comparison_sql( + (test_entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), ) diff --git a/src/sqlbuild/executor/testing/main/_missing_expected_columns.py b/src/sqlbuild/executor/testing/main/_missing_expected_columns.py deleted file mode 100644 index b8536007a2..0000000000 --- a/src/sqlbuild/executor/testing/main/_missing_expected_columns.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Explain expected-output columns that a model does not produce.""" - -from __future__ import annotations - -from sqlbuild.executor.testing.constants import SQL_TEST_IDENTIFIER_QUOTES - - -def describe_missing_expected_columns( - *, - model_name: str, - expected_columns: tuple[str, ...], - available_columns: frozenset[str], -) -> str | None: - """Describe listed expected columns absent from casefolded model output names.""" - - missing: tuple[str, ...] = tuple( - column - for column in expected_columns - if _normalized_identifier(column) not in available_columns - ) - if not missing: - return None - return ( - f"expected output '{model_name}' lists columns that model '{model_name}' does not " - f"output: {', '.join(missing)}. Expected CTEs are compared only on the columns they " - "list, matched by name" - ) - - -def _normalized_identifier(identifier: str) -> str: - closing_quote: str | None = SQL_TEST_IDENTIFIER_QUOTES.get(identifier[:1]) - if closing_quote is None or not identifier.endswith(closing_quote): - return identifier.casefold() - return identifier[1:-1].replace(closing_quote * 2, closing_quote).casefold() diff --git a/src/sqlbuild/executor/testing/main/_resource_id.py b/src/sqlbuild/executor/testing/main/_resource_id.py deleted file mode 100644 index 6d15602496..0000000000 --- a/src/sqlbuild/executor/testing/main/_resource_id.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Canonical resource identity for one SQL test case execution.""" - -from pathlib import Path - - -def sql_test_resource_id( - *, - test_name: str, - source_path: Path | None, - block_index: int | None, - case_name: str | None, -) -> str: - """Return the exact stable identity of one plain or parameterized SQL test.""" - - if source_path is None or block_index is None or case_name is None: - return f"sql_test:{test_name}" - return f"sql_test:{source_path.as_posix()}:{block_index}:{case_name}" diff --git a/src/sqlbuild/executor/testing/main/comparison_sql.py b/src/sqlbuild/executor/testing/main/comparison_sql.py deleted file mode 100644 index 099f3c7613..0000000000 --- a/src/sqlbuild/executor/testing/main/comparison_sql.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Build executable SQL unit-test comparison queries.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.models import SqlTestPlanEntry -from sqlbuild.executor.testing.main.comparison_sql_batch import ( - build_sql_test_comparison_sql_batch, -) - - -def build_sql_test_comparison_sql( - *, - test_entry: SqlTestPlanEntry, - set_difference_operator: str = "EXCEPT", - sql_analysis_dialect: str | None = None, -) -> str: - """Build the single SQL statement used to execute a SQL unit test.""" - - (comparison_sql,) = build_sql_test_comparison_sql_batch( - test_entries=(test_entry,), - set_difference_operator=set_difference_operator, - sql_analysis_dialect=sql_analysis_dialect, - ) - return comparison_sql diff --git a/src/sqlbuild/executor/testing/main/comparison_sql_batch.py b/src/sqlbuild/executor/testing/main/comparison_sql_batch.py deleted file mode 100644 index 1ba1d0d9b0..0000000000 --- a/src/sqlbuild/executor/testing/main/comparison_sql_batch.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Build batches of executable SQL unit-test comparison queries natively.""" - -from __future__ import annotations - -from typing import Any, cast - -import orjson - -import sqlbuild._native as _native -from sqlbuild.compiler.planner.models import SqlTestPlanEntry -from sqlbuild.executor.testing._helpers.native_requests import comparison_request -from sqlbuild.executor.testing.constants import SQL_TEST_NATIVE_RENDER_WORKERS -from sqlbuild.executor.testing.exceptions import SqlTestRenderingError -from sqlbuild.executor.testing.types import NativeSqlTestRenderingModule - - -def build_sql_test_comparison_sql_batch( - *, - test_entries: tuple[SqlTestPlanEntry, ...], - set_difference_operator: str = "EXCEPT", - sql_analysis_dialect: str | None = None, -) -> tuple[str, ...]: - """Build comparison SQL for a deterministic batch of planned SQL tests.""" - - if not test_entries: - return () - requests: list[dict[str, object]] = [ - comparison_request( - test_entry=test_entry, - set_difference_operator=set_difference_operator, - sql_analysis_dialect=sql_analysis_dialect, - ) - for test_entry in test_entries - ] - response: object = orjson.loads( - cast(NativeSqlTestRenderingModule, _native).render_sql_test_comparisons_json( - orjson.dumps( - {"requests": requests, "workers": SQL_TEST_NATIVE_RENDER_WORKERS}, - option=orjson.OPT_SORT_KEYS, - ).decode() - ) - ) - if not isinstance(response, list) or len(response) != len(test_entries): - raise SqlTestRenderingError("native SQL-test rendering returned an invalid batch response") - rendered: list[str] = [] - for item in response: - item_dict: dict[str, Any] | None = ( - cast(dict[str, Any], item) if isinstance(item, dict) else None - ) - sql: object = item_dict.get("sql") if item_dict is not None else None - if not isinstance(sql, str): - raise SqlTestRenderingError("native SQL-test rendering returned an invalid result") - rendered.append(sql) - return tuple(rendered) diff --git a/src/sqlbuild/executor/testing/types.py b/src/sqlbuild/executor/testing/types.py index f26fb0dea4..b0dc160aaa 100644 --- a/src/sqlbuild/executor/testing/types.py +++ b/src/sqlbuild/executor/testing/types.py @@ -3,15 +3,6 @@ from __future__ import annotations from enum import StrEnum -from typing import Protocol - - -class NativeSqlTestRenderingModule(Protocol): - """Native SQL-test comparison-rendering boundary.""" - - def render_sql_test_comparisons_json(self, request_json: str) -> str: ... - - def render_sql_test_difference_sample_json(self, request_json: str) -> str: ... class SqlTestOutcome(StrEnum): diff --git a/src/sqlbuild/integrations/dagster/_helpers/dag.py b/src/sqlbuild/integrations/dagster/_helpers/dag.py deleted file mode 100644 index 3f6bac8882..0000000000 --- a/src/sqlbuild/integrations/dagster/_helpers/dag.py +++ /dev/null @@ -1,44 +0,0 @@ -"""SQLBuild DAG artifact loading helpers for Dagster.""" - -from __future__ import annotations - -import json -from collections.abc import Mapping -from pathlib import Path -from typing import Any - -from sqlbuild.integrations.dagster.exceptions import DagsterDagInputError -from sqlbuild.integrations.dagster.types import SqlBuildDagInput - - -def load_sqlbuild_dag(dag: SqlBuildDagInput) -> Mapping[str, Any]: - """Load a SQLBuild DAG artifact from a mapping or JSON file path.""" - - if not isinstance(dag, (str, Path)): - payload: Mapping[str, Any] = dag - else: - path: Path = Path(dag) - try: - loaded: Any = json.loads(path.read_text(encoding="utf-8")) - except OSError as error: - raise DagsterDagInputError(f"could not read SQLBuild DAG artifact: {path}") from error - except json.JSONDecodeError as error: - raise DagsterDagInputError( - f"SQLBuild DAG artifact is not valid JSON: {path}: {error.msg}" - ) from error - if not isinstance(loaded, Mapping): - raise DagsterDagInputError("SQLBuild DAG artifact must be a JSON object") - payload = loaded - - _validate_dag_payload(payload) - return payload - - -def _validate_dag_payload(payload: Mapping[str, Any]) -> None: - version: object = payload.get("version") - if version != 1: - raise DagsterDagInputError(f"unsupported SQLBuild DAG artifact version: {version!r}") - for key in ("nodes", "edges", "checks"): - value: object = payload.get(key) - if not isinstance(value, list): - raise DagsterDagInputError(f"SQLBuild DAG artifact field '{key}' must be a list") diff --git a/src/sqlbuild/integrations/dagster/_helpers/selection.py b/src/sqlbuild/integrations/dagster/_helpers/selection.py index 25ac702707..9f935ab8ec 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/selection.py +++ b/src/sqlbuild/integrations/dagster/_helpers/selection.py @@ -5,16 +5,7 @@ from collections.abc import Mapping, Sequence from typing import Any -from sqlbuild._native import ( - parse_project_selector, - path_nodes, - transitive_closure_many, - unit_test_selector_error, -) -from sqlbuild.compiler.planner.constants import SELECTOR_KIND_SEPARATOR -from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector -from sqlbuild.compiler.planner.types import SelectorKind -from sqlbuild.integrations.dagster._helpers.dag import load_sqlbuild_dag +from sqlbuild._native import load_sqlbuild_dag, resolve_dag_selection from sqlbuild.integrations.dagster._helpers.imports import load_dagster from sqlbuild.integrations.dagster.classes.sqlbuild_dagster_translator import ( SqlBuildDagsterTranslator, @@ -36,9 +27,9 @@ def build_sqlbuild_asset_selection_impl( if not sqlbuild_assets: raise DagsterDagInputError("SQLBuild asset selection requires an asset definition") - resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag) + resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag, error_class=DagsterDagInputError) resolved_translator: SqlBuildDagsterTranslator = translator or SqlBuildDagsterTranslator() - selected_ids: set[str] = _resolve_dag_selectors( + selected_ids: frozenset[str] = resolve_dag_selection( dag=resolved_dag, select=sqlbuild_select, exclude=sqlbuild_exclude, @@ -55,185 +46,3 @@ def build_sqlbuild_asset_selection_impl( continue asset_keys.append(resolved_translator.get_asset_key(translated_node)) return load_dagster().AssetSelection.assets(*asset_keys) - - -def _resolve_dag_selectors(*, dag: Mapping[str, Any], select: str, exclude: str | None) -> set[str]: - nodes_by_id: dict[str, Mapping[str, Any]] = { - str(node["id"]): node for node in dag.get("nodes", ()) - } - upstream, downstream = _dependency_indexes(dag=dag, node_ids=set(nodes_by_id)) - selected: set[str] = _resolve_expression( - expression=select, - dag=dag, - nodes_by_id=nodes_by_id, - upstream=upstream, - downstream=downstream, - ) - if exclude: - selected.difference_update( - _resolve_expression( - expression=exclude, - dag=dag, - nodes_by_id=nodes_by_id, - upstream=upstream, - downstream=downstream, - ) - ) - return selected - - -def _resolve_expression( - *, - expression: str, - dag: Mapping[str, Any], - nodes_by_id: Mapping[str, Mapping[str, Any]], - upstream: Mapping[str, set[str]], - downstream: Mapping[str, set[str]], -) -> set[str]: - selected: set[str] = set() - for token in expression.split(): - intersections: list[set[str]] = [ - _resolve_atomic( - raw=part, - dag=dag, - nodes_by_id=nodes_by_id, - upstream=upstream, - downstream=downstream, - ) - for part in token.split(",") - ] - token_selection: set[str] = intersections[0] - for intersection in intersections[1:]: - token_selection.intersection_update(intersection) - selected.update(token_selection) - return selected - - -def _resolve_atomic( - *, - raw: str, - dag: Mapping[str, Any], - nodes_by_id: Mapping[str, Mapping[str, Any]], - upstream: Mapping[str, set[str]], - downstream: Mapping[str, set[str]], -) -> set[str]: - parsed: ParsedSelector | PathSelector = parse_project_selector(raw) - if isinstance(parsed, PathSelector): - start_id: str = _node_id_for_name(nodes_by_id=nodes_by_id, name=parsed.start_name) - end_id: str = _node_id_for_name(nodes_by_id=nodes_by_id, name=parsed.end_name) - matched: set[str] = _path_node_ids(start_id=start_id, end_id=end_id, downstream=downstream) - if parsed.upstream: - matched.update( - transitive_closure_many(starts=(start_id,), edges=upstream, include_starts=True) - ) - if parsed.downstream: - matched.update( - transitive_closure_many(starts=(end_id,), edges=downstream, include_starts=True) - ) - return matched - - matched = _match_parsed_selector(parsed=parsed, dag=dag, nodes_by_id=nodes_by_id) - anchors: frozenset[str] = frozenset(matched) - if parsed.upstream: - matched.update(transitive_closure_many(starts=anchors, edges=upstream, include_starts=True)) - if parsed.downstream: - matched.update( - transitive_closure_many(starts=anchors, edges=downstream, include_starts=True) - ) - return matched - - -def _match_parsed_selector( - *, - parsed: ParsedSelector, - dag: Mapping[str, Any], - nodes_by_id: Mapping[str, Mapping[str, Any]], -) -> set[str]: - if parsed.kind == SelectorKind.TEST: - raise unit_test_selector_error( - selector=f"{SelectorKind.TEST}{SELECTOR_KIND_SEPARATOR}{parsed.value}" - ) - if parsed.kind == SelectorKind.TAG: - return { - node_id - for node_id, node in nodes_by_id.items() - if _node_has_tag(node=node, tag=parsed.value) - } - if parsed.kind == SelectorKind.PATH: - folder: str = parsed.value.replace("\\", "/").strip("/") - return { - node_id - for node_id, node in nodes_by_id.items() - if _path_is_under(path=node.get("path"), folder=folder) - } - if parsed.kind == SelectorKind.CHECK: - checked_ids: set[str] = set() - for check in dag.get("checks", ()): - if str(check.get("name")) == parsed.value or str(check.get("id")) == parsed.value: - checked_ids.update(str(asset_id) for asset_id in check.get("checked_asset_ids", ())) - return checked_ids - expected_kind: str | None = None if parsed.kind == SelectorKind.NAME else str(parsed.kind.value) - return { - node_id - for node_id, node in nodes_by_id.items() - if str(node.get("name")) == parsed.value - and (expected_kind is None or str(node.get("kind")) == expected_kind) - } - - -def _node_has_tag(*, node: Mapping[str, Any], tag: str) -> bool: - return any(str(node_tag) == tag for node_tag in node.get("tags", ())) - - -def _path_is_under(*, path: object, folder: str) -> bool: - if not isinstance(path, str): - return False - normalized_path: str = path.replace("\\", "/").strip("/") - candidates: tuple[str, ...] = ( - folder, - f"models/{folder}" if folder and not folder.startswith("models/") else folder, - ) - return any( - normalized_path == candidate or normalized_path.startswith(f"{candidate}/") - for candidate in candidates - if candidate - ) - - -def _dependency_indexes( - *, dag: Mapping[str, Any], node_ids: set[str] -) -> tuple[dict[str, set[str]], dict[str, set[str]]]: - upstream: dict[str, set[str]] = {node_id: set() for node_id in node_ids} - downstream: dict[str, set[str]] = {node_id: set() for node_id in node_ids} - for edge in dag.get("edges", ()): - from_id: str = str(edge.get("from_id")) - to_id: str = str(edge.get("to_id")) - if from_id not in node_ids or to_id not in node_ids: - continue - upstream[to_id].add(from_id) - downstream[from_id].add(to_id) - return upstream, downstream - - -def _node_id_for_name(*, nodes_by_id: Mapping[str, Mapping[str, Any]], name: str) -> str: - matching_ids: list[str] = [ - node_id for node_id, node in nodes_by_id.items() if str(node.get("name")) == name - ] - if len(matching_ids) != 1: - raise DagsterDagInputError( - f"SQLBuild selector name {name!r} matched {len(matching_ids)} DAG nodes" - ) - return matching_ids[0] - - -def _path_node_ids(*, start_id: str, end_id: str, downstream: Mapping[str, set[str]]) -> set[str]: - on_path: frozenset[str] | None = path_nodes( - start=start_id, - end=end_id, - downstream={node_id: tuple(sorted(targets)) for node_id, targets in downstream.items()}, - ) - if on_path is None: - raise DagsterDagInputError( - f"no SQLBuild DAG path exists between {start_id!r} and {end_id!r}" - ) - return set(on_path) diff --git a/src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_resource.py b/src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_resource.py index 78b53c4544..a20825ec60 100644 --- a/src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_resource.py +++ b/src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_resource.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any -from sqlbuild.integrations.dagster._helpers.dag import load_sqlbuild_dag +from sqlbuild._native import load_sqlbuild_dag from sqlbuild.integrations.dagster._helpers.imports import load_dagster from sqlbuild.integrations.dagster._helpers.invocation_factory import start_sqlbuild_cli_invocation from sqlbuild.integrations.dagster._helpers.translation import translate_sqlbuild_dag @@ -14,6 +14,7 @@ from sqlbuild.integrations.dagster.classes.sqlbuild_dagster_translator import ( SqlBuildDagsterTranslator, ) +from sqlbuild.integrations.dagster.exceptions import DagsterDagInputError from sqlbuild.integrations.dagster.models import SqlBuildProject @@ -58,7 +59,7 @@ def cli( loaded_dag: Mapping[str, Any] | None = None if self.dag_path is not None: loaded_dag = translate_sqlbuild_dag( - dag=load_sqlbuild_dag(self.dag_path), + dag=load_sqlbuild_dag(self.dag_path, error_class=DagsterDagInputError), translator=translator or SqlBuildDagsterTranslator(), ) return start_sqlbuild_cli_invocation( diff --git a/src/sqlbuild/integrations/dagster/main/_sqlbuild_assets.py b/src/sqlbuild/integrations/dagster/main/_sqlbuild_assets.py index 95cfb18a85..160502020c 100644 --- a/src/sqlbuild/integrations/dagster/main/_sqlbuild_assets.py +++ b/src/sqlbuild/integrations/dagster/main/_sqlbuild_assets.py @@ -6,8 +6,8 @@ from pathlib import Path from typing import Any +from sqlbuild._native import load_sqlbuild_dag from sqlbuild.integrations.dagster._helpers.assets import build_asset_specs, build_check_specs -from sqlbuild.integrations.dagster._helpers.dag import load_sqlbuild_dag from sqlbuild.integrations.dagster._helpers.imports import load_dagster from sqlbuild.integrations.dagster.classes.sqlbuild_dagster_translator import ( SqlBuildDagsterTranslator, @@ -40,7 +40,7 @@ def sqlbuild_assets( if project is not None else Path("target/sqlbuild_dag.json") ) - resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag_input) + resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag_input, error_class=DagsterDagInputError) resolved_translator: SqlBuildDagsterTranslator = translator or SqlBuildDagsterTranslator() specs: tuple[Any, ...] = build_asset_specs( dag=resolved_dag, diff --git a/src/sqlbuild/integrations/dagster/main/_sqlbuild_scenario_checks.py b/src/sqlbuild/integrations/dagster/main/_sqlbuild_scenario_checks.py index f43c1e4dc9..dc2c17aba6 100644 --- a/src/sqlbuild/integrations/dagster/main/_sqlbuild_scenario_checks.py +++ b/src/sqlbuild/integrations/dagster/main/_sqlbuild_scenario_checks.py @@ -6,8 +6,8 @@ from pathlib import Path from typing import Any +from sqlbuild._native import load_sqlbuild_dag from sqlbuild.integrations.dagster._helpers.assets import build_scenario_check_specs -from sqlbuild.integrations.dagster._helpers.dag import load_sqlbuild_dag from sqlbuild.integrations.dagster._helpers.imports import load_dagster from sqlbuild.integrations.dagster.classes.sqlbuild_dagster_translator import ( SqlBuildDagsterTranslator, @@ -39,7 +39,7 @@ def sqlbuild_scenario_checks( if project is not None else Path("target/sqlbuild_dag.json") ) - resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag_input) + resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag_input, error_class=DagsterDagInputError) resolved_translator: SqlBuildDagsterTranslator = translator or SqlBuildDagsterTranslator() check_specs: tuple[Any, ...] = build_scenario_check_specs( dag=resolved_dag, diff --git a/src/sqlbuild/integrations/rivers/_helpers/dag.py b/src/sqlbuild/integrations/rivers/_helpers/dag.py deleted file mode 100644 index fc5a33e5ca..0000000000 --- a/src/sqlbuild/integrations/rivers/_helpers/dag.py +++ /dev/null @@ -1,44 +0,0 @@ -"""SQLBuild DAG artifact loading helpers for Rivers.""" - -from __future__ import annotations - -import json -from collections.abc import Mapping -from pathlib import Path -from typing import Any - -from sqlbuild.integrations.rivers.exceptions import RiversDagInputError -from sqlbuild.integrations.rivers.types import SqlBuildDagInput - - -def load_sqlbuild_dag(dag: SqlBuildDagInput) -> Mapping[str, Any]: - """Load a SQLBuild DAG artifact from a mapping or JSON file path.""" - - if not isinstance(dag, (str, Path)): - payload: Mapping[str, Any] = dag - else: - path: Path = Path(dag) - try: - loaded: Any = json.loads(path.read_text(encoding="utf-8")) - except OSError as error: - raise RiversDagInputError(f"could not read SQLBuild DAG artifact: {path}") from error - except json.JSONDecodeError as error: - raise RiversDagInputError( - f"SQLBuild DAG artifact is not valid JSON: {path}: {error.msg}" - ) from error - if not isinstance(loaded, Mapping): - raise RiversDagInputError("SQLBuild DAG artifact must be a JSON object") - payload = loaded - - _validate_dag_payload(payload) - return payload - - -def _validate_dag_payload(payload: Mapping[str, Any]) -> None: - version: object = payload.get("version") - if version != 1: - raise RiversDagInputError(f"unsupported SQLBuild DAG artifact version: {version!r}") - for key in ("nodes", "edges", "checks"): - value: object = payload.get(key) - if not isinstance(value, list): - raise RiversDagInputError(f"SQLBuild DAG artifact field '{key}' must be a list") diff --git a/src/sqlbuild/integrations/rivers/main/_sqlbuild_assets.py b/src/sqlbuild/integrations/rivers/main/_sqlbuild_assets.py index 25adc77073..f2a3d86b4f 100644 --- a/src/sqlbuild/integrations/rivers/main/_sqlbuild_assets.py +++ b/src/sqlbuild/integrations/rivers/main/_sqlbuild_assets.py @@ -6,8 +6,8 @@ from pathlib import Path from typing import Any +from sqlbuild._native import load_sqlbuild_dag from sqlbuild.integrations.rivers._helpers.assets import build_asset_defs -from sqlbuild.integrations.rivers._helpers.dag import load_sqlbuild_dag from sqlbuild.integrations.rivers._helpers.imports import load_rivers from sqlbuild.integrations.rivers.classes.sqlbuild_rivers_translator import ( SqlBuildRiversTranslator, @@ -38,7 +38,7 @@ def sqlbuild_assets( if project is not None else Path("target/sqlbuild_dag.json") ) - resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag_input) + resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag_input, error_class=RiversDagInputError) resolved_translator: SqlBuildRiversTranslator = translator or SqlBuildRiversTranslator() output_defs: tuple[Any, ...] = build_asset_defs( dag=resolved_dag, diff --git a/src/sqlbuild/microbatches/main/merge_intervals.py b/src/sqlbuild/microbatches/main/merge_intervals.py deleted file mode 100644 index 2040611c91..0000000000 --- a/src/sqlbuild/microbatches/main/merge_intervals.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Merge canonical microbatch intervals.""" - -from sqlbuild._native import cursor_compare as compare -from sqlbuild._native import cursor_merge as merge -from sqlbuild._native import cursor_parse as parse -from sqlbuild._native import cursor_render as render -from sqlbuild._native import cursor_sort_key -from sqlbuild.compiler.planner.types import CursorType -from sqlbuild.cursor_algebra.models import AlignedInterval, IntegerValue -from sqlbuild.cursor_algebra.types import CursorScalar -from sqlbuild.microbatches.exceptions import MicrobatchStateError -from sqlbuild.microbatches.models import MicrobatchInterval - - -def merge_intervals( - *, intervals: tuple[MicrobatchInterval, ...], cursor_type: str -) -> tuple[MicrobatchInterval, ...]: - """Merge overlapping or adjacent intervals while preserving disjoint sets.""" - - if cursor_type not in {CursorType.TIMESTAMP, CursorType.INTEGER}: - raise MicrobatchStateError(f"unsupported cursor type: {cursor_type}") - if cursor_type == CursorType.INTEGER: - typed_merged: tuple[MicrobatchInterval, ...] | None = _merge_typed_integers( - intervals=intervals - ) - if typed_merged is not None: - return typed_merged - ordered: tuple[MicrobatchInterval, ...] = tuple( - sorted( - intervals, - key=lambda interval: cursor_sort_key(raw=interval.start, cursor_type=cursor_type), - ) - ) - merged: list[MicrobatchInterval] = [] - for interval in ordered: - if not merged or _lt(left=merged[-1].end, right=interval.start, cursor_type=cursor_type): - merged.append(interval) - continue - if _lt(left=merged[-1].end, right=interval.end, cursor_type=cursor_type): - merged[-1] = MicrobatchInterval(start=merged[-1].start, end=interval.end) - return tuple(merged) - - -def _lt(*, left: str, right: str, cursor_type: str) -> bool: - return ( - compare( - left=parse(raw=left, cursor_type=cursor_type), - right=parse(raw=right, cursor_type=cursor_type), - ) - < 0 - ) - - -def _merge_typed_integers( - *, intervals: tuple[MicrobatchInterval, ...] -) -> tuple[MicrobatchInterval, ...] | None: - typed: list[AlignedInterval] = [] - for interval in intervals: - start: CursorScalar = parse(raw=interval.start, cursor_type=CursorType.INTEGER) - end: CursorScalar = parse(raw=interval.end, cursor_type=CursorType.INTEGER) - if ( - not isinstance(start, IntegerValue) - or not isinstance(end, IntegerValue) - or render(value=start) != interval.start - or render(value=end) != interval.end - or start.value >= end.value - ): - return None - typed.append(AlignedInterval(start=start, end=end, grain=None)) - return tuple( - MicrobatchInterval(start=render(value=interval.start), end=render(value=interval.end)) - for interval in merge(intervals=tuple(typed)) - ) diff --git a/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py b/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py index a72e977e61..53435d376d 100644 --- a/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py +++ b/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py @@ -11,14 +11,14 @@ import pytest -from sqlbuild._native import PlaygroundCommandRequest +from sqlbuild._native import PlaygroundCommandRequest, load_sqlbuild_dag from sqlbuild._native import run_playground_command as run_playground from sqlbuild.integrations.rivers import SqlBuildProject, sqlbuild_assets from sqlbuild.integrations.rivers._helpers.assets import build_asset_defs -from sqlbuild.integrations.rivers._helpers.dag import load_sqlbuild_dag from sqlbuild.integrations.rivers.classes.sqlbuild_rivers_translator import ( SqlBuildRiversTranslator, ) +from sqlbuild.integrations.rivers.exceptions import RiversDagInputError from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( REPO_ROOT, prepare_waffle_shop, @@ -76,7 +76,7 @@ def sqlbuild_python_nodes(context: Any) -> Iterator[Any]: yield output_defs: tuple[Any, ...] = build_asset_defs( - dag=load_sqlbuild_dag(sqlbuild_project.dag_path), + dag=load_sqlbuild_dag(sqlbuild_project.dag_path, error_class=RiversDagInputError), translator=SqlBuildRiversTranslator(), ) repo: Any = rs.CodeRepository(assets=[sqlbuild_python_nodes]) diff --git a/tests/integration/src/sqlbuild/executor/testing/helpers.py b/tests/integration/src/sqlbuild/executor/testing/helpers.py index 77801bf3be..2514914c86 100644 --- a/tests/integration/src/sqlbuild/executor/testing/helpers.py +++ b/tests/integration/src/sqlbuild/executor/testing/helpers.py @@ -6,13 +6,13 @@ from pathlib import Path from typing import Any +from sqlbuild._native import render_sql_test_comparison_sql from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from sqlbuild.executor.testing.main._execute import execute_sql_test -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome from tests.integration.src.sqlbuild.compiler.pipeline.helpers import ( @@ -148,11 +148,11 @@ def _project_test_step( ] step: ChainStep = {item.model_name: item for item in entry.chain}[model_name] build_sql: str = {model.name: model.query_sql for model in result.project.models}[model_name] - rendered_sql: str = build_sql_test_comparison_sql( - test_entry=entry, + rendered_sql: str = render_sql_test_comparison_sql( + (entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] return build_sql, step.comparison_body_sql or step.resolved_sql, rendered_sql diff --git a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_build_parity.py b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_build_parity.py index f00fdeaf43..10a88288a7 100644 --- a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_build_parity.py +++ b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_build_parity.py @@ -9,9 +9,9 @@ import pytest +from sqlbuild._native import render_sql_test_comparison_sql from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.planner.models import SqlTestPlanEntry -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.integration.src.sqlbuild.executor.testing._test_types import ( ExpectedBooleanTestCase, SqlTestBuildParityTestCase, @@ -124,8 +124,8 @@ def test_given_shared_cte_name_in_chain_when_rendering_tsql_then_rename_runs_lik ) -> None: entry: SqlTestPlanEntry = build_shared_cte_name_chain_entry() - renamed_sql: str = build_sql_test_comparison_sql(test_entry=entry, sql_analysis_dialect="tsql") - nested_sql: str = build_sql_test_comparison_sql(test_entry=entry, sql_analysis_dialect="duckdb") + renamed_sql: str = render_sql_test_comparison_sql((entry,), sql_analysis_dialect="tsql")[0] + nested_sql: str = render_sql_test_comparison_sql((entry,), sql_analysis_dialect="duckdb")[0] assert "__sqb_cte_0 AS (SELECT id + 1 AS id FROM __ref__stg)" in renamed_sql assert "__ref__mart AS (SELECT __sqb_cte_0.id FROM __sqb_cte_0)" in renamed_sql diff --git a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py index 1bad0a901d..5af2fe7cee 100644 --- a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py +++ b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py @@ -7,18 +7,12 @@ import pytest +from sqlbuild._native import render_sql_test_comparison_sql, render_sql_test_difference_sample_sql from sqlbuild.adapter.contract.types import StatementSizeLimit from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry -from sqlbuild.executor.testing._helpers.difference_samples import ( - build_sql_test_difference_sample_sql, -) from sqlbuild.executor.testing.main._execute import execute_sql_test -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql -from sqlbuild.executor.testing.main.comparison_sql_batch import ( - build_sql_test_comparison_sql_batch, -) from sqlbuild.executor.testing.models import ( SqlTestColumnDifference, SqlTestExecutionResult, @@ -113,12 +107,14 @@ def execute(self, *, connection: Any, sql: str) -> Any: def test_given_sql_test_chain_when_building_comparison_sql_then_uses_readable_ctes( test_case: SqlTestComparisonSqlTestCase, ) -> None: - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=build_sql_test_plan_entry( - name="test_model", - chain_steps=test_case.chain_steps, + comparison_sql: str = render_sql_test_comparison_sql( + ( + build_sql_test_plan_entry( + name="test_model", + chain_steps=test_case.chain_steps, + ), ) - ) + )[0] expected_fragment: str for expected_fragment in test_case.expected_fragments: @@ -196,10 +192,12 @@ def test_given_conflicting_preanalyzed_ctes_when_executing_then_each_step_uses_i adapter=adapter, connection=connection, ) - (native_sql,) = build_sql_test_comparison_sql_batch( - test_entries=(entry,), - set_difference_operator=adapter.render_set_difference_operator(), - sql_analysis_dialect=adapter.sql_analysis_dialect(), + (native_sql,) = tuple( + render_sql_test_comparison_sql( + (entry,), + set_difference_operator=adapter.render_set_difference_operator(), + sql_analysis_dialect=adapter.sql_analysis_dialect(), + ) ) native_rows: list[tuple[object, ...]] = connection.execute(native_sql).fetchall() @@ -229,7 +227,7 @@ def test_given_sqlserver_dialect_when_building_sample_sql_then_uses_bounded_top_ dialect: str | None = SqlServerAdapter().sql_analysis_dialect() assert dialect == test_case.dialect - sample_sql: str = build_sql_test_difference_sample_sql( + sample_sql: str = render_sql_test_difference_sample_sql( test_entry=entry, step=entry.chain[0], direction=SqlTestDifferenceDirection.UNEXPECTED, diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py index fb7ea24f5d..82059d5a6d 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -109,6 +109,7 @@ def test_given_native_source_when_checking_module_paths_then_reports_errors( description="native crates import only existing Python modules and attributes", expected_errors=(), expected_logger_names=( + "sqlbuild.adapter", "sqlbuild.cli", "sqlbuild.cli.commands._helpers.cost.collection", "sqlbuild.cli.commands.classes.prepared_compile_artifacts", diff --git a/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/test_fit_artifact_logical_name.py b/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/test_fit_artifact_logical_name.py index 40c3632495..55e52285af 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/test_fit_artifact_logical_name.py +++ b/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/test_fit_artifact_logical_name.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.adapter.relations.main.fit_artifact_logical_name import fit_artifact_logical_name +from sqlbuild._native import fit_artifact_logical_name from sqlbuild.errors.contracts.exceptions import SharedInputError from tests.unit.src.sqlbuild.adapter.relations.main.fit_artifact_logical_name._test_types import ( FitArtifactLogicalNameErrorTestCase, diff --git a/tests/unit/src/sqlbuild/adapter/relations/main/fit_auxiliary_relation_name/test_fit_auxiliary_relation_name.py b/tests/unit/src/sqlbuild/adapter/relations/main/fit_auxiliary_relation_name/test_fit_auxiliary_relation_name.py index c70d7e4456..16da3b3877 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/main/fit_auxiliary_relation_name/test_fit_auxiliary_relation_name.py +++ b/tests/unit/src/sqlbuild/adapter/relations/main/fit_auxiliary_relation_name/test_fit_auxiliary_relation_name.py @@ -4,9 +4,7 @@ import pytest -from sqlbuild.adapter.relations.main.fit_auxiliary_relation_name import ( - fit_auxiliary_relation_name, -) +from sqlbuild._native import fit_auxiliary_relation_name from sqlbuild.errors.contracts.exceptions import SharedInputError from tests.unit.src.sqlbuild.adapter.relations.main.fit_auxiliary_relation_name._test_types import ( FitAuxiliaryRelationNameErrorTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 4b59180665..80ddcd32f0 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -15,6 +15,7 @@ build_manifest, compiled_model_record, project_with, + render_sql_test_comparison_sql, write_compile_target, write_static_compile_target, ) @@ -27,7 +28,6 @@ DiagnosticPhase, ) from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( ExpectedMessageTestCase, TargetWriterCacheTestCase, @@ -74,7 +74,7 @@ def test_given_plan_output_when_writing_target_then_expected_files_are_written( ) -> None: plan_output: PlanOutput = build_target_writer_plan_output() expected_test_sql: str = ( - build_sql_test_comparison_sql(test_entry=plan_output.test_entries[0]) + "\n" + render_sql_test_comparison_sql((plan_output.test_entries[0],))[0] + "\n" ) manifest: _native.NativeManifest = build_manifest( project=build_test_project(), diff --git a/tests/unit/src/sqlbuild/cli/output/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/output/_helpers/helpers.py index a6be058340..c8143028a1 100644 --- a/tests/unit/src/sqlbuild/cli/output/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/output/_helpers/helpers.py @@ -10,12 +10,12 @@ ExecutionEventWriter, IntegrationResultEnvelope, TerminalEventIndex, + audit_resource_id, format_audit_execution_json, format_build_execution_json, format_test_execution_json, terminal_event_index_scope, ) -from sqlbuild.executor.auditing.main.resource_id import audit_resource_id from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 395b2862df..60fc1cb492 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -16,6 +16,7 @@ plan_compiled_sql_test_artifacts, plan_sql_tests, plan_sql_tests_natively, + render_sql_test_comparison_sql, sql_test_plan_error_messages, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -50,7 +51,6 @@ from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( PlanTestChainTestCase, ) @@ -142,11 +142,11 @@ def assert_native_artifact_matches_runtime_plan( """Assert the compiled artifact matches the runtime comparison SQL for the same plan.""" adapter: DuckDbAdapter = DuckDbAdapter() - expected_sql: str = build_sql_test_comparison_sql( - test_entry=entry, + expected_sql: str = render_sql_test_comparison_sql( + (entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] (artifact,) = plan_compiled_sql_test_artifacts( project=project, tests=(sql_test,), diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index ae3cce2643..6f47e2106d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -13,6 +13,7 @@ build_test_command_plan, plan_compiled_sql_test_artifacts, project_with, + render_sql_test_comparison_sql, resolve_static_command_scope, resolve_static_relation_context, sql_test_plan_error_messages, @@ -41,7 +42,6 @@ PlanOutput, SqlTestPlanEntry, ) -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( AssertionChainCteErrorTestCase, NativePlanningDifferentialTestCase, @@ -1472,11 +1472,11 @@ def test_given_adapter_function_when_resolving_with_analysis_then_keeps_authored entry, _ = plan_single_test( test=compiled_test, project=project, adapter=adapter, sql_analysis_enabled=True ) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=entry, + comparison_sql: str = render_sql_test_comparison_sql( + (entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] assert test_case.expected_sql_fragment in entry.chain[0].resolved_sql assert test_case.expected_sql_fragment in comparison_sql @@ -1523,11 +1523,11 @@ def test_given_tsql_shared_cte_name_when_planning_then_helper_and_model_cte_stay entry, _ = plan_single_test( test=compiled_test, project=project, adapter=adapter, sql_analysis_enabled=True ) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=entry, + comparison_sql: str = render_sql_test_comparison_sql( + (entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] for expected_fragment in test_case.expected_sql_fragments.values(): assert expected_fragment in comparison_sql, comparison_sql diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py index d47675a819..ce7888d745 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py @@ -10,7 +10,11 @@ from types import MappingProxyType, ModuleType from typing import Any, cast -from sqlbuild._native import compiled_graph_keys, resolve_python_sql_run_selection +from sqlbuild._native import ( + build_python_node_graph, + compiled_graph_keys, + resolve_python_sql_run_selection, +) from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -35,7 +39,6 @@ ) from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes._helpers.user_inventory import build_python_node_graph from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunSelection from sqlbuild.providers import Provider from sqlbuild.python_nodes.models import SqlResourceRef diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py index 5bc5f99136..e25150d0a8 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py @@ -9,6 +9,7 @@ import pytest +from sqlbuild._native import build_python_node_graph from sqlbuild.compiler.discovery.models import ( DiscoveredAssetFunction, DiscoveredCheckFunction, @@ -18,10 +19,6 @@ DiscoveredTaskFunction, ) from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.python_nodes._helpers.user_inventory import ( - build_python_node_graph, - build_python_nodes, -) from sqlbuild.compiler.python_nodes.models import ( DiscoveredPythonNode, PythonNodeGraph, @@ -271,63 +268,67 @@ def test_given_fully_configured_python_nodes_when_building_then_identity_config_ retry: RetryPolicy = RetryPolicy(max_attempts=2, retry_on=RuntimeError, jitter=False) columns: tuple[SourceColumnEntry, ...] = (SourceColumnEntry(name="order_id", type="INTEGER"),) - nodes: tuple[DiscoveredPythonNode, ...] = build_python_nodes( - loader_functions=( - DiscoveredLoaderFunction( - file_path=Path("/project/loaders/events.py"), - relative_path=Path("python/loaders/events.py"), - name="load_events", - function=load_events, - destination="raw.events", - cursor_column="updated_at", - unique_key=("event_id",), - columns=columns, - contract="enforced", - connection_mode=LoaderConnectionMode.EXTERNAL, + nodes: tuple[DiscoveredPythonNode, ...] = build_python_node_graph( + discovered_inputs=DiscoveredProjectInputs( + project_config=ProjectConfig(name="demo", adapter="duckdb"), + local_config=LocalConfig(), + loader_functions=( + DiscoveredLoaderFunction( + file_path=Path("/project/loaders/events.py"), + relative_path=Path("python/loaders/events.py"), + name="load_events", + function=load_events, + destination="raw.events", + cursor_column="updated_at", + unique_key=("event_id",), + columns=columns, + contract="enforced", + connection_mode=LoaderConnectionMode.EXTERNAL, + ), ), - ), - task_functions=( - DiscoveredTaskFunction( - file_path=Path("/project/tasks/orders.py"), - relative_path=Path("python/tasks/orders.py"), - name="prepare_orders", - function=prepare_orders, - tags=("orders",), - group="ingestion", - description="Prepare order export inputs.", - meta={"owner": "data-eng"}, - retry=retry, + task_functions=( + DiscoveredTaskFunction( + file_path=Path("/project/tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), + name="prepare_orders", + function=prepare_orders, + tags=("orders",), + group="ingestion", + description="Prepare order export inputs.", + meta={"owner": "data-eng"}, + retry=retry, + ), ), - ), - asset_functions=( - DiscoveredAssetFunction( - file_path=Path("/project/assets/orders.py"), - relative_path=Path("python/assets/orders.py"), - name="export_orders", - function=export_orders, - tags=("orders",), - group="exports", - description="Export orders.", - meta={"owner": "data-eng"}, - columns=columns, - retry=retry, + asset_functions=( + DiscoveredAssetFunction( + file_path=Path("/project/assets/orders.py"), + relative_path=Path("python/assets/orders.py"), + name="export_orders", + function=export_orders, + tags=("orders",), + group="exports", + description="Export orders.", + meta={"owner": "data-eng"}, + columns=columns, + retry=retry, + ), ), - ), - check_functions=( - DiscoveredCheckFunction( - file_path=Path("/project/checks/orders.py"), - relative_path=Path("python/checks/orders.py"), - name="check_orders_export", - function=check_orders_export, - depends_on=(export_orders,), - severity=PythonCheckSeverity.WARN, - tags=("orders",), - group="exports", - description="Check exported orders.", - meta={"owner": "data-eng"}, + check_functions=( + DiscoveredCheckFunction( + file_path=Path("/project/checks/orders.py"), + relative_path=Path("python/checks/orders.py"), + name="check_orders_export", + function=check_orders_export, + depends_on=(export_orders,), + severity=PythonCheckSeverity.WARN, + tags=("orders",), + group="exports", + description="Check exported orders.", + meta={"owner": "data-eng"}, + ), ), - ), - ) + ) + ).nodes assert { f"{node.kind.value}:{node.name}": tuple( diff --git a/tests/unit/src/sqlbuild/executor/diff/test_config.py b/tests/unit/src/sqlbuild/executor/diff/test_config.py index 2880938ca0..5634955258 100644 --- a/tests/unit/src/sqlbuild/executor/diff/test_config.py +++ b/tests/unit/src/sqlbuild/executor/diff/test_config.py @@ -4,13 +4,13 @@ import pytest -from sqlbuild.adapter.contract.models import RowDiffSampling, RowDiffTolerance, RowDiffTolerances -from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.executor.diff._helpers.config import ( +from sqlbuild._native import ( parse_cli_tolerance_overrides, parse_row_diff_tolerances, resolve_row_diff_sampling, ) +from sqlbuild.adapter.contract.models import RowDiffSampling, RowDiffTolerance, RowDiffTolerances +from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.diff.models import RowDiffSamplingOverride from tests.unit.src.sqlbuild.executor.diff._test_types import ( CliToleranceTestCase, diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py index 50b8999d87..c7aefb0ff7 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py @@ -22,6 +22,7 @@ AuditProgressReporter, CostContext, StatementLedgerEntry, + audit_resource_id, read_statement_ledger, record_statement, ) @@ -37,7 +38,6 @@ from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput from sqlbuild.cost.models import CostResourceContext -from sqlbuild.executor.auditing.main.resource_id import audit_resource_id from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.pipeline._helpers import auditing from sqlbuild.executor.pipeline.exceptions import AuditExecutionError, AuditOutcomeError diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index df86a5e314..1d79f6747a 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any, ClassVar, cast -from sqlbuild._native import CostContext, build_python_sql_run_lifecycle +from sqlbuild._native import CostContext, build_python_node_graph, build_python_sql_run_lifecycle from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.assets import AssetContext, asset @@ -23,7 +23,6 @@ DiscoveredProjectInputs, DiscoveredTaskFunction, ) -from sqlbuild.compiler.python_nodes._helpers.user_inventory import build_python_node_graph from sqlbuild.compiler.python_nodes.models import ( DiscoveredPythonNode, PythonNodeGraph, diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/test_expected_comparison_sql.py b/tests/unit/src/sqlbuild/executor/scenario/main/test_expected_comparison_sql.py index 2509e08f22..62200c7067 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/test_expected_comparison_sql.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/test_expected_comparison_sql.py @@ -2,9 +2,7 @@ import pytest -from sqlbuild.executor.scenario.main.expected_comparison_sql import ( - build_scenario_expected_comparison_sql, -) +from sqlbuild._native import build_scenario_expected_comparison_sql from tests.unit.src.sqlbuild.executor.scenario.main._test_types import ( ScenarioExpectedComparisonSqlTestCase, ) diff --git a/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py b/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py index 33923872f2..6f7a19a455 100644 --- a/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py +++ b/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py @@ -4,12 +4,9 @@ import pytest +from sqlbuild._native import render_sql_test_comparison_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql -from sqlbuild.executor.testing.main.comparison_sql_batch import ( - build_sql_test_comparison_sql_batch, -) from tests.unit.src.sqlbuild.executor.testing.main._test_types import ( BuildComparisonSqlTestCase, ExpectedBooleanTestCase, @@ -72,11 +69,11 @@ def test_given_adapter_when_building_comparison_sql_then_it_uses_expected_set_di ) -> None: adapter: BaseAdapter = build_comparison_test_adapter(test_case.adapter_name) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=build_comparison_test_entry(), + comparison_sql: str = render_sql_test_comparison_sql( + (build_comparison_test_entry(),), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] expected_fragment: str for expected_fragment in test_case.expected_fragments: @@ -119,11 +116,11 @@ def test_given_snowflake_function_when_lifting_comparison_ctes_then_keeps_author ), ) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=entry, + comparison_sql: str = render_sql_test_comparison_sql( + (entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] for expected_fragment in test_case.expected_fragments: assert expected_fragment in comparison_sql @@ -161,11 +158,11 @@ def test_given_sql_analysis_disabled_when_building_comparison_sql_then_step_sql_ ), ) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=entry, + comparison_sql: str = render_sql_test_comparison_sql( + (entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] expected_fragment: str for expected_fragment in test_case.expected_fragments: @@ -193,11 +190,11 @@ def test_given_assertion_step_when_building_comparison_sql_then_it_counts_failin ) -> None: adapter: BaseAdapter = build_comparison_test_adapter(test_case.adapter_name) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=build_assertion_test_entry(), + comparison_sql: str = render_sql_test_comparison_sql( + (build_assertion_test_entry(),), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] expected_fragment: str for expected_fragment in test_case.expected_fragments: @@ -224,11 +221,11 @@ def test_given_matching_helper_ctes_when_building_comparison_sql_then_lifts_them ) -> None: adapter: BaseAdapter = build_comparison_test_adapter(test_case.adapter_name) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=build_comparison_test_entry_with_helper_ctes(), + comparison_sql: str = render_sql_test_comparison_sql( + (build_comparison_test_entry_with_helper_ctes(),), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] expected_fragment: str for expected_fragment in test_case.expected_fragments: @@ -250,9 +247,9 @@ def test_given_matching_helper_ctes_when_building_comparison_sql_then_lifts_them def test_given_unasserted_transitive_steps_when_building_comparison_then_emits_only_expected_model( test_case: BuildComparisonSqlTestCase, ) -> None: - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=build_transitive_comparison_test_entry(), - ) + comparison_sql: str = render_sql_test_comparison_sql( + (build_transitive_comparison_test_entry(),) + )[0] for expected_fragment in test_case.expected_fragments: assert expected_fragment in comparison_sql @@ -276,11 +273,11 @@ def test_given_databricks_table_fn_when_building_comparison_sql_then_preserves_c ) -> None: adapter: BaseAdapter = build_comparison_test_adapter(test_case.adapter_name) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=build_table_function_test_entry(), + comparison_sql: str = render_sql_test_comparison_sql( + (build_table_function_test_entry(),), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] expected_fragment: str for expected_fragment in test_case.expected_fragments: @@ -305,13 +302,17 @@ def test_given_bigquery_table_fn_when_building_comparison_sql_then_preserves_bac ) -> None: adapter: BaseAdapter = build_comparison_test_adapter(test_case.adapter_name) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=build_table_function_test_entry( - resolved_sql=("SELECT * FROM `project-d5f92072-d107-4987-9ef.test.customer_orders`(1)") + comparison_sql: str = render_sql_test_comparison_sql( + ( + build_table_function_test_entry( + resolved_sql=( + "SELECT * FROM `project-d5f92072-d107-4987-9ef.test.customer_orders`(1)" + ) + ), ), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] expected_fragment: str for expected_fragment in test_case.expected_fragments: @@ -346,10 +347,7 @@ def test_given_preanalyzed_step_with_authored_cte_when_building_comparison_then_ ), ) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=entry, - sql_analysis_dialect="tsql", - ) + comparison_sql: str = render_sql_test_comparison_sql((entry,), sql_analysis_dialect="tsql")[0] assert ( comparison_sql.index("__ref__raw_orders AS") < comparison_sql.index("picked AS") @@ -373,13 +371,12 @@ def test_given_representative_plans_when_rendering_batch_then_matches_single_ren build_assertion_test_entry(), ) expected: tuple[str, ...] = tuple( - build_sql_test_comparison_sql(test_entry=entry, sql_analysis_dialect="duckdb") + render_sql_test_comparison_sql((entry,), sql_analysis_dialect="duckdb")[0] for entry in entries ) - actual: tuple[str, ...] = build_sql_test_comparison_sql_batch( - test_entries=entries, - sql_analysis_dialect="duckdb", + actual: tuple[str, ...] = tuple( + render_sql_test_comparison_sql(entries, sql_analysis_dialect="duckdb") ) assert (actual == expected) is test_case.expected_result @@ -397,16 +394,18 @@ def test_given_quoted_table_function_when_rendering_batch_then_matches_single_re resolved_sql="SELECT * FROM `project-d5f92072-d107-4987-9ef.test.customer_orders`(1)" ) adapter: BaseAdapter = build_comparison_test_adapter("bigquery") - expected: str = build_sql_test_comparison_sql( - test_entry=entry, + expected: str = render_sql_test_comparison_sql( + (entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] - (actual,) = build_sql_test_comparison_sql_batch( - test_entries=(entry,), - set_difference_operator=adapter.render_set_difference_operator(), - sql_analysis_dialect=adapter.sql_analysis_dialect(), + (actual,) = tuple( + render_sql_test_comparison_sql( + (entry,), + set_difference_operator=adapter.render_set_difference_operator(), + sql_analysis_dialect=adapter.sql_analysis_dialect(), + ) ) assert (actual == expected) is test_case.expected_result