diff --git a/Cargo.lock b/Cargo.lock index 8736887afc..14631c2f4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1354,6 +1354,7 @@ dependencies = [ name = "sqlbuild-state" version = "0.156.1" dependencies = [ + "serde_json", "sha2", "sqlbuild-core", "sqlbuild-cursor", diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/mod.rs index 163629ef76..9c1c3c67c6 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/mod.rs @@ -1 +1,3 @@ +pub(crate) mod resolved_reads; pub(crate) mod schema_order; +pub(crate) mod stub_markers; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs new file mode 100644 index 0000000000..5e70d90bcc --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs @@ -0,0 +1,48 @@ +//! Bounded reuse of native compiler read resolution. + +use crate::planner_fixtures::models::{FixtureReadCache, FixtureReadKey}; +use crate::required_columns::errors::RequiredColumnsError; +use crate::required_columns::main::resolve_required_columns::resolve_required_columns; +use crate::required_columns::models::{ColumnRead, ReferenceInput}; + +/// Cache successful read resolution by the full original Python cache identity. +pub(crate) fn resolved_reads( + mut cache: FixtureReadCache, + key: &FixtureReadKey, +) -> ( + FixtureReadCache, + Result, RequiredColumnsError>, +) { + cache.clock += 1; + let clock = cache.clock; + if let Some((used, reads)) = cache.entries.get_mut(key) { + cache.recency.remove(used); + cache.recency.insert(clock, key.clone()); + *used = clock; + let reads = reads.clone(); + return (cache, Ok(reads)); + } + let references: Vec = key + .references + .iter() + .map(|(kind, name, _, _)| ReferenceInput { + kind: kind.clone(), + name: name.clone(), + }) + .collect(); + cache.misses += 1; + let reads = match resolve_required_columns(&key.sql, &references, key.dialect.as_deref()) { + Ok(reads) => reads, + Err(error) => return (cache, Err(error)), + }; + if cache.entries.len() >= crate::planner_fixtures::constants::READ_CACHE_CAPACITY + && let Some((_, oldest)) = cache.recency.pop_first() + { + cache.entries.remove(&oldest); + } + cache.clock += 1; + let clock = cache.clock; + cache.recency.insert(clock, key.clone()); + cache.entries.insert(key.clone(), (clock, reads.clone())); + (cache, Ok(reads)) +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/stub_markers.rs b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/stub_markers.rs new file mode 100644 index 0000000000..8fd50fa5bc --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/stub_markers.rs @@ -0,0 +1,19 @@ +//! Legacy fixture star-analysis marker substitution. + +use crate::planner_fixtures::constants::MARKER_PATTERNS; +use regex::Regex; +use std::sync::LazyLock; + +static MARKERS: LazyLock, regex::Error>> = + LazyLock::new(|| MARKER_PATTERNS.into_iter().map(Regex::new).collect()); + +/// Substitute exact marker forms, including matches in SQL comments and strings. +pub(crate) fn stub_markers(sql: &str) -> Result { + Ok(MARKERS + .as_ref() + .map_err(Clone::clone)? + .iter() + .fold(sql.to_owned(), |sql, pattern| { + pattern.replace_all(&sql, "$1").into_owned() + })) +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/constants.rs b/crates/sqlbuild-analysis/src/planner_fixtures/constants.rs index ec8f90170a..fdf2ad4716 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/constants.rs @@ -1,3 +1,14 @@ //! Empty fixture marker pattern. +pub(crate) const MARKER_PATTERNS: [&str; 4] = [ + r#"__ref\("([^"]+)"\)"#, + r#"__seed\("([^"]+)"\)"#, + r#"__source\("([^"]+)"\)"#, + r#"__dbt_ref\("[^"]+"[\s\x1c-\x1f]*,[\s\x1c-\x1f]*"([^"]+)"\)"#, +]; + pub(crate) const EMPTY_PATTERN: &str = r"(?i)^[\s\x1c-\x1f]*SELECT[\s\x1c-\x1f]+\*[\s\x1c-\x1f]+FROM[\s\x1c-\x1f]+__empty_fixture[\s\x1c-\x1f]*\([\s\x1c-\x1f]*\)[\s\x1c-\x1f]*$"; +pub(crate) const READ_CACHE_CAPACITY: usize = 4096; +pub(crate) const FIXTURE_WHITESPACE: &str = r"[\s\x1c-\x1f]"; +pub(crate) const COLLECTION_TYPES: [&str; 6] = + ["ARRAY", "LIST", "MAP", "OBJECT", "STRUCT", "VARIANT"]; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/collection_mismatch.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/collection_mismatch.rs new file mode 100644 index 0000000000..6361eeb994 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/collection_mismatch.rs @@ -0,0 +1,19 @@ +//! Static collection-versus-scalar fixture type conflicts. + +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::planner_fixtures::constants::COLLECTION_TYPES; + +/// Whether exactly one of two known fixture types is a collection type. +pub fn collection_mismatch(expected: Option<&str>, inferred: Option<&str>) -> bool { + let (Some(expected), Some(inferred)) = (expected, inferred) else { + return false; + }; + is_collection(expected) != is_collection(inferred) +} + +fn is_collection(sql_type: &str) -> bool { + let upper = sql_type.to_uppercase(); + let base = python_strip(upper.split('(').next().unwrap_or_default()); + COLLECTION_TYPES.contains(&base) +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_line.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_line.rs new file mode 100644 index 0000000000..12915ac28c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_line.rs @@ -0,0 +1,16 @@ +//! Source line of one SQL-test fixture CTE definition. + +use regex::Regex; + +use crate::planner_fixtures::constants::FIXTURE_WHITESPACE; + +/// Return the one-based line that starts a fixture CTE definition, or line one when absent. +pub fn fixture_line(contents: &str, cte_name: &str) -> Result { + let pattern = Regex::new(&format!( + r"(?im)^{FIXTURE_WHITESPACE}*{}{FIXTURE_WHITESPACE}+AS{FIXTURE_WHITESPACE}*\(", + regex::escape(cte_name) + ))?; + Ok(pattern.find(contents).map_or(1, |found| { + contents[..found.start()].matches('\n').count() + 1 + })) +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs index e07b486003..61b86c8338 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/mod.rs @@ -1,10 +1,14 @@ pub mod align_columns; +pub mod collection_mismatch; pub mod empty_fixture_error; pub mod empty_fixture_marker; +pub mod fixture_line; pub mod fixture_schema; pub mod merge_metadata; pub mod propagate_types; pub mod render_completed; pub mod render_empty; +pub mod required_columns; pub mod requires_analysis; +pub mod star_keys; pub mod unknown_columns; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs new file mode 100644 index 0000000000..d8a2c70a58 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs @@ -0,0 +1,97 @@ +//! Required fixture columns derived from compiler read evidence. + +use sqlbuild_core::text::main::python_casefold::python_casefold; +use std::collections::HashMap; + +use crate::planner_fixtures::_helpers::resolved_reads::resolved_reads; +use crate::planner_fixtures::models::{ + FixtureReadCache, FixtureReadModel, FixtureReadRequirements, FixtureReadSchemas, RequiredColumn, +}; +use crate::required_columns::errors::RequiredColumnsError; + +/// Collect direct reads and authoritative star columns in model discovery order. +pub fn required_columns( + mut cache: FixtureReadCache, + models: &[FixtureReadModel], + schemas: FixtureReadSchemas<'_>, +) -> ( + FixtureReadCache, + Result, +) { + let FixtureReadSchemas { + keys: fixture_keys, + authoritative, + relations, + } = schemas; + let mut result = FixtureReadRequirements::default(); + for model in models { + let mut reads = model.high_confidence.clone(); + if model.key.references.iter().any(|(kind, name, _, _)| { + let resource_type = match kind.as_str() { + "ref" => "model", + "source" => "source", + "seed" => "seed", + _ => return false, + }; + fixture_keys.contains(&(resource_type.to_owned(), name.clone())) + }) { + let (updated, resolved) = resolved_reads(cache, &model.key); + cache = updated; + match resolved { + Ok(resolved) => reads.extend(resolved), + Err(error) => return (cache, Err(error)), + } + } + for read in reads { + let key = (read.resource_type, read.resource_name); + if !fixture_keys.contains(&key) + || authoritative + .get(&key) + .is_some_and(|names| !names.contains(&python_casefold(&read.column_name))) + { + continue; + } + result = add_required(result, key, &read.column_name, &model.name); + } + for key in &model.stars { + result.stars.insert(key.clone()); + if let Some(relation) = relations + .iter() + .find(|relation| &relation.key == key && relation.authoritative) + { + for column in &relation.columns { + result = add_required(result, key.clone(), &column.name, &model.name); + } + } + } + } + (cache, Ok(result)) +} + +fn add_required( + mut result: FixtureReadRequirements, + key: (String, String), + name: &str, + model: &str, +) -> FixtureReadRequirements { + let position = *result.indices.entry(key.clone()).or_insert_with(|| { + result.columns.push((key, Vec::new())); + result.column_indices.push(HashMap::new()); + result.columns.len() - 1 + }); + let columns = &mut result.columns[position].1; + if let Some(index) = result.column_indices[position].get(name) { + let column = &mut columns[*index]; + if column.readers.last().is_none_or(|reader| reader != model) { + column.readers.push(model.to_owned()); + } + } else { + result.column_indices[position].insert(name.to_owned(), columns.len()); + columns.push(RequiredColumn { + name: name.to_owned(), + folded_name: python_casefold(name), + readers: vec![model.to_owned()], + }); + } + result +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs new file mode 100644 index 0000000000..5afda5f19d --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs @@ -0,0 +1,125 @@ +//! Fixture dependencies selected by star projections. + +use std::collections::{HashMap, HashSet}; +use std::str::FromStr; + +use polyglot_sql::{ + AnalyzeQueryOptions, DialectType, SchemaColumn, SchemaTable, ValidationSchema, analyze_query, +}; +use sqlbuild_core::text::main::python_casefold::python_casefold; + +use crate::planner_fixtures::_helpers::stub_markers::stub_markers; +use crate::planner_fixtures::models::FixtureStarRelation; + +/// Resolve star inputs using the same semantic analysis as the compiler. +pub fn star_keys( + sql: &str, + relations: &[FixtureStarRelation], + dialect: Option<&str>, + proven_inputs: Option<&[String]>, +) -> Vec<(String, String)> { + if let Some(inputs) = proven_inputs { + let names: HashSet = inputs.iter().map(|name| python_casefold(name)).collect(); + return relations + .iter() + .filter(|relation| names.contains(&python_casefold(&relation.key.1))) + .map(|relation| relation.key.clone()) + .collect(); + } + if !relations + .iter() + .any(|relation| !relation.columns.is_empty()) + { + return Vec::new(); + } + let Ok(dialect) = + DialectType::from_str(dialect.filter(|name| !name.is_empty()).unwrap_or("generic")) + else { + return Vec::new(); + }; + let tables: Vec = relations + .iter() + .filter(|relation| !relation.columns.is_empty()) + .map(schema_table) + .collect(); + let options = AnalyzeQueryOptions { + dialect, + schema: Some(ValidationSchema { + tables, + strict: None, + }), + complexity_guard: None, + }; + let Ok(sql) = stub_markers(sql) else { + return Vec::new(); + }; + let Ok(analysis) = analyze_query(&sql, options) else { + return Vec::new(); + }; + let mut aliases: HashMap = HashMap::new(); + for relation in analysis.relations { + let name = python_casefold(&relation.name); + aliases.insert(name.clone(), name.clone()); + if let Some(alias) = relation.alias { + aliases.insert(python_casefold(&alias), name); + } + } + let mut names: HashSet = HashSet::new(); + for projection in analysis.star_projections { + if let Some(table) = projection.table { + if let Some(name) = aliases.get(&python_casefold(&table)) { + names.insert(name.clone()); + } + } else { + let expanded: HashSet = projection + .expanded_columns + .iter() + .map(|name| python_casefold(name)) + .collect(); + for relation in relations { + if relation + .columns + .iter() + .any(|column| expanded.contains(&column.folded_name)) + { + names.insert(python_casefold(&relation.key.1)); + } + } + } + } + relations + .iter() + .filter(|relation| names.contains(&python_casefold(&relation.key.1))) + .map(|relation| relation.key.clone()) + .collect() +} + +fn schema_table(relation: &FixtureStarRelation) -> SchemaTable { + SchemaTable { + name: relation.key.1.clone(), + schema: None, + columns: relation + .columns + .iter() + .map(|column| SchemaColumn { + name: column.name.clone(), + data_type: column_type(column.sql_type.as_deref()), + nullable: None, + primary_key: false, + unique: false, + references: None, + }) + .collect(), + aliases: Vec::new(), + primary_key: Vec::new(), + unique_keys: Vec::new(), + foreign_keys: Vec::new(), + } +} + +fn column_type(sql_type: Option<&str>) -> String { + sql_type + .filter(|name| !name.is_empty()) + .unwrap_or("UNKNOWN") + .to_owned() +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs b/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs index b010d05a00..42b1da1020 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs @@ -7,3 +7,4 @@ pub mod main; pub mod models; #[cfg(test)] mod tests; +pub mod types; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/models.rs b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs index 2fbeba43f8..e6995f179a 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/models.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs @@ -1,6 +1,61 @@ //! Passive fixture schema and alignment facts. -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +/// Local fixture schemas used by required-read aggregation. +#[derive(Debug)] +pub struct FixtureReadSchemas<'a> { + pub keys: &'a HashSet<(String, String)>, + pub authoritative: &'a HashMap<(String, String), HashSet>, + pub relations: &'a [FixtureStarRelation], +} + +use crate::required_columns::models::ColumnRead; + +use crate::planner_fixtures::types::FixtureReadReference; + +/// The original SQL, complete references and dialect form the read cache key. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct FixtureReadKey { + pub sql: String, + pub references: Vec, + pub dialect: Option, +} + +/// A bounded LRU of successful fixture read analyses. +#[derive(Debug, Default)] +pub struct FixtureReadCache { + pub entries: HashMap)>, + pub recency: BTreeMap, + pub clock: u64, + pub misses: u64, +} + +/// A model's read evidence and star dependencies in planning order. +#[derive(Debug)] +pub struct FixtureReadModel { + pub name: String, + pub key: FixtureReadKey, + pub high_confidence: Vec, + pub stars: Vec<(String, String)>, +} + +/// Required columns retain model and column insertion order. +#[derive(Debug, Default)] +pub struct FixtureReadRequirements { + pub columns: Vec<((String, String), Vec)>, + pub stars: BTreeSet<(String, String)>, + pub(crate) indices: HashMap<(String, String), usize>, + pub(crate) column_indices: Vec>, +} + +/// One fixture dependency and the schema available to star analysis. +#[derive(Debug)] +pub struct FixtureStarRelation { + pub key: (String, String), + pub columns: Vec, + pub authoritative: bool, +} /// One upstream typed column identified by its resource and folded name. #[derive(Debug)] diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs index a1512cca60..3684bc64a0 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests.rs @@ -4,7 +4,11 @@ mod helpers; mod test_alignment; #[path = "tests/test_propagation.rs"] mod test_propagation; +#[path = "tests/test_reads.rs"] +mod test_reads; #[path = "tests/test_schema.rs"] mod test_schema; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_validation.rs"] +mod test_validation; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs new file mode 100644 index 0000000000..8d19e9e57e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs @@ -0,0 +1,112 @@ +use crate::planner_fixtures::_helpers::resolved_reads::resolved_reads; +use crate::planner_fixtures::_helpers::stub_markers::stub_markers; +use crate::planner_fixtures::models::{FixtureReadCache, FixtureReadKey}; +use crate::planner_fixtures::tests::test_types::{FixtureMarkerTestCase, FixtureReadCacheTestCase}; + +#[test] +fn given_repeated_reads_when_resolving_then_full_reference_identity_controls_reuse() { + let test_cases = [ + FixtureReadCacheTestCase { + description: "identical request", + sql: "SELECT order_id FROM __source(\"orders\")", + package: None, + argument_count: None, + dialect: Some("duckdb"), + expected_misses: 1, + }, + FixtureReadCacheTestCase { + description: "package changes", + sql: "SELECT order_id FROM __source(\"orders\")", + package: Some("shop"), + argument_count: None, + dialect: Some("duckdb"), + expected_misses: 2, + }, + FixtureReadCacheTestCase { + description: "argument count changes", + sql: "SELECT order_id FROM __source(\"orders\")", + package: None, + argument_count: Some(1), + dialect: Some("duckdb"), + expected_misses: 2, + }, + FixtureReadCacheTestCase { + description: "dialect changes", + sql: "SELECT order_id FROM __source(\"orders\")", + package: None, + argument_count: None, + dialect: None, + expected_misses: 2, + }, + FixtureReadCacheTestCase { + description: "SQL changes", + sql: "SELECT status FROM __source(\"orders\")", + package: None, + argument_count: None, + dialect: Some("duckdb"), + expected_misses: 2, + }, + ]; + for test_case in test_cases { + let key = FixtureReadKey { + sql: "SELECT order_id FROM __source(\"orders\")".to_owned(), + references: vec![("source".into(), "orders".into(), None, None)], + dialect: Some("duckdb".into()), + }; + let (cache, initial) = resolved_reads(FixtureReadCache::default(), &key); + let (cache, repeated) = resolved_reads(cache, &key); + assert_eq!(initial.ok(), repeated.ok(), "{}", test_case.description); + assert_eq!(cache.misses, 1, "{}", test_case.description); + let changed = FixtureReadKey { + sql: test_case.sql.to_owned(), + references: vec![( + "source".into(), + "orders".into(), + test_case.package.map(str::to_owned), + test_case.argument_count, + )], + dialect: test_case.dialect.map(str::to_owned), + }; + let (cache, reads) = resolved_reads(cache, &changed); + assert!(reads.is_ok(), "{}", test_case.description); + assert_eq!( + cache.misses, test_case.expected_misses, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_marker_text_when_stubbing_for_star_analysis_then_preserves_legacy_lexical_semantics() { + let test_cases = [ + FixtureMarkerTestCase { + description: "Unicode reference", + sql: "SELECT * FROM __source(\"Straße\")", + expected_sql: "SELECT * FROM Straße", + }, + FixtureMarkerTestCase { + description: "string and comment matches", + sql: "SELECT '__ref(\"orders\")' -- __seed(\"customers\")", + expected_sql: "SELECT 'orders' -- customers", + }, + FixtureMarkerTestCase { + description: "packaged reference and block comment", + sql: "SELECT * FROM __dbt_ref(\"shop\", \"orders\") /* __source(\"customers\") */", + expected_sql: "SELECT * FROM orders /* customers */", + }, + FixtureMarkerTestCase { + description: "unmatched marker spellings", + sql: "SELECT * FROM __REF(\"orders\") JOIN __ref( 'customers' ) ON TRUE", + expected_sql: "SELECT * FROM __REF(\"orders\") JOIN __ref( 'customers' ) ON TRUE", + }, + ]; + for test_case in test_cases { + assert_eq!( + stub_markers(test_case.sql).ok().as_deref(), + Some(test_case.expected_sql), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs index 0b02326180..feade452bc 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs @@ -26,3 +26,35 @@ pub(crate) struct AlignmentTestCase { pub(crate) expected_completed: Option>, pub(crate) expected_diagnostics: &'static [&'static str], } +#[derive(Debug)] +pub(super) struct FixtureReadCacheTestCase { + pub description: &'static str, + pub sql: &'static str, + pub package: Option<&'static str>, + pub argument_count: Option, + pub dialect: Option<&'static str>, + pub expected_misses: u64, +} + +#[derive(Debug)] +pub(super) struct FixtureMarkerTestCase { + pub description: &'static str, + pub sql: &'static str, + pub expected_sql: &'static str, +} + +#[derive(Debug)] +pub(crate) struct FixtureLineTestCase { + pub(crate) description: &'static str, + pub(crate) contents: &'static str, + pub(crate) cte_name: &'static str, + pub(crate) expected_line: usize, +} + +#[derive(Debug)] +pub(crate) struct CollectionMismatchTestCase { + pub(crate) description: &'static str, + pub(crate) expected_type: Option<&'static str>, + pub(crate) inferred_type: Option<&'static str>, + pub(crate) expected_mismatch: bool, +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_validation.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_validation.rs new file mode 100644 index 0000000000..a1df4390c6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_validation.rs @@ -0,0 +1,92 @@ +use crate::planner_fixtures::main::{ + collection_mismatch::collection_mismatch, fixture_line::fixture_line, +}; +use crate::planner_fixtures::tests::test_types::{CollectionMismatchTestCase, FixtureLineTestCase}; + +#[test] +fn given_fixture_cte_when_locating_then_reports_python_compatible_line() { + let test_cases = [ + FixtureLineTestCase { + description: "definition line", + contents: "TEST();\n\nWITH\n__source__orders AS (SELECT 1)\n", + cte_name: "__source__orders", + expected_line: 4, + }, + FixtureLineTestCase { + description: "leading blank line starts the match", + contents: "TEST();\nWITH\n\n__source__orders AS (SELECT 1)\n", + cte_name: "__source__orders", + expected_line: 3, + }, + FixtureLineTestCase { + description: "case-insensitive keyword across lines", + contents: "WITH\n__expected__orders\n as\n (SELECT 1)\n", + cte_name: "__expected__orders", + expected_line: 2, + }, + FixtureLineTestCase { + description: "information separator counts as whitespace", + contents: "WITH\n\u{1f}__ref__orders\u{1c}AS(SELECT 1)\n", + cte_name: "__ref__orders", + expected_line: 2, + }, + FixtureLineTestCase { + description: "missing definition falls back to line one", + contents: "WITH /* x */ __seed__codes AS (SELECT 1)\n", + cte_name: "__seed__codes", + expected_line: 1, + }, + FixtureLineTestCase { + description: "name is matched literally", + contents: "WITH\n__ref__aXb AS (SELECT 1)\n__ref__a.b AS (SELECT 1)\n", + cte_name: "__ref__a.b", + expected_line: 3, + }, + ]; + for test_case in test_cases { + assert_eq!( + fixture_line(test_case.contents, test_case.cte_name).ok(), + Some(test_case.expected_line), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_fixture_types_when_comparing_then_flags_collection_scalar_conflicts() { + let test_cases = [ + CollectionMismatchTestCase { + description: "array against scalar", + expected_type: Some("ARRAY"), + inferred_type: Some("VARCHAR"), + expected_mismatch: true, + }, + CollectionMismatchTestCase { + description: "parameterized collections agree", + expected_type: Some("struct(a INTEGER)"), + inferred_type: Some(" MAP (VARCHAR, INTEGER)"), + expected_mismatch: false, + }, + CollectionMismatchTestCase { + description: "scalars agree", + expected_type: Some("INTEGER"), + inferred_type: Some("NUMBER(38, 0)"), + expected_mismatch: false, + }, + CollectionMismatchTestCase { + description: "unknown type never conflicts", + expected_type: None, + inferred_type: Some("ARRAY"), + expected_mismatch: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + collection_mismatch(test_case.expected_type, test_case.inferred_type), + test_case.expected_mismatch, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/types.rs b/crates/sqlbuild-analysis/src/planner_fixtures/types.rs new file mode 100644 index 0000000000..ec569300e0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/types.rs @@ -0,0 +1,4 @@ +//! Fixture analysis identities. + +/// The complete reference identity retained by the fixture read cache. +pub type FixtureReadReference = (String, String, Option, Option); diff --git a/crates/sqlbuild-analysis/src/planner_scenario/_helpers/markers.rs b/crates/sqlbuild-analysis/src/planner_scenario/_helpers/markers.rs new file mode 100644 index 0000000000..4f9ff8fb48 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/_helpers/markers.rs @@ -0,0 +1,25 @@ +//! Exact authored marker recognition at a byte offset. + +use crate::planner_scenario::constants::{DBT_REF_MARKER, MARKERS}; +use sqlbuild_sqltext::resolution::main::quoted_name::quoted_name; + +pub(crate) fn marker_at(sql: &str, kind: usize) -> Option<(usize, String)> { + let tail = sql.strip_prefix(MARKERS[kind])?; + let (first, tail) = quoted_name(tail)?; + if tail.starts_with(')') { + return Some((sql.len() - tail.len() + 1, first.to_owned())); + } + if kind != DBT_REF_MARKER { + return None; + } + let tail = tail + .trim_start_matches(python_whitespace) + .strip_prefix(',')?; + let (second, tail) = quoted_name(tail.trim_start_matches(python_whitespace))?; + tail.starts_with(')') + .then(|| (sql.len() - tail.len() + 1, format!("{first}__{second}"))) +} + +fn python_whitespace(character: char) -> bool { + character.is_whitespace() || matches!(character, '\u{1c}'..='\u{1f}') +} diff --git a/crates/sqlbuild-analysis/src/planner_scenario/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_scenario/_helpers/mod.rs new file mode 100644 index 0000000000..dd0049eb8b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/_helpers/mod.rs @@ -0,0 +1,3 @@ +//! Shared scenario SQL text operations. + +pub(super) mod markers; diff --git a/crates/sqlbuild-analysis/src/planner_scenario/constants.rs b/crates/sqlbuild-analysis/src/planner_scenario/constants.rs new file mode 100644 index 0000000000..6809122df1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/constants.rs @@ -0,0 +1,5 @@ +//! Exact authored scenario marker grammar. + +pub(super) const MARKERS: [&str; 4] = ["__ref(", "__seed(", "__source(", "__dbt_ref("]; +pub(super) const DBT_REF_MARKER: usize = 3; +pub(super) const HASH_LENGTH: usize = 12; diff --git a/crates/sqlbuild-analysis/src/planner_scenario/errors.rs b/crates/sqlbuild-analysis/src/planner_scenario/errors.rs new file mode 100644 index 0000000000..c3203fe1bf --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/errors.rs @@ -0,0 +1,10 @@ +//! Scenario marker callback and lexical failures. + +use sqlbuild_sqltext::sql_scan::models::Unclosed; + +/// Preserve the first callback or lexical failure in authored SQL order. +#[derive(Debug)] +pub enum MarkerError { + Scan(Unclosed), + Resolve(E), +} diff --git a/crates/sqlbuild-analysis/src/planner_scenario/main/hash_prefix.rs b/crates/sqlbuild-analysis/src/planner_scenario/main/hash_prefix.rs new file mode 100644 index 0000000000..bcbc07359c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/main/hash_prefix.rs @@ -0,0 +1,18 @@ +//! Stable scenario artifact identity hashing. + +use sha2::{Digest, Sha256}; + +/// Hash a project, optional namespace and scenario without changing the stored identity format. +pub fn hash_prefix( + project: &str, + scenario: &str, + namespace: Option<&str>, + length: usize, +) -> String { + let input = namespace.map_or_else( + || format!("{project}:{scenario}"), + |namespace| format!("{project}:{namespace}:{scenario}"), + ); + let hash = format!("{:x}", Sha256::digest(input.as_bytes())); + hash[..length.min(hash.len())].to_owned() +} diff --git a/crates/sqlbuild-analysis/src/planner_scenario/main/mod.rs b/crates/sqlbuild-analysis/src/planner_scenario/main/mod.rs index 97c2acc407..cf8afe82ee 100644 --- a/crates/sqlbuild-analysis/src/planner_scenario/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_scenario/main/mod.rs @@ -1,3 +1,6 @@ //! Scenario planning entry points. pub mod graph; +pub mod hash_prefix; +pub mod parse_artifact_name; +pub mod replace_markers; diff --git a/crates/sqlbuild-analysis/src/planner_scenario/main/parse_artifact_name.rs b/crates/sqlbuild-analysis/src/planner_scenario/main/parse_artifact_name.rs new file mode 100644 index 0000000000..0aacd463bc --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/main/parse_artifact_name.rs @@ -0,0 +1,21 @@ +//! Recognize persisted scenario relation names. + +/// Parse a scenario artifact name using the original fixed hash length and accepted kinds. +pub fn parse_artifact_name(name: &str) -> Option<(&str, &str, &str)> { + let (hash, rest) = name.strip_prefix("__sqb_")?.split_once("__")?; + if hash.len() != crate::planner_scenario::constants::HASH_LENGTH + || !hash + .bytes() + .all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c)) + { + return None; + } + let (kind, logical) = rest.split_once("__")?; + if !["model", "source", "ref", "seed", "dbt_ref"].contains(&kind) + || logical.is_empty() + || logical.contains('\n') + { + return None; + } + Some((hash, kind, logical)) +} diff --git a/crates/sqlbuild-analysis/src/planner_scenario/main/replace_markers.rs b/crates/sqlbuild-analysis/src/planner_scenario/main/replace_markers.rs new file mode 100644 index 0000000000..6f105d8713 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/main/replace_markers.rs @@ -0,0 +1,45 @@ +//! Replace exact scenario relation markers without regenerating authored SQL. + +use crate::planner_scenario::{_helpers::markers::marker_at, errors::MarkerError}; +use sqlbuild_sqltext::code_scanner::main::{ + code_scanner::code_scanner, scanner_non_code_end::scanner_non_code_end, +}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +/// Resolve markers in execution order, optionally retaining the legacy unscanned dbt-ref policy. +pub fn replace_markers( + sql: &str, + kind: usize, + syntax: Option, + mut resolve: impl FnMut(&str, &str) -> Result, +) -> Result> { + let scanner = syntax.map(|syntax| code_scanner(sql, Some(syntax))); + let offsets: Vec = sql + .char_indices() + .map(|(offset, _)| offset) + .chain(std::iter::once(sql.len())) + .collect(); + let mut index = 0; + let mut last = 0; + let mut result = String::new(); + while index + 1 < offsets.len() { + if let Some(scanner) = &scanner + && let Some(end) = scanner_non_code_end(scanner, index).map_err(MarkerError::Scan)? + { + index = end; + continue; + } + let start = offsets[index]; + if let Some((end, name)) = marker_at(&sql[start..], kind) { + let found = &sql[start..start + end]; + result.push_str(&sql[last..start]); + result.push_str(&resolve(&name, found).map_err(MarkerError::Resolve)?); + last = start + end; + index += found.chars().count(); + continue; + } + index += 1; + } + result.push_str(&sql[last..]); + Ok(result) +} diff --git a/crates/sqlbuild-analysis/src/planner_scenario/mod.rs b/crates/sqlbuild-analysis/src/planner_scenario/mod.rs index 2c241db51f..6c2bb84a2d 100644 --- a/crates/sqlbuild-analysis/src/planner_scenario/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_scenario/mod.rs @@ -3,5 +3,8 @@ pub mod main; pub mod models; +mod _helpers; +mod constants; +pub mod errors; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-analysis/src/planner_scenario/tests.rs b/crates/sqlbuild-analysis/src/planner_scenario/tests.rs index 59d9eea16f..59074e360f 100644 --- a/crates/sqlbuild-analysis/src/planner_scenario/tests.rs +++ b/crates/sqlbuild-analysis/src/planner_scenario/tests.rs @@ -2,5 +2,7 @@ #[path = "tests/test_graph.rs"] mod test_graph; +#[path = "tests/test_markers.rs"] +mod test_markers; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_scenario/tests/test_markers.rs b/crates/sqlbuild-analysis/src/planner_scenario/tests/test_markers.rs new file mode 100644 index 0000000000..2f4fe2c466 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scenario/tests/test_markers.rs @@ -0,0 +1,69 @@ +//! Scenario marker text and failure-order regression coverage. + +use crate::planner_scenario::main::replace_markers::replace_markers; +use crate::planner_scenario::tests::test_types::MarkerTestCase; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +#[test] +fn given_scenario_markers_when_resolving_then_preserves_authored_text_and_legacy_dbt_policy() { + let test_cases = [ + MarkerTestCase { + description: "Unicode markers preserve comments and aliases", + sql: "SELECT 'é', * FROM __ref(\"订单\") x -- __ref(\"ignored\")", + kind: 0, + scanned: true, + expected_sql: Ok("SELECT 'é', * FROM target_订单 x -- __ref(\"ignored\")"), + }, + MarkerTestCase { + description: "model dbt refs retain legacy unscanned substitution", + sql: "SELECT '__dbt_ref(\"orders\")'", + kind: 3, + scanned: false, + expected_sql: Ok("SELECT 'target_orders'"), + }, + MarkerTestCase { + description: "check dbt refs preserve string literals", + sql: "SELECT '__dbt_ref(\"orders\")'", + kind: 3, + scanned: true, + expected_sql: Ok("SELECT '__dbt_ref(\"orders\")'"), + }, + MarkerTestCase { + description: "dbt refs accept Python whitespace grammar", + sql: "SELECT * FROM __dbt_ref(\"shop\"\u{1c},\u{2003}\"orders\")", + kind: 3, + scanned: true, + expected_sql: Ok("SELECT * FROM target_shop__orders"), + }, + MarkerTestCase { + description: "uppercase dbt calls stay authored", + sql: "SELECT * FROM __DBT_REF(\"orders\")", + kind: 3, + scanned: true, + expected_sql: Ok("SELECT * FROM __DBT_REF(\"orders\")"), + }, + MarkerTestCase { + description: "unterminated strings fail scanning", + sql: "SELECT 'unterminated", + kind: 0, + scanned: true, + expected_sql: Err("failure"), + }, + ]; + for test_case in test_cases { + let syntax = test_case.scanned.then(|| LexicalSyntax { + line_comment_prefixes: vec!["--".to_owned()], + ..LexicalSyntax::default() + }); + let actual = replace_markers(test_case.sql, test_case.kind, syntax, |name, _| { + Ok::<_, ()>(format!("target_{name}")) + }) + .map_err(|_| "failure"); + assert_eq!( + actual.as_deref(), + test_case.expected_sql.map_err(|_| &"failure"), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_scenario/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_scenario/tests/test_types.rs index eb99591ff8..bfdd023dc9 100644 --- a/crates/sqlbuild-analysis/src/planner_scenario/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_scenario/tests/test_types.rs @@ -10,3 +10,11 @@ pub(super) struct GraphTestCase { pub expected_functions: Vec<(&'static str, &'static str)>, pub expected_warnings: Vec<(&'static str, bool)>, } +#[derive(Debug)] +pub(super) struct MarkerTestCase { + pub description: &'static str, + pub sql: &'static str, + pub kind: usize, + pub scanned: bool, + pub expected_sql: Result<&'static str, &'static str>, +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/constants.rs b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs index 9029fe4ae4..2eabec0ab1 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs @@ -76,3 +76,13 @@ pub const DECIMAL_TYPE_NAMES: [&str; 3] = ["DECIMAL", "NUMERIC", "NUMBER"]; pub const POLYGLOT_CUSTOM_TYPE_NAME: &str = "CUSTOM"; /// The dialect cursor type classification parses unknown types with. pub const GENERIC_DIALECT: &str = "generic"; +/// Error code of an authored cursor input column its input does not expose. +pub const CURSOR_INPUT_COLUMN_ERROR_CODE: &str = "S302"; +/// Incremental strategies whose microbatches replay one batch by default. +pub const IDEMPOTENT_MICROBATCH_STRATEGIES: [&str; 2] = ["delete_insert", "merge"]; +/// The table type that drops the adapter's transient DDL keyword. +pub const PERMANENT_TABLE_TYPE: &str = "permanent"; +/// The transient table DDL a permanent table rewrites once. +pub const TRANSIENT_TABLE_DDL: &str = "TRANSIENT TABLE"; +/// The permanent table DDL replacing the transient keyword. +pub const PERMANENT_TABLE_DDL: &str = "TABLE"; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_input_column.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_input_column.rs new file mode 100644 index 0000000000..d64b814a1e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_input_column.rs @@ -0,0 +1,44 @@ +//! Whether an authored cursor input column is exposed by the input it names. + +use crate::planner_strategy::constants::CURSOR_INPUT_COLUMN_ERROR_CODE; +use crate::planner_strategy::models::{CursorInputColumnCheck, CursorInputColumnOwner}; + +/// The planner error message and code when the column is missing from the known columns, matched case-insensitively. +pub fn cursor_input_column_failure( + check: &CursorInputColumnCheck, +) -> Option<(String, &'static str)> { + let column: String = check.column.to_lowercase(); + if check + .known_columns + .iter() + .any(|known| known.to_lowercase() == column) + { + return None; + } + let (kind, exposer, label) = match check.owner { + CursorInputColumnOwner::ModelContract => { + ("model", "that model contract", "Known output columns") + } + CursorInputColumnOwner::SourceContract => ( + "source", + "that source contract", + "Declared contract columns", + ), + CursorInputColumnOwner::SourceWarehouse => { + ("source", "that source", "Known source columns") + } + }; + let known: String = if check.known_columns.is_empty() { + "none".to_owned() + } else { + check.known_columns.join(", ") + }; + Some(( + format!( + "model '{}': cursor_inputs references {kind} '{}' column '{}', but {exposer} does \ + not expose the column. {label}: {known}", + check.model_name, check.input_name, check.column + ), + CURSOR_INPUT_COLUMN_ERROR_CODE, + )) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/microbatch_lookback.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/microbatch_lookback.rs new file mode 100644 index 0000000000..15f2eb1d22 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/microbatch_lookback.rs @@ -0,0 +1,25 @@ +//! The explicit or default lookback of a microbatch model. + +use sqlbuild_core::plan_display::errors::UnknownCursorGrain; +use sqlbuild_core::plan_display::main::effective_microbatch_batch_size::effective_microbatch_batch_size; + +use crate::planner_strategy::constants::IDEMPOTENT_MICROBATCH_STRATEGIES; +use crate::planner_strategy::models::LookbackConfig; + +/// The authored lookback, else one batch at the replay grain for idempotent strategies. +pub fn microbatch_lookback(config: &LookbackConfig) -> Result, UnknownCursorGrain> { + if let Some(lookback) = &config.lookback { + return Ok(Some(lookback.clone())); + } + let idempotent: bool = config + .incremental_strategy + .as_deref() + .is_some_and(|strategy| IDEMPOTENT_MICROBATCH_STRATEGIES.contains(&strategy)); + let (true, Some(batch_size)) = (idempotent, &config.batch_size) else { + return Ok(None); + }; + match &config.grain { + Some(grain) => effective_microbatch_batch_size(batch_size, grain).map(Some), + None => Ok(Some(batch_size.clone())), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs index 533eb51d7b..b7532df463 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs @@ -4,12 +4,15 @@ pub mod capped_watermark_input; pub mod classify_cursor_sql_type; pub mod count_integer_batches; pub mod count_timeline_batches; +pub mod cursor_input_column; pub mod cursor_type_warning; pub mod effective_run_scope; pub mod lineage_reference; pub mod materialization_type; pub mod microbatch_limit_warning; +pub mod microbatch_lookback; pub mod model_plan_action; pub mod model_warnings; pub mod schema_actions; pub mod singular_attachment; +pub mod table_type_ddl; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/table_type_ddl.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/table_type_ddl.rs new file mode 100644 index 0000000000..ee63f64cca --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/table_type_ddl.rs @@ -0,0 +1,13 @@ +//! Logical table DDL for the model's table type. + +use crate::planner_strategy::constants::{ + PERMANENT_TABLE_DDL, PERMANENT_TABLE_TYPE, TRANSIENT_TABLE_DDL, +}; + +/// The adapter's DDL, with its first transient table keyword dropped for permanent tables. +pub fn table_type_ddl(ddl: &str, table_type: &str) -> String { + if table_type == PERMANENT_TABLE_TYPE { + return ddl.replacen(TRANSIENT_TABLE_DDL, PERMANENT_TABLE_DDL, 1); + } + ddl.to_owned() +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/models.rs b/crates/sqlbuild-analysis/src/planner_strategy/models.rs index 5553b77357..959f26c038 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/models.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/models.rs @@ -178,3 +178,30 @@ pub enum CappedBound { End(T), Start(T), } + +/// Which declaration a cursor input column is checked against. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CursorInputColumnOwner { + ModelContract, + SourceContract, + SourceWarehouse, +} + +/// One authored cursor input column and the columns its input is known to expose. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CursorInputColumnCheck { + pub model_name: String, + pub owner: CursorInputColumnOwner, + pub input_name: String, + pub column: String, + pub known_columns: Vec, +} + +/// The configuration a microbatch model's default lookback is resolved from. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LookbackConfig { + pub lookback: Option, + pub incremental_strategy: Option, + pub batch_size: Option, + pub grain: Option, +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests.rs index 30e1350d0c..9cc302c56a 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/tests.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_model_entry.rs"] +mod model_entry; #[path = "tests/test_planner_strategy.rs"] mod planner_strategy; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/helpers.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/helpers.rs index 6d2c4b6c41..1277e868ca 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/helpers.rs @@ -3,7 +3,8 @@ use std::collections::BTreeMap; use crate::planner_changes::models::{SchemaChangeKind, SchemaColumnSource, SchemaFindingRow}; use crate::planner_strategy::errors::LineageFailure; use crate::planner_strategy::models::{ - ChangeFacts, PlanChangeKind, ProducerFacts, ReferenceIdentity, WarningRow, WarningSeverity, + ChangeFacts, CursorInputColumnCheck, CursorInputColumnOwner, LookbackConfig, PlanChangeKind, + ProducerFacts, ReferenceIdentity, WarningRow, WarningSeverity, }; use crate::planner_strategy::tests::test_types::{Lineage, Steps}; use crate::planner_strategy::types::{ReferenceGraph, Timeline}; @@ -113,3 +114,29 @@ impl ReferenceGraph for Lineage { pub(super) fn failure_text(failure: LineageFailure<()>) -> String { format!("{failure:?}") } + +pub(super) fn column_check( + owner: CursorInputColumnOwner, + known: &[&str], +) -> CursorInputColumnCheck { + CursorInputColumnCheck { + model_name: "daily_orders".to_owned(), + owner, + input_name: "orders".to_owned(), + column: "Updated_At".to_owned(), + known_columns: known.iter().map(|name| (*name).to_owned()).collect(), + } +} + +pub(super) fn lookback_config( + lookback: Option<&str>, + strategy: Option<&str>, + grain: Option<&str>, +) -> LookbackConfig { + LookbackConfig { + lookback: lookback.map(str::to_owned), + incremental_strategy: strategy.map(str::to_owned), + batch_size: Some("effective".to_owned()), + grain: grain.map(str::to_owned), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_model_entry.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_model_entry.rs new file mode 100644 index 0000000000..9e9e202fe6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_model_entry.rs @@ -0,0 +1,130 @@ +use sqlbuild_core::plan_display::errors::UnknownCursorGrain; + +use crate::planner_strategy::main::cursor_input_column::cursor_input_column_failure; +use crate::planner_strategy::main::microbatch_lookback::microbatch_lookback; +use crate::planner_strategy::main::table_type_ddl::table_type_ddl; +use crate::planner_strategy::models::CursorInputColumnOwner; +use crate::planner_strategy::tests::helpers::{column_check, lookback_config}; +use crate::planner_strategy::tests::test_types::{ + CursorInputColumnTestCase, LookbackTestCase, TableTypeDdlTestCase, +}; + +#[test] +fn given_cursor_input_column_when_checking_exposure_then_reports_missing_columns() { + let test_cases = [ + CursorInputColumnTestCase { + description: "case-insensitive model contract match", + check: column_check(CursorInputColumnOwner::ModelContract, &["updated_at"]), + expected_failure: None, + }, + CursorInputColumnTestCase { + description: "model contract without the column", + check: column_check(CursorInputColumnOwner::ModelContract, &["order_id"]), + expected_failure: Some(( + "model 'daily_orders': cursor_inputs references model 'orders' column \ + 'Updated_At', but that model contract does not expose the column. Known output \ + columns: order_id" + .to_owned(), + "S302", + )), + }, + CursorInputColumnTestCase { + description: "source contract without columns", + check: column_check(CursorInputColumnOwner::SourceContract, &[]), + expected_failure: Some(( + "model 'daily_orders': cursor_inputs references source 'orders' column \ + 'Updated_At', but that source contract does not expose the column. Declared \ + contract columns: none" + .to_owned(), + "S302", + )), + }, + CursorInputColumnTestCase { + description: "warehouse source without the column", + check: column_check( + CursorInputColumnOwner::SourceWarehouse, + &["order_id", "status"], + ), + expected_failure: Some(( + "model 'daily_orders': cursor_inputs references source 'orders' column \ + 'Updated_At', but that source does not expose the column. Known source \ + columns: order_id, status" + .to_owned(), + "S302", + )), + }, + ]; + for test_case in test_cases { + assert_eq!( + cursor_input_column_failure(&test_case.check), + test_case.expected_failure, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_microbatch_config_when_resolving_lookback_then_matches_python_default() { + let test_cases = [ + LookbackTestCase { + description: "explicit lookback wins", + config: lookback_config(Some("2 days"), Some("append"), None), + expected_lookback: Ok(Some("2 days".to_owned())), + }, + LookbackTestCase { + description: "append has no default", + config: lookback_config(None, Some("append"), Some("day")), + expected_lookback: Ok(None), + }, + LookbackTestCase { + description: "merge without grain keeps the batch size", + config: lookback_config(None, Some("merge"), None), + expected_lookback: Ok(Some("effective".to_owned())), + }, + LookbackTestCase { + description: "delete insert resolves the effective grain", + config: lookback_config(None, Some("delete_insert"), Some("hour")), + expected_lookback: Ok(Some("1h".to_owned())), + }, + LookbackTestCase { + description: "unknown grain", + config: lookback_config(None, Some("merge"), Some("fortnight")), + expected_lookback: Err(UnknownCursorGrain("fortnight".to_owned())), + }, + ]; + for test_case in test_cases { + assert_eq!( + microbatch_lookback(&test_case.config), + test_case.expected_lookback, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_table_type_when_rendering_ddl_then_rewrites_only_permanent_tables() { + let test_cases = [ + TableTypeDdlTestCase { + description: "permanent rewrites the first keyword", + ddl: "CREATE TRANSIENT TABLE a AS SELECT 1; CREATE TRANSIENT TABLE b", + table_type: "permanent", + expected_ddl: "CREATE TABLE a AS SELECT 1; CREATE TRANSIENT TABLE b", + }, + TableTypeDdlTestCase { + description: "transient keeps the adapter DDL", + ddl: "CREATE TRANSIENT TABLE a AS SELECT 1", + table_type: "transient", + expected_ddl: "CREATE TRANSIENT TABLE a AS SELECT 1", + }, + ]; + for test_case in test_cases { + assert_eq!( + table_type_ddl(test_case.ddl, test_case.table_type), + test_case.expected_ddl, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs index e28e6bc03a..325815169d 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs @@ -1,8 +1,11 @@ use std::collections::BTreeSet; +use sqlbuild_core::plan_display::errors::UnknownCursorGrain; + use crate::planner_strategy::models::{ - AuditAttachmentKind, CapDirection, CappedBound, ChangeFacts, CursorType, MaterializationType, - PlanAction, PlanReason, SchemaActionRow, WarningRow, + AuditAttachmentKind, CapDirection, CappedBound, ChangeFacts, CursorInputColumnCheck, + CursorType, LookbackConfig, MaterializationType, PlanAction, PlanReason, SchemaActionRow, + WarningRow, }; pub(super) struct ActionTestCase { @@ -81,3 +84,22 @@ pub(super) struct Steps; pub(super) struct Lineage { pub(super) capped: BTreeSet, } + +pub(super) struct CursorInputColumnTestCase { + pub(super) description: &'static str, + pub(super) check: CursorInputColumnCheck, + pub(super) expected_failure: Option<(String, &'static str)>, +} + +pub(super) struct LookbackTestCase { + pub(super) description: &'static str, + pub(super) config: LookbackConfig, + pub(super) expected_lookback: Result, UnknownCursorGrain>, +} + +pub(super) struct TableTypeDdlTestCase { + pub(super) description: &'static str, + pub(super) ddl: &'static str, + pub(super) table_type: &'static str, + pub(super) expected_ddl: &'static str, +} diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 4ff7b7d80e..bb689cf0d2 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -55,6 +55,11 @@ impl CliStyle { self.apply(BLUE, text) } + #[must_use] + pub fn accent_strong(self, text: &str) -> String { + self.title(text) + } + #[must_use] pub fn plan_section(self, text: &str) -> String { self.apply(BOLD, text) @@ -547,3 +552,67 @@ pub struct DbtInteropPlanText { pub path_translations: Vec<(String, String)>, pub warnings: Vec, } +/// Migration decisions already resolved by the planner's model boundary. +#[derive(Clone, Debug, Default)] +pub struct MigrationDecisionText { + pub value: String, + pub label: String, + pub blocks_build: bool, + pub origin_missing: bool, + pub decision_blocks_build: bool, + pub checks_compatibility: bool, +} + +/// One model identity handover's presentation facts. +#[derive(Clone, Debug, Default)] +pub struct MigrationText { + pub model_name: String, + pub origin: String, + pub destination: String, + pub decision: MigrationDecisionText, + pub compatibility: String, + pub discovery: String, + pub completed_at: Option, + pub target_label: String, + pub transfer: Option, + pub transfer_fallback: Option, + pub storage_transition: Option, + pub promotion: Option, + pub findings: Vec, + pub materialization: Option, +} + +/// One old-name compatibility view's presentation facts. +#[derive(Clone, Debug, Default)] +pub struct OldNameText { + pub model_name: String, + pub origin: String, + pub destination: String, + pub action: String, + pub expires_date: Option, + pub column_aliases: Vec<(String, String)>, + pub grants_supported: bool, + pub grants_copied: Option, + pub reason: Option, +} + +/// One column rename's presentation facts. +#[derive(Clone, Debug, Default)] +pub struct ColumnMigrationText { + pub model_name: String, + pub origin_column: String, + pub destination_column: String, + pub decision: MigrationDecisionText, + pub discovery: String, + pub completed_at: Option, +} + +/// Ordered migration sections appended to pre-build work. +#[derive(Clone, Debug)] +pub struct MigrationTextReport { + pub migrations: Vec, + pub old_names: Vec, + pub columns: Vec, + pub display: DisplayLimit, + pub manual_rename_hint: String, +} diff --git a/crates/sqlbuild-core/src/plan_display/_helpers/migration_sections.rs b/crates/sqlbuild-core/src/plan_display/_helpers/migration_sections.rs new file mode 100644 index 0000000000..3d63718b4c --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/_helpers/migration_sections.rs @@ -0,0 +1,218 @@ +//! Migration rows using the shared plan colors and layout. + +use crate::cli_text::main::split_count_header::split_count_header; + +use crate::cli_text::models::{CliStyle, ColumnMigrationText, MigrationText, OldNameText}; +use crate::plan_display::constants::{ + MIGRATION_AUTOMATIC, MIGRATION_DONE, MIGRATION_INCOMPATIBLE, MIGRATION_MANUAL, + MIGRATION_RENAMED, OLD_NAME_LIVE, OLD_NAME_NONE, OLD_NAME_VIEW_ONLY, +}; +use crate::plan_display::main::old_name_rows::old_name_rows; + +pub(crate) fn property_row(label: &str, value: &str, style: CliStyle) -> String { + format!(" {} {value}", style.muted(label)) +} + +pub(crate) fn old_name_facts(entry: &OldNameText) -> Vec<(&str, String)> { + if entry.action == OLD_NAME_NONE { + return vec![( + "left for janitor", + entry + .reason + .as_deref() + .filter(|value| !value.is_empty()) + .unwrap_or("no compatibility view") + .to_owned(), + )]; + } + let until = entry + .expires_date + .as_ref() + .map_or_else(String::new, |date| format!("until {date}")); + let view = if entry.action == OLD_NAME_LIVE { + format!("live {until}").trim().to_owned() + } else { + until + }; + let mut facts = vec![("view", view)]; + if !entry.column_aliases.is_empty() { + facts.push(( + "columns", + entry + .column_aliases + .iter() + .map(|(old, new)| format!("{old} <- {new}")) + .collect::>() + .join(", "), + )); + } + if entry.grants_supported { + let text = entry.grants_copied.as_ref().map_or_else( + || "copied from the old table".to_owned(), + |count| format!("{count} copied from the old table"), + ); + facts.push(("grants", text)); + } + facts.push(( + "old table", + if entry.action == OLD_NAME_VIEW_ONLY { + "already archived" + } else { + "archived" + } + .to_owned(), + )); + facts +} + +fn decision_text(entry: &MigrationText, style: CliStyle) -> String { + let decision = &entry.decision; + if decision.blocks_build && decision.decision_blocks_build { + return style.error_strong(&decision.label); + } + if decision.origin_missing + || matches!( + decision.value.as_str(), + "superseded_replace" | "forced_replace" + ) + { + return style.warning_strong(&decision.label); + } + if decision.value == MIGRATION_DONE { + return style.muted(&decision.label); + } + style.accent_strong(&decision.label) +} + +pub(crate) fn migration_lines( + entry: &MigrationText, + old_name: Option<&OldNameText>, + style: CliStyle, + hint: &str, +) -> Vec { + let renamed = entry.decision.value == MIGRATION_RENAMED; + let decision = if renamed { + if entry.completed_at.is_some() { + style.muted("done") + } else { + style.accent_strong("migrate") + } + } else { + decision_text(entry, style) + }; + let mut rows = vec![format!( + " {} {decision} {} {}", + style.object_name(&entry.model_name), + style.muted(&format!("{} ->", entry.origin)), + entry.destination + )]; + if renamed && entry.completed_at.is_none() { + rows.push(property_row( + "transfer", + if entry.materialization.as_deref() == Some("view") { + "recreate (view)" + } else { + "rebuild (table)" + }, + style, + )); + } + if entry.decision.checks_compatibility { + let value = match entry.compatibility.as_str() { + "compatible" => style.success(&entry.compatibility), + "incompatible" => style.error(&entry.compatibility), + _ => entry.compatibility.clone(), + }; + rows.push(property_row("compatibility", &value, style)); + } + if let Some(transfer) = &entry.transfer { + let fallback = entry + .transfer_fallback + .as_ref() + .map_or_else(String::new, |value| format!(" ({value} if refused)")); + let mut details = vec![format!("{transfer}{fallback}")]; + if let Some(value) = &entry.storage_transition { + details.push(value.clone()); + } + if let Some(value) = &entry.promotion { + details.push(format!("promote by {value}")); + } + rows.push(property_row( + "transfer", + &details + .into_iter() + .filter(|value| !value.is_empty()) + .collect::>() + .join(", "), + style, + )); + } + if entry.discovery != MIGRATION_MANUAL { + rows.push(property_row("discovery", &entry.discovery, style)); + } + if entry.discovery == MIGRATION_AUTOMATIC && entry.completed_at.is_none() { + rows.push(property_row("hint", hint, style)); + } + if let Some(completed) = &entry.completed_at { + rows.push(property_row( + "completed", + &format!("{completed} on target '{}'", entry.target_label), + style, + )); + } + if let Some(old_name) = old_name { + rows.extend(old_name_rows(old_name, style)); + } + for finding in &entry.findings { + let value = format!("! {finding}"); + let styled = if entry.decision.blocks_build || entry.compatibility == MIGRATION_INCOMPATIBLE + { + style.error(&value) + } else { + style.warning(&value) + }; + rows.push(format!(" {styled}")); + } + rows +} + +pub(crate) fn column_migration_text(entry: &ColumnMigrationText, style: CliStyle) -> String { + let decision = &entry.decision; + let styled = if decision.blocks_build { + style.error_strong(&decision.label) + } else if decision.origin_missing { + style.warning_strong(&decision.label) + } else if decision.value == MIGRATION_DONE { + style.muted(&decision.label) + } else { + style.accent_strong(&decision.label) + }; + let mut details: Vec = Vec::new(); + if entry.discovery != MIGRATION_MANUAL { + details.push(entry.discovery.clone()); + } + if decision.value == MIGRATION_DONE + && let Some(completed) = &entry.completed_at + { + details.push(format!("completed {completed}")); + } + let suffix = if details.is_empty() { + String::new() + } else { + format!(" {}", style.muted(&format!("({})", details.join(", ")))) + }; + format!( + "{} -> {} {styled}{suffix}", + entry.origin_column, entry.destination_column + ) +} +pub(crate) fn section_header(style: CliStyle, text: &str) -> String { + if let Some((title, count)) = split_count_header(text) { + return format!( + "{} {}", + style.plan_section(title), + style.muted(&format!("({count}")) + ); + } + style.plan_section(text) +} diff --git a/crates/sqlbuild-core/src/plan_display/_helpers/mod.rs b/crates/sqlbuild-core/src/plan_display/_helpers/mod.rs index 3715888d75..3d544339f8 100644 --- a/crates/sqlbuild-core/src/plan_display/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/plan_display/_helpers/mod.rs @@ -1 +1,2 @@ pub(crate) mod labels; +pub(crate) mod migration_sections; diff --git a/crates/sqlbuild-core/src/plan_display/constants.rs b/crates/sqlbuild-core/src/plan_display/constants.rs index e771ddcad6..0445540d36 100644 --- a/crates/sqlbuild-core/src/plan_display/constants.rs +++ b/crates/sqlbuild-core/src/plan_display/constants.rs @@ -24,3 +24,12 @@ pub(crate) const GRAIN_BATCH_SIZE: [(&str, &str); 6] = [ ("month", "1mo"), ("year", "1y"), ]; +pub(crate) const OLD_NAME_NONE: &str = "none"; +pub(crate) const OLD_NAME_LIVE: &str = "live"; +pub(crate) const OLD_NAME_VIEW_ONLY: &str = "view_only"; +pub(crate) const MIGRATION_DONE: &str = "done"; +pub(crate) const MIGRATION_RENAMED: &str = "renamed"; +pub(crate) const MIGRATION_MANUAL: &str = "manual"; +pub(crate) const MIGRATION_AUTOMATIC: &str = "automatic"; +pub(crate) const MIGRATION_INCOMPATIBLE: &str = "incompatible"; +pub(crate) const WATERMARK_STRATEGY: &str = "watermark"; diff --git a/crates/sqlbuild-core/src/plan_display/main/effective_timestamp_grain.rs b/crates/sqlbuild-core/src/plan_display/main/effective_timestamp_grain.rs new file mode 100644 index 0000000000..7fa9269f4f --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/effective_timestamp_grain.rs @@ -0,0 +1,42 @@ +//! The coarsest timestamp grain participating in cursor replay. + +use crate::plan_display::constants::{ + DEFAULT_CURSOR_GRAIN, GRAIN_BATCH_SIZE, TIMESTAMP_CURSOR, WATERMARK_STRATEGY, +}; +use crate::plan_display::errors::UnknownCursorGrain; +use crate::plan_display::models::TimestampGrainFacts; + +/// The grain's position from finest to coarsest; an unknown grain is an error. +fn grain_rank(grain: &str) -> Result { + GRAIN_BATCH_SIZE + .iter() + .position(|(known, _)| *known == grain) + .ok_or_else(|| UnknownCursorGrain(grain.to_owned())) +} + +/// The declared grain or `second`, an empty grain counting as absent. +fn grain_or_default(grain: Option<&String>) -> String { + grain + .filter(|grain| !grain.is_empty()) + .map_or_else(|| DEFAULT_CURSOR_GRAIN.to_owned(), Clone::clone) +} + +/// `None` for non-timestamp cursors; the declared grain for watermark replay; else the coarsest declared or input grain. +pub fn effective_timestamp_grain( + facts: &TimestampGrainFacts, +) -> Result, UnknownCursorGrain> { + if facts.cursor_type.as_deref() != Some(TIMESTAMP_CURSOR) { + return Ok(None); + } + let mut effective: String = grain_or_default(facts.downstream_grain.as_ref()); + if facts.microbatch_strategy.as_deref() == Some(WATERMARK_STRATEGY) { + return Ok(Some(effective)); + } + for input_grain in &facts.input_grains { + let candidate: String = grain_or_default(input_grain.as_ref()); + if grain_rank(&candidate)? > grain_rank(&effective)? { + effective = candidate; + } + } + Ok(Some(effective)) +} diff --git a/crates/sqlbuild-core/src/plan_display/main/format_model_migrations.rs b/crates/sqlbuild-core/src/plan_display/main/format_model_migrations.rs new file mode 100644 index 0000000000..d68dade41e --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/format_model_migrations.rs @@ -0,0 +1,95 @@ +//! Migration sections in planner order. + +use crate::cli_text::main::format_overflow_line::format_overflow_line; +use crate::cli_text::models::{CliStyle, MigrationTextReport}; +use crate::plan_display::_helpers::migration_sections::{ + column_migration_text, migration_lines, section_header, +}; +use crate::plan_display::main::old_name_rows::old_name_rows; +use std::collections::{BTreeMap, BTreeSet}; + +/// Append model migrations, resumed old names and grouped column migrations. +pub fn format_model_migrations( + mut lines: Vec, + report: &MigrationTextReport, +) -> Vec { + let style = CliStyle { use_color: true }; + let old_names: BTreeMap<_, _> = report + .old_names + .iter() + .map(|entry| (&entry.model_name, entry)) + .collect(); + if !report.migrations.is_empty() { + lines.push(section_header( + style, + &format!("Migrations ({})", report.migrations.len()), + )); + let visible = report + .display + .max_entries + .map_or(report.migrations.len(), |limit| { + limit.min(report.migrations.len()) + }); + for entry in &report.migrations[..visible] { + lines.extend(migration_lines( + entry, + old_names.get(&entry.model_name).copied(), + style, + &report.manual_rename_hint, + )); + } + lines.extend(format_overflow_line( + (report.migrations.len() - visible) as i64, + " ", + &report.display.overflow_flag, + )); + } + let migrated: BTreeSet<_> = report + .migrations + .iter() + .map(|entry| &entry.model_name) + .collect(); + let resumed: Vec<_> = report + .old_names + .iter() + .filter(|entry| !migrated.contains(&entry.model_name)) + .collect(); + if !resumed.is_empty() { + lines.push(section_header( + style, + &format!("Old names ({})", resumed.len()), + )); + for entry in resumed { + lines.push(format!( + " {} {} {}", + style.object_name(&entry.model_name), + style.muted(&format!("{} ->", entry.origin)), + entry.destination + )); + lines.extend(old_name_rows(entry, style)); + } + } + if !report.columns.is_empty() { + lines.push(section_header( + style, + &format!("Column migrations ({})", report.columns.len()), + )); + let mut seen: BTreeSet<&str> = BTreeSet::new(); + for first in &report.columns { + if !seen.insert(&first.model_name) { + continue; + } + lines.push(format!( + " {} {}", + style.object_name(&first.model_name), + style.muted("migrate columns") + )); + for entry in &report.columns { + if entry.model_name == first.model_name { + lines.push(format!(" {}", column_migration_text(entry, style))); + } + } + } + } + lines +} diff --git a/crates/sqlbuild-core/src/plan_display/main/mod.rs b/crates/sqlbuild-core/src/plan_display/main/mod.rs index b2035bea98..c9d5735ad4 100644 --- a/crates/sqlbuild-core/src/plan_display/main/mod.rs +++ b/crates/sqlbuild-core/src/plan_display/main/mod.rs @@ -1,7 +1,10 @@ pub mod cursor_bound_display; pub mod effective_microbatch_batch_size; +pub mod effective_timestamp_grain; +pub mod format_model_migrations; pub mod format_no_input_rows; pub mod format_waiting_on_empty_inputs; pub mod model_execution_annotation; pub mod model_materialization_label; pub mod model_resource_type; +pub mod old_name_rows; diff --git a/crates/sqlbuild-core/src/plan_display/main/old_name_rows.rs b/crates/sqlbuild-core/src/plan_display/main/old_name_rows.rs new file mode 100644 index 0000000000..aad5d4128f --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/old_name_rows.rs @@ -0,0 +1,19 @@ +//! Render an old relation name and its nested facts. + +use crate::cli_text::main::format_tree_connector::format_tree_connector as tree_connector; +use crate::cli_text::models::{CliStyle, OldNameText}; +use crate::plan_display::_helpers::migration_sections::{old_name_facts, property_row}; + +/// Render the old name as a leaf with one fact per nested row. +pub fn old_name_rows(entry: &OldNameText, style: CliStyle) -> Vec { + let facts = old_name_facts(entry); + let mut rows = vec![property_row("old name", &entry.origin, style)]; + for (index, (label, value)) in facts.iter().enumerate() { + rows.push(format!( + " {} {} {value}", + tree_connector(style, index + 1 == facts.len()), + style.muted(label) + )); + } + rows +} diff --git a/crates/sqlbuild-core/src/plan_display/models.rs b/crates/sqlbuild-core/src/plan_display/models.rs index e1072e7392..ea79ac7945 100644 --- a/crates/sqlbuild-core/src/plan_display/models.rs +++ b/crates/sqlbuild-core/src/plan_display/models.rs @@ -23,3 +23,12 @@ pub struct ParsedBound { /// Hour, minute, second and microsecond are all zero. pub midnight: bool, } + +/// A cursor's type, strategy, declared grain and its inputs' grains, as the replay grain reads them. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TimestampGrainFacts { + pub cursor_type: Option, + pub microbatch_strategy: Option, + pub downstream_grain: Option, + pub input_grains: Vec>, +} diff --git a/crates/sqlbuild-core/src/plan_display/tests/helpers.rs b/crates/sqlbuild-core/src/plan_display/tests/helpers.rs index 047946bc2d..ffb4555267 100644 --- a/crates/sqlbuild-core/src/plan_display/tests/helpers.rs +++ b/crates/sqlbuild-core/src/plan_display/tests/helpers.rs @@ -1,4 +1,4 @@ -use crate::plan_display::models::ModelLabelFacts; +use crate::plan_display::models::{ModelLabelFacts, TimestampGrainFacts}; /// A planned model with only its materialization and action. pub(super) fn facts(materialization_type: &str, action: &str) -> ModelLabelFacts { @@ -8,3 +8,20 @@ pub(super) fn facts(materialization_type: &str, action: &str) -> ModelLabelFacts ..ModelLabelFacts::default() } } + +/// A timestamp cursor with the strategy, declared grain and input grains given. +pub(super) fn timestamp_grains( + microbatch_strategy: Option<&str>, + downstream_grain: Option<&str>, + input_grains: &[Option<&str>], +) -> TimestampGrainFacts { + TimestampGrainFacts { + cursor_type: Some("timestamp".to_owned()), + microbatch_strategy: microbatch_strategy.map(str::to_owned), + downstream_grain: downstream_grain.map(str::to_owned), + input_grains: input_grains + .iter() + .map(|grain| grain.map(str::to_owned)) + .collect(), + } +} diff --git a/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs b/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs index 0e8ed49570..5e41e9cb63 100644 --- a/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs +++ b/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs @@ -1,14 +1,15 @@ use crate::plan_display::main::cursor_bound_display::cursor_bound_display; use crate::plan_display::main::effective_microbatch_batch_size::effective_microbatch_batch_size; +use crate::plan_display::main::effective_timestamp_grain::effective_timestamp_grain; use crate::plan_display::main::format_no_input_rows::format_no_input_rows; use crate::plan_display::main::format_waiting_on_empty_inputs::format_waiting_on_empty_inputs; use crate::plan_display::main::model_execution_annotation::model_execution_annotation; use crate::plan_display::main::model_materialization_label::model_materialization_label; use crate::plan_display::main::model_resource_type::model_resource_type; -use crate::plan_display::models::{ModelLabelFacts, ParsedBound}; -use crate::plan_display::tests::helpers::facts; +use crate::plan_display::models::{ModelLabelFacts, ParsedBound, TimestampGrainFacts}; +use crate::plan_display::tests::helpers::{facts, timestamp_grains}; use crate::plan_display::tests::test_types::{ - BatchSizeTestCase, BoundTestCase, LabelTestCase, NamesTestCase, + BatchSizeTestCase, BoundTestCase, LabelTestCase, NamesTestCase, TimestampGrainTestCase, }; #[test] @@ -207,3 +208,55 @@ fn given_batch_size_when_resolving_effective_size_then_uses_grain_unit() { ); } } + +#[test] +fn given_cursor_grains_when_resolving_replay_grain_then_coarsest_timestamp_grain_wins() { + let test_cases = [ + TimestampGrainTestCase { + description: "an integer cursor has no grain", + facts: TimestampGrainFacts { + cursor_type: Some("integer".to_owned()), + downstream_grain: Some("day".to_owned()), + ..TimestampGrainFacts::default() + }, + expected_grain: Ok(None), + }, + TimestampGrainTestCase { + description: "no declared grain defaults to second", + facts: timestamp_grains(None, None, &[]), + expected_grain: Ok(Some("second")), + }, + TimestampGrainTestCase { + description: "a coarser input grain wins", + facts: timestamp_grains(None, Some("day"), &[Some("month"), None, Some("hour")]), + expected_grain: Ok(Some("month")), + }, + TimestampGrainTestCase { + description: "an empty declared grain counts as second", + facts: timestamp_grains(None, Some(""), &[Some("minute")]), + expected_grain: Ok(Some("minute")), + }, + TimestampGrainTestCase { + description: "watermark replay keeps the declared grain", + facts: timestamp_grains(Some("watermark"), Some("hour"), &[Some("year")]), + expected_grain: Ok(Some("hour")), + }, + TimestampGrainTestCase { + description: "an unknown input grain is an error", + facts: timestamp_grains(None, Some("day"), &[Some("week")]), + expected_grain: Err("week"), + }, + ]; + + for test_case in test_cases { + assert_eq!( + effective_timestamp_grain(&test_case.facts).map_err(|error| error.0), + test_case + .expected_grain + .map(|grain| grain.map(str::to_owned)) + .map_err(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/plan_display/tests/test_types.rs b/crates/sqlbuild-core/src/plan_display/tests/test_types.rs index 08cbf72a61..d924f3052d 100644 --- a/crates/sqlbuild-core/src/plan_display/tests/test_types.rs +++ b/crates/sqlbuild-core/src/plan_display/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::plan_display::models::{ModelLabelFacts, ParsedBound}; +use crate::plan_display::models::{ModelLabelFacts, ParsedBound, TimestampGrainFacts}; pub(super) struct LabelTestCase { pub(super) description: &'static str, @@ -29,3 +29,9 @@ pub(super) struct BatchSizeTestCase { pub(super) grain: &'static str, pub(super) expected_size: Result<&'static str, &'static str>, } + +pub(super) struct TimestampGrainTestCase { + pub(super) description: &'static str, + pub(super) facts: TimestampGrainFacts, + pub(super) expected_grain: Result, &'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs index e942a97baf..332a25bccd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs @@ -31,7 +31,7 @@ pub(crate) fn reserved_cursor_markers() -> Vec { } /// A `CompileInputError` carrying `message`. -fn compile_input_error(py: Python<'_>, message: String) -> PyResult { +pub(crate) fn compile_input_error(py: Python<'_>, message: String) -> PyResult { Ok(PyErr::from_value( imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?.call1((message,))?, )) @@ -128,7 +128,7 @@ fn cursor_intrinsics_analysis_sql( /// Whether `sql` uses cursor intrinsics, and `sql` with both bounds rendered when given. #[pyfunction] #[pyo3(signature = (*, sql, start_sql=None, end_sql=None))] -fn resolve_cursor_intrinsics( +pub(crate) fn resolve_cursor_intrinsics( py: Python<'_>, sql: &str, start_sql: Option<&str>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs index 219cf1d045..ba5e0ba839 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs @@ -365,7 +365,10 @@ fn input_view_classes<'py>(py: Python<'py>) -> PyResult> { } /// `{item.: item for item in items}`. -fn by_attribute<'py>(items: &Bound<'py, PyAny>, attribute: &str) -> PyResult> { +pub(crate) fn by_attribute<'py>( + items: &Bound<'py, PyAny>, + attribute: &str, +) -> PyResult> { let keyed: Bound<'py, PyDict> = PyDict::new(items.py()); for item in items.try_iter()? { let item: Bound<'py, PyAny> = item?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs index 86cb146d1e..272d1aecb9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs @@ -20,7 +20,8 @@ const DECIMAL_KIND: &str = "decimal"; const COLLECTION_KINDS: [&str; 2] = ["list", "set"]; const SCALAR_KINDS: [&str; 5] = ["string", "integer", "boolean", "float", "null"]; -fn replace<'py>( +/// `dataclasses.replace(value, **changes)`. +pub(crate) fn replace<'py>( value: &Bound<'py, PyAny>, changes: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs index 26f364aa21..46cab3f137 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs @@ -54,7 +54,10 @@ pub(crate) fn config_str<'py>( /// An authored boolean, or `None` when the key is absent or `None`. #[pyfunction(name = "get_config_bool")] #[pyo3(signature = (*, values, key))] -fn config_bool<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult>> { +pub(crate) fn config_bool<'py>( + values: &Bound<'py, PyAny>, + key: &str, +) -> PyResult>> { let Some(value) = authored(values, key)? else { return Ok(None); }; @@ -67,7 +70,7 @@ fn config_bool<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult( +pub(crate) fn config_string_tuple<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult>> { @@ -87,7 +90,10 @@ fn config_string_tuple<'py>( /// An authored integer, excluding booleans, or `None` when absent or `None`. #[pyfunction(name = "get_config_int")] #[pyo3(signature = (*, values, key))] -fn config_int<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult>> { +pub(crate) fn config_int<'py>( + values: &Bound<'py, PyAny>, + key: &str, +) -> PyResult>> { let Some(value) = authored(values, key)? else { return Ok(None); }; @@ -100,7 +106,7 @@ fn config_int<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult( +pub(crate) fn config_cursor_bound<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs new file mode 100644 index 0000000000..4e2f85dae8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs @@ -0,0 +1,141 @@ +//! Convert public migration models at the Python boundary for native presentation. + +use crate::bindings::_helpers::plan_output::plan_text::display_limit; +use crate::bindings::_helpers::plan_output::values::{ + iso, items, optional_text, relation_name, text, texts, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::{ + CliStyle, ColumnMigrationText, MigrationDecisionText, MigrationText, MigrationTextReport, + OldNameText, +}; +use sqlbuild_core::plan_display::main::{format_model_migrations, old_name_rows}; +use std::collections::BTreeMap; + +fn label(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + text(&value.getattr("label")?).map(Some) +} + +fn decision(entry: &Bound<'_, PyAny>) -> PyResult { + let value = entry.getattr("decision")?; + Ok(MigrationDecisionText { + value: text(&value)?, + label: text(&value.getattr("label")?)?, + blocks_build: entry.getattr("blocks_build")?.extract()?, + origin_missing: entry.getattr("origin_missing")?.extract()?, + ..MigrationDecisionText::default() + }) +} + +fn migration(entry: &Bound<'_, PyAny>, materialization: Option) -> PyResult { + let mut decision = decision(entry)?; + let value = entry.getattr("decision")?; + decision.decision_blocks_build = value.getattr("blocks_build")?.extract()?; + decision.checks_compatibility = value.getattr("checks_compatibility")?.extract()?; + Ok(MigrationText { + model_name: text(&entry.getattr("model_name")?)?, + origin: text(&relation_name(&entry.getattr("origin")?)?)?, + destination: text(&relation_name(&entry.getattr("destination")?)?)?, + decision, + compatibility: text(&entry.getattr("compatibility")?.getattr("value")?)?, + discovery: text(&entry.getattr("discovery")?.getattr("value")?)?, + completed_at: optional_text(&iso(&entry.getattr("completed_at")?)?)?, + target_label: text(&entry.getattr("target_label")?)?, + transfer: label(&entry.getattr("transfer")?)?, + transfer_fallback: label(&entry.getattr("transfer_fallback")?)?, + storage_transition: optional_text(&entry.getattr("storage_transition")?)?, + promotion: label(&entry.getattr("promotion")?)?, + findings: texts(&entry.getattr("compatibility_findings")?)?, + materialization, + }) +} + +fn old_name(entry: &Bound<'_, PyAny>) -> PyResult { + let expires = entry.getattr("expires_at")?; + let expires_date = if expires.is_none() { + None + } else { + Some(expires.call_method1("strftime", ("%Y-%m-%d",))?.extract()?) + }; + Ok(OldNameText { + model_name: text(&entry.getattr("model_name")?)?, + origin: text(&relation_name(&entry.getattr("origin")?)?)?, + destination: text(&relation_name(&entry.getattr("destination")?)?)?, + action: text(&entry.getattr("action")?)?, + expires_date, + column_aliases: entry.getattr("column_aliases")?.extract()?, + grants_supported: entry.getattr("grants_supported")?.extract()?, + grants_copied: optional_text(&entry.getattr("grants_copied")?)?, + reason: optional_text(&entry.getattr("reason")?)?, + }) +} + +fn column(entry: &Bound<'_, PyAny>) -> PyResult { + Ok(ColumnMigrationText { + model_name: text(&entry.getattr("model_name")?)?, + origin_column: text(&entry.getattr("origin_column")?)?, + destination_column: text(&entry.getattr("destination_column")?)?, + decision: decision(entry)?, + discovery: text(&entry.getattr("discovery")?.getattr("value")?)?, + completed_at: optional_text(&iso(&entry.getattr("completed_at")?)?)?, + }) +} + +#[pyfunction(name = "format_model_migrations", signature = (*, lines, plan, display_options))] +fn format_migrations( + lines: Vec, + plan: &Bound<'_, PyAny>, + display_options: &Bound<'_, PyAny>, +) -> PyResult> { + let mut materializations: BTreeMap = BTreeMap::new(); + for entry in items(&plan.getattr("model_entries")?)? { + materializations.insert( + text(&entry.getattr("name")?)?, + text(&entry.getattr("materialization_type")?)?, + ); + } + let mut migrations: Vec = Vec::new(); + for entry in items(&plan.getattr("migration_entries")?)? { + let name = text(&entry.getattr("model_name")?)?; + migrations.push(migration(&entry, materializations.get(&name).cloned())?); + } + let report = MigrationTextReport { + migrations, + old_names: items(&plan.getattr("old_name_view_entries")?)? + .iter() + .map(old_name) + .collect::>()?, + columns: items(&plan.getattr("column_migration_entries")?)? + .iter() + .map(column) + .collect::>()?, + display: display_limit(display_options)?, + manual_rename_hint: plan + .py() + .import("sqlbuild.compiler.planner.constants")? + .getattr("MANUAL_RENAME_HINT")? + .extract()?, + }; + Ok(format_model_migrations::format_model_migrations( + lines, &report, + )) +} + +#[pyfunction(name = "old_name_rows", signature = (*, entry, style))] +fn render_old_name(entry: &Bound<'_, PyAny>, style: &Bound<'_, PyAny>) -> PyResult> { + Ok(old_name_rows::old_name_rows( + &old_name(entry)?, + CliStyle { + use_color: style.getattr("use_color")?.extract()?, + }, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_migrations, module)?)?; + module.add_function(wrap_pyfunction!(render_old_name, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs index fd834ed5b9..a4063abaf5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod cursor_details; pub(crate) mod display; +pub(crate) mod migrations; pub(crate) mod plan_json; pub(crate) mod plan_text; pub(crate) mod python_identity; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs index 11392a2f4e..e4377d901e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs @@ -391,7 +391,7 @@ fn pruned_models(metadata: &Bound<'_, PyAny>) -> PyResult> { Ok(kept) } -fn display_limit(options: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn display_limit(options: &Bound<'_, PyAny>) -> PyResult { Ok(DisplayLimit { max_entries: options.getattr("max_entries_per_section")?.extract()?, overflow_flag: text(&options.getattr("overflow_flag")?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs index cd0fab2819..96c0593e33 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs @@ -2,10 +2,11 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::plan_output::{display, plan_json, plan_text, safety}; +use crate::bindings::_helpers::plan_output::{display, migrations, plan_json, plan_text, safety}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { display::register(module)?; + migrations::register(module)?; plan_json::register(module)?; plan_text::register(module)?; safety::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs index 15dcb30116..5ac2e6cc45 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs @@ -45,7 +45,10 @@ fn version_identity_config<'py>(config_values: &Bound<'py, PyAny>) -> PyResult(value: Option>, py: Python<'py>) -> PyResult> { +pub(crate) fn or_empty<'py>( + value: Option>, + py: Python<'py>, +) -> PyResult> { match value { Some(value) if value.is_truthy()? => Ok(value), _ => Ok(PyDict::new(py).into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs index 64584d334c..138507dbd4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs @@ -7,6 +7,7 @@ use crate::bindings::_helpers::planner_migrations::column_objects::{ }; use crate::bindings::_helpers::planner_migrations::constants::{AUTOMATIC, DONE, MIGRATION_TYPES}; use crate::bindings::_helpers::planner_migrations::models::{CheckedColumns, ColumnCandidate}; +use crate::bindings::_helpers::state_rows::migration_projections::newest_column; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; use sqlbuild_analysis::planner_migrations::main::column_decision::column_decision; @@ -60,24 +61,18 @@ pub(super) fn decide<'py>( "migration_relation_for_location", )? .call1((candidate.model.getattr("destination")?,))?; - let newest_event = imported( - py, - "sqlbuild.compiler.migrations.main._newest_column_event", - "newest_column_migration_event_mentioning", - )?; let mut entries: Vec> = Vec::new(); for (origin_name, destination, discovery) in pairs { let origin_key = lower(&origin_name)?; let origin = candidate.warehouse.get_item(&origin_key)?; let has_destination = candidate.warehouse.contains(lower(&destination)?)?; - let newest = call_with( - &newest_event, - &[ - ("events", events.clone()), - ("relation", relation.clone()), - ("origin_column", origin_name.clone()), - ("destination_column", destination.clone()), - ], + let newest = newest_column( + events, + &relation, + ( + &origin_name.extract::()?, + &destination.extract::()?, + ), )?; let recorded = !newest.is_none() && call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index 6753114e68..200d78f652 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -48,11 +48,7 @@ fn plan_column_migrations<'py>( &[("runtime", runtime.clone())], )?; let state = call_with( - &imported( - py, - "sqlbuild.compiler.planner.classes.migration_state_inspection", - "MigrationStateInspection", - )?, + &imported(py, "sqlbuild._native", "MigrationStateInspection")?, &[ ("adapter", adapter), ("connection", runtime.getattr("connection")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs index 1715fd4546..b11f9801d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs @@ -2,3 +2,16 @@ pub(super) const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; pub(super) const MIGRATION_TYPES: &str = "sqlbuild.compiler.migrations.types"; pub(super) const AUTOMATIC: &str = "automatic"; pub(super) const DONE: &str = "done"; +pub(super) const PENDING_ARCHIVE: &str = "pending_archive"; +pub(super) const PENDING_VIEW: &str = "pending_view"; +pub(super) const NATIVE_MODULE: &str = "sqlbuild._native"; +pub(super) const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; +pub(super) const NODE_TYPE_MODEL: &str = "model"; +pub(super) const LIST_RELATIONS_FOR_INSPECTION: (&str, &str) = ( + "sqlbuild.adapter.relations.main.list_relations_for_inspection", + "list_relations_for_inspection", +); +pub(super) const GET_COLUMNS_FOR_INSPECTION: (&str, &str) = ( + "sqlbuild.adapter.relations.main.get_columns_for_inspection", + "get_columns_for_inspection", +); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs index 649ca5f5ed..5810366377 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::planner_migrations::column_objects::config; +use crate::bindings::_helpers::state_rows::migration_projections::newest_move; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; @@ -25,14 +26,7 @@ pub(super) fn newest<'py>( "migration_relation_for_location", )? .call1((location,))?; - call_with( - &imported( - location.py(), - "sqlbuild.compiler.migrations.main._newest_event", - "newest_migration_event_mentioning", - )?, - &[("events", events.clone()), ("relation", relation)], - ) + newest_move(events, &relation) } pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index 18301e2647..1d488c57e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -20,9 +20,14 @@ mod model_overlay; mod model_planning; mod model_warnings; mod models; +mod old_name_entries; +mod old_name_objects; +mod old_name_planning; +mod old_name_references; mod old_names; mod planning_boundary; mod planning_execution; mod planning_origin; mod projections; pub(crate) mod registration; +mod state_inspection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs index d426b7231b..4142649951 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs @@ -10,6 +10,7 @@ use crate::bindings::_helpers::planner_migrations::constants::MIGRATION_TYPES; use crate::bindings::_helpers::planner_migrations::model_handover::{ applies, empty_version, handover, }; +use crate::bindings::_helpers::state_rows::migration_projections::newest_move; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyBool, PyTuple}; use sqlbuild_analysis::planner_migrations::main::model_decision::model_decision; @@ -84,17 +85,7 @@ pub(super) fn decide<'py>(args: &KeywordArguments<'py>) -> PyResult( &[("runtime", runtime.clone())], )?; let state = call_with( - &imported( - py, - "sqlbuild.compiler.planner.classes.migration_state_inspection", - "MigrationStateInspection", - )?, + &imported(py, "sqlbuild._native", "MigrationStateInspection")?, &[ ("adapter", runtime.getattr("adapter")?), ("connection", runtime.getattr("connection")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs index a0bf4ad647..1c11e9b2fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs @@ -1,6 +1,17 @@ use pyo3::prelude::{Bound, PyAny}; use pyo3::types::PyDict; +#[derive(Debug)] +pub(crate) struct OldNamePlanner<'py> { + pub runtime: Bound<'py, PyAny>, + pub models: Vec>, + pub histories: Vec>, + pub views: Bound<'py, PyAny>, + pub columns: Bound<'py, PyAny>, + pub retention: Bound<'py, PyAny>, + pub now: Bound<'py, PyAny>, +} + #[derive(Debug)] pub(crate) struct ColumnCandidate<'py> { pub model: Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs new file mode 100644 index 0000000000..b594f187b8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs @@ -0,0 +1,372 @@ +//! Assemble new and resumed old-name view steps from recorded facts. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_migrations::column_objects::{model, native, text}; +use crate::bindings::_helpers::planner_migrations::constants::{PENDING_ARCHIVE, PENDING_VIEW}; +use crate::bindings::_helpers::planner_migrations::models::OldNamePlanner; +use crate::bindings::_helpers::planner_migrations::old_name_objects::{ + expires, location, relation, retention, stores_history, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyBool, PyList, PyTuple}; + +impl<'py> OldNamePlanner<'py> { + pub(super) fn new( + args: &KeywordArguments<'py>, + models: Vec>, + histories: Vec>, + views: Bound<'py, PyAny>, + ) -> PyResult { + let runtime = args.value("runtime")?; + let py = runtime.py(); + let project_config = runtime.getattr("project_config")?; + let config = if project_config.is_none() { + imported(py, "sqlbuild.spec.contracts.models", "MigrationsConfig")?.call0()? + } else { + project_config.getattr("migrations")? + }; + Ok(Self { + models, + histories, + views, + columns: args.value("column_entries")?, + retention: config.getattr("old_name_views")?, + now: runtime.getattr("invocation_time")?, + runtime, + }) + } + + fn newest( + &self, + destination: &Bound<'py, PyAny>, + origin: Option<&Bound<'py, PyAny>>, + ) -> PyResult>> { + let mut newest: Option<(Bound<'py, PyTuple>, Bound<'py, PyAny>)> = None; + for history in &self.histories { + let movement = history.getattr("move")?; + if !movement + .getattr("destination")? + .call_method1("matches", (destination,))? + .is_truthy()? + { + continue; + } + if let Some(origin) = origin + && !movement + .getattr("origin")? + .call_method1("matches", (origin,))? + .is_truthy()? + { + continue; + } + let key = PyTuple::new( + self.runtime.py(), + [ + movement.getattr("created_at")?, + movement.getattr("event_id")?, + ], + )?; + if let Some((previous, _)) = &newest + && !key.gt(previous)? + { + continue; + } + newest = Some((key, history.clone())); + } + Ok(newest.map(|(_, history)| history)) + } + + pub(super) fn for_migration( + &self, + entry: &Bound<'py, PyAny>, + item: &Bound<'py, PyAny>, + ) -> PyResult>> { + if entry.getattr("blocks_build")?.is_truthy()? { + return Ok(None); + } + let decision = entry.getattr("decision")?; + let completed = !entry.getattr("completed_at")?.is_none(); + if decision.getattr("moves_data")?.is_truthy()? || (decision.eq("renamed")? && !completed) { + return self.new_move(entry, item).map(Some); + } + if decision.eq("done")? || (decision.eq("renamed")? && completed) { + let origin = relation(&entry.getattr("origin")?)?; + if let Some(history) = + self.newest(&relation(&entry.getattr("destination")?)?, Some(&origin))? + { + return self.resume(&history, item); + } + } + Ok(None) + } + + pub(super) fn pending_for( + &self, + item: &Bound<'py, PyAny>, + ) -> PyResult>> { + let Some(history) = self.newest(&relation(&item.getattr("destination")?)?, None)? else { + return Ok(None); + }; + if !history + .getattr("move")? + .getattr("destination_model")? + .eq(item.getattr("name")?)? + { + return Ok(None); + } + let Some(planned) = self.resume(&history, item)? else { + return Ok(None); + }; + if planned.getattr("runs_steps")?.is_truthy()? + || (planned.getattr("action")?.eq("none")? + && !planned.getattr("migration_event_id")?.is_none()) + { + Ok(Some(planned)) + } else { + Ok(None) + } + } + + fn skip( + &self, + origin: &Bound<'py, PyAny>, + tracked: Bound<'py, PyAny>, + retention: &Bound<'py, PyAny>, + ) -> PyResult> { + native( + self.runtime.py(), + "old_name_view_skip_reason", + &[ + ("origin", origin.clone()), + ("origin_tracked", tracked), + ("retention", retention.clone()), + ( + "models", + PyTuple::new(self.runtime.py(), &self.models)?.into_any(), + ), + ("views", self.views.clone()), + ], + ) + } + + fn aliases(&self, item: &Bound<'py, PyAny>) -> PyResult> { + let py = self.runtime.py(); + if !stores_history(item)? { + return imported(py, "sqlbuild._native", "declared_column_aliases")?.call1((item,)); + } + let mut aliases: Vec> = Vec::new(); + for entry in self.columns.try_iter()? { + let entry = entry?; + let decision = entry.getattr("decision")?; + if entry.getattr("model_name")?.eq(item.getattr("name")?)? + && (decision.eq("rename")? || decision.eq("record")?) + { + aliases.push(PyTuple::new( + py, + [ + entry.getattr("origin_column")?, + entry.getattr("destination_column")?, + ], + )?); + } + } + let aliases = PyList::new(py, aliases)?; + aliases.call_method0("sort")?; + imported(py, "builtins", "tuple")?.call1((aliases,)) + } + + fn new_move( + &self, + entry: &Bound<'py, PyAny>, + item: &Bound<'py, PyAny>, + ) -> PyResult> { + let py = self.runtime.py(); + let retention = retention(item, &self.retention)?; + let reason = self.skip( + &entry.getattr("origin")?, + entry.getattr("origin_tracked")?, + &retention, + )?; + let mut fields = vec![ + ("model_name", entry.getattr("model_name")?), + ("origin", entry.getattr("origin")?), + ("destination", entry.getattr("destination")?), + ("target_name", entry.getattr("target_name")?), + ]; + if !reason.is_none() || retention.is_none() { + fields.extend([ + ("action", action(&self.runtime, "NONE")?), + ( + "reason", + if reason.is_truthy()? { + reason + } else { + text(py, "old_name_view false") + }, + ), + ]); + } else { + fields.extend([ + ("action", action(&self.runtime, "ARCHIVE_AND_VIEW")?), + ("expires_at", expires(&retention, &self.now)?), + ("retention", retention), + ("column_aliases", self.aliases(item)?), + ( + "stores_history", + PyBool::new(py, stores_history(item)?).to_owned().into_any(), + ), + ( + "records_requirement", + PyBool::new(py, true).to_owned().into_any(), + ), + ( + "grants_supported", + self.runtime + .getattr("adapter")? + .getattr("relation_grants_supported")?, + ), + ]); + } + model(py, "OldNameViewPlanEntry", &fields) + } + + fn resume( + &self, + history: &Bound<'py, PyAny>, + item: &Bound<'py, PyAny>, + ) -> PyResult>> { + let py = self.runtime.py(); + let status: String = + call_with(&history.getattr("status")?, &[("now", self.now.clone())])?.extract()?; + let action_name = match status.as_str() { + "pending_archive" => "ARCHIVE_AND_VIEW", + "pending_view" => "VIEW_ONLY", + "live" | "expired" => "LIVE", + _ => return Ok(None), + }; + if status == PENDING_ARCHIVE || status == PENDING_VIEW { + let reason = self.skip( + &location(&self.runtime, &history.getattr("old")?)?, + py.None().into_bound(py), + &retention(item, &self.retention)?, + )?; + if !reason.is_none() { + return model( + py, + "OldNameViewPlanEntry", + &[ + ( + "model_name", + history.getattr("move")?.getattr("destination_model")?, + ), + ("origin", location(&self.runtime, &history.getattr("old")?)?), + ( + "destination", + location(&self.runtime, &history.getattr("new")?)?, + ), + ("action", action(&self.runtime, "NONE")?), + ( + "target_name", + history.getattr("move")?.getattr("target_name")?, + ), + ( + "migration_event_id", + history.getattr("move")?.getattr("event_id")?, + ), + ("reason", reason), + ], + ) + .map(Some); + } + } + let retention = history.getattr("required")?.getattr("view_retention")?; + let duration = imported(py, "sqlbuild.cursor_algebra.models", "Duration")?.call_method1( + "parse", + (if retention.is_truthy()? { + retention.clone() + } else { + text(py, "") + },), + )?; + let created = history.getattr("created")?; + let expires_at = if !created.is_none() { + created.getattr("expires_at")? + } else if duration.is_none() { + py.None().into_bound(py) + } else { + duration.call_method1("add_to", (&self.now,))? + }; + let grants = if created.is_none() { + py.None().into_bound(py) + } else { + created.getattr("grants_copied")? + }; + model( + py, + "OldNameViewPlanEntry", + &[ + ("model_name", item.getattr("name")?), + ("origin", location(&self.runtime, &history.getattr("old")?)?), + ( + "destination", + location(&self.runtime, &history.getattr("new")?)?, + ), + ("action", action(&self.runtime, action_name)?), + ( + "target_name", + history.getattr("move")?.getattr("target_name")?, + ), + ("retention", retention), + ("expires_at", expires_at), + ( + "column_aliases", + if created.is_none() { + self.aliases(item)? + } else { + created.getattr("column_aliases")? + }, + ), + ( + "stores_history", + PyBool::new(py, stores_history(item)?).to_owned().into_any(), + ), + ( + "migration_event_id", + history.getattr("move")?.getattr("event_id")?, + ), + ( + "grants_copied", + if grants.is_none() { + grants + } else { + imported(py, "builtins", "len")?.call1((grants,))? + }, + ), + ( + "grants_supported", + self.runtime + .getattr("adapter")? + .getattr("relation_grants_supported")?, + ), + ( + "archived", + PyBool::new(py, !history.getattr("archived")?.is_none()) + .to_owned() + .into_any(), + ), + ], + ) + .map(Some) + } +} + +fn action<'py>(runtime: &Bound<'py, PyAny>, name: &str) -> PyResult> { + imported( + runtime.py(), + "sqlbuild.compiler.migrations.types", + "OldNameViewAction", + )? + .getattr(name) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs new file mode 100644 index 0000000000..c9911903b6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs @@ -0,0 +1,114 @@ +//! Old-name planner object conversions and existing native helper calls. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_migrations::column_objects::{config, model}; +use crate::bindings::_helpers::state_rows::migration_projections::old_name_retention; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { + imported( + location.py(), + "sqlbuild.compiler.migrations.main.relation_for_location", + "migration_relation_for_location", + )? + .call1((location,)) +} + +pub(super) fn location<'py>( + runtime: &Bound<'py, PyAny>, + relation: &Bound<'py, PyAny>, +) -> PyResult> { + let mut fields = vec![ + ("database", relation.getattr("database")?), + ("schema", relation.getattr("schema")?), + ("name", relation.getattr("name")?), + ]; + let qualified = call_with( + &runtime + .getattr("adapter")? + .getattr("render_qualified_name")?, + &fields, + )?; + fields.push(("qualified_name", qualified)); + call_with( + &imported( + runtime.py(), + "sqlbuild.compiler.compile.models", + "CompiledRelationLocation", + )?, + &fields, + ) +} + +pub(super) fn stores_history(item: &Bound<'_, PyAny>) -> PyResult { + let materialization = config(item, "materialized")?; + Ok(materialization.eq("incremental")? || materialization.eq("snapshot")?) +} + +pub(super) fn retention<'py>( + item: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + old_name_retention(&item.getattr("config")?.getattr("values")?, project) +} + +pub(super) fn expires<'py>( + retention: &Bound<'py, PyAny>, + now: &Bound<'py, PyAny>, +) -> PyResult> { + let duration = imported(now.py(), "sqlbuild.cursor_algebra.models", "Duration")? + .call_method1("parse", (retention,))?; + if duration.is_none() { + Ok(now.py().None().into_bound(now.py())) + } else { + duration.call_method1("add_to", (now,)) + } +} + +pub(super) fn recorded_view<'py>( + runtime: &Bound<'py, PyAny>, + history: &Bound<'py, PyAny>, + models: &[Bound<'py, PyAny>], +) -> PyResult> { + let py = runtime.py(); + let movement = history.getattr("move")?; + let created = history.getattr("created")?; + let mut reused = py.None().into_bound(py); + for item in models { + if relation(&item.getattr("destination")?)? + .call_method1("matches", (history.getattr("old")?,))? + .is_truthy()? + { + reused = item.getattr("name")?; + break; + } + } + model( + py, + "OldNameView", + &[ + ("destination_model", movement.getattr("destination_model")?), + ("old", location(runtime, &history.getattr("old")?)?), + ("new", location(runtime, &history.getattr("new")?)?), + ( + "expires_at", + if created.is_none() { + py.None().into_bound(py) + } else { + created.getattr("expires_at")? + }, + ), + ( + "column_aliases", + if created.is_none() { + pyo3::types::PyTuple::empty(py).into_any() + } else { + created.getattr("column_aliases")? + }, + ), + ("migration_event_id", movement.getattr("event_id")?), + ("target_name", movement.getattr("target_name")?), + ("name_reused_by", reused), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs new file mode 100644 index 0000000000..aff9f276f8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -0,0 +1,195 @@ +//! Old-name planning orchestration over existing live state inspection. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_migrations::column_objects::{model, native}; +use crate::bindings::_helpers::planner_migrations::old_name_objects::recorded_view; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +fn schemas<'py>( + args: &KeywordArguments<'py>, + selected: &[Bound<'py, PyAny>], +) -> PyResult> { + let runtime = args.value("runtime")?; + let py = runtime.py(); + let snapshot = args.value("snapshot")?; + let needed = PySet::empty(py)?; + for item in selected { + let schema = item.getattr("destination")?.getattr("schema")?; + if !schema.is_none() { + needed.add(schema)?; + } + } + for entry in args.value("migration_entries")?.try_iter()? { + let schema = entry?.getattr("origin")?.getattr("schema")?; + if !schema.is_none() { + needed.add(schema)?; + } + } + let mut include_project = runtime + .getattr("project")? + .getattr("unmatched_literal_sql_relations")? + .is_truthy()?; + if !include_project { + for item in selected { + let existing = snapshot + .getattr("existing_relations")? + .call_method1("get", (item.getattr("name")?,))?; + if !existing.is_none() + && imported( + py, + "sqlbuild.adapter.type_system.main.normalize_relation_type", + "normalize_relation_type", + )? + .call1((existing.getattr("relation_type")?,))? + .eq("view")? + { + include_project = true; + break; + } + } + } + if include_project { + let project = native(py, "migration_project_schemas", &[("runtime", runtime)])?; + for schema in project.try_iter()? { + needed.add(schema?)?; + } + } + native( + py, + "old_name_schemas_to_read", + &[("snapshot", snapshot), ("needed", needed.into_any())], + ) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +fn plan_old_name_views<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "plan_old_name_views", + options, + ( + &[ + "runtime", + "scope", + "snapshot", + "migration_entries", + "column_entries", + ], + &[], + ), + )?; + let runtime = args.value("runtime")?; + let scope = args.value("scope")?; + let project = runtime.getattr("project")?; + let models: Vec> = project + .getattr("models")? + .try_iter()? + .collect::>()?; + let by_name = PyDict::new(py); + for item in &models { + by_name.set_item(item.getattr("name")?, item)?; + } + let mut selected: Vec> = Vec::new(); + for key in scope.getattr("execution_order")?.try_iter()? { + let key = key?; + if scope.getattr("selected_keys")?.contains(&key)? + && key.getattr("resource_type")?.eq("model")? + && let Some(item) = by_name.get_item(key.getattr("name")?)? + { + selected.push(item); + } + } + let state = call_with( + &imported(py, "sqlbuild._native", "MigrationStateInspection")?, + &[ + ("adapter", runtime.getattr("adapter")?), + ("connection", runtime.getattr("connection")?), + ( + "database", + native( + py, + "migration_planning_database", + &[("runtime", runtime.clone())], + )?, + ), + ], + )?; + let needed = schemas(&args, &selected)?; + if needed.is_truthy()? { + call_with( + &state.getattr("inspect_old_name_views")?, + &[("schemas", needed)], + )?; + } + let target = project.getattr("effective_target_name")?; + let mut histories: Vec> = Vec::new(); + let mut views: Vec> = Vec::new(); + for history in state.getattr("old_name_histories")?.try_iter()? { + let history = history?; + let history_target = history.getattr("move")?.getattr("target_name")?; + if !history_target.is_none() && !history_target.eq(&target)? { + continue; + } + if !history.getattr("created")?.is_none() && history.getattr("dropped")?.is_none() { + views.push(recorded_view(&runtime, &history, &models)?); + } + histories.push(history); + } + let views = PyTuple::new(py, views)?.into_any(); + let planner = super::models::OldNamePlanner::new(&args, models, histories, views.clone())?; + let mut entries: Vec> = Vec::new(); + let covered = PySet::empty(py)?; + for entry in args.value("migration_entries")?.try_iter()? { + let entry = entry?; + let name = entry.getattr("model_name")?; + if let Some(item) = by_name.get_item(&name)? + && let Some(planned) = planner.for_migration(&entry, &item)? + { + entries.push(planned); + covered.add(name)?; + } + } + for item in &selected { + if !covered.contains(item.getattr("name")?)? + && let Some(planned) = planner.pending_for(item)? + { + entries.push(planned); + } + } + let entries = PyTuple::new(py, entries)?.into_any(); + let claims = native( + py, + "old_name_claim_errors", + &[ + ("selected", PyTuple::new(py, selected)?.into_any()), + ("snapshot", args.value("snapshot")?), + ("views", views.clone()), + ("now", runtime.getattr("invocation_time")?), + ], + )?; + let mut warnings: Vec> = claims.try_iter()?.collect::>()?; + warnings.extend(super::old_name_references::errors( + &runtime, &views, &entries, + )?); + model( + py, + "OldNameViewPlanning", + &[ + ("entries", entries), + ("views", views), + ("warnings", PyTuple::new(py, warnings)?.into_any()), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(plan_old_name_views, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs new file mode 100644 index 0000000000..c87e7b0639 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs @@ -0,0 +1,143 @@ +//! Diagnose literal references to compatibility-only old names. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_migrations::column_objects::{model, native, text}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; + +fn old_relation<'py>( + old: &Bound<'py, PyAny>, + name: Bound<'py, PyAny>, +) -> PyResult> { + let py = old.py(); + let reference = call_with( + &imported(py, "sqlbuild.python_nodes.models", "SqlResourceRef")?, + &[ + ( + "kind", + imported(py, "sqlbuild.python_nodes.types", "SqlResourceRefKind")? + .getattr("MODEL")?, + ), + ("name", name.clone()), + ], + )?; + let relation = call_with( + &imported(py, "sqlbuild.compiler.references.models", "RelationName")?, + &[ + ("name", old.getattr("name")?), + ("schema", old.getattr("schema")?), + ("database", old.getattr("database")?), + ], + )?; + call_with( + &imported(py, "sqlbuild.compiler.references.models", "ProjectRelation")?, + &[ + ("ref", reference), + ("relation", relation), + ("compatibility_for", name), + ], + ) +} + +pub(super) fn errors<'py>( + runtime: &Bound<'py, PyAny>, + views: &Bound<'py, PyAny>, + entries: &Bound<'py, PyAny>, +) -> PyResult>> { + let py = runtime.py(); + let literals = runtime + .getattr("project")? + .getattr("unmatched_literal_sql_relations")?; + if !literals.is_truthy()? { + return Ok(Vec::new()); + } + let relations = PyDict::new(py); + for view in views.try_iter()? { + let view = view?; + relations.set_item( + old_relation(&view.getattr("old")?, view.getattr("destination_model")?)?, + py.None(), + )?; + } + for entry in entries.try_iter()? { + let entry = entry?; + if entry.getattr("runs_steps")?.is_truthy()? { + relations.set_item( + old_relation(&entry.getattr("origin")?, entry.getattr("model_name")?)?, + py.None(), + )?; + } + } + let index = call_with( + &imported( + py, + "sqlbuild.compiler.references.models", + "ProjectRelationIndex", + )?, + &[("relations", PyTuple::new(py, relations.keys())?.into_any())], + )?; + let mut errors: Vec> = Vec::new(); + for literal in literals.try_iter()? { + let literal = literal?; + let relation = literal.getattr("relation")?; + let matched = native( + py, + "match_project_relation", + &[("index", index.clone()), ("relation", relation.clone())], + )?; + if matched.is_none() || matched.getattr("compatibility_for")?.is_none() { + continue; + } + let mut parts: Vec = Vec::new(); + for field in ["database", "schema", "name"] { + let part = relation.getattr(field)?; + if part.is_truthy()? { + parts.push(part.extract()?); + } + } + let message: String = native( + py, + "old_name_reference_message", + &[ + ("owner_label", literal.getattr("owner_label")?), + ("written", text(py, &parts.join("."))), + ("method", literal.getattr("method")?), + ("model_name", matched.getattr("compatibility_for")?), + ], + )? + .extract()?; + let remedy: String = native( + py, + "hard_coded_relation_remedy", + &[ + ("owner_kind", literal.getattr("owner_kind")?), + ("ref", matched.getattr("ref")?), + ], + )? + .extract()?; + let path = literal + .getattr("relative_path")? + .str()? + .to_str()? + .to_owned(); + let line = literal.getattr("line")?.str()?.to_str()?.to_owned(); + errors.push(model( + py, + "PlanWarning", + &[ + ("model_name", py.None().into_bound(py)), + ( + "severity", + imported(py, "sqlbuild.compiler.planner.types", "WarningSeverity")? + .getattr("ERROR")?, + ), + ( + "message", + text(py, &format!("{path}:{line}: {message}; {remedy}")), + ), + ("code", text(py, "P008")), + ], + )?); + } + Ok(errors) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs index fd047a8ba9..43f7f0a255 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::planner_migrations::old_name_objects::relation; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PySet, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; @@ -93,15 +94,6 @@ fn claim_message(model: &str, view: &Bound<'_, PyAny>, now: &Bound<'_, PyAny>) - )) } -fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { - imported( - location.py(), - "sqlbuild.compiler.migrations.main.relation_for_location", - "migration_relation_for_location", - )? - .call1((location,)) -} - #[pyfunction] #[pyo3(signature = (**options))] fn old_name_claim_errors<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs index b2850d000d..2718ea8c91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs @@ -1,12 +1,13 @@ //! Register pure migration helpers. -use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::discovery::register(module)?; super::compatibility::register(module)?; super::matching::register(module)?; super::old_names::register(module)?; + super::old_name_planning::register(module)?; super::columns::register(module)?; super::column_planning::register(module)?; super::model_planning::register(module)?; @@ -14,5 +15,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::planning_origin::register(module)?; super::planning_execution::register(module)?; super::projections::register(module)?; + module.add_class::()?; super::fingerprint::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs new file mode 100644 index 0000000000..05ea22662c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs @@ -0,0 +1,578 @@ +//! Read-only warehouse evidence gathered once for all model migrations in one plan. + +use std::collections::{HashMap, HashSet}; + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyRef, PyRefMut, PyResult, Python, pyclass, pymethods, +}; +use pyo3::types::{PyBool, PyDict, PyString, PyTuple, PyTupleMethods}; +use sqlbuild_state::events::constants::{ + COLUMN_MIGRATION_TABLE_NAME, MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, +}; +use sqlbuild_state::events::main::newest_fingerprint::newest_fingerprint; +use sqlbuild_state::events::models::{EventRelation, FingerprintKey, FingerprintOrder}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_migrations::constants::{ + FINGERPRINT_TABLE_NAME, GET_COLUMNS_FOR_INSPECTION, LIST_RELATIONS_FOR_INSPECTION, + NATIVE_MODULE, NODE_TYPE_MODEL, +}; +use crate::bindings::_helpers::state_rows::constants::{COLUMN_KIND, MODEL_KIND, OLD_NAME_KIND}; +use crate::bindings::_helpers::state_rows::migration_io::{read_events, stored_old_name_columns}; +use crate::bindings::_helpers::state_rows::migration_objects::layout; +use crate::bindings::_helpers::state_rows::migration_projections::old_name_views; +use crate::bindings::_helpers::state_rows::models::StateAccess; + +/// A lowercased schema and relation name. +type RelationKey = (String, String); + +/// Relations, fingerprints and migration events for one planning database. +#[pyclass(module = "sqlbuild._native", name = "MigrationStateInspection")] +pub(crate) struct MigrationStateInspection { + adapter: Py, + connection: Py, + database: Py, + known_fingerprints: Py, + read_schemas: HashSet, + /// Model fingerprints per lowercased schema, in first-read order. + fingerprints: Vec<(String, Vec>)>, + events: Vec>, + event_schemas: HashSet, + relations: HashMap>, + columns: HashMap>, + column_event_schemas: HashSet, + column_events: Vec>, + old_name_schemas: HashSet, + old_name_moves: Vec>, + old_name_facts: Vec>, +} + +fn lowered(value: &Bound<'_, PyAny>) -> PyResult { + value.call_method0("lower")?.extract() +} + +fn schema_or_empty<'py>(value: Bound<'py, PyAny>) -> Bound<'py, PyAny> { + if value.is_none() { + PyString::new(value.py(), "").into_any() + } else { + value + } +} + +/// `((schema or "").lower(), name.lower())` of a relation or location. +fn relation_key(value: &Bound<'_, PyAny>) -> PyResult { + Ok(( + lowered(&schema_or_empty(value.getattr("schema")?))?, + lowered(&value.getattr("name")?)?, + )) +} + +/// The requested schemas not inspected yet, sorted, each with its lowercased key. +fn pending<'py>( + schemas: &Bound<'py, PyAny>, + inspected: &HashSet, +) -> PyResult, String)>> { + let mut pending: Vec<(String, Bound<'py, PyAny>, String)> = Vec::new(); + for schema in schemas.try_iter()? { + let schema: Bound<'py, PyAny> = schema?; + let key: String = lowered(&schema)?; + if !inspected.contains(&key) { + pending.push((schema.extract()?, schema, key)); + } + } + pending.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(pending + .into_iter() + .map(|(_, schema, key)| (schema, key)) + .collect()) +} + +fn tuple_of<'py>(py: Python<'py>, items: &[Py]) -> PyResult> { + PyTuple::new(py, items.iter().map(|item| item.bind(py))) +} + +fn fingerprint_key(fingerprint: &Bound<'_, PyAny>) -> PyResult { + let target_name: Bound<'_, PyAny> = fingerprint.getattr("target_name")?; + let target: Option = if target_name.is_none() { + None + } else { + Some(EventRelation { + database: fingerprint.getattr("target_database")?.extract()?, + schema: fingerprint.getattr("target_schema")?.extract()?, + name: target_name.extract()?, + }) + }; + Ok(FingerprintKey { + order: FingerprintOrder { + timestamp: fingerprint + .getattr("ts")? + .call_method0("isoformat")? + .extract()?, + run_id: fingerprint.getattr("run_id")?.extract()?, + }, + node_name: fingerprint.getattr("node_name")?.extract()?, + target, + }) +} + +fn newest<'py>( + py: Python<'py>, + candidates: &[Py], + (target, model_name): (Option<&EventRelation>, Option<&str>), +) -> PyResult>> { + let keys: Vec = candidates + .iter() + .map(|candidate| fingerprint_key(candidate.bind(py))) + .collect::>()?; + Ok(newest_fingerprint(&keys, target, model_name) + .map(|index| candidates[index].bind(py).clone())) +} + +impl MigrationStateInspection { + fn access<'py>(&self, py: Python<'py>) -> PyResult> { + let adapter: &Bound<'py, PyAny> = self.adapter.bind(py); + Ok(StateAccess { + connection: self.connection.bind(py).clone(), + execute: adapter.getattr("execute")?, + render_qualified_name: adapter.getattr("render_qualified_name")?, + }) + } + + fn list_relations<'py>( + &self, + py: Python<'py>, + schemas: Vec>, + names: Bound<'py, PyAny>, + ) -> PyResult>> { + let (module, name) = LIST_RELATIONS_FOR_INSPECTION; + call_with( + &imported(py, module, name)?, + &[ + ("adapter", self.adapter.bind(py).clone()), + ("connection", self.connection.bind(py).clone()), + ("database", self.database.bind(py).clone()), + ("schemas", PyTuple::new(py, schemas)?.into_any()), + ("names", names), + ], + )? + .try_iter()? + .collect() + } + + fn model_fingerprints<'py>( + &self, + py: Python<'py>, + schema: &Bound<'py, PyAny>, + key: &str, + ) -> PyResult>> { + let known: Bound<'py, PyAny> = self + .known_fingerprints + .bind(py) + .call_method1("get", (key,))?; + let latest: Bound<'py, PyAny> = if known.is_none() { + let adapter: &Bound<'py, PyAny> = self.adapter.bind(py); + let fingerprint_set: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "read_latest_fingerprints")?, + &[ + ("connection", self.connection.bind(py).clone()), + ("execute", adapter.getattr("execute")?), + ("table_exists", PyBool::new(py, true).to_owned().into_any()), + ("database", self.database.bind(py).clone()), + ("schema", schema.clone()), + ( + "render_qualified_name", + adapter.getattr("render_qualified_name")?, + ), + ( + "render_read_latest_sql", + adapter.getattr("render_read_latest_fingerprints_sql")?, + ), + ], + )?; + let by_identity: Bound<'py, PyAny> = + fingerprint_set.getattr("fingerprints_by_identity")?; + if by_identity.is_truthy()? { + by_identity.call_method0("values")? + } else { + PyTuple::empty(py).into_any() + } + } else { + known + }; + let mut models: Vec> = Vec::new(); + for fingerprint in latest.try_iter()? { + let fingerprint: Bound<'py, PyAny> = fingerprint?; + if fingerprint.getattr("node_type")?.eq(NODE_TYPE_MODEL)? { + models.push(fingerprint.unbind()); + } + } + Ok(models) + } +} + +#[pymethods] +impl MigrationStateInspection { + #[new] + #[pyo3(signature = (*, adapter, connection, database, known_fingerprints = None))] + fn new( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + database: &Bound<'_, PyAny>, + known_fingerprints: Option<&Bound<'_, PyAny>>, + ) -> PyResult { + let py: Python<'_> = adapter.py(); + let known: Bound<'_, PyAny> = match known_fingerprints { + Some(known) + if !known.is_none() && known.getattr("unfiltered_database")?.eq(database)? => + { + known.getattr("unfiltered_schemas")? + } + _ => PyDict::new(py).into_any(), + }; + Ok(Self { + adapter: adapter.clone().unbind(), + connection: connection.clone().unbind(), + database: database.clone().unbind(), + known_fingerprints: known.unbind(), + read_schemas: HashSet::new(), + fingerprints: Vec::new(), + events: Vec::new(), + event_schemas: HashSet::new(), + relations: HashMap::new(), + columns: HashMap::new(), + column_event_schemas: HashSet::new(), + column_events: Vec::new(), + old_name_schemas: HashSet::new(), + old_name_moves: Vec::new(), + old_name_facts: Vec::new(), + }) + } + + /// The database whose namespaces this inspection reads. + #[getter] + fn database(&self, py: Python<'_>) -> Py { + self.database.clone_ref(py) + } + + /// Every migration event read so far. + #[getter] + fn events<'py>(&self, py: Python<'py>) -> PyResult> { + tuple_of(py, &self.events) + } + + /// Every column migration event read so far. + #[getter] + fn column_events<'py>(&self, py: Python<'py>) -> PyResult> { + tuple_of(py, &self.column_events) + } + + /// The old-name history of every move read by `inspect_old_name_views`. + #[getter] + fn old_name_histories<'py>(&self, py: Python<'py>) -> PyResult> { + old_name_views( + tuple_of(py, &self.old_name_moves)?.as_any(), + tuple_of(py, &self.old_name_facts)?.as_any(), + ) + } + + /// Read the column migration state table once for every schema not inspected yet. + #[pyo3(signature = (*, schemas))] + fn inspect_column_events(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let pending = pending(schemas, &slf.borrow().column_event_schemas)?; + if pending.is_empty() { + return Ok(()); + } + slf.borrow_mut() + .column_event_schemas + .extend(pending.iter().map(|(_, key)| key.clone())); + let listed: Vec> = slf.borrow().list_relations( + py, + pending.iter().map(|(schema, _)| schema.clone()).collect(), + PyTuple::new(py, [COLUMN_MIGRATION_TABLE_NAME])?.into_any(), + )?; + let mut present: HashSet = HashSet::new(); + for relation in &listed { + if lowered(&relation.getattr("name")?)? == COLUMN_MIGRATION_TABLE_NAME { + present.insert(lowered(&schema_or_empty(relation.getattr("schema")?))?); + } + } + for (schema, key) in pending { + if !present.contains(&key) { + continue; + } + let (access, database) = { + let this: PyRef<'_, Self> = slf.borrow(); + (this.access(py)?, this.database.bind(py).clone()) + }; + let events = read_events(&access, &layout(COLUMN_KIND)?, (database, schema), None)?; + slf.borrow_mut() + .column_events + .extend(events.iter().map(Bound::unbind)); + } + Ok(()) + } + + /// Read moves and old-name facts once for every schema not inspected yet. + #[pyo3(signature = (*, schemas))] + fn inspect_old_name_views(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let pending = pending(schemas, &slf.borrow().old_name_schemas)?; + if pending.is_empty() { + return Ok(()); + } + slf.borrow_mut() + .old_name_schemas + .extend(pending.iter().map(|(_, key)| key.clone())); + let listed: Vec> = slf.borrow().list_relations( + py, + pending.iter().map(|(schema, _)| schema.clone()).collect(), + PyTuple::new(py, [MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME])?.into_any(), + )?; + let state_tables: HashSet = + listed.iter().map(relation_key).collect::>()?; + for (schema, key) in pending { + if !state_tables.contains(&(key.clone(), OLD_NAME_VIEW_TABLE_NAME.to_owned())) + || !state_tables.contains(&(key, MIGRATION_TABLE_NAME.to_owned())) + { + continue; + } + let (access, adapter, database) = { + let this: PyRef<'_, Self> = slf.borrow(); + ( + this.access(py)?, + this.adapter.bind(py).clone(), + this.database.bind(py).clone(), + ) + }; + let stored: Vec = stored_old_name_columns( + &adapter, + &access.connection, + (database.clone(), schema.clone()), + )?; + let facts = read_events( + &access, + &layout(OLD_NAME_KIND)?, + (database.clone(), schema.clone()), + Some(&stored), + )?; + slf.borrow_mut() + .old_name_facts + .extend(facts.iter().map(Bound::unbind)); + let moves = read_events(&access, &layout(MODEL_KIND)?, (database, schema), None)?; + slf.borrow_mut() + .old_name_moves + .extend(moves.iter().map(Bound::unbind)); + } + Ok(()) + } + + /// Read migration events once from schemas known to hold the migration table. + #[pyo3(signature = (*, schemas))] + fn inspect_migration_events(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let mut sorted: Vec<(String, Bound<'_, PyAny>)> = Vec::new(); + for schema in schemas.try_iter()? { + let schema: Bound<'_, PyAny> = schema?; + sorted.push((schema.extract()?, schema)); + } + sorted.sort_by(|left, right| left.0.cmp(&right.0)); + for (_, schema) in sorted { + let key: String = lowered(&schema)?; + if !slf.borrow_mut().event_schemas.insert(key) { + continue; + } + let (access, database) = { + let this: PyRef<'_, Self> = slf.borrow(); + (this.access(py)?, this.database.bind(py).clone()) + }; + let events = read_events(&access, &layout(MODEL_KIND)?, (database, schema), None)?; + slf.borrow_mut() + .events + .extend(events.iter().map(Bound::unbind)); + } + Ok(()) + } + + /// Read state tables once for every schema not inspected yet. + #[pyo3(signature = (*, schemas))] + fn inspect_schemas(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let pending = pending(schemas, &slf.borrow().read_schemas)?; + if pending.is_empty() { + return Ok(()); + } + slf.borrow_mut() + .read_schemas + .extend(pending.iter().map(|(_, key)| key.clone())); + let listed: Vec> = slf.borrow().list_relations( + py, + pending.iter().map(|(schema, _)| schema.clone()).collect(), + PyTuple::new(py, [MIGRATION_TABLE_NAME, FINGERPRINT_TABLE_NAME])?.into_any(), + )?; + let state_tables: HashSet = + listed.iter().map(relation_key).collect::>()?; + for (schema, key) in pending { + if state_tables.contains(&(key.clone(), MIGRATION_TABLE_NAME.to_owned())) { + let single = PyTuple::new(py, [schema.clone()])?; + Self::inspect_migration_events(slf, single.as_any())?; + } + if state_tables.contains(&(key.clone(), FINGERPRINT_TABLE_NAME.to_owned())) { + let models: Vec> = slf.borrow().model_fingerprints(py, &schema, &key)?; + let mut this: PyRefMut<'_, Self> = slf.borrow_mut(); + match this + .fingerprints + .iter_mut() + .find(|(stored, _)| *stored == key) + { + Some((_, stored)) => *stored = models, + None => this.fingerprints.push((key, models)), + } + } + } + Ok(()) + } + + /// List candidate relations and their columns with one metadata read. + #[pyo3(signature = (*, locations))] + fn inspect_relations(slf: &Bound<'_, Self>, locations: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let mut wanted: HashSet = HashSet::new(); + let mut schemas: Vec = Vec::new(); + let mut names: Vec = Vec::new(); + for location in locations.try_iter()? { + let location: Bound<'_, PyAny> = location?; + if location.getattr("schema")?.is_none() { + continue; + } + let key: RelationKey = relation_key(&location)?; + if slf.borrow().relations.contains_key(&key) { + continue; + } + wanted.insert(key); + schemas.push(location.getattr("schema")?.extract()?); + names.push(location.getattr("name")?.extract()?); + } + if wanted.is_empty() { + return Ok(()); + } + schemas.sort(); + schemas.dedup(); + names.sort(); + names.dedup(); + let listed: Vec> = slf.borrow().list_relations( + py, + schemas + .iter() + .map(|schema| PyString::new(py, schema).into_any()) + .collect(), + PyTuple::new(py, &names)?.into_any(), + )?; + let mut matched: Vec<(RelationKey, Bound<'_, PyAny>)> = Vec::new(); + for relation in listed { + let key: RelationKey = relation_key(&relation)?; + if wanted.contains(&key) { + matched.push((key, relation)); + } + } + let columns: Bound<'_, PyAny> = if matched.is_empty() { + PyDict::new(py).into_any() + } else { + let (module, name) = GET_COLUMNS_FOR_INSPECTION; + let this: PyRef<'_, Self> = slf.borrow(); + let adapter: Bound<'_, PyAny> = this.adapter.bind(py).clone(); + let connection: Bound<'_, PyAny> = this.connection.bind(py).clone(); + drop(this); + call_with( + &imported(py, module, name)?, + &[ + ("adapter", adapter), + ("connection", connection), + ( + "relations", + PyTuple::new(py, matched.iter().map(|(_, relation)| relation))?.into_any(), + ), + ], + )? + }; + let empty: Bound<'_, PyTuple> = PyTuple::empty(py); + for (key, relation) in matched { + let relation_columns: Bound<'_, PyAny> = + columns.call_method1("get", (relation.getattr("identity")?, &empty))?; + let mut this: PyRefMut<'_, Self> = slf.borrow_mut(); + this.relations.insert(key.clone(), relation.unbind()); + this.columns.insert(key, relation_columns.unbind()); + } + Ok(()) + } + + /// The listed relation for a location. + fn relation(&self, py: Python<'_>, location: &Bound<'_, PyAny>) -> PyResult> { + Ok(match self.relations.get(&relation_key(location)?) { + Some(relation) => relation.clone_ref(py), + None => py.None(), + }) + } + + /// Listed columns for a location. + fn relation_columns(&self, py: Python<'_>, location: &Bound<'_, PyAny>) -> PyResult> { + Ok(match self.columns.get(&relation_key(location)?) { + Some(columns) => columns.clone_ref(py), + None => PyTuple::empty(py).into_any().unbind(), + }) + } + + /// The latest model fingerprint that built the location or named model. + #[pyo3(signature = (*, location, model_name))] + fn model_fingerprint<'py>( + &self, + py: Python<'py>, + location: &Bound<'py, PyAny>, + model_name: Option, + ) -> PyResult>> { + let key: String = lowered(&schema_or_empty(location.getattr("schema")?))?; + let Some((_, candidates)) = self.fingerprints.iter().find(|(stored, _)| *stored == key) + else { + return Ok(None); + }; + let target = EventRelation { + database: location.getattr("database")?.extract()?, + schema: location.getattr("schema")?.extract()?, + name: location.getattr("name")?.extract()?, + }; + newest(py, candidates, (Some(&target), model_name.as_deref())) + } + + /// The newest model fingerprint with this node name in any inspected schema. + #[pyo3(signature = (*, model_name))] + fn named_fingerprint<'py>( + &self, + py: Python<'py>, + model_name: &str, + ) -> PyResult>> { + let mut all: Vec> = Vec::new(); + for (_, fingerprints) in &self.fingerprints { + all.extend(fingerprints.iter().map(|item| item.clone_ref(py))); + } + newest(py, &all, (None, Some(model_name))) + } + + /// Latest model fingerprints whose names are no longer in the project. + #[pyo3(signature = (*, project_model_names))] + fn orphan_fingerprints<'py>( + &self, + py: Python<'py>, + project_model_names: &Bound<'py, PyAny>, + ) -> PyResult> { + let mut orphans: Vec> = Vec::new(); + for (_, fingerprints) in &self.fingerprints { + for fingerprint in fingerprints { + let fingerprint: &Bound<'py, PyAny> = fingerprint.bind(py); + if !project_model_names.contains(fingerprint.getattr("node_name")?)? { + orphans.push(fingerprint.clone()); + } + } + } + PyTuple::new(py, orphans) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs new file mode 100644 index 0000000000..20a3aac300 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs @@ -0,0 +1,263 @@ +//! Scenario artifact dataclasses and adapter identifier callbacks. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_scenario::values::{MODELS, TYPES, error, string}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_analysis::planner_scenario::main::{hash_prefix, parse_artifact_name}; + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn compute_scenario_hash_prefix( + py: Python<'_>, + kwargs: Option>, +) -> PyResult { + let args = KeywordArguments::read( + py, + "compute_scenario_hash_prefix", + kwargs, + ( + &["project_name", "scenario_name"], + &["run_namespace", "prefix_length"], + ), + )?; + let project: String = args.value("project_name")?.extract()?; + let scenario: String = args.value("scenario_name")?.extract()?; + let namespace: Option = args.value("run_namespace")?.extract()?; + let length: Option = args.value("prefix_length")?.extract()?; + compute_prefix( + py, + (&project, &scenario, namespace.as_deref()), + length.unwrap_or(12), + ) +} + +fn compute_prefix( + py: Python<'_>, + identity: (&str, &str, Option<&str>), + prefix_length: i64, +) -> PyResult { + if prefix_length < 1 { + return Err(error( + py, + "Scenario hash prefix length must be at least 1", + "SCENARIO_PLAN_INVALID_HASH_PREFIX", + None, + )?); + } + Ok(hash_prefix::hash_prefix( + identity.0, + identity.1, + identity.2, + usize::try_from(prefix_length).unwrap_or(usize::MAX), + )) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn build_scenario_hash_index<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_scenario_hash_index", + kwargs, + ( + &["project_name", "scenarios"], + &["run_namespace", "prefix_length"], + ), + )?; + let project_name: String = args.value("project_name")?.extract()?; + let scenarios = args.value("scenarios")?; + let run_namespace: Option = args.value("run_namespace")?.extract()?; + let prefix_length: Option = args.value("prefix_length")?.extract()?; + let result = PyDict::new(py); + let mut seen: std::collections::HashMap = std::collections::HashMap::new(); + for scenario in scenarios.try_iter()? { + let name: String = scenario?.getattr("name")?.extract()?; + let prefix = compute_prefix( + py, + (&project_name, &name, run_namespace.as_deref()), + prefix_length.unwrap_or(12), + )?; + if let Some(existing) = seen.insert(prefix.clone(), name.clone()) { + return Err(error( + py, + &format!( + "Scenario artifact hash collision: scenarios '{existing}' and '{name}' both map to hash prefix '{prefix}'. Rename one scenario file so SQLBuild can generate distinct warehouse artifact prefixes." + ), + "SCENARIO_PLAN_HASH_COLLISION", + Some( + "Rename one scenario file so SQLBuild can generate distinct artifact prefixes.", + ), + )?); + } + result.set_item(name, prefix)?; + } + Ok(result) +} + +#[pyfunction] +#[pyo3(signature = (*, logical_name, fixed_prefix, identifier_limit))] +pub(crate) fn fit_scenario_artifact_logical_name<'py>( + py: Python<'py>, + logical_name: Bound<'py, PyAny>, + fixed_prefix: Bound<'py, PyAny>, + identifier_limit: Bound<'py, PyAny>, +) -> PyResult> { + call_with( + &imported( + py, + "sqlbuild.adapter.relations.main.fit_artifact_logical_name", + "fit_artifact_logical_name", + )?, + &[ + ("logical_name", logical_name), + ("fixed_prefix", fixed_prefix), + ("identifier_limit", identifier_limit), + ("artifact_label", string(py, "Scenario artifact")), + ], + ) +} + +#[pyfunction] +#[pyo3(signature = (*, hash_prefix, kind, logical_name, identifier_limit))] +pub(crate) fn build_scenario_artifact_physical_name( + hash_prefix: &str, + kind: &str, + logical_name: Bound<'_, PyAny>, + identifier_limit: Bound<'_, PyAny>, +) -> PyResult { + let py = logical_name.py(); + let prefix = format!("__sqb_{hash_prefix}__{kind}__"); + let logical: String = fit_scenario_artifact_logical_name( + py, + logical_name, + string(py, &prefix), + identifier_limit, + )? + .extract()?; + Ok(format!("{prefix}{logical}")) +} + +#[pyfunction] +#[pyo3(signature = (*, hash_prefix, kind, logical_name, identifier_limit=63))] +pub(crate) fn build_scenario_artifact_name( + hash_prefix: &str, + kind: Bound<'_, PyAny>, + logical_name: Bound<'_, PyAny>, + identifier_limit: i64, +) -> PyResult { + let py = kind.py(); + let kind: String = imported(py, TYPES, "ScenarioArtifactKind")? + .call1((kind,))? + .getattr("value")? + .extract()?; + let limit = pyo3::IntoPyObjectExt::into_bound_py_any(identifier_limit, py)?; + build_scenario_artifact_physical_name(hash_prefix, &kind, logical_name, limit) +} + +#[pyfunction] +pub(crate) fn parse_scenario_artifact_physical_name<'py>( + py: Python<'py>, + name: &str, +) -> PyResult> { + let Some((hash, kind, logical)) = parse_artifact_name::parse_artifact_name(name) else { + return Ok(py.None().into_bound(py)); + }; + call_with( + &imported(py, MODELS, "ParsedScenarioArtifactName")?, + &[ + ("hash_prefix", string(py, hash)), + ("kind", string(py, kind)), + ("logical_name", string(py, logical)), + ], + ) +} + +#[pyfunction] +pub(crate) fn is_scenario_artifact_physical_name(name: &str) -> bool { + parse_artifact_name::parse_artifact_name(name).is_some() +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn build_scenario_relation_map<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_scenario_relation_map", + kwargs, + ( + &["scenario_name", "hash_prefix", "artifacts"], + &["identifier_limit", "normalize_identifier"], + ), + )?; + let scenario: String = args.value("scenario_name")?.extract()?; + let hash: String = args.value("hash_prefix")?.extract()?; + let limit = args.value("identifier_limit")?; + let limit = if limit.is_none() { + 63 + } else { + limit.extract()? + }; + let normalize = args.value("normalize_identifier")?; + let seen = PyDict::new(py); + let mut artifacts: Vec> = Vec::new(); + for artifact in args.value("artifacts")?.try_iter()? { + let artifact = artifact?; + let physical = build_scenario_artifact_name( + &hash, + artifact.getattr("kind")?, + artifact.getattr("logical_name")?, + limit, + )?; + let normalized = if normalize.is_none() { + string(py, &physical) + } else { + normalize.call1((&physical,))? + }; + if let Some(existing) = seen.get_item(&normalized)? { + let kind = imported(py, TYPES, "ScenarioArtifactKind")?; + let old_kind: String = kind + .call1((existing.getattr("kind")?,))? + .getattr("value")? + .extract()?; + let new_kind: String = kind + .call1((artifact.getattr("kind")?,))? + .getattr("value")? + .extract()?; + let old_name: String = existing.getattr("logical_name")?.extract()?; + let new_name: String = artifact.getattr("logical_name")?.extract()?; + return Err(error( + py, + &format!( + "Scenario relation name collision in scenario '{scenario}'. Both scenario artifacts map to '{physical}': {old_kind} {old_name} and {new_kind} {new_name}. Rename one artifact or shorten one model/source name." + ), + "SCENARIO_PLAN_RELATION_COLLISION", + Some("Rename or shorten one of the colliding scenario artifacts."), + )?); + } + seen.set_item(normalized, &artifact)?; + artifacts.push(call_with( + &imported(py, MODELS, "ScenarioArtifactName")?, + &[ + ("identity", artifact), + ("physical_name", string(py, &physical)), + ], + )?); + } + call_with( + &imported(py, MODELS, "ScenarioRelationMap")?, + &[ + ("scenario_name", args.value("scenario_name")?), + ("hash_prefix", args.value("hash_prefix")?), + ("artifacts", PyTuple::new(py, artifacts)?.into_any()), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index e07ff15c71..7d7bc1c889 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -9,7 +9,7 @@ use crate::bindings::_helpers::planner_scenario::values::{CONSTANTS, MODELS, TYP use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -const ARTIFACTS: &str = "sqlbuild.compiler.planner._helpers.scenario.artifacts"; +const ARTIFACTS: &str = "sqlbuild._native"; #[pyfunction] #[pyo3(signature = (**kwargs))] @@ -98,11 +98,7 @@ pub(crate) fn build_cli_scenario_plan<'py>( } let relations = build_scenario_relation_plan(py, Some(relation_args))?; let result = call_with( - &imported( - py, - "sqlbuild.compiler.planner._helpers.scenario.relations", - "build_scenario_execution_plan", - )?, + &imported(py, "sqlbuild._native", "build_scenario_execution_plan")?, &[ ("scenario", scenario), ("project", project), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs index bdb5f14345..22fd61edbd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs @@ -2,3 +2,5 @@ pub(super) const REF: &str = "ref"; pub(super) const SOURCE: &str = "source"; +pub(super) const SOURCE_MARKER: usize = 2; +pub(super) const DBT_REF: &str = "dbt_ref"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs new file mode 100644 index 0000000000..1cf94c2c32 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs @@ -0,0 +1,154 @@ +//! Existing seed and function plan dataclass construction. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::compiled_project::loop_classes::by_attribute; +use crate::bindings::_helpers::planner_scenario::{ + models::SqlContext, + sql::required_target, + values::{MODELS, error}, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +pub(super) fn seed_entries<'py>( + project: &Bound<'py, PyAny>, + graph: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let seeds = by_attribute(&project.getattr("seeds")?, "name")?; + let fixtures = graph.getattr("seed_fixture_names")?; + let mut result: Vec> = Vec::new(); + for name in graph.getattr("seed_names")?.try_iter()? { + let name = name?; + if fixtures.contains(&name)? { + continue; + } + let Some(seed) = seeds.get_item(&name)? else { + let scenario: String = graph.getattr("name")?.extract()?; + let name: String = name.extract()?; + return Err(error( + py, + &format!("Scenario '{scenario}' requires unknown seed '{name}'"), + "SCENARIO_PLAN_UNKNOWN_SEED", + None, + )?); + }; + let destination = required_target( + relations, + "seed_locations", + &name.extract::()?, + "seed", + )?; + result.push(call_with( + &imported(py, MODELS, "SeedPlanEntry")?, + &[ + ("key", seed.getattr("key")?), + ("name", seed.getattr("name")?), + ("destination", destination), + ( + "file_path", + seed.getattr("seed_file")?.getattr("file_path")?, + ), + ( + "columns", + imported(py, "sqlbuild._native", "extract_seed_columns")?.call1((&seed,))?, + ), + ( + "csv_settings", + seed.getattr("schema_entry")?.getattr("csv_settings")?, + ), + ], + )?); + } + PyTuple::new(py, result) +} + +pub(super) fn function_entry<'py>( + function: &Bound<'py, PyAny>, + context: &SqlContext<'py>, + locations: &Bound<'py, PyAny>, + warehouse_columns: &Bound<'py, PyAny>, +) -> PyResult> { + let py = function.py(); + let body = call_with( + &imported( + py, + "sqlbuild.compiler.planner._helpers.resolve.resolve", + "resolve_function_sql", + )?, + &[ + ("adapter", context.adapter.clone()), + ("function", function.clone()), + ( + "model_locations", + context.relations.getattr("model_locations")?, + ), + ( + "seed_locations", + context.relations.getattr("seed_locations")?, + ), + ("function_locations", locations.clone()), + ("source_map", context.relations.getattr("source_map")?), + ("source_warehouse_columns", warehouse_columns.clone()), + ( + "star_exclude_keyword", + context.adapter.call_method0("star_exclude_keyword")?, + ), + ], + )?; + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = Vec::new(); + for name in [ + "key", + "name", + "relative_path", + "destination", + "arguments", + "returns", + ] { + fields.push((name, function.getattr(name)?)); + } + fields.push(("body_sql", body)); + fields.push(( + "fingerprint_query_sql", + imported( + py, + "sqlbuild._native", + "build_compiled_function_fingerprint_sql", + )? + .call1((function,))?, + )); + for name in [ + "fingerprint_destination", + "return_columns", + "language", + "source_file_path", + "runtime_version", + "entry_point", + "packages", + ] { + fields.push((name, function.getattr(name)?)); + } + call_with(&imported(py, MODELS, "FunctionPlanEntry")?, &fields) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn build_scenario_seed_entries<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_scenario_seed_entries", + kwargs, + (&["project", "graph_plan", "relation_plan"], &[]), + )?; + seed_entries( + &args.value("project")?, + &args.value("graph_plan")?, + &args.value("relation_plan")?, + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs new file mode 100644 index 0000000000..88eb774c88 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -0,0 +1,336 @@ +//! Assemble scenario execution plans through existing planner and adapter APIs. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::compiled_project::loop_classes::by_attribute; +use crate::bindings::_helpers::planner_scenario::{ + entries::{function_entry, seed_entries}, + fixtures::{extract_fixtures, fixture_plans}, + models::SqlContext, + sql::{required_target, resolve_model_dbt_refs, resolve_sql, source_label}, + values::{MODELS, error, string}, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn build_scenario_execution_plan<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_scenario_execution_plan", + kwargs, + ( + &[ + "scenario", + "project", + "adapter", + "graph_plan", + "relation_plan", + "source_lexical_syntax", + ], + &[ + "snapshot", + "source_warehouse_columns", + "sql_analysis_enabled", + ], + ), + )?; + let scenario = args.value("scenario")?; + let project = args.value("project")?; + let graph = args.value("graph_plan")?; + let context = SqlContext { + adapter: args.value("adapter")?, + relations: args.value("relation_plan")?, + syntax: args.value("source_lexical_syntax")?, + label: source_label(&scenario)?, + }; + let snapshot = args.value("snapshot")?; + let snapshot = if snapshot.is_truthy()? { + snapshot + } else { + imported(py, MODELS, "WarehouseSnapshot")?.call0()? + }; + let columns = args.value("source_warehouse_columns")?; + let columns = if columns.is_truthy()? { + columns + } else { + PyDict::new(py).into_any() + }; + let analysis = args.value("sql_analysis_enabled")?; + let analysis = if analysis.is_none() { + true + } else { + analysis.is_truthy()? + }; + let models = by_attribute(&project.getattr("models")?, "name")?; + let functions = by_attribute(&project.getattr("functions")?, "key")?; + let locations = imported(py, "sqlbuild._native", "build_function_locations")? + .call1((project.getattr("functions")?,))?; + let upstream = + imported(py, "sqlbuild._native", "build_execution_upstream_deps")?.call1((&project,))?; + let ordered = call_with( + &imported(py, "sqlbuild._native", "topologically_order_keys")?, + &[("upstream", upstream)], + )?; + let mut model_keys: Vec> = Vec::new(); + for key in ordered.try_iter()? { + let key = key?; + if key.getattr("resource_type")?.eq("model")? + && graph + .getattr("model_names")? + .contains(key.getattr("name")?)? + { + model_keys.push(key); + } + } + let overrides = if analysis { + complete_fixtures(&scenario, &project, (&context, &model_keys))? + } else { + py.None().into_bound(py) + }; + let fixtures = fixture_plans(&scenario, &graph, &context, &overrides)?; + let seeds = seed_entries(&project, &graph, &context.relations)?; + let mut function_entries: Vec> = Vec::new(); + for key in ordered.try_iter()? { + let key = key?; + if graph.getattr("function_deps")?.contains(&key)? + && let Some(function) = functions.get_item(key)? + { + function_entries.push(function_entry(&function, &context, &locations, &columns)?); + } + } + let mut entries: Vec> = Vec::new(); + let mut warnings: Vec> = Vec::new(); + for key in model_keys { + let model = models + .get_item(key.getattr("name")?)? + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err("scenario model"))?; + let destination = required_target( + &context.relations, + "model_locations", + &model.getattr("name")?.extract::()?, + "model", + )?; + let query = resolve_model_dbt_refs( + &model.getattr("query_sql")?.extract::()?, + &context.relations, + )?; + let record = imported(py, "sqlbuild._native", "compiled_model_record")?.call1((&model,))?; + let changes = PyDict::new(py); + changes.set_item("destination", destination)?; + changes.set_item("query_sql", query)?; + let model = imported(py, "dataclasses", "replace")?.call((record,), Some(&changes))?; + let model_context = model_context(&project, &context, (&models, &locations, &columns))?; + let result = call_with( + &imported( + py, + "sqlbuild.compiler.planner._helpers.output.plan_entry", + "plan_model", + )?, + &[ + ("model", model), + ("snapshot", snapshot.clone()), + ("adapter", context.adapter.clone()), + ("context", model_context), + ( + "sql_analysis_enabled", + PyBool::new(py, analysis).to_owned().into_any(), + ), + ( + "query_change_tracking", + PyBool::new(py, false).to_owned().into_any(), + ), + ("full_refresh", PyBool::new(py, true).to_owned().into_any()), + ( + "cursor_overrides", + imported(py, MODELS, "CursorOverridePair")?.call0()?, + ), + ], + )?; + entries.push(result.get_item(0)?); + for warning in result.get_item(1)?.try_iter()? { + warnings.push(warning?); + } + } + let (expected, assertions) = expectations(&scenario, &context)?; + let plan = call_with( + &imported(py, MODELS, "ScenarioExecutionPlan")?, + &[ + ("key", scenario.getattr("key")?), + ("name", scenario.getattr("name")?), + ("graph_plan", graph), + ("relation_plan", context.relations), + ("fixture_plans", fixtures.into_any()), + ("seed_entries", seeds.into_any()), + ( + "function_entries", + PyTuple::new(py, function_entries)?.into_any(), + ), + ("model_entries", PyTuple::new(py, entries)?.into_any()), + ("hook_functions", project.getattr("hook_functions")?), + ("expected_expectations", expected), + ("assertion_expectations", assertions), + ], + )?; + Ok(PyTuple::new(py, [plan, PyTuple::new(py, warnings)?.into_any()])?.into_any()) +} + +fn complete_fixtures<'py>( + scenario: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, + inputs: (&SqlContext<'py>, &[Bound<'py, PyAny>]), +) -> PyResult> { + let (context, keys) = inputs; + let py = scenario.py(); + let mut groups: Vec> = Vec::new(); + for (kind, prefix) in [ + ("SOURCE", "__source__"), + ("MODEL", "__ref__"), + ("SEED", "__seed__"), + ] { + let kind = imported( + py, + "sqlbuild.compiler.compile.types", + "CompiledResourceType", + )? + .getattr(kind)?; + groups.push(PyTuple::new( + py, + [kind, extract_fixtures(scenario, prefix)?.into_any()], + )?); + } + let names = keys + .iter() + .map(|key| key.getattr("name")) + .collect::>>()?; + let completion = call_with( + &imported(py, "sqlbuild._native", "build_relation_fixture_completion")?, + &[ + ("project", project.clone()), + ("adapter", context.adapter.clone()), + ( + "request", + PyTuple::new( + py, + [ + PyTuple::new(py, names)?.into_any(), + PyTuple::new(py, groups)?.into_any(), + ], + )? + .into_any(), + ), + ], + )?; + let diagnostics = completion.getattr("diagnostics")?; + if diagnostics.is_truthy()? { + let mut details: Vec = Vec::new(); + for diagnostic in diagnostics.try_iter()? { + details.push(format!( + "- {}", + diagnostic?.getattr("message")?.extract::()? + )); + } + let name: String = scenario.getattr("name")?.extract()?; + return Err(error( + py, + &format!( + "Scenario '{name}' has invalid relation fixtures in {}:\n{}", + context.label, + details.join("\n") + ), + "SCENARIO_PLAN_INVALID_FIXTURE", + None, + )?); + } + completion.getattr("fixture_sql_by_key") +} + +fn model_context<'py>( + project: &Bound<'py, PyAny>, + context: &SqlContext<'py>, + inputs: (&Bound<'py, PyDict>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let (models, locations, columns) = inputs; + let py = project.py(); + call_with( + &imported(py, MODELS, "ModelPlanContext")?, + &[ + ( + "model_locations", + context.relations.getattr("model_locations")?, + ), + ("models_by_name", models.clone().into_any()), + ( + "functions_by_name", + by_attribute(&project.getattr("functions")?, "name")?.into_any(), + ), + ( + "seed_locations", + context.relations.getattr("seed_locations")?, + ), + ("function_locations", locations.clone()), + ("source_map", context.relations.getattr("source_map")?), + ("source_warehouse_columns", columns.clone()), + ( + "star_exclude_keyword", + context.adapter.call_method0("star_exclude_keyword")?, + ), + ], + ) +} + +fn expectations<'py>( + scenario: &Bound<'py, PyAny>, + context: &SqlContext<'py>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py = scenario.py(); + let mut expected: Vec> = Vec::new(); + for cte in scenario.getattr("expected_ctes")?.try_iter()? { + let cte = cte?; + let name: String = cte.getattr("name")?.extract()?; + let name = name.strip_prefix("__expected__").unwrap_or(&name); + let target = required_target(&context.relations, "model_locations", name, "model")?; + let sql = resolve_sql( + context, + &cte.getattr("sql_body")?.extract::()?, + false, + )?; + expected.push(call_with( + &imported(py, MODELS, "ScenarioExpectedExpectationPlan")?, + &[ + ("model_name", string(py, name)), + ("actual_destination", target), + ("expected_sql", string(py, &sql)), + ], + )?); + } + let mut assertions: Vec> = Vec::new(); + for cte in scenario.getattr("assertion_ctes")?.try_iter()? { + let cte = cte?; + let name: String = cte.getattr("name")?.extract()?; + let sql = resolve_sql( + context, + &cte.getattr("sql_body")?.extract::()?, + false, + )?; + assertions.push(call_with( + &imported(py, MODELS, "ScenarioAssertionExpectationPlan")?, + &[ + ( + "name", + string(py, name.strip_prefix("__assert__").unwrap_or(&name)), + ), + ("sql", string(py, &sql)), + ], + )?); + } + Ok(( + PyTuple::new(py, expected)?.into_any(), + PyTuple::new(py, assertions)?.into_any(), + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs new file mode 100644 index 0000000000..786df6d4ac --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs @@ -0,0 +1,142 @@ +//! Materialize fixture SQL plans while preserving adapter and completion boundaries. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_scenario::{ + models::SqlContext, + sql::{required_target, resolve_sql, source_label}, + values::{MODELS, TYPES, error, string}, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +pub(super) fn extract_fixtures<'py>( + scenario: &Bound<'py, PyAny>, + prefix: &str, +) -> PyResult> { + crate::bindings::_helpers::sql_test_glue::authored_ctes::prefixed_ctes( + &scenario.getattr("authored_ctes")?, + prefix, + ) +} + +pub(super) fn fixture_plans<'py>( + scenario: &Bound<'py, PyAny>, + graph: &Bound<'py, PyAny>, + context: &SqlContext<'py>, + overrides: &Bound<'py, PyAny>, +) -> PyResult> { + let py = scenario.py(); + let mut helpers: Vec = Vec::new(); + for cte in scenario.getattr("authored_ctes")?.try_iter()? { + let cte = cte?; + let name: String = cte.getattr("name")?.extract()?; + if ["__source__", "__ref__", "__seed__", "__dbt_ref__"] + .iter() + .any(|prefix| name.starts_with(prefix)) + { + continue; + } + let sql = resolve_sql( + context, + &cte.getattr("sql_body")?.extract::()?, + true, + )?; + helpers.push(format!("{name} AS ({sql})")); + } + let mut result: Vec> = Vec::new(); + for (kind, resource, names, locations) in [ + ( + "source", + "source", + "source_fixture_names", + "source_fixture_locations", + ), + ("ref", "model", "ref_fixture_names", "ref_fixture_locations"), + ( + "seed", + "seed", + "seed_fixture_names", + "seed_fixture_locations", + ), + ( + "dbt_ref", + "dbt_ref", + "dbt_ref_fixture_names", + "dbt_ref_fixture_locations", + ), + ] { + let fixtures = extract_fixtures(scenario, &format!("__{kind}__"))?; + for name in graph.getattr(names)?.try_iter()? { + let name: String = name?.extract()?; + let target = required_target(&context.relations, locations, &name, kind)?; + let Some(mut sql) = fixtures.get_item(&name)? else { + return Err(error( + py, + &format!("Scenario is missing {kind} fixture SQL '{name}'"), + "SCENARIO_PLAN_MISSING_FIXTURE_SQL", + None, + )?); + }; + if kind != super::constants::DBT_REF && !overrides.is_none() { + sql = overrides.call_method1("get", ((resource, &name), sql))?; + } + let sql = resolve_sql(context, &sql.extract::()?, true)?; + let sql = if helpers.is_empty() { + sql + } else { + format!("WITH {} {sql}", helpers.join(", ")) + }; + result.push(call_with( + &imported(py, MODELS, "ScenarioFixturePlan")?, + &[ + ( + "kind", + imported(py, TYPES, "ScenarioArtifactKind")?.call1((kind,))?, + ), + ("logical_name", string(py, &name)), + ("destination", target), + ("sql", string(py, &sql)), + ], + )?); + } + } + PyTuple::new(py, result) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn build_scenario_fixture_plans<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_scenario_fixture_plans", + kwargs, + ( + &[ + "scenario", + "graph_plan", + "relation_plan", + "adapter", + "source_lexical_syntax", + ], + &["fixture_sql_overrides"], + ), + )?; + let scenario = args.value("scenario")?; + let context = SqlContext { + adapter: args.value("adapter")?, + relations: args.value("relation_plan")?, + syntax: args.value("source_lexical_syntax")?, + label: source_label(&scenario)?, + }; + fixture_plans( + &scenario, + &args.value("graph_plan")?, + &context, + &args.value("fixture_sql_overrides")?, + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs index 05781c3595..fcbd7bb887 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs @@ -1,8 +1,14 @@ //! Python boundaries for native scenario planning. +mod artifacts; mod cli; mod constants; +mod entries; +mod execution; +mod fixtures; mod graph; +mod models; pub(crate) mod registration; mod relations; +mod sql; mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs new file mode 100644 index 0000000000..0c97f32fd3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs @@ -0,0 +1,11 @@ +//! Borrowed Python callback boundaries for scenario SQL assembly. + +use pyo3::prelude::{Bound, PyAny}; + +#[derive(Debug)] +pub(crate) struct SqlContext<'py> { + pub adapter: Bound<'py, PyAny>, + pub relations: Bound<'py, PyAny>, + pub syntax: Bound<'py, PyAny>, + pub label: String, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs index c7c943ec48..a6e7cbe60c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs @@ -1,10 +1,57 @@ //! Register scenario planning bindings. -use crate::bindings::_helpers::planner_scenario::{cli, graph, relations}; +use crate::bindings::_helpers::planner_scenario::{ + artifacts, cli, entries, execution, fixtures, graph, relations, sql, +}; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + execution::build_scenario_execution_plan, + module + )?)?; + module.add_function(wrap_pyfunction!( + fixtures::build_scenario_fixture_plans, + module + )?)?; + module.add_function(wrap_pyfunction!( + entries::build_scenario_seed_entries, + module + )?)?; + module.add_function(wrap_pyfunction!(sql::resolve_scenario_check_sql, module)?)?; + module.add_function(wrap_pyfunction!( + artifacts::compute_scenario_hash_prefix, + module + )?)?; + module.add_function(wrap_pyfunction!( + artifacts::build_scenario_hash_index, + module + )?)?; + module.add_function(wrap_pyfunction!( + artifacts::build_scenario_artifact_name, + module + )?)?; + module.add_function(wrap_pyfunction!( + artifacts::build_scenario_artifact_physical_name, + module + )?)?; + module.add_function(wrap_pyfunction!( + artifacts::fit_scenario_artifact_logical_name, + module + )?)?; + module.add_function(wrap_pyfunction!( + artifacts::parse_scenario_artifact_physical_name, + module + )?)?; + module.add_function(wrap_pyfunction!( + artifacts::is_scenario_artifact_physical_name, + module + )?)?; + module.add_function(wrap_pyfunction!( + artifacts::build_scenario_relation_map, + module + )?)?; module.add_function(wrap_pyfunction!(graph::plan_scenario_graph, module)?)?; module.add_function(wrap_pyfunction!( relations::build_scenario_relation_plan, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs new file mode 100644 index 0000000000..e3b8b69279 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs @@ -0,0 +1,144 @@ +//! Resolve scenario SQL through the shared native scanner and adapter callbacks. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_scenario::{ + models::SqlContext, + values::{error, string}, +}; +use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::{PyDict, PyStringMethods}; +use sqlbuild_analysis::planner_scenario::{ + errors::MarkerError, main::replace_markers::replace_markers, +}; +use sqlbuild_sqltext::sql_scan::models::Unclosed; + +pub(super) fn resolve_sql( + context: &SqlContext<'_>, + sql: &str, + project_sources: bool, +) -> PyResult { + let py = context.adapter.py(); + let syntax: LexicalSyntaxInput = context.syntax.getattr("native_mapping")?.extract()?; + let mut result = sql.to_owned(); + for (kind, field) in [ + (0, "model_locations"), + (1, "seed_locations"), + ( + 2, + if project_sources { + "project_source_map" + } else { + "source_map" + }, + ), + (3, "dbt_ref_fixture_locations"), + ] { + if project_sources && kind != super::constants::SOURCE_MARKER { + continue; + } + let locations = context.relations.getattr(field)?; + result = replace_markers(&result, kind, Some(syntax.clone().into()), |name, marker| { + let target = locations.call_method1("get", (name,))?; + let target = if target.is_none() {target} else if kind == super::constants::SOURCE_MARKER { + call_with(&imported(py, "sqlbuild._native", "render_source_relation")?, &[("entry", target), ("adapter", context.adapter.clone())])? + } else {target.getattr("qualified_name")?}; + if target.is_none() { + let marker = string(py, marker).repr()?.to_str()?.to_owned(); + return Err(error(py, &format!("Scenario SQL in {} references {marker}, which is not a relation in this scenario", context.label), + "SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER", Some("Reference a model, seed, source, or dbt ref the scenario builds or provides as a fixture."))?); + } + target.extract::() + }).map_err(|failure| match failure { + MarkerError::Resolve(error) => error, + MarkerError::Scan(unclosed) => { + let construct = match unclosed {Unclosed::BlockComment => "block comment", Unclosed::Quote => "quoted string", Unclosed::Parenthesis => "parenthesis"}; + error(py, &format!("Scenario SQL in {} could not be scanned for relation markers: SQL reference resolution contains an unclosed {construct}", context.label), + "SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER", Some("Close every quoted string and block comment using the project dialect's quoting and comment rules.")).unwrap_or_else(|error| error) + } + })?; + } + Ok(result) +} + +pub(super) fn resolve_model_dbt_refs(sql: &str, relations: &Bound<'_, PyAny>) -> PyResult { + let locations = relations.getattr("dbt_ref_fixture_locations")?; + replace_markers(sql, 3, None, |name, marker| { + let target = locations.call_method1("get", (name,))?; + let qualified = if target.is_none() { + target + } else { + target.getattr("qualified_name")? + }; + Ok(if qualified.is_none() { + marker.to_owned() + } else { + qualified.extract()? + }) + }) + .map_err(|failure| match failure { + MarkerError::Resolve(error) => error, + MarkerError::Scan(_) => { + pyo3::exceptions::PyRuntimeError::new_err("unscanned marker replacement failed") + } + }) +} + +pub(super) fn source_label(scenario: &Bound<'_, PyAny>) -> PyResult { + let source = scenario.getattr("source_path")?; + if source.is_truthy()? { + Ok(source.str()?.to_str()?.to_owned()) + } else { + scenario.getattr("name")?.extract() + } +} + +pub(super) fn required_target<'py>( + relations: &Bound<'py, PyAny>, + field: &str, + name: &str, + kind: &str, +) -> PyResult> { + let target = relations.getattr(field)?.call_method1("get", (name,))?; + if target.is_none() { + return Err(error( + relations.py(), + &format!("Scenario relation plan is missing {kind} target '{name}'"), + "SCENARIO_PLAN_MISSING_RELATION_TARGET", + Some("This is likely a SQLBuild bug. Please file an issue with the scenario name."), + )?); + } + Ok(target) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn resolve_scenario_check_sql( + py: Python<'_>, + kwargs: Option>, +) -> PyResult { + let args = KeywordArguments::read( + py, + "resolve_scenario_check_sql", + kwargs, + ( + &[ + "sql", + "relation_plan", + "adapter", + "source_label", + "source_lexical_syntax", + ], + &[], + ), + )?; + let context = SqlContext { + adapter: args.value("adapter")?, + relations: args.value("relation_plan")?, + syntax: args.value("source_lexical_syntax")?, + label: args.value("source_label")?.extract()?, + }; + resolve_sql(&context, &args.value("sql")?.extract::()?, false) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs new file mode 100644 index 0000000000..d4d74b7dde --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs @@ -0,0 +1,240 @@ +//! Assemble audit plan dataclasses using native rendering and scheduling. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{ + PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple, PyTupleMethods, +}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +use crate::bindings::_helpers::planner_strategy::constants::{ + AUDIT_TYPES_MODULE, COMPILE_MODELS_MODULE, COMPILE_TYPES_MODULE, MODEL_ATTACHMENT, + NATIVE_MODULE, PLAN_MODELS_MODULE, SOURCE_ATTACHMENT, +}; + +/// Render and validate one audit query with the supplied adapter callbacks. +fn render_query<'py>( + request: &Request<'_, 'py>, + sql: Bound<'py, PyAny>, + context: &str, +) -> PyResult> { + let py = request.py(); + let resolved = call_with( + &imported(py, NATIVE_MODULE, "render_audit_sql")?, + &[ + ("unresolved_sql", sql), + ("model_locations", request.get("model_locations")?), + ("seed_locations", request.get("seed_locations")?), + ("source_map", request.get("source_map")?), + ("adapter", request.get("adapter")?), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "assert_no_unresolved_sql_markers")?, + &[ + ("sql", resolved.clone()), + ("context", PyString::new(py, context).into_any()), + ], + )?; + Ok(resolved) +} + +/// Resolve an audit enum, applying the default only to absent values. +fn audit_enum<'py>( + audit: &Bound<'py, PyAny>, + attribute: &str, + enum_name: &str, + default: &str, +) -> PyResult> { + let py = audit.py(); + let value = audit.getattr(attribute)?; + imported(py, AUDIT_TYPES_MODULE, enum_name)?.call1((if value.is_none() { + PyString::new(py, default).into_any() + } else { + value + },)) +} + +/// Build an audit entry while retaining the public Python plan dataclass. +#[pyfunction] +#[pyo3(signature = (**request))] +pub(crate) fn plan_audit<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("plan_audit", request)?; + let py = request.py(); + let audit = request.get("audit")?; + let name: String = audit.getattr("name")?.extract()?; + let resolved = render_query( + &request, + audit.getattr("sql_body")?, + &format!("audit '{name}' planned SQL"), + )?; + let evidence = audit.getattr("evidence_sql")?; + let evidence_resolved = if evidence.is_none() { + py.None().into_bound(py) + } else { + render_query( + &request, + evidence, + &format!("audit '{name}' planned evidence SQL"), + )? + }; + let attachment = call_with( + &imported(py, NATIVE_MODULE, "resolve_attachment_kind")?, + &[ + ("audit", audit.clone()), + ("downstream_deps", request.get("lineage_downstream_deps")?), + ], + )?; + let attachment = attachment.cast::()?; + let kind = attachment.get_item(0)?; + let target = attachment.get_item(1)?; + let kind_text: String = kind.extract()?; + let materialization = if kind_text == MODEL_ATTACHMENT && !target.is_none() { + request + .get("model_materializations")? + .call_method1("get", (&target,))? + } else { + py.None().into_bound(py) + }; + let requested = audit_enum(&audit, "run_scope", "AuditRunScope", "final")?; + let severity = audit_enum(&audit, "severity", "AuditSeverity", "warn")?; + let effective = call_with( + &imported(py, NATIVE_MODULE, "resolve_effective_run_scope")?, + &[ + ("requested_run_scope", requested.clone()), + ("attached_model_materialization", materialization), + ], + )?; + let mut target_kind = audit.getattr("attached_target_kind")?; + if target_kind.is_none() && (kind_text == MODEL_ATTACHMENT || kind_text == SOURCE_ATTACHMENT) { + target_kind = + imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind_text,))?; + } + let values = PyDict::new(py); + for field in [ + "key", + "name", + "definition_name", + "evaluation_mode", + "thresholds", + "minimum_samples", + "evidence_limit", + "scope_deps", + "attached_column_name", + "always_run", + "description", + ] { + values.set_item(field, audit.getattr(field)?)?; + } + let measurement = audit.getattr("measurement_contract")?; + for field in ["value_column", "sample_count_column", "sample_unit"] { + values.set_item( + field, + if measurement.is_none() { + py.None().into_bound(py) + } else { + measurement.getattr(field)? + }, + )?; + } + for (field, value) in [ + ("resolved_sql", resolved), + ("unresolved_sql", audit.getattr("sql_body")?), + ("attachment_kind", kind), + ("severity", severity), + ("requested_run_scope", requested), + ("effective_run_scope", effective), + ("evidence_resolved_sql", evidence_resolved), + ("evidence_unresolved_sql", audit.getattr("evidence_sql")?), + ("attached_target_kind", target_kind), + ("attached_target_name", target), + ( + "reads_outside_target_lineage", + call_with( + &imported(py, NATIVE_MODULE, "reads_outside_target_lineage")?, + &[ + ("audit", audit), + ("upstream_deps", request.get("lineage_upstream_deps")?), + ], + )?, + ), + ] { + values.set_item(field, value)?; + } + imported(py, PLAN_MODELS_MODULE, "AuditPlanEntry")?.call((), Some(&values)) +} + +/// Select attached audits by their logical target and singular audits by dependency overlap. +#[pyfunction] +#[pyo3(signature = (*, audit, selected_keys))] +pub(crate) fn audit_is_selected( + audit: &Bound<'_, PyAny>, + selected_keys: &Bound<'_, PyAny>, +) -> PyResult { + let py = audit.py(); + let kind = audit.getattr("attached_target_kind")?; + let name = audit.getattr("attached_target_name")?; + if !kind.is_none() && !name.is_none() { + let resource_type = imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")? + .call1((kind,))? + .getattr("resource_type")?; + let key = call_with( + &imported(py, COMPILE_MODELS_MODULE, "CompiledObjectKey")?, + &[("resource_type", resource_type), ("name", name)], + )?; + return selected_keys.contains(key); + } + for dependency in audit.getattr("scope_deps")?.try_iter()? { + if selected_keys.contains(dependency?)? { + return Ok(true); + } + } + Ok(false) +} + +/// Assemble selected audits in compiled-project order without querying lineage for an empty selection. +#[pyfunction] +#[pyo3(signature = (**request))] +pub(crate) fn build_selected_audit_entries<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("build_selected_audit_entries", request)?; + let py = request.py(); + let project = request.get("project")?; + let selected_keys = request.get("scope")?.getattr("selected_keys")?; + let mut selected: Vec> = Vec::new(); + for audit in project.getattr("audits")?.try_iter()? { + let audit = audit?; + if audit_is_selected(&audit, &selected_keys)? { + selected.push(audit); + } + } + let entries = PyList::empty(py); + if selected.is_empty() { + return Ok(entries); + } + let lineage = imported(py, NATIVE_MODULE, "project_lineage_views")?.call1((project,))?; + let relations = request.get("relations")?; + let arguments = PyDict::new(py); + for field in ["model_locations", "seed_locations"] { + arguments.set_item(field, relations.getattr(field)?)?; + } + arguments.set_item("source_map", relations.getattr("source_read_map")?)?; + arguments.set_item("adapter", request.get("adapter")?)?; + arguments.set_item( + "model_materializations", + request.get("model_materializations")?, + )?; + arguments.set_item("lineage_upstream_deps", lineage.getattr("upstream_deps")?)?; + arguments.set_item( + "lineage_downstream_deps", + lineage.getattr("downstream_deps")?, + )?; + for audit in selected { + arguments.set_item("audit", audit)?; + entries.append(plan_audit(Some(&arguments))?)?; + } + Ok(entries) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs new file mode 100644 index 0000000000..d46eb709b9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs @@ -0,0 +1,21 @@ +//! Audit plan boundary modules and attachment values. + +pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; +pub(crate) const AUDIT_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; +pub(crate) const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +pub(crate) const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +pub(crate) const PLAN_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +pub(crate) const MODEL_ATTACHMENT: &str = "model"; +pub(crate) const SOURCE_ATTACHMENT: &str = "source"; +pub(crate) const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +pub(crate) const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; +pub(crate) const CURSOR_TYPES_MODULE: &str = "sqlbuild.cursor_algebra.types"; +pub(crate) const INCREMENTAL_MATERIALIZATION: &str = "incremental"; +pub(crate) const MICROBATCH_MODE: &str = "microbatch"; +pub(crate) const CUSTOM_MATERIALIZATION: &str = "custom"; +pub(crate) const ENFORCED_CONTRACT: &str = "enforced"; +pub(crate) const TIMESTAMP_CURSOR: &str = "timestamp"; +pub(crate) const MODEL_REFERENCE: &str = "ref"; +pub(crate) const SEED_REFERENCE: &str = "seed"; +pub(crate) const SOURCE_REFERENCE: &str = "source"; +pub(crate) const DDL_SEPARATOR: &str = ";\n\n"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs new file mode 100644 index 0000000000..2a266b95c7 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs @@ -0,0 +1,314 @@ +//! Cursor input relations and authored cursor input column checks of model plan entries. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyBool, PyDict, PyFrozenSet, PyString, PyTuple}; + +use sqlbuild_analysis::planner_strategy::main::cursor_input_column::cursor_input_column_failure; +use sqlbuild_analysis::planner_strategy::models::{CursorInputColumnCheck, CursorInputColumnOwner}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cursor::scalars::parse_value; +use crate::bindings::_helpers::model_config::config_values::{config_cursor_bound, config_str}; +use crate::bindings::_helpers::planner_identity::metadata::or_empty; +use crate::bindings::_helpers::planner_strategy::constants::{ + ENFORCED_CONTRACT, INCREMENTAL_MATERIALIZATION, MODEL_REFERENCE, PLAN_MODELS_MODULE, + PLANNER_TYPES_MODULE, SEED_REFERENCE, SOURCE_REFERENCE, TIMESTAMP_CURSOR, +}; +use crate::bindings::_helpers::planner_strategy::lineage::resolve_lineage_reference; +use crate::bindings::_helpers::planner_strategy::values::{ + PLANNER_EXCEPTIONS_MODULE, input_error, text, +}; +use crate::bindings::_helpers::references::source_relations::render_source_relation; + +/// A model's authored config values. +pub(crate) fn model_values<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + model.getattr("config")?.getattr("values") +} + +/// Whether a model is incremental with a cursor column. +fn incremental_cursor_flag( + model: &Bound<'_, PyAny>, + cursor_column: &Bound<'_, PyAny>, +) -> PyResult { + let materialized: Option> = config_str(&model_values(model)?, "materialized")?; + Ok(!cursor_column.is_none() + && match materialized { + Some(materialized) => materialized.eq(INCREMENTAL_MATERIALIZATION)?, + None => false, + }) +} + +/// The names of a sequence of objects with a `name`. +fn names(items: &Bound<'_, PyAny>) -> PyResult> { + items + .try_iter()? + .map(|item| text(&item?.getattr("name")?)) + .collect() +} + +/// Raise the planner error of a cursor input column its input does not expose. +fn check_column(check: CursorInputColumnCheck, py: Python<'_>) -> PyResult<()> { + match cursor_input_column_failure(&check) { + Some((message, code)) => Err(input_error( + py, + (PLANNER_EXCEPTIONS_MODULE, "PlannerInputError"), + &message, + Some(code), + )), + None => Ok(()), + } +} + +/// Validate authored cursor input columns against contracts, then source warehouse metadata. +#[pyfunction(name = "validate_source_cursor_input_columns")] +#[pyo3(signature = (**request))] +pub(crate) fn reject_unexposed_cursor_input_columns<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult<()> { + let request = Request::new("validate_source_cursor_input_columns", request)?; + let py: Python<'py> = request.py(); + let model: Bound<'py, PyAny> = request.get("model")?; + if !incremental_cursor_flag(&model, &request.get("cursor_column")?)? { + return Ok(()); + } + let roles: Bound<'py, PyAny> = resolve_cursor_input_roles(&model)?; + let models_by_name = or_empty(request.optional("models_by_name")?, py)?; + let functions_by_name = or_empty(request.optional("functions_by_name")?, py)?; + let source_map = or_empty(request.optional("source_map")?, py)?; + let warehouse_columns: Bound<'py, PyAny> = request.get("source_warehouse_columns")?; + let model_name: String = text(&model.getattr("name")?)?; + for role in ["filter_inputs", "watermark_inputs"] { + for item in roles.getattr(role)?.call_method0("items")?.try_iter()? { + let (input_name, column): (String, String) = item?.extract()?; + let reference: Bound<'py, PyAny> = resolve_lineage_reference( + &model, + &input_name, + models_by_name.clone(), + functions_by_name.clone(), + )?; + let kind: String = text(&reference.getattr("ref_kind")?)?; + let name: String = text(&reference.getattr("ref_name")?)?; + let check = |owner, known_columns| CursorInputColumnCheck { + model_name: model_name.clone(), + owner, + input_name: name.clone(), + column: column.clone(), + known_columns, + }; + if kind == MODEL_REFERENCE { + let upstream: Bound<'py, PyAny> = models_by_name.call_method1("get", (&name,))?; + if upstream.is_none() + || !model_values(&upstream)? + .call_method1("get", ("contract",))? + .eq(ENFORCED_CONTRACT)? + { + continue; + } + let entry: Bound<'py, PyAny> = upstream.getattr("schema_entry")?; + let declared: Vec = if entry.is_none() { + Vec::new() + } else { + names(&entry.getattr("columns")?)? + }; + check_column(check(CursorInputColumnOwner::ModelContract, declared), py)?; + continue; + } + if kind != SOURCE_REFERENCE { + continue; + } + let source: Bound<'py, PyAny> = source_map.call_method1("get", (&name,))?; + if !source.is_none() && source.getattr("contract")?.eq(ENFORCED_CONTRACT)? { + let declared: Vec = names(&source.getattr("columns")?)?; + check_column(check(CursorInputColumnOwner::SourceContract, declared), py)?; + continue; + } + let known: Bound<'py, PyAny> = warehouse_columns.call_method1("get", (&name,))?; + if known.is_none() { + continue; + } + check_column( + check(CursorInputColumnOwner::SourceWarehouse, names(&known)?), + py, + )?; + } + } + Ok(()) +} + +/// The qualified relation one cursor input reference reads, when it resolves. +fn input_relation<'py>( + request: &Request<'_, 'py>, + (kind, name): (&str, &str), +) -> PyResult> { + let py: Python<'py> = request.py(); + let location: Bound<'py, PyAny> = match kind { + MODEL_REFERENCE => { + let model: Bound<'py, PyAny> = request + .get("model_locations")? + .call_method1("get", (name,))?; + if model.is_none() { + request + .get("seed_locations")? + .call_method1("get", (name,))? + } else { + model + } + } + SEED_REFERENCE => request + .get("seed_locations")? + .call_method1("get", (name,))?, + SOURCE_REFERENCE => { + let source: Bound<'py, PyAny> = + request.get("source_map")?.call_method1("get", (name,))?; + if source.is_none() { + return Ok(None); + } + let adapter: Bound<'py, PyAny> = request.get("adapter")?; + return render_source_relation(&source, Some(&adapter)).map(Some); + } + _ => py.None().into_bound(py), + }; + if location.is_none() { + return Ok(None); + } + location.getattr("qualified_name")?.extract() +} + +/// The resource kind a resolved cursor input reference reads. +fn input_kind<'py>( + py: Python<'py>, + (kind, model_backed): (&str, bool), +) -> PyResult> { + let value: &str = match kind { + MODEL_REFERENCE if model_backed => "model", + MODEL_REFERENCE | SEED_REFERENCE => "seed", + SOURCE_REFERENCE => "source", + _ => return Ok(py.None().into_bound(py)), + }; + imported(py, PLANNER_TYPES_MODULE, "GraphResourceKind")?.call1((value,)) +} + +/// A producer model's terminal cursor bound parsed with the consumer's cursor type. +fn terminal_bound<'py>( + producer: Option<&Bound<'py, PyAny>>, + (key, cursor_type): (&str, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = cursor_type.py(); + let Some(producer) = producer else { + return Ok(py.None().into_bound(py)); + }; + match config_cursor_bound(&model_values(producer)?, key)? { + Some(raw) => parse_value(&raw, cursor_type), + None => Ok(py.None().into_bound(py)), + } +} + +/// Cursor-bearing input relations of an incremental model, for runtime range discovery. +#[pyfunction] +#[pyo3(signature = (**request))] +pub(crate) fn build_cursor_input_relations<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("build_cursor_input_relations", request)?; + let py: Python<'py> = request.py(); + let model: Bound<'py, PyAny> = request.get("model")?; + if !incremental_cursor_flag(&model, &request.get("cursor_column")?)? { + return Ok(PyTuple::empty(py)); + } + let models_by_name: Bound<'py, PyAny> = request.get("models_by_name")?; + let functions_by_name = or_empty(request.optional("functions_by_name")?, py)?; + let model_locations: Bound<'py, PyAny> = request.get("model_locations")?; + let producers: Bound<'py, PyAny> = request.get("runtime_cursor_producer_names")?; + let cursor_type: Bound<'py, PyAny> = match config_str(&model_values(&model)?, "cursor_type")? { + Some(cursor_type) if cursor_type.is_truthy()? => cursor_type, + _ => PyString::new(py, TIMESTAMP_CURSOR).into_any(), + }; + let class: Bound<'py, PyAny> = imported(py, PLAN_MODELS_MODULE, "CursorInputRelation")?; + let mut relations: Vec> = Vec::new(); + let watermarks: Bound<'py, PyAny> = + resolve_cursor_input_roles(&model)?.getattr("watermark_inputs")?; + for item in watermarks.call_method0("items")?.try_iter()? { + let (input_name, column): (String, Bound<'py, PyAny>) = item?.extract()?; + let reference: Bound<'py, PyAny> = resolve_lineage_reference( + &model, + &input_name, + models_by_name.clone(), + functions_by_name.clone(), + )?; + let kind: String = text(&reference.getattr("ref_kind")?)?; + let name: Bound<'py, PyAny> = reference.getattr("ref_name")?; + let name_text: String = text(&name)?; + let Some(relation) = input_relation(&request, (&kind, &name_text))? else { + continue; + }; + let producer: Option> = if kind == MODEL_REFERENCE { + Some(models_by_name.call_method1("get", (&name,))?).filter(|model| !model.is_none()) + } else { + None + }; + let grain: Bound<'py, PyAny> = match &producer { + Some(producer) => config_str(&model_values(producer)?, "cursor_grain")? + .unwrap_or_else(|| py.None().into_bound(py)), + None => py.None().into_bound(py), + }; + let model_backed: bool = kind == MODEL_REFERENCE && model_locations.contains(&name)?; + relations.push(call_with( + &class, + &[ + ("relation", PyString::new(py, &relation).into_any()), + ("cursor_column", column), + ("cursor_grain", grain), + ( + "is_model_backed", + PyBool::new(py, model_backed).to_owned().into_any(), + ), + ( + "is_runtime_produced", + PyBool::new(py, producers.contains(&name)?) + .to_owned() + .into_any(), + ), + ("input_name", name.clone()), + ("input_kind", input_kind(py, (&kind, model_backed))?), + ( + "terminal_cursor_start", + terminal_bound(producer.as_ref(), ("cursor_start", &cursor_type))?, + ), + ( + "terminal_cursor_end", + terminal_bound(producer.as_ref(), ("cursor_end", &cursor_type))?, + ), + ], + )?); + } + PyTuple::new(py, relations) +} + +/// Selected models and seeds, and selected sources with loaders, whose relations this invocation produces. +#[pyfunction] +#[pyo3(signature = (*, scope, source_map))] +pub(crate) fn runtime_cursor_producer_names<'py>( + scope: &Bound<'py, PyAny>, + source_map: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = scope.py(); + let mut names: Vec> = Vec::new(); + for key in scope.getattr("selected_keys")?.try_iter()? { + let key: Bound<'py, PyAny> = key?; + let name: Bound<'py, PyAny> = key.getattr("name")?; + match text(&key.getattr("resource_type")?)?.as_str() { + "model" | "seed" => names.push(name), + "source" => { + let source: Bound<'py, PyAny> = source_map.call_method1("get", (&name,))?; + if !source.is_none() && !source.getattr("loader")?.is_none() { + names.push(name); + } + } + _ => {} + } + } + PyFrozenSet::new(py, names) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs index 488767569c..6833326347 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs @@ -1,46 +1,35 @@ //! Native-hosted relation fixture completion over Python boundary models. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::align_columns::align_columns; use sqlbuild_analysis::planner_fixtures::models::{AlignmentInput, RequiredColumn}; +use crate::bindings::_helpers::planner_strategy::fixture_context::build_relation_fixture_context; use crate::bindings::_helpers::planner_strategy::fixture_preparation::{PreparedFixtures, prepare}; +use crate::bindings::_helpers::planner_strategy::fixture_reads::required_fixture_columns; use crate::bindings::_helpers::planner_strategy::fixture_selection::{ fixture_keys_requiring_analysis, unknown_fixture_column_diagnostics, }; use crate::bindings::_helpers::planner_strategy::fixture_sql::{CompletionSql, render_completed}; use crate::bindings::_helpers::planner_strategy::fixture_types::expected_fixture_types; use crate::bindings::_helpers::planner_strategy::fixture_values::{ - COMPLETION, columns, diagnostic, folded, label, record, strings, + columns, diagnostic, folded, label, record, strings, }; -fn helper<'py>( - py: Python<'py>, - name: &str, - kwargs: &Bound<'py, PyDict>, -) -> PyResult> { - py.import(COMPLETION)?.getattr(name)?.call((), Some(kwargs)) -} - #[pyfunction] #[pyo3(signature = (*, project, adapter, request, planning_context = None))] -fn build_relation_fixture_completion<'py>( +pub(crate) fn build_relation_fixture_completion<'py>( project: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, request: (Bound<'py, PyAny>, Bound<'py, PyAny>), planning_context: Option>, ) -> PyResult> { let (ordered_model_names, fixture_groups) = request; - let py = project.py(); let context = match planning_context { Some(context) if context.is_truthy()? => context, - _ => { - let kwargs = PyDict::new(py); - kwargs.set_item("project", project)?; - helper(py, "build_relation_fixture_context", &kwargs)? - } + _ => build_relation_fixture_context(project, None)?, }; let relations = context.getattr("relations")?.cast_into::()?; let prepared = prepare( @@ -55,19 +44,20 @@ fn build_relation_fixture_completion<'py>( .getattr("authoritative_columns")? .cast_into::()?, )?; - let kwargs = PyDict::new(py); - kwargs.set_item("ordered_model_names", &ordered_model_names)?; - kwargs.set_item("model_map", context.getattr("models_by_name")?)?; - kwargs.set_item("fixture_keys", fixture_keys)?; - kwargs.set_item("relations", &relations)?; - kwargs.set_item( - "authoritative_columns", - context.getattr("authoritative_columns")?, + let (required, stars) = required_fixture_columns( + ( + ordered_model_names.extract()?, + context.getattr("models_by_name")?.cast_into::()?, + ), + ( + fixture_keys.into_any(), + relations.clone(), + context + .getattr("authoritative_columns")? + .cast_into::()?, + ), + adapter, )?; - kwargs.set_item("adapter", adapter)?; - let required_and_stars = helper(py, "_required_fixture_columns", &kwargs)?; - let required = required_and_stars.get_item(0)?.cast_into::()?; - let stars = required_and_stars.get_item(1)?; let expected = expected_fixture_types( &context.getattr("expected_types")?.cast_into::()?, ordered_model_names.extract()?, @@ -75,7 +65,7 @@ fn build_relation_fixture_completion<'py>( )?; complete_prepared( prepared, - (relations, required, stars), + (relations, required, stars.into_any()), expected.into_any(), adapter, ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs new file mode 100644 index 0000000000..674d1d0a2a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs @@ -0,0 +1,230 @@ +//! Boundary conversion for native fixture read analysis. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_fixtures::main::required_columns::required_columns; +use sqlbuild_analysis::planner_fixtures::main::star_keys::star_keys; +use sqlbuild_analysis::planner_fixtures::models::{ + FixtureReadKey, FixtureReadModel, FixtureReadSchemas, FixtureStarRelation, +}; +use sqlbuild_analysis::planner_fixtures::types::FixtureReadReference; +use sqlbuild_analysis::required_columns::models::ColumnRead; +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; +use std::collections::{HashMap, HashSet}; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::planner_strategy::fixture_values::columns; +use crate::bindings::_helpers::planner_strategy::models::FixtureReadCacheHost; +use crate::bindings::types::CompilerDetach; + +#[pyfunction] +#[pyo3(signature = (*, model_inputs, fixture_inputs, adapter))] +pub(crate) fn required_fixture_columns<'py>( + model_inputs: (Vec, Bound<'py, PyDict>), + fixture_inputs: (Bound<'py, PyAny>, Bound<'py, PyDict>, Bound<'py, PyDict>), + adapter: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyFrozenSet>)> { + let (ordered_model_names, model_map) = model_inputs; + let (fixture_keys, relations, authoritative_columns) = fixture_inputs; + let py = adapter.py(); + let mut keys: HashMap<(String, String), Bound<'py, PyAny>> = HashMap::new(); + for key in fixture_keys.try_iter()? { + let key = key?; + keys.insert(key.extract::<(String, String)>()?, key); + } + let mut schema: Vec = Vec::new(); + let mut authoritative: HashMap<(String, String), HashSet> = HashMap::new(); + for (key, original) in &keys { + if let Some(value) = relations.get_item(original)? { + schema.push(FixtureStarRelation { + key: key.clone(), + columns: columns(&value.getattr("columns")?, false)?, + authoritative: value.getattr("authoritative_names")?.extract()?, + }); + } else { + schema.push(FixtureStarRelation { + key: key.clone(), + columns: Vec::new(), + authoritative: false, + }); + } + if let Some(value) = authoritative_columns.get_item(original)? { + authoritative.insert(key.clone(), value.extract()?); + } + } + let dialect = adapter + .call_method0("expression_inference_profile")? + .getattr("sql_analysis_dialect")? + .extract::>()?; + let mut models: Vec = Vec::new(); + for name in ordered_model_names { + let Some(model) = model_map.get_item(&name)? else { + continue; + }; + let mut references: Vec = Vec::new(); + for reference in model.getattr("references")?.try_iter()? { + let reference = reference?; + references.push(( + reference.getattr("ref_kind")?.extract()?, + reference.getattr("ref_name")?.extract()?, + reference.getattr("ref_package")?.extract()?, + reference.getattr("call_argument_count")?.extract()?, + )); + } + models.push(FixtureReadModel { + name, + key: FixtureReadKey { + sql: model.getattr("query_sql")?.extract()?, + references, + dialect: dialect.clone(), + }, + high_confidence: lineage_reads(&model)?, + stars: model_stars(&model, &schema, adapter)?, + }); + } + let owner = py + .import("sqlbuild._native")? + .getattr("_fixture_read_cache")? + .cast_into::()?; + let mut owner = owner.borrow_mut(); + let cache = std::mem::take(&mut owner.cache); + let fixture_keys: HashSet<(String, String)> = keys.keys().cloned().collect(); + let (cache, result) = catch_compiler_panic(|| { + Ok(required_columns( + cache, + &models, + FixtureReadSchemas { + keys: &fixture_keys, + authoritative: &authoritative, + relations: &schema, + }, + )) + }) + .map_err(value_error)?; + owner.cache = cache; + let result = result.map_err(|error| value_error(format!("{error:?}")))?; + let required = PyDict::new(py); + for (key, columns) in result.columns { + let values = PyDict::new(py); + for column in columns { + values.set_item(column.name, PySet::new(py, column.readers)?)?; + } + required.set_item(&keys[&key], values)?; + } + let stars = PyFrozenSet::new(py, result.stars.iter().map(|key| &keys[key]))?; + Ok((required, stars)) +} + +fn lineage_reads(model: &Bound<'_, PyAny>) -> PyResult> { + let mut reads: Vec = Vec::new(); + let facts = model.getattr("fast_lineage_columns")?; + if facts.is_none() { + return Ok(reads); + } + for fact in facts.try_iter()? { + let fact = fact?; + if !fact.getattr("confidence")?.eq(model + .py() + .import("sqlbuild.compiler.lineage.types")? + .getattr("ColumnLineageConfidence")? + .getattr("HIGH")?)? + { + continue; + } + for source in fact.getattr("upstream_columns")?.try_iter()? { + let source = source?; + reads.push(ColumnRead { + resource_type: source.getattr("resource_type")?.extract()?, + resource_name: source.getattr("resource_name")?.extract()?, + column_name: source.getattr("column_name")?.extract()?, + }); + } + } + Ok(reads) +} + +fn model_stars( + model: &Bound<'_, PyAny>, + schema: &[FixtureStarRelation], + adapter: &Bound<'_, PyAny>, +) -> PyResult> { + if !model.getattr("fast_lineage_has_star")?.is_truthy()? { + return Ok(Vec::new()); + } + let mut deps: Vec<(String, String)> = Vec::new(); + for dependency in model.getattr("deps")?.try_iter()? { + let dependency = dependency?; + let key = ( + dependency.getattr("resource_type")?.extract::()?, + dependency.getattr("name")?.extract::()?, + ); + if schema.iter().any(|relation| relation.key == key) { + deps.push(key); + } + } + let dependencies = PyFrozenSet::new(model.py(), deps)?; + let mut selected: Vec = Vec::new(); + for dependency in dependencies.try_iter()? { + let dependency: (String, String) = dependency?.extract()?; + if let Some(relation) = schema.iter().find(|relation| relation.key == dependency) { + selected.push(FixtureStarRelation { + key: relation.key.clone(), + columns: relation.columns.clone(), + authoritative: relation.authoritative, + }); + } + } + let contract = model.getattr("dynamic_column_contract")?; + let inputs: Option> = + if !contract.is_none() && contract.getattr("output_proven")?.is_truthy()? { + Some(contract.getattr("input_relations")?.extract()?) + } else { + None + }; + let dialect: Option = + if inputs.is_none() && selected.iter().any(|relation| !relation.columns.is_empty()) { + adapter + .call_method0("expression_inference_profile")? + .getattr("sql_analysis_dialect")? + .extract()? + } else { + None + }; + let sql: String = model.getattr("query_sql")?.extract()?; + model + .py() + .compiler_detach(|| { + Ok(star_keys( + &sql, + &selected, + dialect.as_deref(), + inputs.as_deref(), + )) + }) + .map_err(value_error) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add( + "_fixture_read_cache", + pyo3::Py::new(module.py(), FixtureReadCacheHost::default())?, + )?; + module.add_function(wrap_pyfunction!(fixture_read_cache_info, module)?)?; + module.add_function(wrap_pyfunction!(required_fixture_columns, module)?) +} + +/// Report resolver misses and retained entries, optionally clearing prior fixture reuse. +#[pyfunction] +#[pyo3(signature = (*, reset = false))] +fn fixture_read_cache_info(py: pyo3::Python<'_>, reset: bool) -> PyResult<(u64, usize)> { + let owner = py + .import("sqlbuild._native")? + .getattr("_fixture_read_cache")? + .cast_into::()?; + let mut owner = owner.borrow_mut(); + if reset { + owner.cache = Default::default(); + } + Ok((owner.cache.misses, owner.cache.entries.len())) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs index 0f35b03b71..0bc5231dda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs @@ -5,7 +5,6 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; pub(crate) const MODELS: &str = "sqlbuild.compiler.planner.models"; -pub(crate) const COMPLETION: &str = "sqlbuild.compiler.planner._helpers.fixtures.completion"; pub(crate) fn folded(value: &Bound<'_, PyAny>) -> PyResult { value.call_method0("casefold")?.extract() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs index 47bd1cb509..1a7d6f905c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs @@ -116,7 +116,7 @@ fn lineage_error(py: Python<'_>, failure: LineageFailure) -> PyErr { /// The reference one named input resolves to through model and SQL function references. #[pyfunction] #[pyo3(signature = (*, model, input_name, models_by_name, functions_by_name))] -fn resolve_lineage_reference<'py>( +pub(crate) fn resolve_lineage_reference<'py>( model: &Bound<'py, PyAny>, input_name: &str, models_by_name: Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs new file mode 100644 index 0000000000..a5a0018752 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs @@ -0,0 +1,266 @@ +//! Microbatch limits applied to planned model entries in CLI, project and model precedence. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction}; + +use sqlbuild_analysis::planner_strategy::constants::MICROBATCH_LIMIT_WARNING_CODE; +use sqlbuild_analysis::planner_strategy::main::microbatch_limit_warning::microbatch_limit_warning; +use sqlbuild_core::plan_display::errors::UnknownCursorGrain; +use sqlbuild_core::plan_display::main::effective_timestamp_grain::effective_timestamp_grain; +use sqlbuild_core::plan_display::models::TimestampGrainFacts; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compiled_project::records::replace; +use crate::bindings::_helpers::plan_output::display::resolve_effective_microbatch_batch_size; +use crate::bindings::_helpers::planner_changes::results::planner_enum; +use crate::bindings::_helpers::planner_strategy::constants::{ + PLAN_MODELS_MODULE, PLANNER_TYPES_MODULE, +}; +use crate::bindings::_helpers::planner_strategy::microbatches::{ + CAP_ACTIONS, CONTRACT_TYPES_MODULE, capped_microbatch_bounds, count_microbatches, +}; +use crate::bindings::_helpers::planner_strategy::values::{ + COMPILE_EXCEPTIONS_MODULE, input_error, optional_string, text, +}; + +const ERROR_ACTION: &str = "error"; +const SKIP_ACTION: &str = "skip"; +const CREATE_TABLE_ACTION: &str = "create_table"; +const ROLLING_WINDOW_STRATEGY: &str = "rolling_window"; +const FULL_REFRESH_REASON: &str = "full_refresh"; + +/// A planned entry and the warning its limit raised. +type LimitedEntry<'py> = (Bound<'py, PyAny>, Option>); + +/// `CursorGrain(grain)`'s error for a grain outside the known grains. +pub(crate) fn unknown_grain(py: Python<'_>, error: UnknownCursorGrain) -> PyErr { + imported(py, PLANNER_TYPES_MODULE, "CursorGrain") + .and_then(|class| class.call1((error.0,))) + .err() + .unwrap_or_else(|| pyo3::exceptions::PyValueError::new_err("unknown cursor grain")) +} + +/// The coarsest timestamp grain of a cursor and the `cursor_grain` of its input relations. +pub(crate) fn replay_grain( + (cursor_type, microbatch_strategy, downstream_grain): ( + Option, + Option, + Option, + ), + relations: &Bound<'_, PyAny>, +) -> PyResult> { + let mut input_grains: Vec> = Vec::new(); + for relation in relations.try_iter()? { + input_grains.push(optional_string(&relation?.getattr("cursor_grain")?)?); + } + effective_timestamp_grain(&TimestampGrainFacts { + cursor_type, + microbatch_strategy, + downstream_grain, + input_grains, + }) + .map_err(|error| unknown_grain(relations.py(), error)) +} + +/// The entry's effective batch size and its batch count over the planned range. +fn batch_count(entry: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = entry.py(); + let range: Bound<'_, PyAny> = entry.getattr("microbatch_range")?; + let batch_size: Bound<'_, PyAny> = entry.getattr("batch_size")?; + let cursor_type: Bound<'_, PyAny> = entry.getattr("cursor_type")?; + if range.is_none() || batch_size.is_none() || cursor_type.is_none() { + return Ok(None); + } + let (batch_size, cursor_type): (String, String) = (text(&batch_size)?, text(&cursor_type)?); + let grain: Option = replay_grain( + ( + Some(cursor_type.clone()), + optional_string(&entry.getattr("microbatch_strategy")?)?, + optional_string(&entry.getattr("cursor_grain")?)?, + ), + &entry.getattr("cursor_input_relations")?, + )?; + let effective_size: String = match grain { + Some(grain) => resolve_effective_microbatch_batch_size(py, &batch_size, &grain)?, + None => batch_size, + }; + let creates: bool = entry.getattr("action")?.eq(CREATE_TABLE_ACTION)?; + let count: i128 = count_microbatches(&range, &effective_size, &cursor_type, creates)?; + Ok(Some((effective_size, count))) +} + +/// The entry with one batch limit applied, and the warning when a non-error limit is exceeded. +fn limited<'py>( + entry: &Bound<'py, PyAny>, + max_batches: Bound<'py, PyAny>, + action: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = entry.py(); + if entry.getattr("action")?.eq(SKIP_ACTION)? { + return Ok((entry.clone(), None)); + } + let action_text: String = text(&action)?; + let maximum: Option = if max_batches.is_none() { + None + } else { + Some(max_batches.extract()?) + }; + let counted: Option<(String, i128)> = batch_count(entry)?; + let name: Bound<'py, PyAny> = entry.getattr("name")?; + let cap: bool = CAP_ACTIONS.contains(&action_text.as_str()); + let warning: Option = match &counted { + Some((_, count)) => { + microbatch_limit_warning(&text(&name)?, (maximum, *count), (&action_text, cap)) + } + None => None, + }; + let mut range: Bound<'py, PyAny> = entry.getattr("microbatch_range")?; + if let (Some(_), Some((size, _)), Some(maximum), true) = + (&warning, &counted, maximum, cap && !range.is_none()) + { + let cursor_type: String = + optional_string(&entry.getattr("cursor_type")?)?.unwrap_or_default(); + range = capped_microbatch_bounds(range, (size, &cursor_type), maximum, &action_text)?; + } + let none: Bound<'py, PyAny> = py.None().into_bound(py); + let limited_entry: Bound<'py, PyAny> = replace( + entry, + &[ + ("microbatch_range", range), + ( + "microbatch_limit_count", + counted + .as_ref() + .map(|(_, count)| *count) + .into_pyobject(py)? + .into_any(), + ), + ( + "microbatch_limit_action", + if max_batches.is_none() { none } else { action }, + ), + ("microbatch_limit", max_batches), + ( + "microbatch_limit_warning", + warning.clone().into_pyobject(py)?.into_any(), + ), + ], + )?; + let Some(message) = warning else { + return Ok((limited_entry, None)); + }; + if action_text == ERROR_ACTION { + return Err(input_error( + py, + (COMPILE_EXCEPTIONS_MODULE, "CompileInputError"), + &message, + None, + )); + } + let plan_warning: Bound<'py, PyAny> = call_with( + &imported(py, PLAN_MODELS_MODULE, "PlanWarning")?, + &[ + ("model_name", name), + ("severity", planner_enum(py, "WarningSeverity", "warning")?), + ( + "code", + PyString::new(py, MICROBATCH_LIMIT_WARNING_CODE).into_any(), + ), + ("message", PyString::new(py, &message).into_any()), + ], + )?; + Ok((limited_entry, Some(plan_warning))) +} + +/// Apply one batch limit to a planned microbatch entry, failing when an `error` limit is exceeded. +#[pyfunction] +#[pyo3(signature = (*, entry, max_batches, action))] +pub(crate) fn apply_microbatch_limit<'py>( + entry: &Bound<'py, PyAny>, + max_batches: Bound<'py, PyAny>, + action: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = entry.py(); + let (limited_entry, warning) = limited(entry, max_batches, action)?; + PyTuple::new( + py, + [ + limited_entry, + warning.unwrap_or_else(|| py.None().into_bound(py)), + ], + ) +} + +/// `(entry, warnings)` with the limit outcome's warning, if any. +fn with_warnings<'py>( + (entry, warning): LimitedEntry<'py>, + mut warnings: Vec>, +) -> PyResult> { + let py: Python<'py> = entry.py(); + warnings.extend(warning); + PyTuple::new(py, [entry, PyTuple::new(py, warnings)?.into_any()]) +} + +/// The default limit: none for an incremental rolling window, else the model's limit or the configured ceiling. +fn default_limit<'py>( + entry: &Bound<'py, PyAny>, + configured: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = entry.py(); + if entry + .getattr("microbatch_strategy")? + .eq(ROLLING_WINDOW_STRATEGY)? + && !entry.getattr("reason")?.eq(FULL_REFRESH_REASON)? + && entry.getattr("start_cursor_override")?.is_none() + && entry.getattr("end_cursor_override")?.is_none() + { + return Ok(py.None().into_bound(py)); + } + let model_limit: Bound<'py, PyAny> = entry.getattr("microbatch_limit")?; + Ok(if model_limit.is_truthy()? { + model_limit + } else { + configured + }) +} + +/// Apply CLI, project, and model batch limits in their documented precedence order. +#[pyfunction] +#[pyo3(signature = (*, entry, inputs))] +pub(crate) fn apply_configured_microbatch_limits<'py>( + entry: &Bound<'py, PyAny>, + inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = entry.py(); + let configured: Bound<'py, PyAny> = inputs.getattr("max_microbatches")?; + let configured_action: Bound<'py, PyAny> = inputs.getattr("microbatch_limit_action")?; + if inputs + .getattr("max_microbatches_is_override")? + .is_truthy()? + { + let error: Bound<'py, PyAny> = + imported(py, CONTRACT_TYPES_MODULE, "MicrobatchLimitAction")?.getattr("ERROR")?; + return with_warnings(limited(entry, configured, error)?, Vec::new()); + } + let declared: Bound<'py, PyAny> = entry.getattr("declared_microbatch_limit_action")?; + if declared.is_none() { + let maximum: Bound<'py, PyAny> = default_limit(entry, configured)?; + return with_warnings(limited(entry, maximum, configured_action)?, Vec::new()); + } + let mut warnings: Vec> = Vec::new(); + let mut entry: Bound<'py, PyAny> = entry.clone(); + if !configured.is_none() { + let (_, safety_warning) = limited(&entry, configured.clone(), configured_action.clone())?; + entry = replace( + &entry, + &[ + ("microbatch_safety_limit", configured), + ("microbatch_safety_limit_action", configured_action), + ], + )?; + warnings.extend(safety_warning); + } + let model_limit: Bound<'py, PyAny> = entry.getattr("microbatch_limit")?; + with_warnings(limited(&entry, model_limit, declared)?, warnings) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs index 916e4d3de5..ea42c15438 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs @@ -18,16 +18,14 @@ use sqlbuild_analysis::planner_strategy::types::Timeline; use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; use sqlbuild_cursor::algebra::models::{DurationDigits, DurationSpan}; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::cursor::durations::shifted_moment; use crate::bindings::_helpers::plan_output::display::aligned_timestamp_bounds; use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; use crate::bindings::_helpers::planner_strategy::values::text; const CURSOR_MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; -const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +pub(crate) const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; const INTEGER_CURSOR: &str = "integer"; const TIMESTAMP_CURSOR: &str = "timestamp"; const MONTHS_PER_YEAR: i64 = 12; @@ -131,9 +129,7 @@ fn timeline_bounds<'py>( } /// Batches in one concrete cursor range. -#[pyfunction] -#[pyo3(signature = (*, bounds, batch_size, cursor_type, equal_bounds_are_batch = false))] -fn count_microbatches( +pub(crate) fn count_microbatches( bounds: &Bound<'_, PyAny>, batch_size: &str, cursor_type: &str, @@ -177,33 +173,15 @@ fn cursor_value<'py>( ) } -/// One concrete range capped to its earliest or latest batch windows. -#[pyfunction] -#[pyo3(signature = (**options))] -fn cap_microbatch_bounds<'py>( - py: Python<'py>, - options: Option>, +/// One concrete range capped to `max_batches` batch windows from the end the action keeps. +pub(crate) fn capped_microbatch_bounds<'py>( + bounds: Bound<'py, PyAny>, + (batch_size, cursor_type): (&str, &str), + max_batches: i128, + action: &str, ) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "cap_microbatch_bounds", - options, - ( - &[ - "bounds", - "batch_size", - "cursor_type", - "max_batches", - "action", - ], - &[], - ), - )?; - let bounds: Bound<'py, PyAny> = arguments.value("bounds")?; - let batch_size: String = text(&arguments.value("batch_size")?)?; - let cursor_type: String = text(&arguments.value("cursor_type")?)?; - let max_batches: i128 = arguments.value("max_batches")?.extract()?; - let direction: CapDirection = match text(&arguments.value("action")?)?.as_str() { + let py: Python<'py> = bounds.py(); + let direction: CapDirection = match action { CAP_FROM_START => CapDirection::FromStart, CAP_FROM_END => CapDirection::FromEnd, _ => return Ok(bounds), @@ -213,7 +191,7 @@ fn cap_microbatch_bounds<'py>( let Some(range) = integer_bounds(&bounds)? else { return Ok(bounds); }; - let Some(capped) = cap_integer_bounds(range, &batch_size, max_batches, direction) else { + let Some(capped) = cap_integer_bounds(range, batch_size, max_batches, direction) else { return Ok(bounds); }; match capped { @@ -232,7 +210,7 @@ fn cap_microbatch_bounds<'py>( if cursor_type != TIMESTAMP_CURSOR { return Ok(bounds); } - let Some((timeline, range)) = timeline_bounds(&bounds, &batch_size)? else { + let Some((timeline, range)) = timeline_bounds(&bounds, batch_size)? else { return Ok(bounds); }; match cap_timeline_bounds(&timeline, range, max_batches, direction)? { @@ -317,8 +295,6 @@ fn resolve_microbatch_limit_config<'py>( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(count_microbatches, module)?)?; - module.add_function(wrap_pyfunction!(cap_microbatch_bounds, module)?)?; module.add_function(wrap_pyfunction!(microbatch_limit_warning, module)?)?; module.add_function(wrap_pyfunction!(resolve_microbatch_limit_config, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs index d1023a4c18..ca7401805f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs @@ -1,15 +1,22 @@ +pub(crate) mod audit_entries; pub(crate) mod audits; +pub(crate) mod constants; +pub(crate) mod cursor_inputs; pub(crate) mod cursor_types; pub(crate) mod fixture_completion; pub(crate) mod fixture_context; pub(crate) mod fixture_metadata; pub(crate) mod fixture_preparation; +pub(crate) mod fixture_reads; pub(crate) mod fixture_selection; pub(crate) mod fixture_sql; pub(crate) mod fixture_types; pub(crate) mod fixture_values; pub(crate) mod lineage; +pub(crate) mod microbatch_limits; pub(crate) mod microbatches; +pub(crate) mod model_entries; +pub(crate) mod models; pub(crate) mod registration; pub(crate) mod strategy; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs new file mode 100644 index 0000000000..8ed737996f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -0,0 +1,565 @@ +//! Assemble model plan dataclasses from resolved plan decisions and authored model config. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction}; + +use sqlbuild_analysis::planner_strategy::main::microbatch_lookback::microbatch_lookback; +use sqlbuild_analysis::planner_strategy::main::table_type_ddl::table_type_ddl; +use sqlbuild_analysis::planner_strategy::models::LookbackConfig; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; +use crate::bindings::_helpers::model_config::config_values::{ + config_bool, config_cursor_bound, config_int, config_str, config_string_tuple, +}; +use crate::bindings::_helpers::planner_strategy::constants::{ + ADAPTER_MODELS_MODULE, CURSOR_TYPES_MODULE, CUSTOM_MATERIALIZATION, DDL_SEPARATOR, + ENFORCED_CONTRACT, MICROBATCH_MODE, NATIVE_MODULE, PLAN_MODELS_MODULE, PLANNER_TYPES_MODULE, +}; +use crate::bindings::_helpers::planner_strategy::cursor_inputs::model_values; +use crate::bindings::_helpers::planner_strategy::microbatch_limits::{replay_grain, unknown_grain}; +use crate::bindings::_helpers::planner_strategy::microbatches::microbatch_limit_config; +use crate::bindings::_helpers::planner_strategy::values::{ + PLANNER_EXCEPTIONS_MODULE, input_error, text, +}; + +/// Model config keys copied to the plan entry as optional strings, with their entry fields. +const STRING_FIELDS: [(&str, &str); 15] = [ + ("incremental_strategy", "incremental_strategy"), + ("incremental_mode", "incremental_mode"), + ("microbatch_strategy", "microbatch_strategy"), + ("cursor_watermark_mode", "cursor_watermark_mode"), + ("cursor_type", "cursor_type"), + ("cursor_grain", "cursor_grain"), + ("batch_size", "batch_size"), + ( + "unaccounted_partition_policy", + "unaccounted_partition_policy", + ), + ("snapshot_strategy", "snapshot_strategy"), + ("updated_at", "updated_at_column"), + ("observed_at", "observed_at_column"), + ("historical_input", "historical_input"), + ("valid_from_column", "valid_from_column"), + ("valid_to_column", "valid_to_column"), + ("initial_valid_from", "initial_valid_from"), +]; +/// Model config keys copied as optional strings that complete the snapshot settings. +const SNAPSHOT_POLICY_FIELDS: [&str; 2] = ["snapshot_full_refresh", "snapshot_schema_change"]; +/// Plan decisions passed through to the entry unchanged. +const PASSED_FIELDS: [&str; 16] = [ + "action", + "reason", + "materialization_type", + "resolved_sql", + "cursor_column", + "cursor_bounds", + "empty_cursor_inputs", + "waiting_on_empty_inputs", + "cursor_input_relations", + "microbatch_range", + "start_cursor_override", + "end_cursor_override", + "future_cursor_config", + "start_cursor_config", + "invocation_time", + "backfill", +]; +/// Change-detection facts copied to the entry. +const CHANGE_FIELDS: [&str; 8] = [ + "fingerprint_metadata_json", + "previous_metadata_json", + "fingerprint_version_hash", + "previous_version_hash", + "query_changed", + "config_changed", + "schema_findings", + "changed_functions", +]; + +fn none(py: Python<'_>) -> Bound<'_, PyAny> { + py.None().into_bound(py) +} + +fn or_none<'py>(py: Python<'py>, value: Option>) -> Bound<'py, PyAny> { + value.unwrap_or_else(|| none(py)) +} + +/// `ColumnInfo(name=..., type=...)`. +fn column_info<'py>( + class: &Bound<'py, PyAny>, + (name, column_type): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + call_with(class, &[("name", name), ("type", column_type)]) +} + +/// Declared schema columns as `ColumnInfo`: typed columns only, or every column with an empty default type. +fn schema_columns<'py>( + model: &Bound<'py, PyAny>, + typed_only: bool, +) -> PyResult> { + let py: Python<'py> = model.py(); + let entry: Bound<'py, PyAny> = model.getattr("schema_entry")?; + if entry.is_none() { + return Ok(PyTuple::empty(py)); + } + let class: Bound<'py, PyAny> = imported(py, ADAPTER_MODELS_MODULE, "ColumnInfo")?; + let mut columns: Vec> = Vec::new(); + for column in entry.getattr("columns")?.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let column_type: Bound<'py, PyAny> = column.getattr("type")?; + if typed_only && column_type.is_none() { + continue; + } + let column_type: Bound<'py, PyAny> = if typed_only || column_type.is_truthy()? { + column_type + } else { + PyString::new(py, "").into_any() + }; + columns.push(column_info(&class, (column.getattr("name")?, column_type))?); + } + PyTuple::new(py, columns) +} + +/// A string or list-of-strings config value as a tuple of strings. +fn string_list<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult> { + let py: Python<'py> = values.py(); + let raw: Bound<'py, PyAny> = values.call_method1("get", (key,))?; + if raw.is_instance_of::() { + return PyTuple::new(py, [raw]); + } + if raw.is_instance_of::() { + let items: Vec> = raw + .try_iter()? + .filter(|item| { + item.as_ref() + .map_or(true, |item| item.is_instance_of::()) + }) + .collect::>()?; + return PyTuple::new(py, items); + } + Ok(PyTuple::empty(py)) +} + +/// The model's `on_schema_change` policy, rejecting unknown values. +fn on_schema_change<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + let raw: Bound<'py, PyAny> = model_values(model)?.call_method1("get", ("on_schema_change",))?; + if raw.is_none() { + return Ok(raw); + } + if raw.is_instance_of::() { + match imported(py, PLANNER_TYPES_MODULE, "OnSchemaChange")?.call1((&raw,)) { + Ok(policy) => return Ok(policy), + Err(error) if !error.is_instance_of::(py) => return Err(error), + Err(_) => {} + } + } + Err(input_error( + py, + (PLANNER_EXCEPTIONS_MODULE, "PlannerInputError"), + &format!( + "model '{}': unknown on_schema_change '{}'", + text(&model.getattr("name")?)?, + raw.str()? + ), + None, + )) +} + +/// Whether the model's schema enforces declared column types. +fn type_enforcement<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + let entry: Bound<'py, PyAny> = model.getattr("schema_entry")?; + if !entry.is_none() { + let enforcement: Bound<'py, PyAny> = entry.getattr("type_enforcement")?; + if !enforcement.is_none() { + return Ok(enforcement); + } + } + Ok(PyBool::new(py, false).to_owned().into_any()) +} + +/// The explicit or default lookback of a microbatch model at the effective grain. +fn lookback_of( + model: &Bound<'_, PyAny>, + effective_grain: Option, +) -> PyResult> { + let values: Bound<'_, PyAny> = model_values(model)?; + let read = |key: &str| -> PyResult> { + config_str(&values, key)? + .map(|value| text(&value)) + .transpose() + }; + let config = LookbackConfig { + lookback: read("lookback")?, + incremental_strategy: read("incremental_strategy")?, + batch_size: read("batch_size")?, + grain: match effective_grain.filter(|grain| !grain.is_empty()) { + Some(grain) => Some(grain), + None => read("cursor_grain")?, + }, + }; + microbatch_lookback(&config).map_err(|error| unknown_grain(model.py(), error)) +} + +/// Resolve explicit or default lookback for one microbatch model. +#[pyfunction] +#[pyo3(signature = (*, model, effective_grain = None))] +pub(crate) fn resolve_microbatch_lookback( + model: &Bound<'_, PyAny>, + effective_grain: Option, +) -> PyResult> { + lookback_of(model, effective_grain) +} + +/// Declared seed columns with types, as `ColumnInfo`. +#[pyfunction] +#[pyo3(signature = (seed))] +pub(crate) fn extract_seed_columns<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + schema_columns(seed, true) +} + +/// The adapter-rendered logical DDL of one plan action. +fn logical_ddl<'py>( + request: &Request<'_, 'py>, + (unique_key, merge_exclude): (&Bound<'py, PyTuple>, &Bound<'py, PyTuple>), + (warehouse_columns, cursor_bounds): (&Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult { + let py: Python<'py> = request.py(); + let adapter: Bound<'py, PyAny> = request.get("adapter")?; + let model: Bound<'py, PyAny> = request.get("model")?; + let destination: Bound<'py, PyAny> = model.getattr("destination")?; + let qualified: Bound<'py, PyAny> = destination.getattr("qualified_name")?; + let qualified: Bound<'py, PyAny> = if qualified.is_truthy()? { + qualified + } else { + destination.getattr("name")? + }; + let sql: Bound<'py, PyAny> = request.get("resolved_sql")?; + let mut pairs: Vec<(&str, Bound<'py, PyAny>)> = vec![("destination", qualified), ("sql", sql)]; + let cursor_column: Bound<'py, PyAny> = request.get("cursor_column")?; + let method: &str = match text(&request.get("action")?)?.as_str() { + "create_view" => "render_create_view_as", + "create_table" => "render_create_table_as", + "incremental_append" => "render_append", + "incremental_delete_insert" if !cursor_column.is_none() && !cursor_bounds.is_none() => { + let values: Bound<'py, PyAny> = model_values(&model)?; + let start = sentinel_to_token(&cursor_bounds.getattr("start")?)?; + let end = sentinel_to_token(&cursor_bounds.getattr("end")?)?; + pairs.push(("cursor_column", cursor_column)); + pairs.push(("cursor_start", PyString::new(py, &start).into_any())); + pairs.push(("cursor_end", PyString::new(py, &end).into_any())); + pairs.push(( + "cursor_type", + or_none(py, config_str(&values, "cursor_type")?), + )); + "render_delete_insert_cursor" + } + "incremental_delete_insert" => { + pairs.push(("unique_key", unique_key.clone().into_any())); + "render_delete_insert" + } + "incremental_merge" => { + let mut source_columns: Vec> = Vec::new(); + for column in warehouse_columns.try_iter()? { + source_columns.push(column?.getattr("name")?); + } + pairs.push(("unique_key", unique_key.clone().into_any())); + pairs.push(( + "source_columns", + PyTuple::new(py, source_columns)?.into_any(), + )); + pairs.push(("exclude_columns", merge_exclude.clone().into_any())); + "render_merge" + } + _ => return Ok(String::new()), + }; + let statements: Vec = call_with(&adapter.getattr(method)?, &pairs)? + .try_iter()? + .map(|statement| statement?.extract()) + .collect::>()?; + Ok(statements.join(DDL_SEPARATOR)) +} + +/// The custom materialization name, config and placeholders of a custom model. +fn custom_fields<'py>( + values: &Bound<'py, PyAny>, + custom: bool, + entry: &Bound<'py, PyDict>, +) -> PyResult<()> { + let py: Python<'py> = values.py(); + let config: Bound<'py, PyDict> = PyDict::new(py); + let placeholders: Bound<'py, PyDict> = PyDict::new(py); + let mut name: Bound<'py, PyAny> = none(py); + if custom { + let raw_name: Bound<'py, PyAny> = values.call_method1("get", ("materialized",))?; + if raw_name.is_instance_of::() { + name = raw_name; + } + if let Ok(raw) = values + .call_method1("get", ("config",))? + .downcast::() + { + for (key, value) in raw.iter() { + config.set_item(key.str()?, value)?; + } + } + if let Ok(raw) = values + .call_method1("get", ("placeholders",))? + .downcast::() + { + for (key, value) in raw.iter() { + placeholders.set_item(key.str()?, value.str()?)?; + } + } + } + entry.set_item("custom_materialization_name", name)?; + entry.set_item("custom_config", config)?; + entry.set_item("custom_placeholders", placeholders) +} + +/// The model's base warnings followed by its cursor type warning. +fn model_warnings<'py>( + request: &Request<'_, 'py>, + (schema_actions, policy): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (enforcement, warehouse_columns): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = request.py(); + let model: Bound<'py, PyAny> = request.get("model")?; + let name: Bound<'py, PyAny> = model.getattr("name")?; + let mut warnings: Vec> = call_with( + &imported(py, NATIVE_MODULE, "build_model_warnings")?, + &[ + ("model_name", name.clone()), + ("materialization_type", request.get("materialization_type")?), + ("change_result", request.get("change_result")?), + ("schema_actions", schema_actions.clone()), + ("on_schema_change", policy.clone()), + ("type_enforcement", enforcement.clone()), + ], + )? + .try_iter()? + .collect::>()?; + let cursor_warning: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "check_cursor_type_consistency")?, + &[ + ("model_name", name), + ("cursor_column", request.get("cursor_column")?), + ( + "cursor_type", + or_none(py, config_str(&model_values(&model)?, "cursor_type")?), + ), + ("warehouse_columns", warehouse_columns.clone()), + ("sql_analysis_enabled", request.get("sql_analysis_enabled")?), + ], + )?; + if !cursor_warning.is_none() { + warnings.push(cursor_warning); + } + PyTuple::new(py, warnings) +} + +/// The cursor bounds the logical DDL renders: placeholders for runtime-owned windows, else the batch range or cursor bounds. +fn ddl_cursor_bounds<'py>(request: &Request<'_, 'py>) -> PyResult> { + let py: Python<'py> = request.py(); + let microbatch_range: Bound<'py, PyAny> = request.get("microbatch_range")?; + Ok( + if request.get("runtime_owned_cursor_bounds")?.is_truthy()? + && !request.get("cursor_column")?.is_none() + { + let sentinel: Bound<'py, PyAny> = imported(py, CURSOR_TYPES_MODULE, "BoundSentinel")?; + call_with( + &imported(py, PLAN_MODELS_MODULE, "CursorBounds")?, + &[ + ("start", sentinel.getattr("START")?), + ("end", sentinel.getattr("END")?), + ], + )? + } else if microbatch_range.is_truthy()? { + microbatch_range.clone() + } else { + request.get("cursor_bounds")? + }, + ) +} + +/// Copy the model's authored strategy, cursor, batch and snapshot settings onto the entry. +fn config_fields<'py>( + request: &Request<'_, 'py>, + values: &Bound<'py, PyAny>, + entry: &Bound<'py, PyDict>, +) -> PyResult<()> { + let py: Python<'py> = request.py(); + let model: Bound<'py, PyAny> = request.get("model")?; + for (key, field) in STRING_FIELDS { + entry.set_item(field, or_none(py, config_str(values, key)?))?; + } + for key in SNAPSHOT_POLICY_FIELDS { + entry.set_item(key, or_none(py, config_str(values, key)?))?; + } + for key in ["cursor_start", "cursor_end"] { + entry.set_item(key, or_none(py, config_cursor_bound(values, key)?))?; + } + let microbatch: bool = match config_str(values, "incremental_mode")? { + Some(mode) => mode.eq(MICROBATCH_MODE)?, + None => false, + }; + let lookback: Option = if microbatch { + let read = |key: &str| -> PyResult> { + config_str(values, key)? + .map(|value| text(&value)) + .transpose() + }; + let grain: Option = replay_grain( + ( + read("cursor_type")?, + read("microbatch_strategy")?, + read("cursor_grain")?, + ), + &request.get("cursor_input_relations")?, + )?; + lookback_of(&model, grain)? + } else { + config_str(values, "lookback")? + .map(|value| text(&value)) + .transpose()? + }; + entry.set_item("lookback", lookback)?; + let concurrency: Bound<'py, PyAny> = match config_int(values, "batch_concurrency")? { + Some(concurrency) if concurrency.is_truthy()? => concurrency, + _ => 1_i64.into_pyobject(py)?.into_any(), + }; + entry.set_item("batch_concurrency", concurrency)?; + let (limit, limit_action) = microbatch_limit_config(values)?; + entry.set_item("microbatch_limit", or_none(py, limit))?; + entry.set_item( + "declared_microbatch_limit_action", + or_none(py, limit_action), + )?; + entry.set_item("check_columns", string_list(values, "check_columns")?)?; + let hard_deletes: Bound<'py, PyAny> = match config_bool(values, "invalidate_hard_deletes")? { + Some(flag) if flag.is_truthy()? => flag, + _ => PyBool::new(py, false).to_owned().into_any(), + }; + entry.set_item("invalidate_hard_deletes", hard_deletes)?; + Ok(()) +} + +/// Copy the model's declared columns and contract onto the entry. +fn contract_fields<'py>( + model: &Bound<'py, PyAny>, + values: &Bound<'py, PyAny>, + entry: &Bound<'py, PyDict>, +) -> PyResult<()> { + let py: Python<'py> = model.py(); + entry.set_item("declared_columns", schema_columns(model, true)?)?; + entry.set_item( + "contract_enforced", + values + .call_method1("get", ("contract",))? + .eq(ENFORCED_CONTRACT)?, + )?; + entry.set_item("contract_columns", schema_columns(model, false)?)?; + let schema_entry: Bound<'py, PyAny> = model.getattr("schema_entry")?; + entry.set_item( + "contract_dynamic_columns", + if schema_entry.is_none() { + PyTuple::empty(py).into_any() + } else { + schema_entry.getattr("dynamic_columns")? + }, + )?; + Ok(()) +} + +/// Build a model plan entry and its warnings from resolved plan decisions, keeping the Python dataclasses. +#[pyfunction] +#[pyo3(signature = (**request))] +pub(crate) fn assemble_model_plan_entry<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("assemble_model_plan_entry", request)?; + let py: Python<'py> = request.py(); + let model: Bound<'py, PyAny> = request.get("model")?; + let values: Bound<'py, PyAny> = model_values(&model)?; + let change: Bound<'py, PyAny> = request.get("change_result")?; + let snapshot: Bound<'py, PyAny> = request.get("snapshot")?; + let name: Bound<'py, PyAny> = model.getattr("name")?; + let policy: Bound<'py, PyAny> = on_schema_change(&model)?; + let schema_actions: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_schema_actions")?, + &[ + ("schema_findings", change.getattr("schema_findings")?), + ("on_schema_change", policy.clone()), + ], + )?; + let enforcement: Bound<'py, PyAny> = type_enforcement(&model)?; + let warehouse_columns: Bound<'py, PyAny> = snapshot + .getattr("existing_columns")? + .call_method1("get", (&name, PyTuple::empty(py)))?; + let warnings: Bound<'py, PyTuple> = model_warnings( + &request, + (&schema_actions, &policy), + (&enforcement, &warehouse_columns), + )?; + let unique_key: Bound<'py, PyTuple> = string_list(&values, "unique_key")?; + let merge_exclude: Bound<'py, PyTuple> = config_string_tuple(&values, "merge_exclude_columns")? + .unwrap_or_else(|| PyTuple::empty(py)); + let table_type: Bound<'py, PyAny> = model + .getattr("config")? + .getattr("table_type")? + .getattr("value")? + .getattr("value")?; + let ddl: String = logical_ddl( + &request, + (&unique_key, &merge_exclude), + (&warehouse_columns, ddl_cursor_bounds(&request)?), + )?; + let entry: Bound<'py, PyDict> = PyDict::new(py); + for field in ["key", "name", "relative_path", "destination"] { + entry.set_item(field, model.getattr(field)?)?; + } + entry.set_item("fingerprint_query_sql", model.getattr("query_sql")?)?; + entry.set_item("logical_ddl", table_type_ddl(&ddl, &text(&table_type)?))?; + for field in PASSED_FIELDS { + entry.set_item(field, request.get(field)?)?; + } + for field in CHANGE_FIELDS { + entry.set_item(field, change.getattr(field)?)?; + } + config_fields(&request, &values, &entry)?; + entry.set_item("unique_key", unique_key)?; + entry.set_item("merge_exclude_columns", merge_exclude)?; + entry.set_item("table_type", table_type)?; + entry.set_item("on_schema_change", policy)?; + entry.set_item("type_enforcement", enforcement)?; + contract_fields(&model, &values, &entry)?; + for field in ["pre_hooks", "post_hooks"] { + entry.set_item(field, request.get(field)?)?; + } + let fingerprint: Bound<'py, PyAny> = snapshot + .getattr("fingerprints")? + .getattr("models")? + .call_method1("get", (&name,))?; + entry.set_item( + "previous_query_sql", + if fingerprint.is_none() { + fingerprint + } else { + fingerprint.getattr("definition")? + }, + )?; + entry.set_item("schema_actions", schema_actions)?; + custom_fields( + &values, + text(&request.get("materialization_type")?)? == CUSTOM_MATERIALIZATION, + &entry, + )?; + let plan_entry: Bound<'py, PyAny> = + imported(py, PLAN_MODELS_MODULE, "ModelPlanEntry")?.call((), Some(&entry))?; + PyTuple::new(py, [plan_entry, warnings.into_any()]) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs new file mode 100644 index 0000000000..f3abaa0b26 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs @@ -0,0 +1,11 @@ +//! Extension-owned fixture analysis reuse, with pure state passed into analysis. + +use pyo3::pyclass; +use sqlbuild_analysis::planner_fixtures::models::FixtureReadCache; + +/// Per-extension cache replacing the Python fixture resolver's LRU. +#[pyclass] +#[derive(Debug, Default)] +pub(crate) struct FixtureReadCacheHost { + pub(crate) cache: FixtureReadCache, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs index c04fcde07b..c5bdbebbc5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -1,6 +1,7 @@ //! Register the planner strategy. use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::{prelude::PyModuleMethods, wrap_pyfunction}; use crate::bindings::_helpers::planner_strategy::{ audits, cursor_types, fixture_completion, fixture_metadata, fixture_sql, lineage, microbatches, @@ -8,12 +9,54 @@ use crate::bindings::_helpers::planner_strategy::{ }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(super::audit_entries::plan_audit, module)?)?; + module.add_function(wrap_pyfunction!( + super::audit_entries::audit_is_selected, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::audit_entries::build_selected_audit_entries, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::cursor_inputs::reject_unexposed_cursor_input_columns, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::cursor_inputs::build_cursor_input_relations, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::cursor_inputs::runtime_cursor_producer_names, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::model_entries::assemble_model_plan_entry, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::model_entries::resolve_microbatch_lookback, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::model_entries::extract_seed_columns, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::microbatch_limits::apply_microbatch_limit, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::microbatch_limits::apply_configured_microbatch_limits, + module + )?)?; audits::register(module)?; cursor_types::register(module)?; fixture_completion::register(module)?; fixture_metadata::register(module)?; fixture_sql::register(module)?; super::fixture_context::register(module)?; + super::fixture_reads::register(module)?; super::fixture_selection::register(module)?; super::fixture_types::register(module)?; lineage::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs index 1546c8daee..d0adb4dbcd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::PyDict; +use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::main::text_position::text_position; use sqlbuild_analysis::compiled_project::models::TextPosition; @@ -18,6 +18,22 @@ const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_INPUT_ERROR: &str = "CompileInputError"; const MODEL_MODE: &str = "model"; +/// Extract fixture bodies from authored test or scenario CTEs in authored order. +pub(crate) fn prefixed_ctes<'py>( + ctes: &Bound<'py, PyAny>, + prefix: &str, +) -> PyResult> { + let result = PyDict::new(ctes.py()); + for cte in ctes.try_iter()? { + let cte = cte?; + let name: String = cte.getattr("name")?.extract()?; + if let Some(name) = name.strip_prefix(prefix) { + result.set_item(name, cte.getattr("sql_body")?)?; + } + } + Ok(result) +} + /// One authored SQL body to read: its text, file contents, relative path and body span. struct AuthoredBody { sql: String, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs new file mode 100644 index 0000000000..b78aa93e68 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs @@ -0,0 +1,26 @@ +//! SQL-test plan resource and severity decisions. + +pub(super) const UDF: &str = "udf"; +pub(super) const TABLE_FN: &str = "table_fn"; +pub(super) const ERROR_SEVERITY: &str = "error"; +pub(super) const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; +pub(super) const PLAN_MODELS: &str = "sqlbuild.compiler.planner.models"; +pub(super) const NATIVE_PLANNING: &str = "sqlbuild._native"; +pub(super) const FIXTURE_PREFIXES: [(&str, &str); 4] = [ + ("model", "__ref__"), + ("source", "__source__"), + ("seed", "__seed__"), + ("sql_test", "__expected__"), +]; +pub(super) const CURSOR_ALGEBRA_EXCEPTIONS: &str = "sqlbuild.cursor_algebra.exceptions"; +pub(super) const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; +pub(super) const INTEGER_CURSOR_TYPE: &str = "integer"; +pub(super) const TIMESTAMP_CURSOR_TYPE: &str = "timestamp"; +pub(super) const DEFAULT_TIMESTAMP_START: &str = "1900-01-01 00:00:00"; +pub(super) const DEFAULT_TIMESTAMP_END: &str = "2999-12-31 00:00:00"; +pub(super) const DEFAULT_INTEGER_START: &str = "-1000000000000000"; +pub(super) const DEFAULT_INTEGER_END: &str = "1000000000000000"; +pub(super) const CURSOR_START_KEY: &str = "cursor_start"; +pub(super) const CURSOR_END_KEY: &str = "cursor_end"; +pub(super) const SENTINEL: &str = "__SQLBUILD_CALL_SUFFIX__"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs new file mode 100644 index 0000000000..6c1ae72432 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs @@ -0,0 +1,270 @@ +//! Cursor windows rendered for models that SQL tests evaluate from their real SQL. + +use std::cmp::Ordering; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_cursor::algebra::main::compare::compare; +use sqlbuild_cursor::algebra::main::floor_to_grain::floor_to_grain; +use sqlbuild_cursor::algebra::models::{Grain, Scalar}; + +use crate::bindings::_helpers::attachments::cursor_intrinsics::{ + compile_input_error, resolve_cursor_intrinsics, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cursor::conversion::CursorSession; +use crate::bindings::_helpers::cursor::scalars::{parse_value, render_value}; +use crate::bindings::_helpers::model_config::config_values::config_str; +use crate::bindings::_helpers::sql_test_glue::constants::{ + CURSOR_ALGEBRA_EXCEPTIONS, CURSOR_END_KEY, CURSOR_START_KEY, DEFAULT_INTEGER_END, + DEFAULT_INTEGER_START, DEFAULT_TIMESTAMP_END, DEFAULT_TIMESTAMP_START, INTEGER_CURSOR_TYPE, + PLANNER_TYPES, TIMESTAMP_CURSOR_TYPE, +}; + +/// One model's cursor name, type and timestamp grain. +struct ModelCursor<'py> { + name: String, + cursor_type: Bound<'py, PyAny>, + type_name: String, + grain: Option<(String, Grain)>, +} + +/// One declared bound with the defaults and key used when it is absent or invalid. +struct BoundRequest<'a, 'py> { + raw: Option>, + default: &'a str, + key: &'a str, +} + +/// Whether SQL calls `__cursor_start()` or `__cursor_end()`. +pub(super) fn uses_cursor_intrinsics(py: Python<'_>, sql: &str) -> PyResult { + Ok(resolve_cursor_intrinsics(py, sql, None, None)?.1) +} + +/// Whether a SQL test header declares its own cursor window. +pub(super) fn declares_cursor_window(test: &Bound<'_, PyAny>) -> PyResult { + let block = test.getattr("test_block")?; + Ok(!block.getattr("cursor_start")?.is_none() || !block.getattr("cursor_end")?.is_none()) +} + +/// SQL with intrinsics rendered from the test's declared window, or the wide default window. +pub(super) fn render_test_cursor_intrinsics<'py>( + sql: Bound<'py, PyAny>, + model: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + test: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let py = sql.py(); + let text: String = sql.extract()?; + if !uses_cursor_intrinsics(py, &text)? { + return Ok(sql); + } + let cursor = model_cursor(model)?; + let (start, end) = test_window(&cursor, test)?; + let literal = |value: &Bound<'py, PyAny>| -> PyResult { + call_with( + &adapter.getattr("render_cursor_bound_literal")?, + &[ + ("value", PyString::new(py, &render_value(value)?).into_any()), + ("cursor_type", cursor.cursor_type.clone()), + ], + )? + .extract() + }; + let (start_sql, end_sql) = (literal(&start)?, literal(&end)?); + let (rendered, _) = resolve_cursor_intrinsics(py, &text, Some(&start_sql), Some(&end_sql))?; + Ok(PyString::new(py, &rendered).into_any()) +} + +/// Chain models using intrinsics, after validating the test's declared window against each. +pub(super) fn declared_window_cursor_models<'py>( + test: &Bound<'py, PyAny>, + chain_models: Vec>, +) -> PyResult>> { + let py = test.py(); + let mut cursor_models: Vec> = Vec::new(); + for model in chain_models { + if uses_cursor_intrinsics(py, &model.getattr("query_sql")?.extract::()?)? { + cursor_models.push(model); + } + } + if cursor_models.is_empty() { + return Err(compile_input_error( + py, + format!( + "{} declares cursor_start or cursor_end, but no model it evaluates uses __cursor_start() or __cursor_end()", + test_label(test)? + ), + )?); + } + for model in &cursor_models { + test_window(&model_cursor(model)?, Some(test))?; + } + Ok(cursor_models) +} + +/// The start and exclusive end of one model's window, validating declared bounds. +fn test_window<'py>( + cursor: &ModelCursor<'py>, + test: Option<&Bound<'py, PyAny>>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let block = test.map(|test| test.getattr("test_block")).transpose()?; + let declared = |name: &str| -> PyResult>> { + let Some(block) = &block else { + return Ok(None); + }; + let value = block.getattr(name)?; + Ok((!value.is_none()).then_some(value)) + }; + let integer = cursor.type_name == INTEGER_CURSOR_TYPE; + let start = bound( + BoundRequest { + raw: declared(CURSOR_START_KEY)?, + default: if integer { + DEFAULT_INTEGER_START + } else { + DEFAULT_TIMESTAMP_START + }, + key: CURSOR_START_KEY, + }, + cursor, + test, + )?; + let end = bound( + BoundRequest { + raw: declared(CURSOR_END_KEY)?, + default: if integer { + DEFAULT_INTEGER_END + } else { + DEFAULT_TIMESTAMP_END + }, + key: CURSOR_END_KEY, + }, + cursor, + test, + )?; + if let Some(test) = test + && ordering(&start, &end)? != Ordering::Less + { + return Err(compile_input_error( + test.py(), + format!( + "{}: cursor_start '{}' must be before the exclusive cursor_end '{}' for model '{}'", + test_label(test)?, + render_value(&start)?, + render_value(&end)?, + cursor.name + ), + )?); + } + Ok((start, end)) +} + +/// One parsed bound, or the parsed default when the test declares none. +fn bound<'py>( + request: BoundRequest<'_, 'py>, + cursor: &ModelCursor<'py>, + test: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let py = cursor.cursor_type.py(); + let (Some(raw), Some(test)) = (request.raw, test) else { + return parse_value( + &PyString::new(py, request.default).into_any(), + &cursor.cursor_type, + ); + }; + let value = match parse_value(&raw, &cursor.cursor_type) { + Ok(value) => value, + Err(error) + if error.value(py).is_instance(&imported( + py, + CURSOR_ALGEBRA_EXCEPTIONS, + "CursorAlgebraError", + )?)? => + { + let expected = if cursor.type_name == INTEGER_CURSOR_TYPE { + "an integer" + } else { + "an ISO timestamp or date" + }; + return Err(compile_input_error( + py, + format!( + "{}: {} '{}' must be {expected} because model '{}' uses cursor_type {}", + test_label(test)?, + request.key, + raw.str()?, + cursor.name, + cursor.type_name + ), + )?); + } + Err(error) => return Err(error), + }; + if let Some((grain_name, grain)) = &cursor.grain { + let session = CursorSession::new(py)?; + let scalar: Scalar = session.scalar(&value)?; + let floored: Scalar = floor_to_grain(&scalar, Some(*grain)); + if session.checked(compare(&floored, &scalar, &session))? != Ordering::Equal { + return Err(compile_input_error( + py, + format!( + "{}: {} '{}' is not aligned to cursor_grain {grain_name} of model '{}'", + test_label(test)?, + request.key, + raw.str()?, + cursor.name + ), + )?); + } + } + Ok(value) +} + +/// How two parsed cursor values order. +fn ordering(left: &Bound<'_, PyAny>, right: &Bound<'_, PyAny>) -> PyResult { + let session = CursorSession::new(left.py())?; + let (left, right) = (session.scalar(left)?, session.scalar(right)?); + session.checked(compare(&left, &right, &session)) +} + +/// The cursor type and timestamp grain a model's config declares. +fn model_cursor<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py = model.py(); + let values = model.getattr("config")?.getattr("values")?; + let declared = config_str(&values, "cursor_type")?.filter(|value| { + value + .extract::() + .is_ok_and(|value| !value.is_empty()) + }); + let cursor_type = imported(py, PLANNER_TYPES, "CursorType")?.call1(( + declared.unwrap_or_else(|| PyString::new(py, TIMESTAMP_CURSOR_TYPE).into_any()), + ))?; + let type_name: String = cursor_type.getattr("value")?.extract()?; + let grain = match config_str(&values, "cursor_grain")? { + Some(grain) if type_name == TIMESTAMP_CURSOR_TYPE => { + let grain = imported(py, PLANNER_TYPES, "CursorGrain")?.call1((grain,))?; + let session = CursorSession::new(py)?; + session + .grain(&grain)? + .map(|known| PyResult::Ok((grain.getattr("value")?.extract()?, known))) + .transpose()? + } + _ => None, + }; + Ok(ModelCursor { + name: model.getattr("name")?.extract()?, + cursor_type, + type_name, + grain, + }) +} + +/// `SQL test '' in ''`. +fn test_label(test: &Bound<'_, PyAny>) -> PyResult { + Ok(format!( + "SQL test '{}' in '{}'", + test.getattr("name")?.str()?, + test.getattr("test_file")?.getattr("relative_path")?.str()? + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs new file mode 100644 index 0000000000..6915959b3b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs @@ -0,0 +1,204 @@ +//! Static SQL-test fixture shape validation over native fixture completion. + +use std::collections::BTreeSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_analysis::planner_fixtures::main::{ + collection_mismatch::collection_mismatch, fixture_line::fixture_line, +}; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::planner_strategy::fixture_completion::build_relation_fixture_completion; +use crate::bindings::_helpers::planner_strategy::fixture_context::build_relation_fixture_context; +use crate::bindings::_helpers::planner_strategy::fixture_sql::{ + complete_empty_fixture_sql, complete_typed_null_columns, +}; +use crate::bindings::_helpers::sql_test_glue::constants::{COMPILE_TYPES, FIXTURE_PREFIXES}; + +/// Report every statically provable fixture defect or return the completed fixture SQL groups. +pub(super) fn validated_fixtures<'py>( + test: &Bound<'py, PyAny>, + request: (&Bound<'py, PyAny>, [Bound<'py, PyDict>; 4]), + inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + context: &Bound<'py, PyAny>, +) -> PyResult; 4], Vec>> { + let py = test.py(); + let (chain, [refs, sources, seeds, expected]) = request; + let (project, adapter) = inputs; + let resource_types = imported(py, COMPILE_TYPES, "CompiledResourceType")?; + let model = resource_types.getattr("MODEL")?; + let kinds = [ + model.clone(), + resource_types.getattr("SOURCE")?, + resource_types.getattr("SEED")?, + ]; + let groups = PyTuple::new( + py, + [ + PyTuple::new(py, [kinds[0].clone(), refs.clone().into_any()])?, + PyTuple::new(py, [kinds[1].clone(), sources.clone().into_any()])?, + PyTuple::new(py, [kinds[2].clone(), seeds.clone().into_any()])?, + ], + )?; + let context = if context.is_truthy()? { + context.clone() + } else { + build_relation_fixture_context(project, None)? + }; + let completion = build_relation_fixture_completion( + project, + adapter, + (chain.clone(), groups.into_any()), + Some(context.clone()), + )?; + let mut errors: BTreeSet = BTreeSet::new(); + for diagnostic in completion.getattr("diagnostics")?.try_iter()? { + let diagnostic = diagnostic?; + let key = diagnostic.getattr("key")?; + errors.insert(format!( + "{}: {}", + fixture_location( + test, + &key.get_item(0)?, + &key.get_item(1)?.extract::()? + )?, + diagnostic.getattr("message")?.extract::()? + )); + } + let expected_types = completion.getattr("expected_types")?; + let inferred_by_fixture = completion.getattr("inferred_by_fixture")?; + let completed_expected = expected.copy()?; + let sql_test = resource_types.getattr("SQL_TEST")?; + let type_groups = [ + (&kinds[0], &refs), + (&kinds[1], &sources), + (&kinds[2], &seeds), + (&sql_test, &expected), + ]; + for (kind, fixtures) in type_groups { + let expected_output = kind.is(&sql_test); + for (key_name, sql) in fixtures.iter() { + let name: String = key_name.extract()?; + let location = || fixture_location(test, kind, &name); + let inferred = if expected_output { + let model_key = PyTuple::new(py, [&model, &key_name])?; + let relation = context + .getattr("relations")? + .call_method1("get", (&model_key,))?; + let (completed_sql, empty_error) = + complete_empty_fixture_sql(&sql.extract::()?, &relation, adapter)?; + if let Some(error) = empty_error { + errors.insert(format!("{}: expected output '{name}' {error}", location()?)); + } + completed_expected.set_item(&name, &completed_sql)?; + let kwargs = PyDict::new(py); + kwargs.set_item("query_sql", &completed_sql)?; + kwargs.set_item( + "inference_profile", + adapter.call_method0("expression_inference_profile")?, + )?; + let inference = context.getattr("fixture_inferences")?.call_method( + "infer", + (), + Some(&kwargs), + )?; + if inference.is_none() { + None + } else { + let types = expected_types + .call_method1("get", (&model_key, PyDict::new(py)))? + .cast_into::()?; + completed_expected.set_item( + &name, + complete_typed_null_columns(&completed_sql, &inference, &types, adapter)?, + )?; + Some(inference.getattr("columns")?) + } + } else { + let key = PyTuple::new(py, [kind, &key_name])?; + Some(inferred_by_fixture.call_method1("get", (key,))?) + .filter(|value| !value.is_none()) + }; + let Some(columns) = inferred else { + continue; + }; + let type_kind = if expected_output { &model } else { kind }; + let type_key = PyTuple::new(py, [type_kind, &key_name])?; + let types = expected_types.call_method1("get", (type_key, PyDict::new(py)))?; + for column in columns.try_iter()? { + let column = column?; + let column_name = column.getattr("name")?; + let expected_type: Option = types + .call_method1("get", (column_name.call_method0("casefold")?,))? + .extract()?; + let inferred_type: Option = column.getattr("type")?.extract()?; + if !collection_mismatch(expected_type.as_deref(), inferred_type.as_deref()) { + continue; + } + let label = if expected_output { + "expected output".to_owned() + } else { + format!("mock {}", kind.getattr("value")?.extract::()?) + }; + errors.insert(format!( + "{}: {label} '{name}' column '{}' has incompatible type {}; resource type is {}. Use an explicit CAST, ARRAY_CONSTRUCT, or PARSE_JSON as appropriate", + location()?, + column_name.extract::()?, + inferred_type.unwrap_or_default(), + expected_type.unwrap_or_default(), + )); + } + } + } + if !errors.is_empty() { + let test_name: String = test.getattr("name")?.extract()?; + return Ok(Err(errors + .into_iter() + .map(|error| format!("SQL test '{test_name}': {error}")) + .collect())); + } + let fixture_sql = completion.getattr("fixture_sql_by_key")?; + let completed = + |kind: &Bound<'py, PyAny>, fixtures: &Bound<'py, PyDict>| -> PyResult> { + let result = PyDict::new(py); + for (name, sql) in fixtures.iter() { + let key = PyTuple::new(py, [kind.clone(), name.clone()])?; + result.set_item(&name, fixture_sql.call_method1("get", (key, sql))?)?; + } + Ok(result) + }; + Ok(Ok([ + completed(&kinds[0], &refs)?, + completed(&kinds[1], &sources)?, + completed(&kinds[2], &seeds)?, + completed_expected, + ])) +} + +/// Return the test file and line that defines one fixture or expected-output CTE. +pub(super) fn fixture_location( + test: &Bound<'_, PyAny>, + resource_type: &Bound<'_, PyAny>, + name: &str, +) -> PyResult { + let value: String = resource_type.getattr("value")?.extract()?; + let prefix = FIXTURE_PREFIXES + .iter() + .find_map(|(kind, prefix)| (*kind == value).then_some(*prefix)) + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(value))?; + let test_file = test.getattr("test_file")?; + let line = fixture_line( + &test_file.getattr("contents")?.extract::()?, + &format!("{prefix}{name}"), + ) + .map_err(value_error)?; + Ok(format!( + "{}:{line}", + test_file + .getattr("relative_path")? + .call_method0("as_posix")? + .extract::()? + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index 2964e1bb08..e1a4f4766c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -1,5 +1,17 @@ pub(crate) mod assembly; pub(crate) mod authored_ctes; mod compiled_tests; +mod constants; +mod cursor_window; +mod fixture_validation; +mod plan_assembly; +mod plan_driver; +mod plan_entries; +mod plan_errors; +mod plan_fixtures; +mod plan_rendering; +mod plan_rows; +mod plan_values; pub(crate) mod planning; pub(crate) mod registration; +mod types; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs new file mode 100644 index 0000000000..6ecfaaac8d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs @@ -0,0 +1,129 @@ +//! Assemble per-test outcomes around native planning and adapter fixture validation. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::sql_test_glue::constants::{NATIVE_PLANNING, PLAN_MODELS}; + +#[pyfunction(name = "plan_sql_tests")] +#[pyo3(signature = (**kwargs))] +pub(crate) fn assemble_sql_test_plans<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "plan_sql_tests", + kwargs, + ( + &["tests", "project", "adapter"], + &[ + "sql_analysis_enabled", + "validate_fixtures", + "fixture_planning_context", + ], + ), + )?; + let tests: Vec> = args.value("tests")?.extract()?; + let project = args.value("project")?; + let adapter = args.value("adapter")?; + let mut context = py.None().into_bound(py); + let (planned, diagnostics) = + if args.flag("sql_analysis_enabled")? && args.flag("validate_fixtures")? { + context = args.value("fixture_planning_context")?; + if !context.is_truthy()? { + context = call_with( + &imported(py, "sqlbuild._native", "build_relation_fixture_context")?, + &[("project", project.clone())], + )?; + } + super::plan_fixtures::validate(&tests, (&project, &adapter), &context)? + } else { + (tests.clone(), Default::default()) + }; + let plannable: Vec<_> = planned + .into_iter() + .enumerate() + .filter_map(|(index, test)| (!diagnostics.contains_key(&index)).then_some(test)) + .collect(); + let plans = call_with( + &imported(py, NATIVE_PLANNING, "plan_sql_tests_natively")?, + &[ + ("project", project.clone()), + ("tests", PyTuple::new(py, plannable)?.into_any()), + ("adapter", adapter), + ( + "sql_analysis_enabled", + args.value_or_false("sql_analysis_enabled")?, + ), + ( + "render_sql", + pyo3::types::PyBool::new(py, false).to_owned().into_any(), + ), + ], + )?; + let plans: Vec> = plans.extract()?; + if plans.len() != tests.len() - diagnostics.len() { + return Err(pyo3::exceptions::PyValueError::new_err( + "zip() argument lengths differ", + )); + } + let mut plans = plans.into_iter(); + let models = PyDict::new(py); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + models.set_item(model.getattr("name")?, model)?; + } + let result_class = imported(py, PLAN_MODELS, "SqlTestPlanResult")?; + let mut results: Vec> = Vec::new(); + for (index, test) in tests.iter().enumerate() { + if let Some(diagnostic) = diagnostics.get(&index) { + results.push(call_with( + &result_class, + &[ + ("entry", py.None().into_bound(py)), + ("fixture_diagnostics", diagnostic.clone()), + ], + )?); + continue; + } + let Some(plan) = plans.next() else { + return Err(pyo3::exceptions::PyValueError::new_err( + "missing SQL-test plan", + )); + }; + let mut errors: Vec = plan.getattr("error_messages")?.extract()?; + errors.extend(super::plan_entries::expected_diagnostics( + test, &plan, &context, + )?); + if !errors.is_empty() { + results.push(call_with( + &result_class, + &[ + ("entry", py.None().into_bound(py)), + ("fixture_diagnostics", PyTuple::new(py, errors)?.into_any()), + ], + )?); + continue; + } + results.push(call_with( + &result_class, + &[ + ( + "entry", + super::plan_entries::entry( + test, + &plan, + (&project, &models), + &args.value_or_false("sql_analysis_enabled")?, + )?, + ), + ("warnings", plan.getattr("warnings")?), + ], + )?); + } + PyTuple::new(py, results) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs new file mode 100644 index 0000000000..deb6b75c96 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs @@ -0,0 +1,264 @@ +//! Host SQL-test request preparation, native planning, and Python result conversion. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::sql_test_glue::constants::PLAN_MODELS; +use crate::bindings::_helpers::sql_test_glue::plan_values::tuple; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; +use pyo3::types::{PyBool, PyDict, PyTuple}; + +pub(super) fn chains<'py>( + project: &Bound<'py, PyAny>, + tests: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + if !tests.is_truthy()? { + return Ok(PyTuple::empty(py).into_any()); + } + let request = call_with( + &imported( + py, + "sqlbuild.compiler.sql_test_glue.models", + "NativeSqlTestChainRequest", + )?, + &[ + ("models", project.getattr("models")?), + ("tests", tests.clone()), + ( + "lexical_syntax", + project + .getattr("sql_lexical_syntax")? + .getattr("native_mapping")?, + ), + ], + )?; + let native = + imported(py, "sqlbuild._native", "resolve_compiled_sql_test_chains")?.call1((request,)); + let native = match native { + Ok(value) => value, + Err(error) if error.is_instance_of::(py) => { + return Err(super::plan_errors::planning_error(py, error, tests)?); + } + Err(error) => return Err(error), + }; + let mut result: Vec> = Vec::new(); + for chain in native.try_iter()? { + result.push(tuple(&chain?)?); + } + Ok(PyTuple::new(py, result)?.into_any()) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn resolve_sql_test_model_chains<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "resolve_sql_test_model_chains", + kwargs, + (&["project", "tests"], &[]), + )?; + chains(&args.value("project")?, &args.value("tests")?) +} + +fn planned<'py>(args: &KeywordArguments<'py>, artifact: bool) -> PyResult> { + let project = args.value("project")?; + let py = project.py(); + let tests = args.value("tests")?; + if !tests.is_truthy()? { + return Ok(PyTuple::empty(py)); + } + let start: u64 = imported(py, "time", "perf_counter_ns")? + .call0()? + .extract()?; + let adapter = args.value("adapter")?; + let model_sql = super::plan_rendering::models(&project, &adapter)?; + let functions = super::plan_rendering::functions(&project, &adapter)?; + let overrides = super::plan_rendering::overrides(&project, &tests, &adapter)?; + let request = call_with( + &imported( + py, + "sqlbuild.compiler.sql_test_glue.models", + "NativeSqlTestPlanningRequest", + )?, + &[ + ("models", project.getattr("models")?), + ("rendered_model_sql", model_sql.into_any()), + ("functions", functions.into_any()), + ("tests", tests.clone()), + ("rendered_test_overrides", overrides.into_any()), + ("sql_analysis_enabled", args.value("sql_analysis_enabled")?), + ( + "sql_analysis_dialect", + adapter.call_method0("sql_analysis_dialect")?, + ), + ( + "set_difference_operator", + adapter.call_method0("render_set_difference_operator")?, + ), + ( + "requires_derived_table_aliases", + adapter.call_method0("requires_derived_table_aliases")?, + ), + ( + "lexical_syntax", + project + .getattr("sql_lexical_syntax")? + .getattr("native_mapping")?, + ), + ( + "render_sql", + PyBool::new(py, artifact || args.flag("render_sql")?) + .to_owned() + .into_any(), + ), + ( + "include_plan", + PyBool::new( + py, + !artifact && (!args.passed("include_plan")? || args.flag("include_plan")?), + ) + .to_owned() + .into_any(), + ), + ], + )?; + let native = imported(py, "sqlbuild._native", "plan_compiled_sql_tests")?.call1((request,)); + let native = match native { + Ok(value) => value, + Err(error) if error.is_instance_of::(py) => { + return Err(super::plan_errors::planning_error(py, error, &tests)?); + } + Err(error) => return Err(error), + }; + let planning_ns: u64 = native.get_item(1)?.extract()?; + let rendering_ns: u64 = native.get_item(2)?.extract()?; + let end: u64 = imported(py, "time", "perf_counter_ns")? + .call0()? + .extract()?; + let collector = imported( + py, + "sqlbuild.compiler.profiling.classes.context", + "CompileTimingContext", + )? + .getattr("active")? + .call_method0("get")?; + if !collector.is_none() { + use pyo3::IntoPyObject; + for (phase, elapsed) in [ + ( + "test_planning_ms", + planning_ns + + end + .saturating_sub(start) + .saturating_sub(planning_ns) + .saturating_sub(rendering_ns), + ), + ("comparison_render_ms", rendering_ns), + ] { + call_with( + &collector.getattr("add")?, + &[ + ("phase", super::plan_values::string(py, phase)), + ("elapsed_ns", elapsed.into_pyobject(py)?.into_any()), + ], + )?; + } + } + let mut plans: Vec> = Vec::new(); + for row in native.get_item(0)?.try_iter()? { + let plan = super::plan_rows::plan(&row?)?; + if artifact { + let sql = plan.getattr("sql")?; + if sql.is_none() { + return Err(PyErr::from_value( + imported( + py, + "sqlbuild.compiler.planner.exceptions", + "NativeSqlTestPlanningError", + )? + .call1(("native SQL-test planning omitted rendered SQL",))?, + )); + } + plans.push(call_with( + &imported(py, PLAN_MODELS, "NativeSqlTestArtifact")?, + &[ + ("sql", sql), + ("model_names", plan.getattr("model_names")?), + ("error_messages", plan.getattr("error_messages")?), + ], + )?); + } else { + plans.push(plan); + } + } + PyTuple::new(py, plans) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn plan_sql_tests_natively<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "plan_sql_tests_natively", + kwargs, + ( + &[ + "project", + "tests", + "adapter", + "sql_analysis_enabled", + "render_sql", + ], + &["include_plan"], + ), + )?; + planned(&args, false) +} + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub(crate) fn plan_compiled_sql_test_artifacts<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "plan_compiled_sql_test_artifacts", + kwargs, + ( + &["project", "tests", "adapter", "sql_analysis_enabled"], + &[], + ), + )?; + planned(&args, true) +} + +#[pyfunction] +#[pyo3(signature = (*, warnings))] +pub(crate) fn sql_test_plan_error_messages<'py>( + warnings: Bound<'py, PyAny>, +) -> PyResult> { + let mut messages: Vec = Vec::new(); + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + for warning in warnings.try_iter()? { + let warning = warning?; + let message: String = warning.getattr("message")?.extract()?; + if warning + .getattr("severity")? + .getattr("value")? + .extract::()? + == super::constants::ERROR_SEVERITY + && seen.insert(message.clone()) + { + messages.push(message); + } + } + PyTuple::new(warnings.py(), messages) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs new file mode 100644 index 0000000000..4f39f527d9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs @@ -0,0 +1,178 @@ +//! Assemble existing SQL-test plan dataclasses without changing executor ownership. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::sql_test_glue::constants::{COMPILE_TYPES, PLAN_MODELS}; +use crate::bindings::_helpers::sql_test_glue::constants::{TABLE_FN, UDF}; +use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; + +use crate::bindings::_helpers::sql_test_glue::authored_ctes::prefixed_ctes; + +fn prefixed_names<'py>(test: &Bound<'py, PyAny>, prefix: &str) -> PyResult> { + let py = test.py(); + let mut names: Vec = if model_payload(test)? { + prefixed_ctes(&test.getattr("payload")?.getattr("authored_ctes")?, prefix)? + .keys() + .extract()? + } else { + Vec::new() + }; + names.sort(); + Ok(PyTuple::new(py, names)?.into_any()) +} + +fn function_deps<'py>( + test: &Bound<'py, PyAny>, + plan: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, + inputs: (&Bound<'py, PyAny>, &Bound<'py, PyDict>), +) -> PyResult> { + let py = test.py(); + let deps = PyDict::new(py); + if model_payload(test)? { + let names: Vec = plan.getattr("model_names")?.extract()?; + let mocked: std::collections::HashSet = + inputs.0.extract::>()?.into_iter().collect(); + let models = inputs.1; + for name in names { + let Some(model) = models.get_item(name)? else { + continue; + }; + for dep in model.getattr("deps")?.try_iter()? { + let dep = dep?; + let resource: String = dep.getattr("resource_type")?.getattr("value")?.extract()?; + if resource != UDF && resource != TABLE_FN { + continue; + } + if resource == TABLE_FN + && mocked.contains(&dep.getattr("name")?.extract::()?) + { + continue; + } + deps.set_item(dep, py.None())?; + } + } + } else { + let payload = test.getattr("payload")?; + let mode: String = payload.getattr("mode")?.getattr("value")?.extract()?; + if mode == UDF || mode == TABLE_FN { + let names: std::collections::HashSet = payload + .getattr("tested_resource_names")? + .extract::>()? + .into_iter() + .collect(); + for function in project.getattr("functions")?.try_iter()? { + let function = function?; + let key = function.getattr("key")?; + if key + .getattr("resource_type")? + .getattr("value")? + .extract::()? + == mode + && names.contains(&function.getattr("name")?.extract::()?) + { + deps.set_item(key, py.None())?; + } + } + } + } + Ok(PyTuple::new(py, deps.keys().iter())?.into_any()) +} + +pub(super) fn entry<'py>( + test: &Bound<'py, PyAny>, + plan: &Bound<'py, PyAny>, + inputs: (&Bound<'py, PyAny>, &Bound<'py, PyDict>), + analysis: &Bound<'py, PyAny>, +) -> PyResult> { + let py = test.py(); + let fields = PyDict::new(py); + for name in [ + "key", + "name", + "source_path", + "block_index", + "parent_name", + "case_name", + "case_index", + "case_fingerprint", + "parameter_schema", + "parameter_values", + "scope_deps", + ] { + fields.set_item(name, test.getattr(name)?)?; + } + for (field, prefix) in [ + ("mock_ref_names", "__ref__"), + ("mock_source_names", "__source__"), + ("mock_seed_names", "__seed__"), + ("mock_dbt_ref_names", "__dbt_ref__"), + ("mock_table_function_names", "__table_fn__"), + ] { + fields.set_item(field, prefixed_names(test, prefix)?)?; + } + let mocks = fields + .get_item("mock_table_function_names")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("missing mock names"))?; + fields.set_item( + "function_deps", + function_deps(test, plan, inputs.0, (&mocks, inputs.1))?, + )?; + fields.set_item("chain", plan.getattr("chain")?)?; + fields.set_item("assertions", plan.getattr("assertions")?)?; + fields.set_item("sql_analysis_enabled", analysis)?; + imported(py, PLAN_MODELS, "SqlTestPlanEntry")?.call((), Some(&fields)) +} + +pub(super) fn expected_diagnostics( + test: &Bound<'_, PyAny>, + plan: &Bound<'_, PyAny>, + context: &Bound<'_, PyAny>, +) -> PyResult> { + let mut result: Vec = Vec::new(); + if context.is_none() { + return Ok(result); + } + let py = test.py(); + let resource_types = imported(py, COMPILE_TYPES, "CompiledResourceType")?; + for step in plan.getattr("chain")?.try_iter()? { + let step = step?; + let name = step.getattr("model_name")?; + let available = context + .getattr("authoritative_columns")? + .call_method1("get", ((resource_types.getattr("MODEL")?, &name),))?; + let expected = step.getattr("expected_columns")?; + if expected.is_none() || available.is_none() { + continue; + } + let message = call_with( + &imported( + py, + "sqlbuild.executor.testing.main._missing_expected_columns", + "describe_missing_expected_columns", + )?, + &[ + ("model_name", name.clone()), + ("expected_columns", expected), + ("available_columns", available), + ], + )?; + if message.is_none() { + continue; + } + let location = super::fixture_validation::fixture_location( + test, + &resource_types.getattr("SQL_TEST")?, + &name.extract::()?, + )?; + result.push(format!( + "SQL test '{}': {}: {}", + test.getattr("name")?.extract::()?, + location, + message.extract::()? + )); + } + Ok(result) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs new file mode 100644 index 0000000000..a90e0ead0e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs @@ -0,0 +1,101 @@ +//! Project native planning failures into existing located diagnostic exceptions. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::sql_test_glue::plan_values::string; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; + +pub(super) fn planning_error( + py: Python<'_>, + error: PyErr, + tests: &Bound<'_, PyAny>, +) -> PyResult { + let message = error.value(py).str()?.to_string(); + if let Some(payload) = message.strip_prefix("sql_test_reference:") { + let payload = imported(py, "orjson", "loads")?.call1((payload,))?; + let mut found = None; + for test in tests.try_iter()? { + let test = test?; + if test.getattr("name")?.eq(payload.get_item("testName")?)? + && test + .getattr("test_file")? + .getattr("relative_path")? + .str()? + .into_any() + .eq(payload.get_item("fileLabel")?)? + { + found = Some(test); + break; + } + } + let cte = payload.get_item("cteName")?; + let call = payload.get_item("call")?; + let message = format!( + "SQL test CTE '{}' calls {}, which the test query cannot resolve", + cte.extract::()?, + call.extract::()? + ); + let help = format!( + "Mock it in the test, for example {} AS (SELECT ...), or call __ref(\"\") on a model the test runs.", + payload.get_item("mockCte")?.extract::()? + ); + let kwargs = pyo3::types::PyDict::new(py); + use pyo3::types::PyDictMethods; + kwargs.set_item("help", help)?; + let class = if let Some(test) = found { + kwargs.set_item( + "location", + call_with( + &imported( + py, + "sqlbuild.compiler.compile.classes.sql_test_cte_locator", + "SqlTestCteLocator", + )? + .getattr("locate")?, + &[ + ("test_file", test.getattr("test_file")?), + ("test_block", test.getattr("test_block")?), + ("cte_name", cte), + ("call", call), + ], + )?, + )?; + "SqlTestReferenceError" + } else { + kwargs.set_item( + "code", + imported( + py, + "sqlbuild.compiler.compile.constants", + "SQL_TEST_HELPER_REFERENCE_CODE", + )?, + )?; + "CompileInputError" + }; + return Ok(PyErr::from_value( + imported(py, "sqlbuild.compiler.compile.exceptions", class)? + .call((message,), Some(&kwargs))?, + )); + } + let (module, class, message) = if let Some(message) = message.strip_prefix("compile_input:") { + ( + "sqlbuild.compiler.compile.exceptions", + "CompileInputError", + message.to_owned(), + ) + } else if let Some(message) = message.strip_prefix("planner_input:") { + ( + "sqlbuild.compiler.planner.exceptions", + "PlannerInputError", + message.to_owned(), + ) + } else { + ( + "sqlbuild.compiler.planner.exceptions", + "NativeSqlTestPlanningError", + format!("native SQL-test planning failed: {message}"), + ) + }; + Ok(PyErr::from_value( + imported(py, module, class)?.call1((string(py, &message),))?, + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs new file mode 100644 index 0000000000..df8734947d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs @@ -0,0 +1,119 @@ +//! Complete SQL-test fixtures through the native fixture validation boundary. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::sql_test_glue::authored_ctes::prefixed_ctes; +use crate::bindings::_helpers::sql_test_glue::constants::NATIVE_PLANNING; +use crate::bindings::_helpers::sql_test_glue::fixture_validation::validated_fixtures; +use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; + +pub(super) fn validated_test<'py>( + test: &Bound<'py, PyAny>, + chain: &Bound<'py, PyAny>, + inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + context: &Bound<'py, PyAny>, +) -> PyResult, Vec>> { + let py = test.py(); + let payload = test.getattr("payload")?; + let ctes = payload.getattr("authored_ctes")?; + let expected = payload.getattr("expected_ctes")?; + let completed = match validated_fixtures( + test, + ( + chain, + [ + prefixed_ctes(&ctes, "__ref__")?, + prefixed_ctes(&ctes, "__source__")?, + prefixed_ctes(&ctes, "__seed__")?, + prefixed_ctes(&expected, "__expected__")?, + ], + ), + inputs, + context, + )? { + Ok(completed) => completed, + Err(diagnostics) => return Ok(Err(diagnostics)), + }; + let replace = imported(py, "dataclasses", "replace")?; + let mut authored: Vec> = Vec::new(); + for cte in ctes.try_iter()? { + let cte = cte?; + let name: String = cte.getattr("name")?.extract()?; + let mut replacement = cte.clone(); + for (index, prefix) in ["__ref__", "__source__", "__seed__"].iter().enumerate() { + if let Some(name) = name.strip_prefix(prefix) { + let sql = completed[index].call_method1("get", (name, cte.getattr("sql_body")?))?; + let kwargs = PyDict::new(py); + kwargs.set_item("sql_body", sql)?; + replacement = replace.call((&cte,), Some(&kwargs))?; + break; + } + } + authored.push(replacement); + } + let mut expected_ctes: Vec> = Vec::new(); + for cte in expected.try_iter()? { + let cte = cte?; + let name: String = cte.getattr("name")?.extract()?; + let name = name.strip_prefix("__expected__").unwrap_or(&name); + let kwargs = PyDict::new(py); + kwargs.set_item( + "sql_body", + completed[3].call_method1("get", (name, cte.getattr("sql_body")?))?, + )?; + expected_ctes.push(replace.call((&cte,), Some(&kwargs))?); + } + let kwargs = PyDict::new(py); + kwargs.set_item("authored_ctes", PyTuple::new(py, authored)?)?; + kwargs.set_item("expected_ctes", PyTuple::new(py, expected_ctes)?)?; + let payload = replace.call((payload,), Some(&kwargs))?; + let kwargs = PyDict::new(py); + kwargs.set_item("payload", payload)?; + replace.call((test,), Some(&kwargs)).map(Ok) +} + +pub(super) fn validate<'py>( + tests: &[Bound<'py, PyAny>], + inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + context: &Bound<'py, PyAny>, +) -> PyResult> { + let py = context.py(); + let mut model_tests: Vec> = Vec::new(); + let mut indexes: Vec = Vec::new(); + for (index, test) in tests.iter().enumerate() { + if model_payload(test)? { + indexes.push(index); + model_tests.push(test.clone()); + } + } + if indexes.is_empty() { + return Ok((tests.to_vec(), Default::default())); + } + let chains = call_with( + &imported(py, NATIVE_PLANNING, "resolve_sql_test_model_chains")?, + &[ + ("project", inputs.0.clone()), + ("tests", PyTuple::new(py, model_tests)?.into_any()), + ], + )?; + let chains: Vec> = chains.extract()?; + if chains.len() != indexes.len() { + return Err(pyo3::exceptions::PyValueError::new_err( + "zip() argument lengths differ", + )); + } + let mut validated = tests.to_vec(); + let mut diagnostics: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for (index, chain) in indexes.into_iter().zip(chains) { + match validated_test(&tests[index], &chain, inputs, context)? { + Ok(test) => validated[index] = test, + Err(errors) => { + diagnostics.insert(index, PyTuple::new(py, errors)?.into_any()); + } + } + } + Ok((validated, diagnostics)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs new file mode 100644 index 0000000000..f39de53e1e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs @@ -0,0 +1,177 @@ +//! Adapter rendering and declared cursor-window boundaries for native test planning. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::sql_test_glue::plan_values::{model_payload, string}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +use crate::bindings::_helpers::sql_test_glue::constants::SENTINEL; +use crate::bindings::_helpers::sql_test_glue::cursor_window::{ + declared_window_cursor_models, declares_cursor_window, render_test_cursor_intrinsics, +}; + +pub(super) fn mentions(sql: &str) -> bool { + ["__cursor_start", "__cursor_end"] + .iter() + .any(|name| sql.contains(name)) +} + +pub(super) fn models<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let result = PyDict::new(py); + for (index, model) in project.getattr("models")?.try_iter()?.enumerate() { + let model = model?; + let sql = model.getattr("query_sql")?; + if mentions(&sql.extract::()?) { + result.set_item( + index, + render_test_cursor_intrinsics(sql, &model, adapter, None)?, + )?; + } + } + Ok(result) +} + +pub(super) fn overrides<'py>( + project: &Bound<'py, PyAny>, + tests: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let models = PyDict::new(py); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + models.set_item(model.getattr("name")?, model)?; + } + let tests: Vec> = tests.extract()?; + let mut indexes: Vec = Vec::new(); + let mut windowed: Vec> = Vec::new(); + for (index, test) in tests.iter().enumerate() { + if model_payload(test)? && declares_cursor_window(test)? { + indexes.push(index); + windowed.push(test.clone()); + } + } + let chains = if indexes.is_empty() { + Vec::new() + } else { + super::plan_driver::chains(project, &PyTuple::new(py, windowed)?.into_any())? + .extract::>>()? + }; + if indexes.len() != chains.len() { + return Err(pyo3::exceptions::PyValueError::new_err( + "zip() argument lengths differ", + )); + } + let chains: std::collections::BTreeMap<_, _> = indexes.into_iter().zip(chains).collect(); + let result = PyDict::new(py); + for (index, test) in tests.iter().enumerate() { + if !model_payload(test)? { + continue; + } + let overrides = test + .getattr("payload")? + .getattr("model_query_overrides")? + .cast_into::()?; + let chain = chains.get(&index); + if chain.is_none() + && !overrides + .values() + .try_iter()? + .map(|value| Ok(mentions(&value?.extract::()?))) + .collect::>>()? + .contains(&true) + { + continue; + } + let chain_names: Vec = chain + .map(|value| value.extract()) + .transpose()? + .unwrap_or_default(); + let rendered = PyDict::new(py); + for (name, sql) in overrides.iter() { + let name: String = name.extract()?; + let sql = if let Some(model) = models.get_item(&name)? { + render_test_cursor_intrinsics( + sql, + &model, + adapter, + chain_names.contains(&name).then_some(test), + )? + } else { + sql + }; + rendered.set_item(name, sql)?; + } + if chain.is_some() { + let mut chain_models: Vec> = Vec::new(); + for name in chain_names { + if let Some(model) = models.get_item(name)? { + chain_models.push(model); + } + } + for model in declared_window_cursor_models(test, chain_models)? { + let name = model.getattr("name")?; + if !rendered.contains(&name)? { + rendered.set_item( + name, + render_test_cursor_intrinsics( + model.getattr("query_sql")?, + &model, + adapter, + Some(test), + )?, + )?; + } + } + } + result.set_item(index, rendered)?; + } + Ok(result) +} + +pub(super) fn functions<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let mut result: Vec> = Vec::new(); + for function in project.getattr("functions")?.try_iter()? { + let function = function?; + let target = function.getattr("destination")?.getattr("qualified_name")?; + if target.is_none() { + continue; + } + let mut fields = vec![function.getattr("name")?]; + for method in ["render_udf_call", "render_table_function_call"] { + let rendered: String = call_with( + &adapter.getattr(method)?, + &[ + ("target", target.clone()), + ("call_suffix_sql", string(py, SENTINEL)), + ], + )? + .extract()?; + let Some((prefix, suffix)) = rendered + .split_once(SENTINEL) + .filter(|(_, suffix)| !suffix.contains(SENTINEL)) + else { + return Err(PyErr::from_value( + imported( + py, + "sqlbuild.compiler.planner.exceptions", + "NativeSqlTestPlanningError", + )? + .call1(("adapter SQL function call template is not deterministic",))?, + )); + }; + fields.push(string(py, prefix)); + fields.push(string(py, suffix)); + } + result.push(PyTuple::new(py, fields)?); + } + PyTuple::new(py, result) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs new file mode 100644 index 0000000000..2d599dabcb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs @@ -0,0 +1,84 @@ +//! Materialize native plan rows as the existing executor-facing dataclasses. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::sql_test_glue::constants::PLAN_MODELS; +use crate::bindings::_helpers::sql_test_glue::plan_values::tuple; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +fn step<'py>(row: &Bound<'py, PyAny>, chain: bool) -> PyResult> { + let py = row.py(); + let kwargs = PyDict::new(py); + let fields: &[&str] = if chain { + &[ + "model_name", + "resolved_sql", + "expected_cte_sql", + "lifted_ctes", + "comparison_body_sql", + "expected_columns", + "expected_lifted_ctes", + ] + } else { + &["name", "resolved_sql", "lifted_ctes", "comparison_body_sql"] + }; + for (index, name) in fields.iter().enumerate() { + let value = row.get_item(index)?; + kwargs.set_item( + *name, + if ["lifted_ctes", "expected_columns", "expected_lifted_ctes"].contains(name) + && !value.is_none() + { + tuple(&value)? + } else { + value + }, + )?; + } + imported( + py, + PLAN_MODELS, + if chain { + "ChainStep" + } else { + "SqlTestAssertionStep" + }, + )? + .call((), Some(&kwargs)) +} + +pub(super) fn plan<'py>(row: &Bound<'py, PyAny>) -> PyResult> { + let py = row.py(); + let mut chain: Vec> = Vec::new(); + for value in row.get_item(1)?.try_iter()? { + chain.push(step(&value?, true)?); + } + let mut assertions: Vec> = Vec::new(); + for value in row.get_item(2)?.try_iter()? { + assertions.push(step(&value?, false)?); + } + let mut warnings: Vec> = Vec::new(); + let severity = imported(py, "sqlbuild.compiler.planner.types", "WarningSeverity")?; + for value in row.get_item(4)?.try_iter()? { + let value = value?; + warnings.push(call_with( + &imported(py, PLAN_MODELS, "PlanWarning")?, + &[ + ("model_name", value.get_item(0)?), + ("severity", severity.call1((value.get_item(1)?,))?), + ("message", value.get_item(2)?), + ], + )?); + } + call_with( + &imported(py, PLAN_MODELS, "NativeSqlTestPlan")?, + &[ + ("chain", PyTuple::new(py, chain)?.into_any()), + ("assertions", PyTuple::new(py, assertions)?.into_any()), + ("model_names", tuple(&row.get_item(3)?)?), + ("warnings", PyTuple::new(py, warnings)?.into_any()), + ("error_messages", tuple(&row.get_item(5)?)?), + ("sql", row.get_item(0)?), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs new file mode 100644 index 0000000000..1a3617db47 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs @@ -0,0 +1,24 @@ +//! Python value boundaries shared by SQL-test plan assembly. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +use crate::bindings::_helpers::sql_test_glue::constants::COMPILE_MODELS; + +pub(super) fn string<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +pub(super) fn model_payload(test: &Bound<'_, PyAny>) -> PyResult { + test.getattr("payload")?.is_instance(&imported( + test.py(), + COMPILE_MODELS, + "CompiledModelSqlTestPayload", + )?) +} + +pub(super) fn tuple<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + imported(value.py(), "builtins", "tuple")?.call1((value,)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs index 2c18eb70c4..c6df114194 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs @@ -1,10 +1,31 @@ //! Register the native SQL test planning and assembly functions. -use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; use crate::bindings::_helpers::sql_test_glue::{authored_ctes, compiled_tests, planning}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + super::plan_assembly::assemble_sql_test_plans, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::plan_driver::plan_sql_tests_natively, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::plan_driver::resolve_sql_test_model_chains, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::plan_driver::plan_compiled_sql_test_artifacts, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::plan_driver::sql_test_plan_error_messages, + module + )?)?; authored_ctes::register(module)?; planning::register(module)?; compiled_tests::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/types.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/types.rs new file mode 100644 index 0000000000..707f3f2975 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/types.rs @@ -0,0 +1,6 @@ +//! SQL-test fixture validation boundary values. + +use pyo3::prelude::{Bound, PyAny}; +use std::collections::BTreeMap; + +pub(super) type ValidatedTests<'py> = (Vec>, BTreeMap>); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs new file mode 100644 index 0000000000..79a3b60291 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs @@ -0,0 +1,19 @@ +//! Python modules, names and messages of migration state rows. + +pub(crate) const MIGRATION_MODELS: &str = "sqlbuild.compiler.migrations.models"; +pub(crate) const MIGRATION_TYPES: &str = "sqlbuild.compiler.migrations.types"; +pub(crate) const MIGRATION_EXCEPTIONS: &str = "sqlbuild.compiler.migrations.exceptions"; +pub(crate) const STATE_TABLE_DDL: &str = + "sqlbuild.adapter.state_sql.main.render_state_table_create_sql"; +pub(crate) const SQL_VALUE_TYPES: &str = "sqlbuild.sql_values.types"; +pub(crate) const OLD_NAME_VIEW_CONFIG_KEY: &str = "old_name_view"; +pub(crate) const MODEL_KIND: &str = "model"; +pub(crate) const COLUMN_KIND: &str = "column"; +pub(crate) const OLD_NAME_KIND: &str = "old_name"; +pub(crate) const MIGRATION_LOGGER: &str = "sqlbuild.migrations"; +pub(crate) const LOGGING_MODULE: &str = "logging"; +pub(crate) const TIME_MODULE: &str = "time"; +pub(crate) const MIGRATION_WRITE_ATTEMPTS: i64 = 3; +pub(crate) const MIGRATION_WRITE_RETRY_BASE_SECONDS: f64 = 0.05; +pub(crate) const MIGRATION_WRITE_RETRY_MESSAGE: &str = + "migration event write attempt %s/%s failed for '%s'; retrying: %s"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs index 5b0c5d4001..dded2008a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs @@ -1,4 +1,4 @@ -//! Migration event IDs and the SQL that appends and looks up one event row. +//! Migration event IDs and the typed literals of one appended event row. use pyo3::exceptions::PyValueError; use pyo3::prelude::{ @@ -9,7 +9,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::json::errors::JsonEmitError; use sqlbuild_state::events::errors::StateSqlError; use sqlbuild_state::events::main::column_migration_event_id::column_migration_event_id; -use sqlbuild_state::events::main::event_exists_sql::event_exists_sql; +use sqlbuild_state::events::main::event_cells::event_cells; use sqlbuild_state::events::main::event_insert_sql::event_insert_sql; use sqlbuild_state::events::main::migration_event_id::migration_event_id; use sqlbuild_state::events::main::old_name_view_event_id::old_name_view_event_id; @@ -20,14 +20,12 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::state_rows::timestamps::{date_time_typed, presumed_utc}; const SQL_VALUES_EXCEPTIONS: &str = "sqlbuild.sql_values.exceptions"; -const STRING_TYPE: &str = "string"; -const TEXT_TIMESTAMP_TYPE: &str = "text_timestamp"; fn emit_error(error: JsonEmitError) -> PyErr { PyValueError::new_err(format!("{error:?}")) } -fn relation(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn relation(value: &Bound<'_, PyAny>) -> PyResult { Ok(EventRelation { database: value.getattr("database")?.extract()?, schema: value.getattr("schema")?.extract()?, @@ -77,13 +75,6 @@ fn old_name_id(event_type: &Bound<'_, PyAny>, migration_event_id: &str) -> PyRes old_name_view_event_id(&event_type, migration_event_id).map_err(emit_error) } -/// The lookup that makes an append-only event write idempotent. -#[pyfunction(name = "render_event_exists_sql")] -#[pyo3(signature = (*, table, event_id))] -fn exists_sql(table: &str, event_id: &str) -> String { - event_exists_sql(table, event_id) -} - fn state_value(value: &Bound<'_, PyAny>, column_type: StateColumnType) -> PyResult { if value.is_none() { return Ok(StateValue::Null); @@ -97,49 +88,23 @@ fn state_value(value: &Bound<'_, PyAny>, column_type: StateColumnType) -> PyResu Ok(StateValue::Other) } -fn column_type(declared: &Bound<'_, PyAny>) -> PyResult { - let name: String = declared.str()?.extract()?; - match name.as_str() { - STRING_TYPE => Ok(StateColumnType::String), - TEXT_TIMESTAMP_TYPE => Ok(StateColumnType::TextTimestamp), - _ => Err(PyValueError::new_err(format!( - "migration event columns are string or text timestamps, not {name}" - ))), - } -} - -/// One append-only event row as `INSERT ... VALUES` with typed literals. -#[pyfunction(name = "render_event_insert_sql")] -#[pyo3(signature = (*, table, columns, column_types, values))] -fn insert_sql( +/// One append-only event row as `INSERT ... VALUES` with typed literals, failing like Python. +pub(crate) fn typed_insert_sql( + py: Python<'_>, table: &str, - columns: Vec, - column_types: &Bound<'_, PyAny>, - values: Vec>, + (columns, values): (&[&str], &[Bound<'_, PyAny>]), ) -> PyResult { - if columns.len() != values.len() { - return Err(PyValueError::new_err( - "event columns and values must have the same length", - )); - } - let mut cells: Vec = Vec::with_capacity(columns.len()); - for (column, value) in columns.into_iter().zip(&values) { - let column_type: StateColumnType = column_type(&column_types.get_item(&column)?)?; - let converted: PyResult = state_value(value, column_type); - let value: StateValue = match converted { - Ok(value) => value, + let mut cells: Vec = event_cells(columns, vec![StateValue::Null; values.len()]); + for (index, value) in values.iter().enumerate() { + match state_value(value, cells[index].column_type) { + Ok(converted) => cells[index].value = converted, Err(error) => { - event_insert_sql(table, &cells).map_err(rendering_error(value.py()))?; + event_insert_sql(table, &cells[..index]).map_err(rendering_error(py))?; return Err(error); } - }; - cells.push(StateCell { - column, - column_type, - value, - }); + } } - event_insert_sql(table, &cells).map_err(rendering_error(column_types.py())) + event_insert_sql(table, &cells).map_err(rendering_error(py)) } /// `SqlValueRenderingError(message)` for an ill-typed value. @@ -158,7 +123,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(migration_id, module)?)?; module.add_function(wrap_pyfunction!(column_id, module)?)?; module.add_function(wrap_pyfunction!(old_name_id, module)?)?; - module.add_function(wrap_pyfunction!(exists_sql, module)?)?; - module.add_function(wrap_pyfunction!(insert_sql, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs new file mode 100644 index 0000000000..732a2326b3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs @@ -0,0 +1,326 @@ +//! Read and append migration state events through the adapter's execute callable. + +use pyo3::exceptions::PyException; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyString, PyTuple, PyTupleMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_state::events::constants::OLD_NAME_VIEW_TABLE_NAME; +use sqlbuild_state::events::main::recorded_renames::recorded_renames; +use sqlbuild_state::events::models::RecordedRenameKey; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::state_rows::constants::{ + COLUMN_KIND, LOGGING_MODULE, MIGRATION_LOGGER, MIGRATION_WRITE_ATTEMPTS, + MIGRATION_WRITE_RETRY_BASE_SECONDS, MIGRATION_WRITE_RETRY_MESSAGE, MODEL_KIND, OLD_NAME_KIND, + TIME_MODULE, +}; +use crate::bindings::_helpers::state_rows::events::relation as event_relation; +use crate::bindings::_helpers::state_rows::migration_objects::{event_order, layout, state_error}; +use crate::bindings::_helpers::state_rows::migration_projections::old_name_views; +use crate::bindings::_helpers::state_rows::migration_rows::{ + decoded_events, read_statement, write_statements, +}; +use crate::bindings::_helpers::state_rows::models::{EventTable, EventTableLayout, StateAccess}; +use crate::bindings::_helpers::state_rows::timestamps::presumed_utc; + +fn execute<'py>(access: &StateAccess<'py>, sql: &str) -> PyResult> { + let py: Python<'py> = access.execute.py(); + call_with( + &access.execute, + &[ + ("connection", access.connection.clone()), + ("sql", PyString::new(py, sql).into_any()), + ], + ) +} + +fn unreadable(layout: &EventTableLayout, schema: &Bound<'_, PyAny>, error: PyErr) -> PyErr { + let py: Python<'_> = schema.py(); + let describe = |value: &Bound<'_, PyAny>| -> String { + value.str().map(|text| text.to_string()).unwrap_or_default() + }; + let message: String = format!( + "Unable to read {} from schema '{}': {}", + layout.read_subject, + describe(schema), + describe(error.value(py).as_any()), + ); + let wrapped: PyErr = state_error(py, &message); + wrapped.set_cause(py, Some(error)); + wrapped +} + +/// Every event of `layout`'s table this version reads from one schema, from `stored` columns when known. +pub(crate) fn read_events<'py>( + access: &StateAccess<'py>, + layout: &EventTableLayout, + (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), + stored: Option<&[String]>, +) -> PyResult> { + let py: Python<'py> = schema.py(); + let (sql, columns): (String, Vec) = read_statement( + layout, + &access.render_qualified_name, + (database, schema.clone()), + stored, + )?; + let rows: Bound<'py, PyAny> = + match execute(access, &sql).and_then(|cursor| cursor.call_method0("fetchall")) { + Ok(rows) => rows, + Err(error) if error.is_instance_of::(py) => { + return Err(unreadable(layout, &schema, error)); + } + Err(error) => return Err(error), + }; + decoded_events(layout.table, &rows, &columns) +} + +/// The column names of one schema's old-name view state table, as the adapter lists them. +pub(crate) fn stored_old_name_columns<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let columns: Bound<'py, PyAny> = call_with( + &adapter.getattr("get_columns")?, + &[ + ("connection", connection.clone()), + ("database", database), + ("schema", schema), + ( + "name", + PyString::new(py, OLD_NAME_VIEW_TABLE_NAME).into_any(), + ), + ], + )?; + columns + .try_iter()? + .map(|column| column?.getattr("name")?.extract()) + .collect() +} + +fn subject(table: EventTable, event: &Bound<'_, PyAny>) -> PyResult { + let text = |value: Bound<'_, PyAny>| -> PyResult { Ok(value.str()?.to_string()) }; + Ok(match table { + EventTable::Model => text(event.getattr("destination_model")?)?, + EventTable::Column => format!( + "{}.{}", + text(event.getattr("model_name")?)?, + text(event.getattr("destination_column")?)? + ), + EventTable::OldName => format!( + "{} old name {}", + text(event.getattr("destination_model")?)?, + text(event.getattr("old")?.getattr("name")?)? + ), + }) +} + +fn append_once( + access: &StateAccess<'_>, + create_table_sql: Option<&str>, + (existing_sql, insert_sql): (&str, &str), +) -> PyResult<()> { + if let Some(create_table_sql) = create_table_sql { + execute(access, create_table_sql)?; + } + if !execute(access, existing_sql)? + .call_method0("fetchall")? + .is_truthy()? + { + execute(access, insert_sql)?; + } + Ok(()) +} + +fn retry_pause( + py: Python<'_>, + (attempt, attempts): (i64, i64), + subject: &str, + error: &PyErr, +) -> PyResult<()> { + let logger: Bound<'_, PyAny> = py + .import(LOGGING_MODULE)? + .call_method1("getLogger", (MIGRATION_LOGGER,))?; + logger.call_method1( + "warning", + ( + MIGRATION_WRITE_RETRY_MESSAGE, + attempt + 1, + attempts, + subject, + error.value(py), + ), + )?; + let step: f64 = f64::from(i32::try_from(attempt + 1).unwrap_or(i32::MAX)); + py.import(TIME_MODULE)? + .call_method1("sleep", (MIGRATION_WRITE_RETRY_BASE_SECONDS * step,))?; + Ok(()) +} + +/// Create the state table when given DDL, then insert the event unless its ID is recorded. +fn append_event( + access: &StateAccess<'_>, + (kind, event): (&str, &Bound<'_, PyAny>), + create_table_sql: Option<&str>, + attempts: i64, +) -> PyResult<()> { + let py: Python<'_> = event.py(); + let layout: EventTableLayout = layout(kind)?; + let (existing_sql, insert_sql): (String, String) = + write_statements(&layout, event, &access.render_qualified_name)?; + let subject: String = subject(layout.table, event)?; + for attempt in 0..attempts { + let Err(error) = append_once(access, create_table_sql, (&existing_sql, &insert_sql)) else { + return Ok(()); + }; + if attempt + 1 == attempts || !error.is_instance_of::(py) { + return Err(error); + } + retry_pause(py, (attempt, attempts), &subject, &error)?; + } + Ok(()) +} + +fn access<'py>(request: &Request<'_, 'py>) -> PyResult> { + Ok(StateAccess { + connection: request.get("connection")?, + execute: request.get("execute")?, + render_qualified_name: request.get("render_qualified_name")?, + }) +} + +fn stored_names(stored: Option>) -> PyResult>> { + match stored { + Some(stored) if !stored.is_none() => stored + .try_iter()? + .map(|column| column?.extract()) + .collect::>>() + .map(Some), + _ => Ok(None), + } +} + +/// Every event of one migration state table this version reads from one schema. +#[pyfunction(name = "read_migration_state_events")] +#[pyo3(signature = (**request))] +fn read_state_events<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("read_migration_state_events", request)?; + let stored: Option> = stored_names(request.optional("stored_columns")?)?; + read_events( + &access(&request)?, + &layout(&request.get("kind")?.extract::()?)?, + (request.get("database")?, request.get("schema")?), + stored.as_deref(), + ) +} + +/// The old-name view history of every recorded move in a schema that holds both state tables. +#[pyfunction(name = "read_old_name_view_histories")] +#[pyo3(signature = (**request))] +fn read_histories<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("read_old_name_view_histories", request)?; + let access: StateAccess<'py> = access(&request)?; + let location = || -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + Ok((request.get("database")?, request.get("schema")?)) + }; + let stored: Option> = stored_names(Some(request.get("stored_columns")?))?; + let moves: Bound<'py, PyTuple> = read_events(&access, &layout(MODEL_KIND)?, location()?, None)?; + let facts: Bound<'py, PyTuple> = read_events( + &access, + &layout(OLD_NAME_KIND)?, + location()?, + stored.as_deref(), + )?; + old_name_views(moves.as_any(), facts.as_any()) +} + +/// Renames recorded on the relation at or after `since`, oldest first. +#[pyfunction(name = "read_recorded_column_renames")] +#[pyo3(signature = (**request))] +fn read_renames<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("read_recorded_column_renames", request)?; + let py: Python<'py> = request.py(); + let relation: Bound<'py, PyAny> = request.get("relation")?; + let since = presumed_utc(&request.get("since")?)?; + let schema: Bound<'py, PyAny> = relation.getattr("schema")?; + let schema: Bound<'py, PyAny> = if schema.is_none() { + PyString::new(py, "").into_any() + } else { + schema + }; + let events: Bound<'py, PyTuple> = read_events( + &access(&request)?, + &layout(COLUMN_KIND)?, + (relation.getattr("database")?, schema), + None, + )?; + let keys: Vec = events + .iter() + .map(|event| { + Ok(RecordedRenameKey { + order: event_order(&event)?, + relation: event_relation(&event.getattr("relation")?)?, + records_event: event + .getattr("decision")? + .getattr("records_event")? + .is_truthy()?, + }) + }) + .collect::>()?; + let target = event_relation(&relation)?; + let selected: Vec> = recorded_renames(&keys, &target, &since) + .into_iter() + .map(|index| events.get_item(index)) + .collect::>()?; + PyTuple::new(py, selected) +} + +/// The column names of one schema's old-name view state table. +#[pyfunction(name = "stored_old_name_view_columns")] +#[pyo3(signature = (*, adapter, connection, database, schema))] +fn stored_columns_binding<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + database: Bound<'py, PyAny>, + schema: Bound<'py, PyAny>, +) -> PyResult> { + PyTuple::new( + adapter.py(), + stored_old_name_columns(adapter, connection, (database, schema))?, + ) +} + +/// Create the state table from adapter DDL when given, then append the event unless recorded. +#[pyfunction(name = "write_migration_state_event")] +#[pyo3(signature = (**request))] +fn write_state_event(request: Option<&Bound<'_, PyDict>>) -> PyResult<()> { + let request = Request::new("write_migration_state_event", request)?; + let create_table_sql: Option = request.get("create_table_sql")?.extract()?; + let attempts: i64 = match request.optional("attempts")? { + Some(attempts) => attempts.extract()?, + None => MIGRATION_WRITE_ATTEMPTS, + }; + append_event( + &access(&request)?, + ( + &request.get("kind")?.extract::()?, + &request.get("event")?, + ), + create_table_sql.as_deref(), + attempts, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(read_state_events, module)?)?; + module.add_function(wrap_pyfunction!(read_histories, module)?)?; + module.add_function(wrap_pyfunction!(read_renames, module)?)?; + module.add_function(wrap_pyfunction!(stored_columns_binding, module)?)?; + module.add_function(wrap_pyfunction!(write_state_event, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs new file mode 100644 index 0000000000..43cec94da4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs @@ -0,0 +1,92 @@ +//! Migration state table layouts and the Python objects stored in them. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_state::events::constants::{ + COLUMN_MIGRATION_COLUMNS, COLUMN_MIGRATION_TABLE_NAME, MIGRATION_COLUMNS, MIGRATION_TABLE_NAME, + OLD_NAME_VIEW_COLUMNS, OLD_NAME_VIEW_TABLE_NAME, +}; +use sqlbuild_state::events::models::EventOrder; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::state_rows::constants::{ + COLUMN_KIND, MIGRATION_EXCEPTIONS, MODEL_KIND, OLD_NAME_KIND, +}; +use crate::bindings::_helpers::state_rows::models::{EventTable, EventTableLayout}; +use crate::bindings::_helpers::state_rows::timestamps::presumed_utc; + +/// The layout of the state table `kind` names: `model`, `column` or `old_name`. +pub(crate) fn layout(kind: &str) -> PyResult { + match kind { + MODEL_KIND => Ok(EventTableLayout { + table: EventTable::Model, + name: MIGRATION_TABLE_NAME, + columns: &MIGRATION_COLUMNS, + relation_attribute: "destination", + state_message: "model migration state requires a target schema", + event_message: "model migration events require a destination schema", + read_subject: "model migration events", + }), + COLUMN_KIND => Ok(EventTableLayout { + table: EventTable::Column, + name: COLUMN_MIGRATION_TABLE_NAME, + columns: &COLUMN_MIGRATION_COLUMNS, + relation_attribute: "relation", + state_message: "column migration state requires a target schema", + event_message: "column migration events require a relation schema", + read_subject: "column migration events", + }), + OLD_NAME_KIND => Ok(EventTableLayout { + table: EventTable::OldName, + name: OLD_NAME_VIEW_TABLE_NAME, + columns: &OLD_NAME_VIEW_COLUMNS, + relation_attribute: "new", + state_message: "old-name view state requires a target schema", + event_message: "old-name view facts require a destination schema", + read_subject: "old-name view facts", + }), + _ => Err(PyValueError::new_err(format!( + "unknown migration state table kind: {kind}" + ))), + } +} + +/// `MigrationStateError(message)`. +pub(crate) fn state_error(py: Python<'_>, message: &str) -> PyErr { + match imported(py, MIGRATION_EXCEPTIONS, "MigrationStateError") + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// The adapter-qualified name of `layout`'s table in one schema. +pub(crate) fn qualified_table<'py>( + layout: &EventTableLayout, + render_qualified_name: &Bound<'py, PyAny>, + (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult { + let py: Python<'py> = render_qualified_name.py(); + let table: Bound<'py, PyAny> = call_with( + render_qualified_name, + &[ + ("database", database), + ("schema", schema), + ("name", PyString::new(py, layout.name).into_any()), + ], + )?; + if table.is_none() { + return Err(state_error(py, layout.state_message)); + } + table.extract() +} + +/// The history order of a stored event: its creation time in UTC, then its ID. +pub(crate) fn event_order(event: &Bound<'_, PyAny>) -> PyResult { + Ok(EventOrder { + created_at: presumed_utc(&event.getattr("created_at")?)?, + event_id: event.getattr("event_id")?.extract()?, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_projections.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_projections.rs new file mode 100644 index 0000000000..cb87db8e4c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_projections.rs @@ -0,0 +1,191 @@ +//! Current-state projections over recorded migration events. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_state::events::main::newest_column_event::newest_column_event; +use sqlbuild_state::events::main::newest_move_event::newest_move_event; +use sqlbuild_state::events::main::old_name_histories::old_name_histories; +use sqlbuild_state::events::models::{ + ColumnEventKey, EventOrder, EventRelation, MoveEventKey, OldNameFactKey, OldNameStep, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::state_rows::constants::{ + MIGRATION_MODELS, OLD_NAME_VIEW_CONFIG_KEY, +}; +use crate::bindings::_helpers::state_rows::events::relation as event_relation; +use crate::bindings::_helpers::state_rows::migration_objects::event_order; + +fn collected<'py>(events: &Bound<'py, PyAny>) -> PyResult>> { + events.try_iter()?.collect() +} + +fn chosen<'py>( + py: Python<'py>, + events: &[Bound<'py, PyAny>], + index: Option, +) -> Bound<'py, PyAny> { + index + .and_then(|index| events.get(index).cloned()) + .unwrap_or_else(|| py.None().into_bound(py)) +} + +/// The latest recorded move naming `relation` as origin or destination, or `None`. +pub(crate) fn newest_move<'py>( + events: &Bound<'py, PyAny>, + relation: &Bound<'py, PyAny>, +) -> PyResult> { + let events: Vec> = collected(events)?; + let keys: Vec = events + .iter() + .map(|event| { + Ok(MoveEventKey { + order: event_order(event)?, + origin: event_relation(&event.getattr("origin")?)?, + destination: event_relation(&event.getattr("destination")?)?, + }) + }) + .collect::>()?; + let index: Option = newest_move_event(&keys, &event_relation(relation)?); + Ok(chosen(relation.py(), &events, index)) +} + +/// The latest recorded rename on `relation` naming either column, or `None`. +pub(crate) fn newest_column<'py>( + events: &Bound<'py, PyAny>, + relation: &Bound<'py, PyAny>, + (origin_column, destination_column): (&str, &str), +) -> PyResult> { + let events: Vec> = collected(events)?; + let keys: Vec = events + .iter() + .map(|event| { + Ok(ColumnEventKey { + order: event_order(event)?, + relation: event_relation(&event.getattr("relation")?)?, + origin_column: event.getattr("origin_column")?.extract()?, + destination_column: event.getattr("destination_column")?.extract()?, + }) + }) + .collect::>()?; + let target: EventRelation = event_relation(relation)?; + let index: Option = + newest_column_event(&keys, &target, (origin_column, destination_column)); + Ok(chosen(relation.py(), &events, index)) +} + +fn step(fact: &Bound<'_, PyAny>) -> PyResult { + let value: String = fact.getattr("event_type")?.getattr("value")?.extract()?; + match value.as_str() { + "required" => Ok(OldNameStep::Required), + "origin_archived" => Ok(OldNameStep::OriginArchived), + "view_created" => Ok(OldNameStep::ViewCreated), + "view_dropped" => Ok(OldNameStep::ViewDropped), + _ => Err(PyValueError::new_err(format!( + "unknown old-name view event type: {value}" + ))), + } +} + +/// One history per recorded move whose old name needs a compatibility view, in move order. +pub(crate) fn old_name_views<'py>( + moves: &Bound<'py, PyAny>, + facts: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = moves.py(); + let moves: Vec> = collected(moves)?; + let facts: Vec> = collected(facts)?; + let move_orders: Vec = moves.iter().map(event_order).collect::>()?; + let fact_keys: Vec = facts + .iter() + .map(|fact| { + Ok(OldNameFactKey { + order: event_order(fact)?, + migration_event_id: fact.getattr("migration_event_id")?.extract()?, + step: step(fact)?, + }) + }) + .collect::>()?; + let history = imported(py, MIGRATION_MODELS, "OldNameViewHistory")?; + let fact_at = |index: Option| chosen(py, &facts, index); + let histories: Vec> = old_name_histories(&move_orders, &fact_keys) + .into_iter() + .map(|indexes| { + call_with( + &history, + &[ + ("move", moves[indexes.moved].clone()), + ("required", facts[indexes.required].clone()), + ("archived", fact_at(indexes.archived)), + ("created", fact_at(indexes.created)), + ("dropped", fact_at(indexes.dropped)), + ], + ) + }) + .collect::>()?; + PyTuple::new(py, histories) +} + +/// The model's old-name view retention, the project's when unset, or `None` when disabled. +pub(crate) fn old_name_retention<'py>( + config_values: &Bound<'py, PyAny>, + project_retention: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = config_values.py(); + if !config_values.contains(OLD_NAME_VIEW_CONFIG_KEY)? { + return Ok(project_retention.clone()); + } + let value: Bound<'py, PyAny> = config_values.get_item(OLD_NAME_VIEW_CONFIG_KEY)?; + if value.is(PyBool::new(py, false)) { + return Ok(py.None().into_bound(py)); + } + value.str()?.call_method0("strip") +} + +#[pyfunction(name = "newest_migration_event_mentioning")] +#[pyo3(signature = (*, events, relation))] +fn newest_move_binding<'py>( + events: &Bound<'py, PyAny>, + relation: &Bound<'py, PyAny>, +) -> PyResult> { + newest_move(events, relation) +} + +#[pyfunction(name = "newest_column_migration_event_mentioning")] +#[pyo3(signature = (*, events, relation, origin_column, destination_column))] +fn newest_column_binding<'py>( + events: &Bound<'py, PyAny>, + relation: &Bound<'py, PyAny>, + origin_column: &str, + destination_column: &str, +) -> PyResult> { + newest_column(events, relation, (origin_column, destination_column)) +} + +#[pyfunction(name = "project_old_name_views")] +#[pyo3(signature = (*, moves, facts))] +fn old_name_views_binding<'py>( + moves: &Bound<'py, PyAny>, + facts: &Bound<'py, PyAny>, +) -> PyResult> { + old_name_views(moves, facts) +} + +#[pyfunction(name = "resolve_old_name_view_retention")] +#[pyo3(signature = (*, config_values, project_retention))] +fn retention_binding<'py>( + config_values: &Bound<'py, PyAny>, + project_retention: &Bound<'py, PyAny>, +) -> PyResult> { + old_name_retention(config_values, project_retention) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(newest_move_binding, module)?)?; + module.add_function(wrap_pyfunction!(newest_column_binding, module)?)?; + module.add_function(wrap_pyfunction!(old_name_views_binding, module)?)?; + module.add_function(wrap_pyfunction!(retention_binding, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs new file mode 100644 index 0000000000..04c014d2d3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs @@ -0,0 +1,418 @@ +//! Create, read and append SQL of migration state tables, and stored rows decoded to events. + +use pyo3::exceptions::{PyKeyError, PyValueError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_state::events::constants::TIMESTAMP_COLUMNS; +use sqlbuild_state::events::main::column_aliases_json::column_aliases_json; +use sqlbuild_state::events::main::decoded_column_aliases::decoded_column_aliases; +use sqlbuild_state::events::main::decoded_grants::decoded_grants; +use sqlbuild_state::events::main::event_exists_sql::event_exists_sql; +use sqlbuild_state::events::main::event_read_sql::event_read_sql; +use sqlbuild_state::events::main::grants_json::grants_json; +use sqlbuild_state::events::main::readable_columns::readable_columns; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; +use crate::bindings::_helpers::state_rows::constants::{ + MIGRATION_MODELS, MIGRATION_TYPES, SQL_VALUE_TYPES, STATE_TABLE_DDL, +}; +use crate::bindings::_helpers::state_rows::events::typed_insert_sql; +use crate::bindings::_helpers::state_rows::migration_objects::{ + layout, qualified_table, state_error, +}; +use crate::bindings::_helpers::state_rows::models::{EventTable, EventTableLayout}; +use crate::bindings::_helpers::state_rows::timestamps::date_time_typed; + +/// DDL that creates one migration state table when it is missing. +#[pyfunction(name = "render_migration_state_create_sql")] +#[pyo3(signature = (**request))] +fn create_sql<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("render_migration_state_create_sql", request)?; + let py: Python<'py> = request.py(); + let layout: EventTableLayout = layout(&request.get("kind")?.extract::()?)?; + let table: String = qualified_table( + &layout, + &request.get("render_qualified_name")?, + (request.get("database")?, request.get("schema")?), + )?; + let value_types: Bound<'py, PyAny> = imported(py, SQL_VALUE_TYPES, "StateSqlValueType")?; + let column_types: Bound<'py, PyDict> = PyDict::new(py); + for column in layout.columns { + let member: &str = if TIMESTAMP_COLUMNS.contains(column) { + "TEXT_TIMESTAMP" + } else { + "STRING" + }; + column_types.set_item(*column, value_types.getattr(member)?)?; + } + call_with( + &imported(py, STATE_TABLE_DDL, "render_state_table_create_sql")?, + &[ + ("qualified_name", PyString::new(py, &table).into_any()), + ("columns", PyTuple::new(py, layout.columns)?.into_any()), + ("column_types", column_types.into_any()), + ("required_columns", PyFrozenSet::empty(py)?.into_any()), + ( + "render_framework_type", + request.get("render_framework_type")?, + ), + ], + ) +} + +/// The SQL reading every stored event of one schema, and the columns it selects. +pub(crate) fn read_statement<'py>( + layout: &EventTableLayout, + render_qualified_name: &Bound<'py, PyAny>, + location: (Bound<'py, PyAny>, Bound<'py, PyAny>), + stored: Option<&[String]>, +) -> PyResult<(String, Vec)> { + let columns: Vec<&str> = match stored { + Some(stored) => readable_columns(layout.columns, stored), + None => layout.columns.to_vec(), + }; + let table: String = qualified_table(layout, render_qualified_name, location)?; + Ok(( + event_read_sql(&table, &columns), + columns.into_iter().map(str::to_owned).collect(), + )) +} + +fn optional<'py>(value: Bound<'py, PyAny>, attribute: &str) -> PyResult> { + if value.is_none() { + Ok(value) + } else { + value.getattr(attribute) + } +} + +fn text<'py>(py: Python<'py>, value: Option) -> Bound<'py, PyAny> { + match value { + Some(value) => PyString::new(py, &value).into_any(), + None => py.None().into_bound(py), + } +} + +fn row_values<'py>( + table: EventTable, + event: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = event.py(); + let field = |name: &str| event.getattr(name); + let relation = |name: &str| -> PyResult<[Bound<'py, PyAny>; 3]> { + let relation: Bound<'py, PyAny> = event.getattr(name)?; + Ok([ + relation.getattr("database")?, + relation.getattr("schema")?, + relation.getattr("name")?, + ]) + }; + let mut values: Vec> = vec![field("event_id")?, field("target_name")?]; + match table { + EventTable::Model => { + values.push(field("origin_model")?); + values.extend(relation("origin")?); + values.push(field("destination_model")?); + values.extend(relation("destination")?); + values.push(field("origin_version_hash")?); + } + EventTable::Column => { + values.push(field("model_name")?); + values.extend(relation("relation")?); + values.push(field("origin_column")?); + values.push(field("destination_column")?); + } + EventTable::OldName => { + values.push(field("event_type")?.getattr("value")?); + values.push(field("migration_event_id")?); + values.push(field("destination_model")?); + values.extend(relation("old")?); + values.extend(relation("new")?); + values.push(field("view_retention")?); + values.push(field("archive_name")?); + let aliases: Vec<(String, String)> = field("column_aliases")?.extract()?; + let encoded = column_aliases_json(&aliases).map_err(json_error)?; + values.push(text(py, encoded)); + let grants: Option> = field("grants_copied")?.extract()?; + values.push(text( + py, + grants_json(grants.as_deref()).map_err(json_error)?, + )); + values.push(field("view_sql")?); + values.push(field("expires_at")?); + values.push(optional(field("drop_reason")?, "value")?); + values.push(field("run_id")?); + values.push(field("created_at")?); + return Ok(values); + } + } + values.push(field("discovery")?.getattr("value")?); + values.push(field("decision")?.getattr("value")?); + values.push(field("run_id")?); + values.push(field("created_at")?); + Ok(values) +} + +fn json_error(error: sqlbuild_core::json::errors::JsonEmitError) -> PyErr { + PyValueError::new_err(format!("{error:?}")) +} + +/// The idempotency lookup and the insert that append one event to its schema's state table. +pub(crate) fn write_statements<'py>( + layout: &EventTableLayout, + event: &Bound<'py, PyAny>, + render_qualified_name: &Bound<'py, PyAny>, +) -> PyResult<(String, String)> { + let py: Python<'py> = event.py(); + let relation: Bound<'py, PyAny> = event.getattr(layout.relation_attribute)?; + let schema: Bound<'py, PyAny> = relation.getattr("schema")?; + if schema.is_none() { + return Err(state_error(py, layout.event_message)); + } + let table: String = qualified_table( + layout, + render_qualified_name, + (relation.getattr("database")?, schema), + )?; + let event_id: String = event.getattr("event_id")?.extract()?; + let values: Vec> = row_values(layout.table, event)?; + let insert: String = typed_insert_sql(py, &table, (layout.columns, &values))?; + Ok((event_exists_sql(&table, &event_id), insert)) +} + +/// One stored row keyed by the columns its query selected. +struct StoredRow<'a, 'py> { + columns: &'a [String], + values: Vec>, +} + +impl<'py> StoredRow<'_, 'py> { + fn get(&self, column: &str) -> Option<&Bound<'py, PyAny>> { + self.columns + .iter() + .position(|name| name == column) + .and_then(|index| self.values.get(index)) + } + + fn required(&self, column: &str) -> PyResult> { + self.get(column) + .cloned() + .ok_or_else(|| PyKeyError::new_err(column.to_owned())) + } + + fn text(&self, column: &str) -> PyResult> { + Ok(self.required(column)?.str()?.into_any()) + } + + fn optional_text(&self, column: &str) -> PyResult>> { + match self.get(column) { + Some(value) if !value.is_none() => Ok(Some(value.str()?.into_any())), + _ => Ok(None), + } + } + + fn nullable(&self, py: Python<'py>, column: &str) -> PyResult> { + Ok(self + .optional_text(column)? + .unwrap_or_else(|| py.None().into_bound(py))) + } + + fn relation(&self, py: Python<'py>, prefix: &str) -> PyResult> { + call_with( + &imported(py, MIGRATION_MODELS, "MigrationRelation")?, + &[ + ( + "database", + self.nullable(py, &format!("{prefix}_database"))?, + ), + ("schema", self.nullable(py, &format!("{prefix}_schema"))?), + ("name", self.text(&format!("{prefix}_name"))?), + ], + ) + } +} + +/// `decode_event_timestamp`: a stored timestamp as an aware datetime, naive values read as UTC. +fn timestamp<'py>(raw: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = raw.py(); + let parsed: Bound<'py, PyAny> = if date_time_typed(raw)? { + raw.clone() + } else { + imported(py, DATETIME_MODULE, "datetime")?.call_method1("fromisoformat", (raw.str()?,))? + }; + if !parsed.getattr("tzinfo")?.is_none() { + return Ok(parsed); + } + let utc: Bound<'py, PyDict> = PyDict::new(py); + utc.set_item("tzinfo", imported(py, DATETIME_MODULE, "UTC")?)?; + parsed.call_method("replace", (), Some(&utc)) +} + +fn enumerated<'py>( + py: Python<'py>, + class: &str, + value: Bound<'py, PyAny>, +) -> PyResult> { + imported(py, MIGRATION_TYPES, class)?.call1((value,)) +} + +fn known_member<'py>( + py: Python<'py>, + class: &str, + value: Bound<'py, PyAny>, +) -> PyResult>> { + match enumerated(py, class, value) { + Ok(member) => Ok(Some(member)), + Err(error) if error.is_instance_of::(py) => Ok(None), + Err(error) => Err(error), + } +} + +fn model_event<'py>(py: Python<'py>, row: &StoredRow<'_, 'py>) -> PyResult> { + let created_at: Bound<'py, PyAny> = timestamp(&row.required("created_at")?)?; + let version_hash: Bound<'py, PyAny> = row.required("origin_version_hash")?; + let version_hash: Bound<'py, PyAny> = if version_hash.is_truthy()? { + version_hash.str()?.into_any() + } else { + PyString::new(py, "").into_any() + }; + call_with( + &imported(py, MIGRATION_MODELS, "MigrationEvent")?, + &[ + ("event_id", row.text("event_id")?), + ("target_name", row.nullable(py, "target_name")?), + ("origin_model", row.nullable(py, "origin_model")?), + ("origin", row.relation(py, "origin")?), + ("destination_model", row.text("destination_model")?), + ("destination", row.relation(py, "destination")?), + ("origin_version_hash", version_hash), + ( + "discovery", + enumerated(py, "MigrationDiscovery", row.text("discovery")?)?, + ), + ( + "decision", + enumerated(py, "MigrationDecision", row.text("decision")?)?, + ), + ("run_id", row.text("run_id")?), + ("created_at", created_at), + ], + ) +} + +fn column_event<'py>(py: Python<'py>, row: &StoredRow<'_, 'py>) -> PyResult> { + call_with( + &imported(py, MIGRATION_MODELS, "ColumnMigrationEvent")?, + &[ + ("event_id", row.text("event_id")?), + ("target_name", row.nullable(py, "target_name")?), + ("model_name", row.text("model_name")?), + ("relation", row.relation(py, "relation")?), + ("origin_column", row.text("origin_column")?), + ("destination_column", row.text("destination_column")?), + ( + "discovery", + enumerated(py, "MigrationDiscovery", row.text("discovery")?)?, + ), + ( + "decision", + enumerated(py, "ColumnMigrationDecision", row.text("decision")?)?, + ), + ("run_id", row.text("run_id")?), + ("created_at", timestamp(&row.required("created_at")?)?), + ], + ) +} + +fn stored_text(row: &StoredRow<'_, '_>, column: &str) -> PyResult> { + row.optional_text(column)? + .map(|value| value.extract()) + .transpose() +} + +fn old_name_event<'py>( + py: Python<'py>, + row: &StoredRow<'_, 'py>, +) -> PyResult>> { + let raw_type: Bound<'py, PyAny> = match row.get("event_type") { + Some(value) => value.str()?.into_any(), + None => PyString::new(py, "None").into_any(), + }; + let Some(event_type) = known_member(py, "OldNameViewEventType", raw_type)? else { + return Ok(None); + }; + let drop_reason: Option> = match row.optional_text("drop_reason")? { + Some(raw) => known_member(py, "OldNameViewDropReason", raw)?, + None => None, + }; + let none = || py.None().into_bound(py); + let expires_at: Bound<'py, PyAny> = match row.get("expires_at") { + Some(raw) if !raw.is_none() => timestamp(raw)?, + _ => none(), + }; + let aliases: Vec<(String, String)> = + decoded_column_aliases(stored_text(row, "column_aliases")?.as_deref()); + let grants: Bound<'py, PyAny> = + match decoded_grants(stored_text(row, "grants_copied")?.as_deref()) { + Some(grants) => PyTuple::new(py, grants)?.into_any(), + None => none(), + }; + let fields: [(&str, Bound<'py, PyAny>); 16] = [ + ("event_id", row.text("event_id")?), + ("target_name", row.nullable(py, "target_name")?), + ("event_type", event_type), + ("migration_event_id", row.text("migration_event_id")?), + ("destination_model", row.text("destination_model")?), + ("old", row.relation(py, "old")?), + ("new", row.relation(py, "new")?), + ("run_id", row.text("run_id")?), + ("created_at", timestamp(&row.required("created_at")?)?), + ("view_retention", row.nullable(py, "view_retention")?), + ("archive_name", row.nullable(py, "archive_name")?), + ("column_aliases", PyTuple::new(py, aliases)?.into_any()), + ("expires_at", expires_at), + ("drop_reason", drop_reason.unwrap_or_else(none)), + ("grants_copied", grants), + ("view_sql", row.nullable(py, "view_sql")?), + ]; + call_with( + &imported(py, MIGRATION_MODELS, "OldNameViewEvent")?, + &fields, + ) + .map(Some) +} + +/// Stored rows of the selected `columns` as events; rows this version cannot read are skipped. +pub(crate) fn decoded_events<'py>( + table: EventTable, + rows: &Bound<'py, PyAny>, + columns: &[String], +) -> PyResult> { + let py: Python<'py> = rows.py(); + let mut events: Vec> = Vec::new(); + for row in rows.try_iter()? { + let values: Vec> = row?.try_iter()?.collect::>()?; + let stored = StoredRow { columns, values }; + let decoded: Option> = match table { + EventTable::Model => Some(model_event(py, &stored)?), + EventTable::Column => match column_event(py, &stored) { + Ok(event) => Some(event), + Err(error) if error.is_instance_of::(py) => None, + Err(error) => return Err(error), + }, + EventTable::OldName => old_name_event(py, &stored)?, + }; + events.extend(decoded); + } + PyTuple::new(py, events) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(create_sql, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs index 936e0ba0fa..a130979144 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs @@ -1,4 +1,10 @@ +pub(crate) mod constants; pub(crate) mod events; pub(crate) mod freshness; +pub(crate) mod migration_io; +pub(crate) mod migration_objects; +pub(crate) mod migration_projections; +pub(crate) mod migration_rows; +pub(crate) mod models; pub(crate) mod registration; pub(crate) mod timestamps; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/models.rs new file mode 100644 index 0000000000..b69ffba5c7 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/models.rs @@ -0,0 +1,31 @@ +//! The layout of one append-only migration state table as the bindings use it. + +use pyo3::prelude::{Bound, PyAny}; + +/// Which migration state table a request reads or writes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum EventTable { + Model, + Column, + OldName, +} + +/// Name, columns, event relation attribute and messages of one migration state table. +#[derive(Clone, Copy, Debug)] +pub(crate) struct EventTableLayout { + pub table: EventTable, + pub name: &'static str, + pub columns: &'static [&'static str], + pub relation_attribute: &'static str, + pub state_message: &'static str, + pub event_message: &'static str, + pub read_subject: &'static str, +} + +/// How migration state reaches the warehouse: the connection, execute callable and name renderer. +#[derive(Debug)] +pub(crate) struct StateAccess<'py> { + pub connection: Bound<'py, PyAny>, + pub execute: Bound<'py, PyAny>, + pub render_qualified_name: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs index c06548943e..df9f69c0f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs @@ -2,10 +2,15 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::state_rows::{events, freshness}; +use crate::bindings::_helpers::state_rows::{ + events, freshness, migration_io, migration_projections, migration_rows, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { events::register(module)?; freshness::register(module)?; + migration_rows::register(module)?; + migration_io::register(module)?; + migration_projections::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-sqltext/src/resolution/main/mod.rs b/crates/sqlbuild-sqltext/src/resolution/main/mod.rs index 8b223a84d6..4b60837f85 100644 --- a/crates/sqlbuild-sqltext/src/resolution/main/mod.rs +++ b/crates/sqlbuild-sqltext/src/resolution/main/mod.rs @@ -1,6 +1,7 @@ //! Public SQL resolution decisions. pub mod cursor_subquery; +pub mod quoted_name; pub mod ref_alias; pub mod reference_spans; pub mod source_alias; diff --git a/crates/sqlbuild-sqltext/src/resolution/main/quoted_name.rs b/crates/sqlbuild-sqltext/src/resolution/main/quoted_name.rs new file mode 100644 index 0000000000..2bd599f113 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/main/quoted_name.rs @@ -0,0 +1,11 @@ +//! Shared exact double-quoted reference marker argument parsing. + +/// Read a nonempty marker argument and return the unconsumed text. +pub fn quoted_name(text: &str) -> Option<(&str, &str)> { + let text = text.strip_prefix('"')?; + let end = text.find('"')?; + if end == 0 { + return None; + } + Some((&text[..end], &text[end + 1..])) +} diff --git a/crates/sqlbuild-sqltext/src/resolution/main/reference_spans.rs b/crates/sqlbuild-sqltext/src/resolution/main/reference_spans.rs index 7f1d881d41..9970c9f2ac 100644 --- a/crates/sqlbuild-sqltext/src/resolution/main/reference_spans.rs +++ b/crates/sqlbuild-sqltext/src/resolution/main/reference_spans.rs @@ -1,6 +1,7 @@ //! Locate planner reference markers with their historical lexical policy. use crate::resolution::constants::EXTERNAL_REFERENCE; +use crate::resolution::main::quoted_name::quoted_name; use crate::resolution::models::ReferenceSpan; /// Locate exact reference markers, retaining the external-reference whitespace grammar. @@ -49,12 +50,3 @@ pub fn reference_spans(sql: &str, kind: &str) -> Vec { } spans } - -fn quoted_name(text: &str) -> Option<(&str, &str)> { - let text = text.strip_prefix('"')?; - let end = text.find('"')?; - if end == 0 { - return None; - } - Some((&text[..end], &text[end + 1..])) -} diff --git a/crates/sqlbuild-state/Cargo.toml b/crates/sqlbuild-state/Cargo.toml index b8436c7922..17f6e3272b 100644 --- a/crates/sqlbuild-state/Cargo.toml +++ b/crates/sqlbuild-state/Cargo.toml @@ -8,6 +8,7 @@ publish.workspace = true rust-version.workspace = true [dependencies] +serde_json.workspace = true sha2.workspace = true sqlbuild-core.workspace = true sqlbuild-cursor.workspace = true diff --git a/crates/sqlbuild-state/src/events/_helpers/mod.rs b/crates/sqlbuild-state/src/events/_helpers/mod.rs index 059db07285..9280126ad8 100644 --- a/crates/sqlbuild-state/src/events/_helpers/mod.rs +++ b/crates/sqlbuild-state/src/events/_helpers/mod.rs @@ -1,2 +1,4 @@ pub(crate) mod identity; pub(crate) mod literals; +pub(crate) mod python_text; +pub(crate) mod relations; diff --git a/crates/sqlbuild-state/src/events/_helpers/python_text.rs b/crates/sqlbuild-state/src/events/_helpers/python_text.rs new file mode 100644 index 0000000000..e198799b8c --- /dev/null +++ b/crates/sqlbuild-state/src/events/_helpers/python_text.rs @@ -0,0 +1,21 @@ +//! `str()` of a decoded JSON value, as Python prints the value `json.loads` returns. + +use serde_json::Value; +use sqlbuild_core::json::main::python_float_repr::python_float_repr; + +use crate::events::constants::{PYTHON_FALSE, PYTHON_NONE, PYTHON_TRUE}; + +/// Python's `str()` of a JSON scalar; containers keep their compact JSON text. +pub(crate) fn python_text(value: &Value) -> String { + match value { + Value::String(text) => text.clone(), + Value::Bool(true) => PYTHON_TRUE.to_owned(), + Value::Bool(false) => PYTHON_FALSE.to_owned(), + Value::Null => PYTHON_NONE.to_owned(), + Value::Number(number) => match number.as_f64() { + Some(float) if number.is_f64() => python_float_repr(float), + _ => number.to_string(), + }, + Value::Array(_) | Value::Object(_) => value.to_string(), + } +} diff --git a/crates/sqlbuild-state/src/events/_helpers/relations.rs b/crates/sqlbuild-state/src/events/_helpers/relations.rs new file mode 100644 index 0000000000..0589b429be --- /dev/null +++ b/crates/sqlbuild-state/src/events/_helpers/relations.rs @@ -0,0 +1,34 @@ +//! Case-insensitive relation identity, treating an unknown database as a wildcard. + +use crate::events::models::EventRelation; + +/// `MigrationRelation.matches`: equal schema and name, and databases equal unless one is unknown. +pub(crate) fn relations_match(left: &EventRelation, right: &EventRelation) -> bool { + let schema = |relation: &EventRelation| -> String { + relation + .schema + .as_deref() + .unwrap_or_default() + .to_lowercase() + }; + if schema(left) != schema(right) || left.name.to_lowercase() != right.name.to_lowercase() { + return false; + } + match (&left.database, &right.database) { + (Some(left), Some(right)) => left.to_lowercase() == right.to_lowercase(), + _ => true, + } +} + +/// The index of the greatest order among the selected items, keeping the first of equal ones. +pub(crate) fn newest_index<'a, T: Ord + 'a>( + orders: impl Iterator, +) -> Option { + let mut newest: Option<(usize, &T)> = None; + for (index, order) in orders { + if newest.is_none_or(|(_, current)| order > current) { + newest = Some((index, order)); + } + } + newest.map(|(index, _)| index) +} diff --git a/crates/sqlbuild-state/src/events/constants.rs b/crates/sqlbuild-state/src/events/constants.rs index e6992b279b..f0cf958db1 100644 --- a/crates/sqlbuild-state/src/events/constants.rs +++ b/crates/sqlbuild-state/src/events/constants.rs @@ -10,3 +10,66 @@ pub(crate) const NULL_LITERAL: &str = "NULL"; pub(crate) const UTC_SUFFIX: &str = "+00:00"; pub const STRING_VALUE_REQUIRED: &str = "state string columns require str values"; pub const TIMESTAMP_VALUE_REQUIRED: &str = "state timestamp columns require datetime values"; +pub(crate) const ALIAS_ITEM_SEPARATOR: &str = ","; +pub(crate) const ALIAS_KEY_SEPARATOR: &str = ":"; +pub(crate) const PYTHON_TRUE: &str = "True"; +pub(crate) const PYTHON_FALSE: &str = "False"; +pub(crate) const PYTHON_NONE: &str = "None"; +pub const MIGRATION_TABLE_NAME: &str = "_sqlbuild_migrations"; +pub const COLUMN_MIGRATION_TABLE_NAME: &str = "_sqlbuild_column_migrations"; +pub const OLD_NAME_VIEW_TABLE_NAME: &str = "_sqlbuild_old_name_views"; +/// Event columns stored as text timestamps; every other event column is a string. +pub const TIMESTAMP_COLUMNS: [&str; 2] = ["expires_at", "created_at"]; +pub const MIGRATION_COLUMNS: [&str; 15] = [ + "event_id", + "target_name", + "origin_model", + "origin_database", + "origin_schema", + "origin_name", + "destination_model", + "destination_database", + "destination_schema", + "destination_name", + "origin_version_hash", + "discovery", + "decision", + "run_id", + "created_at", +]; +pub const COLUMN_MIGRATION_COLUMNS: [&str; 12] = [ + "event_id", + "target_name", + "model_name", + "relation_database", + "relation_schema", + "relation_name", + "origin_column", + "destination_column", + "discovery", + "decision", + "run_id", + "created_at", +]; +pub const OLD_NAME_VIEW_COLUMNS: [&str; 20] = [ + "event_id", + "target_name", + "event_type", + "migration_event_id", + "destination_model", + "old_database", + "old_schema", + "old_name", + "new_database", + "new_schema", + "new_name", + "view_retention", + "archive_name", + "column_aliases", + "grants_copied", + "view_sql", + "expires_at", + "drop_reason", + "run_id", + "created_at", +]; diff --git a/crates/sqlbuild-state/src/events/main/column_aliases_json.rs b/crates/sqlbuild-state/src/events/main/column_aliases_json.rs new file mode 100644 index 0000000000..3287a7d788 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/column_aliases_json.rs @@ -0,0 +1,31 @@ +//! `column_aliases_json`: the stored text of an old-name view's column aliases. + +use std::collections::BTreeMap; + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::events::constants::{ALIAS_ITEM_SEPARATOR, ALIAS_KEY_SEPARATOR}; + +/// Compact sorted ASCII JSON of `dict(aliases)`, or `None` when there are no aliases. +pub fn column_aliases_json(aliases: &[(String, String)]) -> Result, JsonEmitError> { + if aliases.is_empty() { + return Ok(None); + } + let mapping: BTreeMap<&str, &str> = aliases + .iter() + .map(|(old, new)| (old.as_str(), new.as_str())) + .collect(); + let value = JsonValue::Object( + mapping + .into_iter() + .map(|(old, new)| (old.to_owned(), JsonValue::String(new.to_owned()))) + .collect(), + ); + let options = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None).with_separators(ALIAS_ITEM_SEPARATOR, ALIAS_KEY_SEPARATOR) + }; + dumps(&value, &JsonDialect::Stdlib(options)).map(Some) +} diff --git a/crates/sqlbuild-state/src/events/main/decoded_column_aliases.rs b/crates/sqlbuild-state/src/events/main/decoded_column_aliases.rs new file mode 100644 index 0000000000..9ba2e6cd1d --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/decoded_column_aliases.rs @@ -0,0 +1,22 @@ +//! `decoded_column_aliases`: stored column alias text back to sorted pairs. + +use serde_json::Value; + +use crate::events::_helpers::python_text::python_text; + +/// Sorted `(old, new)` pairs of a stored JSON object; empty, invalid or non-object text reads as none. +#[must_use] +pub fn decoded_column_aliases(raw: Option<&str>) -> Vec<(String, String)> { + let Some(raw) = raw.filter(|raw| !raw.is_empty()) else { + return Vec::new(); + }; + let Ok(Value::Object(mapping)) = serde_json::from_str::(raw) else { + return Vec::new(); + }; + let mut pairs: Vec<(String, String)> = mapping + .iter() + .map(|(old, new)| (old.clone(), python_text(new))) + .collect(); + pairs.sort(); + pairs +} diff --git a/crates/sqlbuild-state/src/events/main/decoded_grants.rs b/crates/sqlbuild-state/src/events/main/decoded_grants.rs new file mode 100644 index 0000000000..0c69ecd53c --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/decoded_grants.rs @@ -0,0 +1,14 @@ +//! `decoded_grants`: stored grant text back to the copied statements. + +use serde_json::Value; + +use crate::events::_helpers::python_text::python_text; + +/// The statements of a stored JSON array; absent, invalid or non-array text reads as unknown. +#[must_use] +pub fn decoded_grants(raw: Option<&str>) -> Option> { + match serde_json::from_str::(raw?) { + Ok(Value::Array(statements)) => Some(statements.iter().map(python_text).collect()), + _ => None, + } +} diff --git a/crates/sqlbuild-state/src/events/main/event_cells.rs b/crates/sqlbuild-state/src/events/main/event_cells.rs new file mode 100644 index 0000000000..0efb609b79 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/event_cells.rs @@ -0,0 +1,22 @@ +//! `event_cells`: one event row's values with their declared state column types. + +use crate::events::constants::TIMESTAMP_COLUMNS; +use crate::events::models::{StateCell, StateColumnType, StateValue}; + +/// Pair each column with its value, typing `expires_at` and `created_at` as text timestamps. +#[must_use] +pub fn event_cells(columns: &[&str], values: Vec) -> Vec { + columns + .iter() + .zip(values) + .map(|(column, value)| StateCell { + column: (*column).to_owned(), + column_type: if TIMESTAMP_COLUMNS.contains(column) { + StateColumnType::TextTimestamp + } else { + StateColumnType::String + }, + value, + }) + .collect() +} diff --git a/crates/sqlbuild-state/src/events/main/event_read_sql.rs b/crates/sqlbuild-state/src/events/main/event_read_sql.rs new file mode 100644 index 0000000000..8cfb6ae19f --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/event_read_sql.rs @@ -0,0 +1,10 @@ +//! `event_read_sql`: every stored event of one state table in history order. + +/// `SELECT FROM ORDER BY created_at, event_id`. +#[must_use] +pub fn event_read_sql(table: &str, columns: &[&str]) -> String { + format!( + "SELECT {} FROM {table} ORDER BY created_at, event_id", + columns.join(", ") + ) +} diff --git a/crates/sqlbuild-state/src/events/main/grants_json.rs b/crates/sqlbuild-state/src/events/main/grants_json.rs new file mode 100644 index 0000000000..b3964032ee --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/grants_json.rs @@ -0,0 +1,14 @@ +//! `grants_json`: the stored text of the grant statements copied to an old-name view. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +/// `json.dumps(list(grants))`, or `None` when grants were not recorded. +pub fn grants_json(grants: Option<&[String]>) -> Result, JsonEmitError> { + let Some(grants) = grants else { + return Ok(None); + }; + let value = JsonValue::Array(grants.iter().cloned().map(JsonValue::String).collect()); + dumps(&value, &JsonDialect::Stdlib(StdlibJsonOptions::new(None))).map(Some) +} diff --git a/crates/sqlbuild-state/src/events/main/mod.rs b/crates/sqlbuild-state/src/events/main/mod.rs index 3870be1589..9b543bf5fb 100644 --- a/crates/sqlbuild-state/src/events/main/mod.rs +++ b/crates/sqlbuild-state/src/events/main/mod.rs @@ -1,5 +1,17 @@ +pub mod column_aliases_json; pub mod column_migration_event_id; +pub mod decoded_column_aliases; +pub mod decoded_grants; +pub mod event_cells; pub mod event_exists_sql; pub mod event_insert_sql; +pub mod event_read_sql; +pub mod grants_json; pub mod migration_event_id; +pub mod newest_column_event; +pub mod newest_fingerprint; +pub mod newest_move_event; +pub mod old_name_histories; pub mod old_name_view_event_id; +pub mod readable_columns; +pub mod recorded_renames; diff --git a/crates/sqlbuild-state/src/events/main/newest_column_event.rs b/crates/sqlbuild-state/src/events/main/newest_column_event.rs new file mode 100644 index 0000000000..888d381ff2 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/newest_column_event.rs @@ -0,0 +1,28 @@ +//! `newest_column_event`: the latest recorded rename of either column of a pair. + +use crate::events::_helpers::relations::{newest_index, relations_match}; +use crate::events::models::{ColumnEventKey, EventRelation}; + +/// The index of the latest rename on `relation` naming either column; the first wins ties. +#[must_use] +pub fn newest_column_event( + events: &[ColumnEventKey], + relation: &EventRelation, + (origin_column, destination_column): (&str, &str), +) -> Option { + let columns: [String; 2] = [ + origin_column.to_lowercase(), + destination_column.to_lowercase(), + ]; + newest_index( + events + .iter() + .enumerate() + .filter(|(_, event)| { + relations_match(&event.relation, relation) + && (columns.contains(&event.origin_column.to_lowercase()) + || columns.contains(&event.destination_column.to_lowercase())) + }) + .map(|(index, event)| (index, &event.order)), + ) +} diff --git a/crates/sqlbuild-state/src/events/main/newest_fingerprint.rs b/crates/sqlbuild-state/src/events/main/newest_fingerprint.rs new file mode 100644 index 0000000000..11332aaf4d --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/newest_fingerprint.rs @@ -0,0 +1,38 @@ +//! `newest_fingerprint`: the latest model fingerprint that built a relation or carries a name. + +use crate::events::_helpers::relations::{newest_index, relations_match}; +use crate::events::models::{EventRelation, FingerprintKey}; + +/// The newest fingerprint that built `target`, else the newest named `model_name`; the first wins ties. +#[must_use] +pub fn newest_fingerprint( + fingerprints: &[FingerprintKey], + target: Option<&EventRelation>, + model_name: Option<&str>, +) -> Option { + if let Some(target) = target { + let built: Option = newest_index( + fingerprints + .iter() + .enumerate() + .filter(|(_, fingerprint)| { + fingerprint + .target + .as_ref() + .is_some_and(|built| relations_match(target, built)) + }) + .map(|(index, fingerprint)| (index, &fingerprint.order)), + ); + if built.is_some() { + return built; + } + } + let model_name: &str = model_name?; + newest_index( + fingerprints + .iter() + .enumerate() + .filter(|(_, fingerprint)| fingerprint.node_name == model_name) + .map(|(index, fingerprint)| (index, &fingerprint.order)), + ) +} diff --git a/crates/sqlbuild-state/src/events/main/newest_move_event.rs b/crates/sqlbuild-state/src/events/main/newest_move_event.rs new file mode 100644 index 0000000000..f90d6cbd73 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/newest_move_event.rs @@ -0,0 +1,19 @@ +//! `newest_move_event`: the latest recorded move naming one relation. + +use crate::events::_helpers::relations::{newest_index, relations_match}; +use crate::events::models::{EventRelation, MoveEventKey}; + +/// The index of the latest move naming `relation` as origin or destination; the first wins ties. +#[must_use] +pub fn newest_move_event(events: &[MoveEventKey], relation: &EventRelation) -> Option { + newest_index( + events + .iter() + .enumerate() + .filter(|(_, event)| { + relations_match(&event.origin, relation) + || relations_match(&event.destination, relation) + }) + .map(|(index, event)| (index, &event.order)), + ) +} diff --git a/crates/sqlbuild-state/src/events/main/old_name_histories.rs b/crates/sqlbuild-state/src/events/main/old_name_histories.rs new file mode 100644 index 0000000000..8523358f0c --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/old_name_histories.rs @@ -0,0 +1,41 @@ +//! `old_name_histories`: one history per recorded move whose old name needs a compatibility view. + +use std::collections::HashMap; + +use crate::events::models::{EventOrder, OldNameFactKey, OldNameHistoryIndexes, OldNameStep}; + +/// Histories in move order, keeping the earliest fact of each step; moves without a requirement are skipped. +#[must_use] +pub fn old_name_histories( + moves: &[EventOrder], + facts: &[OldNameFactKey], +) -> Vec { + let mut fact_order: Vec = (0..facts.len()).collect(); + fact_order.sort_by(|left, right| facts[*left].order.cmp(&facts[*right].order)); + let mut by_move: HashMap<(&str, OldNameStep), usize> = HashMap::new(); + for index in fact_order { + let fact: &OldNameFactKey = &facts[index]; + by_move + .entry((fact.migration_event_id.as_str(), fact.step)) + .or_insert(index); + } + let mut move_order: Vec = (0..moves.len()).collect(); + move_order.sort_by(|left, right| moves[*left].cmp(&moves[*right])); + move_order + .into_iter() + .filter_map(|moved| { + let step = |step: OldNameStep| { + by_move + .get(&(moves[moved].event_id.as_str(), step)) + .copied() + }; + Some(OldNameHistoryIndexes { + moved, + required: step(OldNameStep::Required)?, + archived: step(OldNameStep::OriginArchived), + created: step(OldNameStep::ViewCreated), + dropped: step(OldNameStep::ViewDropped), + }) + }) + .collect() +} diff --git a/crates/sqlbuild-state/src/events/main/readable_columns.rs b/crates/sqlbuild-state/src/events/main/readable_columns.rs new file mode 100644 index 0000000000..248ce6e4e2 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/readable_columns.rs @@ -0,0 +1,14 @@ +//! `readable_columns`: the known columns a stored state table has, in their declared order. + +use std::collections::HashSet; + +/// The `known` columns present in `stored` (compared case-insensitively); later additions may be absent. +#[must_use] +pub fn readable_columns<'a>(known: &[&'a str], stored: &[String]) -> Vec<&'a str> { + let stored: HashSet = stored.iter().map(|column| column.to_lowercase()).collect(); + known + .iter() + .copied() + .filter(|column| stored.contains(*column)) + .collect() +} diff --git a/crates/sqlbuild-state/src/events/main/recorded_renames.rs b/crates/sqlbuild-state/src/events/main/recorded_renames.rs new file mode 100644 index 0000000000..e35245b2a0 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/recorded_renames.rs @@ -0,0 +1,27 @@ +//! `recorded_renames`: the renames recorded on one relation since a point in time. + +use sqlbuild_cursor::algebra::models::LocalDateTime; + +use crate::events::_helpers::relations::relations_match; +use crate::events::models::{EventRelation, RecordedRenameKey}; + +/// Indexes of recorded renames on `relation` created at or after `since`, oldest first. +#[must_use] +pub fn recorded_renames( + events: &[RecordedRenameKey], + relation: &EventRelation, + since: &LocalDateTime, +) -> Vec { + let mut indexes: Vec = events + .iter() + .enumerate() + .filter(|(_, event)| { + relations_match(relation, &event.relation) + && event.records_event + && event.order.created_at >= *since + }) + .map(|(index, _)| index) + .collect(); + indexes.sort_by(|left, right| events[*left].order.cmp(&events[*right].order)); + indexes +} diff --git a/crates/sqlbuild-state/src/events/models.rs b/crates/sqlbuild-state/src/events/models.rs index 9f850af603..f806f49e0f 100644 --- a/crates/sqlbuild-state/src/events/models.rs +++ b/crates/sqlbuild-state/src/events/models.rs @@ -35,3 +35,77 @@ pub struct StateCell { pub column_type: StateColumnType, pub value: StateValue, } + +/// The order of an event in its history: UTC creation time, then event ID. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct EventOrder { + pub created_at: LocalDateTime, + pub event_id: String, +} + +/// The fields of a recorded relation move that projections read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MoveEventKey { + pub order: EventOrder, + pub origin: EventRelation, + pub destination: EventRelation, +} + +/// The fields of a recorded column rename that projections read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ColumnEventKey { + pub order: EventOrder, + pub relation: EventRelation, + pub origin_column: String, + pub destination_column: String, +} + +/// One step of an old-name view's lifecycle. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum OldNameStep { + Required, + OriginArchived, + ViewCreated, + ViewDropped, +} + +/// The fields of one old-name fact that the history projection reads. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OldNameFactKey { + pub order: EventOrder, + pub migration_event_id: String, + pub step: OldNameStep, +} + +/// One projected old-name history as indexes into the moves and facts it was built from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct OldNameHistoryIndexes { + pub moved: usize, + pub required: usize, + pub archived: Option, + pub created: Option, + pub dropped: Option, +} + +/// The fields of a recorded column rename that the since-filter reads. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RecordedRenameKey { + pub order: EventOrder, + pub relation: EventRelation, + pub records_event: bool, +} + +/// The order of a stored fingerprint: its ISO timestamp text, then its run ID. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct FingerprintOrder { + pub timestamp: String, + pub run_id: String, +} + +/// The fields of a stored model fingerprint that the newest-fingerprint lookup reads. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FingerprintKey { + pub order: FingerprintOrder, + pub node_name: String, + pub target: Option, +} diff --git a/crates/sqlbuild-state/src/events/tests.rs b/crates/sqlbuild-state/src/events/tests.rs index b55f12f730..abf04f2330 100644 --- a/crates/sqlbuild-state/src/events/tests.rs +++ b/crates/sqlbuild-state/src/events/tests.rs @@ -1,6 +1,12 @@ +#[path = "tests/test_codecs.rs"] +mod codecs; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_insert_sql.rs"] mod insert_sql; +#[path = "tests/test_lookups.rs"] +mod lookups; +#[path = "tests/test_projections.rs"] +mod projections; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-state/src/events/tests/helpers.rs b/crates/sqlbuild-state/src/events/tests/helpers.rs index 3ad5209a10..47e1632627 100644 --- a/crates/sqlbuild-state/src/events/tests/helpers.rs +++ b/crates/sqlbuild-state/src/events/tests/helpers.rs @@ -1,4 +1,9 @@ -use crate::events::models::{StateCell, StateColumnType, StateValue}; +use sqlbuild_cursor::algebra::models::{CivilDate, LocalDateTime}; + +use crate::events::models::{ + ColumnEventKey, EventOrder, EventRelation, FingerprintKey, FingerprintOrder, MoveEventKey, + OldNameFactKey, OldNameStep, RecordedRenameKey, StateCell, StateColumnType, StateValue, +}; pub(crate) fn cell(column: &str, column_type: StateColumnType, value: StateValue) -> StateCell { StateCell { @@ -7,3 +12,91 @@ pub(crate) fn cell(column: &str, column_type: StateColumnType, value: StateValue value, } } + +pub(crate) fn relation(database: Option<&str>, schema: &str, name: &str) -> EventRelation { + EventRelation { + database: database.map(str::to_owned), + schema: Some(schema.to_owned()), + name: name.to_owned(), + } +} + +pub(crate) fn order(hour: u32, event_id: &str) -> EventOrder { + EventOrder { + created_at: LocalDateTime { + date: CivilDate { + year: 2026, + month: 1, + day: 1, + }, + hour, + minute: 0, + second: 0, + microsecond: 0, + }, + event_id: event_id.to_owned(), + } +} + +pub(crate) fn moved(order: EventOrder, (origin, destination): (&str, &str)) -> MoveEventKey { + MoveEventKey { + order, + origin: relation(None, "main", origin), + destination: relation(None, "main", destination), + } +} + +pub(crate) fn renamed(order: EventOrder, (origin, destination): (&str, &str)) -> ColumnEventKey { + ColumnEventKey { + order, + relation: relation(None, "main", "orders"), + origin_column: origin.to_owned(), + destination_column: destination.to_owned(), + } +} + +pub(crate) fn fact( + order: EventOrder, + migration_event_id: &str, + step: OldNameStep, +) -> OldNameFactKey { + OldNameFactKey { + order, + migration_event_id: migration_event_id.to_owned(), + step, + } +} + +pub(crate) fn pairs(items: &[(&str, &str)]) -> Vec<(String, String)> { + items + .iter() + .map(|(old, new)| ((*old).to_owned(), (*new).to_owned())) + .collect() +} + +pub(crate) fn recorded( + order: EventOrder, + relation: EventRelation, + records_event: bool, +) -> RecordedRenameKey { + RecordedRenameKey { + order, + relation, + records_event, + } +} + +pub(crate) fn fingerprint( + (timestamp, run_id): (&str, &str), + node_name: &str, + target: Option, +) -> FingerprintKey { + FingerprintKey { + order: FingerprintOrder { + timestamp: timestamp.to_owned(), + run_id: run_id.to_owned(), + }, + node_name: node_name.to_owned(), + target, + } +} diff --git a/crates/sqlbuild-state/src/events/tests/test_codecs.rs b/crates/sqlbuild-state/src/events/tests/test_codecs.rs new file mode 100644 index 0000000000..3d52c6484e --- /dev/null +++ b/crates/sqlbuild-state/src/events/tests/test_codecs.rs @@ -0,0 +1,159 @@ +use crate::events::constants::OLD_NAME_VIEW_COLUMNS; +use crate::events::main::column_aliases_json::column_aliases_json; +use crate::events::main::decoded_column_aliases::decoded_column_aliases; +use crate::events::main::decoded_grants::decoded_grants; +use crate::events::main::event_read_sql::event_read_sql; +use crate::events::main::grants_json::grants_json; +use crate::events::main::readable_columns::readable_columns; +use crate::events::tests::helpers::pairs; +use crate::events::tests::test_types::{ + AliasesTestCase, DecodedAliasesTestCase, DecodedGrantsTestCase, GrantsTestCase, ReadSqlTestCase, +}; + +#[test] +fn given_aliases_when_encoding_then_matches_python_compact_sorted_json() { + let test_cases = [ + AliasesTestCase { + description: "sorted keys, last duplicate wins and non-ASCII is escaped", + aliases: pairs(&[("z", "\u{e4}"), ("a", "b"), ("a", "c")]), + expected_json: Some("{\"a\":\"c\",\"z\":\"\\u00e4\"}".to_owned()), + }, + AliasesTestCase { + description: "no aliases store null", + aliases: vec![], + expected_json: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + column_aliases_json(&test_case.aliases).ok(), + Some(test_case.expected_json), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_grants_when_encoding_then_matches_python_default_json() { + let test_cases = [ + GrantsTestCase { + description: "quotes kept and non-ASCII escaped with default separators", + grants: Some(vec!["GRANT 'x'".to_owned(), "\u{e9}".to_owned()]), + expected_json: Some("[\"GRANT 'x'\", \"\\u00e9\"]".to_owned()), + }, + GrantsTestCase { + description: "unrecorded grants store null", + grants: None, + expected_json: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + grants_json(test_case.grants.as_deref()).ok(), + Some(test_case.expected_json), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_stored_alias_text_when_decoding_then_reads_sorted_pairs_or_none() { + let test_cases = [ + DecodedAliasesTestCase { + description: "object values are printed as Python would", + raw: Some("{\"k\": 1, \"j\": true, \"i\": null, \"h\": 1.5, \"g\": \"x\"}"), + expected_aliases: pairs(&[ + ("g", "x"), + ("h", "1.5"), + ("i", "None"), + ("j", "True"), + ("k", "1"), + ]), + }, + DecodedAliasesTestCase { + description: "invalid JSON", + raw: Some("{not json"), + expected_aliases: vec![], + }, + DecodedAliasesTestCase { + description: "array is not a mapping", + raw: Some("[\"a\"]"), + expected_aliases: vec![], + }, + DecodedAliasesTestCase { + description: "empty text", + raw: Some(""), + expected_aliases: vec![], + }, + ]; + + for test_case in test_cases { + assert_eq!( + decoded_column_aliases(test_case.raw), + test_case.expected_aliases, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_stored_grant_text_when_decoding_then_reads_statements_or_unknown() { + let test_cases = [ + DecodedGrantsTestCase { + description: "array items are printed as Python would", + raw: Some("[\"GRANT\", 3, null]"), + expected_grants: Some(vec!["GRANT".to_owned(), "3".to_owned(), "None".to_owned()]), + }, + DecodedGrantsTestCase { + description: "object is not a statement list", + raw: Some("{\"a\": 1}"), + expected_grants: None, + }, + DecodedGrantsTestCase { + description: "absent column", + raw: None, + expected_grants: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + decoded_grants(test_case.raw), + test_case.expected_grants, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_stored_columns_when_reading_old_name_facts_then_selects_known_columns_in_order() { + let test_cases = [ReadSqlTestCase { + description: "unknown and missing columns are left out", + stored: vec![ + "EVENT_ID".to_owned(), + "zzz".to_owned(), + "created_at".to_owned(), + "event_type".to_owned(), + "view_sql".to_owned(), + ], + expected_sql: "SELECT event_id, event_type, view_sql, created_at FROM s.t ORDER BY created_at, event_id".to_owned(), + }]; + + for test_case in test_cases { + assert_eq!( + event_read_sql( + "s.t", + &readable_columns(&OLD_NAME_VIEW_COLUMNS, &test_case.stored) + ), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/events/tests/test_lookups.rs b/crates/sqlbuild-state/src/events/tests/test_lookups.rs new file mode 100644 index 0000000000..8c1b670195 --- /dev/null +++ b/crates/sqlbuild-state/src/events/tests/test_lookups.rs @@ -0,0 +1,107 @@ +use crate::events::main::newest_fingerprint::newest_fingerprint; +use crate::events::main::recorded_renames::recorded_renames; +use crate::events::tests::helpers::{fingerprint, order, recorded, relation}; +use crate::events::tests::test_types::{NewestFingerprintTestCase, RecordedRenamesTestCase}; + +#[test] +fn given_column_renames_when_reading_since_then_recorded_matches_return_oldest_first() { + let test_cases = [ + RecordedRenamesTestCase { + description: "renames at or after the cutoff sort by time then event ID", + events: vec![ + recorded(order(5, "b"), relation(None, "main", "orders"), true), + recorded(order(3, "z"), relation(Some("db"), "MAIN", "Orders"), true), + recorded(order(5, "a"), relation(None, "main", "orders"), true), + recorded(order(2, "c"), relation(None, "main", "orders"), true), + ], + relation: relation(None, "main", "orders"), + since: order(3, "").created_at, + expected_indexes: vec![1, 2, 0], + }, + RecordedRenamesTestCase { + description: "unrecorded decisions and other relations are skipped", + events: vec![ + recorded(order(4, "a"), relation(None, "main", "orders"), false), + recorded(order(4, "b"), relation(None, "main", "customers"), true), + recorded(order(4, "c"), relation(Some("one"), "main", "orders"), true), + ], + relation: relation(Some("two"), "main", "orders"), + since: order(1, "").created_at, + expected_indexes: vec![], + }, + ]; + + for test_case in test_cases { + assert_eq!( + recorded_renames(&test_case.events, &test_case.relation, &test_case.since), + test_case.expected_indexes, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_fingerprints_when_finding_newest_then_built_relation_wins_before_name() { + let test_cases = [ + NewestFingerprintTestCase { + description: "a fingerprint that built the relation wins over a newer named one", + fingerprints: vec![ + fingerprint(("2026-01-03", "r3"), "orders", None), + fingerprint( + ("2026-01-01", "r1"), + "legacy", + Some(relation(None, "MAIN", "ORDERS")), + ), + ], + target: Some(relation(Some("db"), "main", "orders")), + model_name: Some("orders"), + expected_index: Some(1), + }, + NewestFingerprintTestCase { + description: "without a built match the newest named fingerprint wins", + fingerprints: vec![ + fingerprint(("2026-01-01", "r1"), "orders", None), + fingerprint(("2026-01-02", "r0"), "orders", None), + fingerprint(("2026-01-02", "r0"), "orders", None), + ], + target: Some(relation(None, "main", "orders")), + model_name: Some("orders"), + expected_index: Some(1), + }, + NewestFingerprintTestCase { + description: "no built match and no model name", + fingerprints: vec![fingerprint(("2026-01-01", "r1"), "orders", None)], + target: Some(relation(None, "main", "orders")), + model_name: None, + expected_index: None, + }, + NewestFingerprintTestCase { + description: "a name lookup alone ignores built relations", + fingerprints: vec![ + fingerprint( + ("2026-01-05", "r5"), + "customers", + Some(relation(None, "main", "orders")), + ), + fingerprint(("2026-01-01", "r1"), "orders", None), + ], + target: None, + model_name: Some("orders"), + expected_index: Some(1), + }, + ]; + + for test_case in test_cases { + assert_eq!( + newest_fingerprint( + &test_case.fingerprints, + test_case.target.as_ref(), + test_case.model_name, + ), + test_case.expected_index, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/events/tests/test_projections.rs b/crates/sqlbuild-state/src/events/tests/test_projections.rs new file mode 100644 index 0000000000..33bc1d3a9c --- /dev/null +++ b/crates/sqlbuild-state/src/events/tests/test_projections.rs @@ -0,0 +1,138 @@ +use crate::events::main::newest_column_event::newest_column_event; +use crate::events::main::newest_move_event::newest_move_event; +use crate::events::main::old_name_histories::old_name_histories; +use crate::events::models::{OldNameHistoryIndexes, OldNameStep}; +use crate::events::tests::helpers::{fact, moved, order, relation, renamed}; +use crate::events::tests::test_types::{HistoryTestCase, NewestColumnTestCase, NewestMoveTestCase}; + +#[test] +fn given_moves_when_projecting_newest_mention_then_latest_first_tied_event_wins() { + let test_cases = [ + NewestMoveTestCase { + description: "later event wins by time", + events: vec![ + moved(order(2, "a"), ("orders", "orders_v2")), + moved(order(1, "z"), ("orders_v2", "orders")), + ], + relation: relation(None, "MAIN", "Orders"), + expected_index: Some(0), + }, + NewestMoveTestCase { + description: "equal times order by event ID", + events: vec![ + moved(order(1, "b"), ("orders", "x")), + moved(order(1, "a"), ("x", "orders")), + ], + relation: relation(Some("db"), "main", "orders"), + expected_index: Some(0), + }, + NewestMoveTestCase { + description: "identical orders keep the first", + events: vec![ + moved(order(1, "a"), ("orders", "x")), + moved(order(1, "a"), ("orders", "y")), + ], + relation: relation(None, "main", "orders"), + expected_index: Some(0), + }, + NewestMoveTestCase { + description: "unrelated relation", + events: vec![moved(order(1, "a"), ("orders", "x"))], + relation: relation(None, "main", "customers"), + expected_index: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + newest_move_event(&test_case.events, &test_case.relation), + test_case.expected_index, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_column_renames_when_projecting_newest_mention_then_either_column_matches() { + let test_cases = [ + NewestColumnTestCase { + description: "destination column of a later rename matches case-insensitively", + events: vec![ + renamed(order(1, "a"), ("amount", "total")), + renamed(order(2, "b"), ("TOTAL", "net")), + ], + columns: ("x", "Total"), + expected_index: Some(1), + }, + NewestColumnTestCase { + description: "no rename names either column", + events: vec![renamed(order(1, "a"), ("amount", "total"))], + columns: ("x", "y"), + expected_index: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + newest_column_event( + &test_case.events, + &relation(None, "main", "orders"), + test_case.columns + ), + test_case.expected_index, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_moves_and_facts_when_projecting_histories_then_earliest_fact_of_each_step_is_kept() { + let test_cases = [ + HistoryTestCase { + description: "moves sort by time and keep the earliest fact per step", + moves: vec![order(3, "m2"), order(1, "m1"), order(1, "m0")], + facts: vec![ + fact(order(2, "f3"), "m1", OldNameStep::Required), + fact(order(2, "f2"), "m1", OldNameStep::Required), + fact(order(1, "f9"), "m1", OldNameStep::ViewDropped), + fact(order(1, "f1"), "m2", OldNameStep::Required), + fact(order(1, "f4"), "m2", OldNameStep::OriginArchived), + fact(order(1, "f5"), "m2", OldNameStep::ViewCreated), + fact(order(1, "f6"), "m0", OldNameStep::ViewCreated), + ], + expected_histories: vec![ + OldNameHistoryIndexes { + moved: 1, + required: 1, + archived: None, + created: None, + dropped: Some(2), + }, + OldNameHistoryIndexes { + moved: 0, + required: 3, + archived: Some(4), + created: Some(5), + dropped: None, + }, + ], + }, + HistoryTestCase { + description: "no moves", + moves: vec![], + facts: vec![fact(order(1, "f1"), "m1", OldNameStep::Required)], + expected_histories: vec![], + }, + ]; + + for test_case in test_cases { + assert_eq!( + old_name_histories(&test_case.moves, &test_case.facts), + test_case.expected_histories, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/events/tests/test_types.rs b/crates/sqlbuild-state/src/events/tests/test_types.rs index 46f300b779..e40b5bc07c 100644 --- a/crates/sqlbuild-state/src/events/tests/test_types.rs +++ b/crates/sqlbuild-state/src/events/tests/test_types.rs @@ -1,8 +1,80 @@ use crate::events::errors::StateSqlError; -use crate::events::models::StateCell; +use sqlbuild_cursor::algebra::models::LocalDateTime; + +use crate::events::models::{ + ColumnEventKey, EventOrder, EventRelation, FingerprintKey, MoveEventKey, OldNameFactKey, + OldNameHistoryIndexes, RecordedRenameKey, StateCell, +}; pub(crate) struct InsertTestCase { pub(crate) description: &'static str, pub(crate) cells: Vec, pub(crate) expected_sql: Result, } + +pub(crate) struct NewestMoveTestCase { + pub(crate) description: &'static str, + pub(crate) events: Vec, + pub(crate) relation: EventRelation, + pub(crate) expected_index: Option, +} + +pub(crate) struct NewestColumnTestCase { + pub(crate) description: &'static str, + pub(crate) events: Vec, + pub(crate) columns: (&'static str, &'static str), + pub(crate) expected_index: Option, +} + +pub(crate) struct HistoryTestCase { + pub(crate) description: &'static str, + pub(crate) moves: Vec, + pub(crate) facts: Vec, + pub(crate) expected_histories: Vec, +} + +pub(crate) struct AliasesTestCase { + pub(crate) description: &'static str, + pub(crate) aliases: Vec<(String, String)>, + pub(crate) expected_json: Option, +} + +pub(crate) struct DecodedAliasesTestCase { + pub(crate) description: &'static str, + pub(crate) raw: Option<&'static str>, + pub(crate) expected_aliases: Vec<(String, String)>, +} + +pub(crate) struct DecodedGrantsTestCase { + pub(crate) description: &'static str, + pub(crate) raw: Option<&'static str>, + pub(crate) expected_grants: Option>, +} + +pub(crate) struct ReadSqlTestCase { + pub(crate) description: &'static str, + pub(crate) stored: Vec, + pub(crate) expected_sql: String, +} + +pub(crate) struct GrantsTestCase { + pub(crate) description: &'static str, + pub(crate) grants: Option>, + pub(crate) expected_json: Option, +} + +pub(crate) struct RecordedRenamesTestCase { + pub(crate) description: &'static str, + pub(crate) events: Vec, + pub(crate) relation: EventRelation, + pub(crate) since: LocalDateTime, + pub(crate) expected_indexes: Vec, +} + +pub(crate) struct NewestFingerprintTestCase { + pub(crate) description: &'static str, + pub(crate) fingerprints: Vec, + pub(crate) target: Option, + pub(crate) model_name: Option<&'static str>, + pub(crate) expected_index: Option, +} diff --git a/python-boundary.toml b/python-boundary.toml index e8fb54d60b..3449eeb06b 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 33506 +remaining_lines = 27784 [[stays]] path = "executor" diff --git a/scripts/planner_differential/_helpers/capturing/canonical.py b/scripts/planner_differential/_helpers/capturing/canonical.py index 574825e90f..927fb20359 100644 --- a/scripts/planner_differential/_helpers/capturing/canonical.py +++ b/scripts/planner_differential/_helpers/capturing/canonical.py @@ -9,12 +9,15 @@ from scripts.compiler_differential.classes.capture_file import expand_capture_text from scripts.planner_differential.constants import ( COMPACT_TIMESTAMP_PATTERN, + DATE_PATTERN, ELAPSED_TIME_MASK, ELAPSED_TIME_PATTERN, ISO_TIMESTAMP_PATTERN, MAPPING_MARKER, RECENT_TIMESTAMP_DAYS, RECENT_TIMESTAMP_MASK, + RUN_RELATIVE_FUTURE_DAYS, + RUN_RELATIVE_PAST_DAYS, VOLATILE_KEY_SUFFIXES, VOLATILE_KEYS, VOLATILE_VALUE_MASK, @@ -24,7 +27,7 @@ def mask_text(*, text: str, masks: TextMasks) -> str: - """Mask invocation ids, run paths and timestamps near the run date.""" + """Mask invocation ids, run paths, timestamps near the run date and run-relative dates.""" masked: str = mask_capture_noise(text) for prefix, replacement in masks.replacements: @@ -32,9 +35,12 @@ def mask_text(*, text: str, masks: TextMasks) -> str: masked = ISO_TIMESTAMP_PATTERN.sub( lambda match: _recent_or_kept(match=match, today=masks.today, iso=True), masked ) - return COMPACT_TIMESTAMP_PATTERN.sub( + masked = COMPACT_TIMESTAMP_PATTERN.sub( lambda match: _recent_or_kept(match=match, today=masks.today, iso=False), masked ) + return DATE_PATTERN.sub( + lambda match: _run_relative_date(match=match, today=masks.today), masked + ) def mask_console_text(*, text: str, masks: TextMasks) -> str: @@ -93,3 +99,16 @@ def _recent_or_kept(*, match: re.Match[str], today: date, iso: bool) -> str: return match.group(0) recent: bool = day >= today - timedelta(days=RECENT_TIMESTAMP_DAYS) return RECENT_TIMESTAMP_MASK if recent else match.group(0) + + +def _run_relative_date(*, match: re.Match[str], today: date) -> str: + """Write dates the planner derives from the run date, such as replay ranges, as offsets.""" + + try: + day: date = date.fromisoformat(match.group(1)) + except ValueError: + return match.group(0) + offset: int = (day - today).days + if -RUN_RELATIVE_PAST_DAYS <= offset <= RUN_RELATIVE_FUTURE_DAYS: + return f"" + return match.group(0) diff --git a/scripts/planner_differential/constants.py b/scripts/planner_differential/constants.py index 2ee124ef40..58d1c11d62 100644 --- a/scripts/planner_differential/constants.py +++ b/scripts/planner_differential/constants.py @@ -117,6 +117,9 @@ ) COMPACT_TIMESTAMP_PATTERN: re.Pattern[str] = re.compile(r"(? tuple[SqlTestPlanResult, ...]: ... +def plan_sql_tests_natively( + *, + project: CompiledProjectView, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + sql_analysis_enabled: bool, + render_sql: bool, + include_plan: bool = True, +) -> tuple[NativeSqlTestPlan, ...]: ... +def plan_compiled_sql_test_artifacts( + *, + project: CompiledProjectView, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + sql_analysis_enabled: bool, +) -> tuple[NativeSqlTestArtifact, ...]: ... +def resolve_sql_test_model_chains( + *, + project: CompiledProjectView, + tests: tuple[CompiledSqlTest, ...], +) -> tuple[tuple[str, ...], ...]: ... +def sql_test_plan_error_messages(*, warnings: tuple[PlanWarning, ...]) -> tuple[str, ...]: ... + STAGE_CAPTURE_OMITTED_ATTRIBUTES: dict[str, frozenset[str]] def migration_planning_database(*, runtime: PlannerRuntime) -> str | None: ... @@ -2502,6 +2553,20 @@ def expected_fixture_types( ordered_model_names: tuple[str, ...], model_map: dict[str, CompiledModelView], ) -> Mapping[tuple[CompiledResourceType, str], dict[str, str]]: ... +def fixture_read_cache_info(*, reset: bool = False) -> tuple[int, int]: ... +def required_fixture_columns( + *, + model_inputs: tuple[tuple[str, ...], dict[str, CompiledModelView]], + fixture_inputs: tuple[ + frozenset[tuple[CompiledResourceType, str]], + dict[tuple[CompiledResourceType, str], FixtureRelationMetadata], + dict[tuple[CompiledResourceType, str], frozenset[str]], + ], + adapter: BaseAdapter, +) -> tuple[ + dict[tuple[CompiledResourceType, str], dict[str, set[str]]], + frozenset[tuple[CompiledResourceType, str]], +]: ... def build_relation_fixture_context( *, project: CompiledProjectView, models_by_name: dict[str, CompiledModelView] | None = None ) -> RelationFixturePlanningContext: ... @@ -3957,6 +4022,74 @@ def build_cli_scenario_plan( source_lexical_syntax: SqlLexicalSyntax, ) -> ScenarioExecutionPlan: ... def run_scenario_test_command(*, request: ScenarioTestCommandRequest) -> int: ... +def compute_scenario_hash_prefix( + *, + project_name: str, + scenario_name: str, + run_namespace: str | None = None, + prefix_length: int = 12, +) -> str: ... +def build_scenario_hash_index( + *, + project_name: str, + scenarios: tuple[CompiledSqlScenario, ...], + run_namespace: str | None = None, + prefix_length: int = 12, +) -> dict[str, str]: ... +def build_scenario_artifact_name( + *, hash_prefix: str, kind: str, logical_name: str, identifier_limit: int = 63 +) -> str: ... +def build_scenario_artifact_physical_name( + *, hash_prefix: str, kind: str, logical_name: str, identifier_limit: int +) -> str: ... +def fit_scenario_artifact_logical_name( + *, logical_name: str, fixed_prefix: str, identifier_limit: int +) -> str: ... +def parse_scenario_artifact_physical_name(name: str) -> ParsedScenarioArtifactName | None: ... +def is_scenario_artifact_physical_name(name: str) -> bool: ... +def build_scenario_relation_map( + *, + scenario_name: str, + hash_prefix: str, + artifacts: tuple[ScenarioArtifactIdentity, ...], + identifier_limit: int = 63, + normalize_identifier: Callable[[str], str] | None = None, +) -> ScenarioRelationMap: ... +def resolve_scenario_check_sql( + *, + sql: str, + relation_plan: ScenarioRelationPlan, + adapter: BaseAdapter, + source_label: str, + source_lexical_syntax: SqlLexicalSyntax, +) -> str: ... +def build_scenario_fixture_plans( + *, + scenario: CompiledSqlScenario, + graph_plan: ScenarioGraphPlan, + relation_plan: ScenarioRelationPlan, + adapter: BaseAdapter, + source_lexical_syntax: SqlLexicalSyntax, + fixture_sql_overrides: dict[tuple[CompiledResourceType, str], str] | None = None, +) -> tuple[ScenarioFixturePlan, ...]: ... +def build_scenario_seed_entries( + *, + project: CompiledProjectView, + graph_plan: ScenarioGraphPlan, + relation_plan: ScenarioRelationPlan, +) -> tuple[SeedPlanEntry, ...]: ... +def build_scenario_execution_plan( + *, + scenario: CompiledSqlScenario, + project: CompiledProjectView, + adapter: BaseAdapter, + graph_plan: ScenarioGraphPlan, + relation_plan: ScenarioRelationPlan, + source_lexical_syntax: SqlLexicalSyntax, + snapshot: WarehouseSnapshot | None = None, + source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]] | None = None, + sql_analysis_enabled: bool = True, +) -> tuple[ScenarioExecutionPlan, tuple[PlanWarning, ...]]: ... def run_scenario_capture_command(*, request: ScenarioCaptureCommandRequest) -> int: ... def sql_analysis_off_guidance( *, discovered_inputs: DiscoveredProjectInputs, purpose: str @@ -4303,14 +4436,105 @@ def deterministic_column_migration_event_id( def deterministic_old_name_view_event_id( *, event_type: OldNameViewEventType, migration_event_id: str ) -> str: ... -def render_event_exists_sql(*, table: str, event_id: str) -> str: ... -def render_event_insert_sql( +def render_migration_state_create_sql( *, - table: str, - columns: tuple[str, ...], - column_types: Mapping[str, StateSqlValueType], - values: tuple[object | None, ...], + kind: Literal["model", "column", "old_name"], + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], ) -> str: ... +def read_migration_state_events( + *, + kind: Literal["model", "column", "old_name"], + connection: Any, + execute: AdapterExecute[Any, Any], + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + stored_columns: Iterable[str] | None = None, +) -> tuple[Any, ...]: ... +def read_old_name_view_histories( + *, + connection: Any, + execute: AdapterExecute[Any, Any], + database: str | None, + schema: str, + stored_columns: Iterable[str], + render_qualified_name: Callable[..., str | None], +) -> tuple[OldNameViewHistory, ...]: ... +def read_recorded_column_renames( + *, + connection: Any, + execute: AdapterExecute[Any, Any], + relation: MigrationRelation, + since: datetime, + render_qualified_name: Callable[..., str | None], +) -> tuple[ColumnMigrationEvent, ...]: ... +def stored_old_name_view_columns( + *, adapter: BaseAdapter, connection: Any, database: str | None, schema: str +) -> tuple[str, ...]: ... +def write_migration_state_event( + *, + kind: Literal["model", "column", "old_name"], + connection: Any, + execute: AdapterExecute[Any, Any], + event: MigrationEvent | ColumnMigrationEvent | OldNameViewEvent, + render_qualified_name: Callable[..., str | None], + create_table_sql: str | None, + attempts: int = 3, +) -> None: ... + +class MigrationStateInspection: + """Relations, fingerprints, and migration events for one planning database.""" + + def __init__( + self, + *, + adapter: BaseAdapter, + connection: Any, + database: str | None, + known_fingerprints: WarehouseFingerprints | None = None, + ) -> None: ... + @property + def database(self) -> str | None: ... + @property + def events(self) -> tuple[MigrationEvent, ...]: ... + @property + def column_events(self) -> tuple[ColumnMigrationEvent, ...]: ... + @property + def old_name_histories(self) -> tuple[OldNameViewHistory, ...]: ... + def inspect_column_events(self, *, schemas: set[str]) -> None: ... + def inspect_old_name_views(self, *, schemas: set[str]) -> None: ... + def inspect_migration_events(self, *, schemas: set[str]) -> None: ... + def inspect_schemas(self, *, schemas: set[str]) -> None: ... + def inspect_relations(self, *, locations: tuple[CompiledRelationLocation, ...]) -> None: ... + def relation(self, location: CompiledRelationLocation) -> RelationInfo | None: ... + def relation_columns(self, location: CompiledRelationLocation) -> tuple[ColumnInfo, ...]: ... + def model_fingerprint( + self, *, location: CompiledRelationLocation, model_name: str | None + ) -> Fingerprint | None: ... + def named_fingerprint(self, *, model_name: str) -> Fingerprint | None: ... + def orphan_fingerprints( + self, *, project_model_names: frozenset[str] + ) -> tuple[Fingerprint, ...]: ... + +def newest_migration_event_mentioning( + *, events: Iterable[MigrationEvent], relation: MigrationRelation +) -> MigrationEvent | None: ... +def newest_column_migration_event_mentioning( + *, + events: Iterable[ColumnMigrationEvent], + relation: MigrationRelation, + origin_column: str, + destination_column: str, +) -> ColumnMigrationEvent | None: ... +def project_old_name_views( + *, moves: Iterable[MigrationEvent], facts: Iterable[OldNameViewEvent] +) -> tuple[OldNameViewHistory, ...]: ... +def resolve_old_name_view_retention( + *, config_values: Mapping[str, object], project_retention: str | None +) -> str | None: ... def source_freshness_data_version_hash( *, source_name: str, @@ -4667,6 +4891,14 @@ def migration_destination_fingerprints( ) -> dict[str, tuple[str, ...]]: ... def stored_migration_fingerprint(fingerprint: Fingerprint) -> str | None: ... def declared_column_aliases(model: CompiledModelView) -> tuple[tuple[str, str], ...]: ... +def plan_old_name_views( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + migration_entries: tuple[ModelMigrationPlanEntry, ...], + column_entries: tuple[ColumnMigrationPlanEntry, ...], +) -> OldNameViewPlanning: ... def manual_migration_requests(*, scope: PlannerScope) -> tuple[ModelMigrationRequest, ...]: ... def discover_model_migrations( *, @@ -4793,6 +5025,83 @@ def build_model_warnings( on_schema_change: OnSchemaChange | None, type_enforcement: bool, ) -> tuple[PlanWarning, ...]: ... +def validate_source_cursor_input_columns( + *, + model: CompiledModelView, + cursor_column: str | None, + models_by_name: dict[str, CompiledModelView] | None = None, + functions_by_name: dict[str, CompiledFunctionView] | None = None, + source_map: dict[str, SourceEntry] | None = None, + source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]], +) -> None: ... +def build_cursor_input_relations( + *, + model: CompiledModelView, + adapter: BaseAdapter, + model_locations: dict[str, CompiledRelationLocation], + models_by_name: dict[str, CompiledModelView], + functions_by_name: dict[str, CompiledFunctionView] | None = None, + seed_locations: dict[str, CompiledRelationLocation], + source_map: dict[str, SourceEntry], + cursor_column: str | None, + runtime_cursor_producer_names: frozenset[str], +) -> tuple[CursorInputRelation, ...]: ... +def runtime_cursor_producer_names( + *, scope: PlannerScope, source_map: dict[str, SourceEntry] +) -> frozenset[str]: ... +def resolve_microbatch_lookback( + *, model: CompiledModelView, effective_grain: str | None = None +) -> str | None: ... +def extract_seed_columns(seed: CompiledSeedView) -> tuple[ColumnInfo, ...]: ... +def assemble_model_plan_entry( + *, + model: CompiledModelView, + adapter: BaseAdapter, + snapshot: WarehouseSnapshot, + change_result: ChangeDetectionResult, + action: PlanAction, + reason: PlanReason, + materialization_type: MaterializationType, + resolved_sql: str, + sql_analysis_enabled: bool, + pre_hooks: object, + post_hooks: object, + cursor_column: str | None, + cursor_input_relations: tuple[CursorInputRelation, ...], + runtime_owned_cursor_bounds: bool, + cursor_bounds: CursorBounds | None, + microbatch_range: CursorBounds | None, + empty_cursor_inputs: tuple[str, ...], + waiting_on_empty_inputs: bool, + backfill: BackfillResult, + start_cursor_override: str | None, + end_cursor_override: str | None, + future_cursor_config: FutureCursorsConfig | None, + start_cursor_config: StartCursorsConfig | None, + invocation_time: datetime | None, +) -> tuple[ModelPlanEntry, tuple[PlanWarning, ...]]: ... +def plan_audit( + *, + audit: CompiledAuditView, + model_locations: dict[str, CompiledRelationLocation], + seed_locations: dict[str, CompiledRelationLocation], + source_map: dict[str, SourceEntry], + adapter: BaseAdapter, + lineage_upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + lineage_downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + model_materializations: dict[str, str], +) -> AuditPlanEntry: ... +def audit_is_selected( + *, audit: CompiledAuditView, selected_keys: frozenset[CompiledObjectKey] +) -> bool: ... +def build_selected_audit_entries( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + scope: PlannerScope, + relations: PlannerRelationsContext, + model_materializations: dict[str, str], +) -> list[AuditPlanEntry]: ... def resolve_attachment_kind( *, audit: CompiledAuditView, @@ -4815,21 +5124,12 @@ def check_cursor_type_consistency( warehouse_columns: tuple[ColumnInfo, ...], sql_analysis_enabled: bool, ) -> PlanWarning | None: ... -def count_microbatches( - *, - bounds: CursorBounds, - batch_size: str, - cursor_type: str, - equal_bounds_are_batch: bool = False, -) -> int: ... -def cap_microbatch_bounds( - *, - bounds: CursorBounds, - batch_size: str, - cursor_type: str, - max_batches: int, - action: MicrobatchLimitAction, -) -> CursorBounds: ... +def apply_microbatch_limit( + *, entry: ModelPlanEntry, max_batches: int | None, action: MicrobatchLimitAction +) -> tuple[ModelPlanEntry, PlanWarning | None]: ... +def apply_configured_microbatch_limits( + *, entry: ModelPlanEntry, inputs: PlanEntryBuildInputs +) -> tuple[ModelPlanEntry, tuple[PlanWarning, ...]]: ... def microbatch_limit_warning( *, model_name: str, max_batches: int | None, batch_count: int, action: MicrobatchLimitAction ) -> str | None: ... @@ -5021,3 +5321,7 @@ def build_source_load_entries( source_map: dict[str, SourceEntry], is_reload: bool, ) -> tuple[SourceLoadPlanEntry, ...]: ... +def format_model_migrations( + *, lines: list[str], plan: PlanOutput, display_options: DisplayOptions +) -> list[str]: ... +def old_name_rows(*, entry: OldNameViewPlanEntry, style: CliStyle) -> list[str]: ... diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 43980ea07a..9578d2fc0b 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -2295,11 +2295,10 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: """Render DDL that creates the model migration event table when it is missing.""" - from sqlbuild.compiler.migrations.main.create_table_sql import ( - build_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="model", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -2311,11 +2310,10 @@ def render_create_old_name_view_state_table_sql( ) -> str: """Render DDL that creates the old-name view fact table when it is missing.""" - from sqlbuild.compiler.migrations.main.old_name_view_create_table_sql import ( - build_old_name_view_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_old_name_view_state_create_table_sql( + return render_migration_state_create_sql( + kind="old_name", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -2327,11 +2325,10 @@ def render_create_column_migration_state_table_sql( ) -> str: """Render DDL that creates the column migration event table when it is missing.""" - from sqlbuild.compiler.migrations.main.column_create_table_sql import ( - build_column_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_column_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="column", database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 2f85f55996..cf6a389d47 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -909,11 +909,10 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: ) def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.migrations.main.create_table_sql import ( - build_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="model", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -923,11 +922,10 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem def render_create_old_name_view_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.old_name_view_create_table_sql import ( - build_old_name_view_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_old_name_view_state_create_table_sql( + return render_migration_state_create_sql( + kind="old_name", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -937,11 +935,10 @@ def render_create_old_name_view_state_table_sql( def render_create_column_migration_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.column_create_table_sql import ( - build_column_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_column_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="column", database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 67557b5a56..8184bbba30 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -363,11 +363,10 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: ) def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.migrations.main.create_table_sql import ( - build_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="model", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -377,11 +376,10 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem def render_create_old_name_view_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.old_name_view_create_table_sql import ( - build_old_name_view_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_old_name_view_state_create_table_sql( + return render_migration_state_create_sql( + kind="old_name", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -391,11 +389,10 @@ def render_create_old_name_view_state_table_sql( def render_create_column_migration_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.column_create_table_sql import ( - build_column_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_column_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="column", database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 7e295f05f7..478eaf5c33 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -459,11 +459,10 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: ) def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.migrations.main.create_table_sql import ( - build_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="model", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -473,11 +472,10 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem def render_create_old_name_view_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.old_name_view_create_table_sql import ( - build_old_name_view_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_old_name_view_state_create_table_sql( + return render_migration_state_create_sql( + kind="old_name", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -487,11 +485,10 @@ def render_create_old_name_view_state_table_sql( def render_create_column_migration_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.column_create_table_sql import ( - build_column_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_column_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="column", database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 035c742a9a..8eb92916f0 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -1728,11 +1728,10 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: ) def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.migrations.main.create_table_sql import ( - build_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="model", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -1742,11 +1741,10 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem def render_create_old_name_view_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.old_name_view_create_table_sql import ( - build_old_name_view_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_old_name_view_state_create_table_sql( + return render_migration_state_create_sql( + kind="old_name", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -1756,11 +1754,10 @@ def render_create_old_name_view_state_table_sql( def render_create_column_migration_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.column_create_table_sql import ( - build_column_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql - return build_column_migration_state_create_table_sql( + return render_migration_state_create_sql( + kind="column", database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 0c0324a0fe..826a015573 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -679,12 +679,11 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: ) def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.migrations.main.create_table_sql import ( - build_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql return self._with_state_table_retention( - create_sql=build_migration_state_create_table_sql( + create_sql=render_migration_state_create_sql( + kind="model", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -697,12 +696,11 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem def render_create_old_name_view_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.old_name_view_create_table_sql import ( - build_old_name_view_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql return self._with_state_table_retention( - create_sql=build_old_name_view_state_create_table_sql( + create_sql=render_migration_state_create_sql( + kind="old_name", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -715,12 +713,11 @@ def render_create_old_name_view_state_table_sql( def render_create_column_migration_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild.compiler.migrations.main.column_create_table_sql import ( - build_column_migration_state_create_table_sql, - ) + from sqlbuild._native import render_migration_state_create_sql return self._with_state_table_retention( - create_sql=build_column_migration_state_create_table_sql( + create_sql=render_migration_state_create_sql( + kind="column", database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 3c47f98943..2dcddfb1e7 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -657,12 +657,11 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: ) def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild._native import render_migration_state_create_sql from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME - from sqlbuild.compiler.migrations.main.create_table_sql import ( - build_migration_state_create_table_sql, - ) - create_sql: str = build_migration_state_create_table_sql( + create_sql: str = render_migration_state_create_sql( + kind="model", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -678,12 +677,11 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem def render_create_old_name_view_state_table_sql( self, *, database: str | None, schema: str ) -> str: + from sqlbuild._native import render_migration_state_create_sql from sqlbuild.compiler.migrations.constants import OLD_NAME_VIEW_TABLE_NAME - from sqlbuild.compiler.migrations.main.old_name_view_create_table_sql import ( - build_old_name_view_state_create_table_sql, - ) - create_sql: str = build_old_name_view_state_create_table_sql( + create_sql: str = render_migration_state_create_sql( + kind="old_name", database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -699,12 +697,11 @@ def render_create_old_name_view_state_table_sql( def render_create_column_migration_state_table_sql( self, *, database: str | None, schema: str ) -> str: + from sqlbuild._native import render_migration_state_create_sql from sqlbuild.compiler.migrations.constants import COLUMN_MIGRATION_TABLE_NAME - from sqlbuild.compiler.migrations.main.column_create_table_sql import ( - build_column_migration_state_create_table_sql, - ) - create_sql: str = build_column_migration_state_create_table_sql( + create_sql: str = render_migration_state_create_sql( + kind="column", database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/compiler/migrations/_helpers/__init__.py b/src/sqlbuild/compiler/migrations/_helpers/__init__.py deleted file mode 100644 index 114aa7085f..0000000000 --- a/src/sqlbuild/compiler/migrations/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Model migration state helper package.""" diff --git a/src/sqlbuild/compiler/migrations/_helpers/column_sql.py b/src/sqlbuild/compiler/migrations/_helpers/column_sql.py deleted file mode 100644 index 4cacd717c9..0000000000 --- a/src/sqlbuild/compiler/migrations/_helpers/column_sql.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Portable SQL for append-only column migration events.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild._native import render_event_exists_sql, render_event_insert_sql -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( - render_state_table_create_sql, -) -from sqlbuild.compiler.migrations._helpers.sql import ( - decode_event_timestamp, - optional_text, -) -from sqlbuild.compiler.migrations.constants import ( - COLUMN_MIGRATION_COLUMN_TYPES, - COLUMN_MIGRATION_COLUMNS, - COLUMN_MIGRATION_TABLE_NAME, -) -from sqlbuild.compiler.migrations.exceptions import MigrationStateError -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation -from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDiscovery - - -def qualified_column_migration_table( - *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] -) -> str: - table: str | None = render_qualified_name( - database=database, schema=schema, name=COLUMN_MIGRATION_TABLE_NAME - ) - if table is None: - raise MigrationStateError("column migration state requires a target schema") - return table - - -def build_column_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - return render_state_table_create_sql( - qualified_name=qualified_column_migration_table( - database=database, schema=schema, render_qualified_name=render_qualified_name - ), - columns=COLUMN_MIGRATION_COLUMNS, - column_types=COLUMN_MIGRATION_COLUMN_TYPES, - required_columns=frozenset(), - render_framework_type=render_framework_type, - ) - - -def build_column_existing_event_sql( - *, event: ColumnMigrationEvent, render_qualified_name: Callable[..., str | None] -) -> str: - return render_event_exists_sql( - table=_event_table(event=event, render_qualified_name=render_qualified_name), - event_id=event.event_id, - ) - - -def build_column_insert_sql( - *, event: ColumnMigrationEvent, render_qualified_name: Callable[..., str | None] -) -> str: - return render_event_insert_sql( - table=_event_table(event=event, render_qualified_name=render_qualified_name), - columns=COLUMN_MIGRATION_COLUMNS, - column_types=COLUMN_MIGRATION_COLUMN_TYPES, - values=( - event.event_id, - event.target_name, - event.model_name, - event.relation.database, - event.relation.schema, - event.relation.name, - event.origin_column, - event.destination_column, - event.discovery.value, - event.decision.value, - event.run_id, - event.created_at, - ), - ) - - -def build_column_read_sql( - *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] -) -> str: - table: str = qualified_column_migration_table( - database=database, schema=schema, render_qualified_name=render_qualified_name - ) - return ( - f"SELECT {', '.join(COLUMN_MIGRATION_COLUMNS)} FROM {table} ORDER BY created_at, event_id" - ) - - -def decode_column_event_row(row: tuple[Any, ...]) -> ColumnMigrationEvent: - values: dict[str, Any] = dict(zip(COLUMN_MIGRATION_COLUMNS, row, strict=False)) - return ColumnMigrationEvent( - event_id=str(values["event_id"]), - target_name=optional_text(values["target_name"]), - model_name=str(values["model_name"]), - relation=MigrationRelation( - database=optional_text(values["relation_database"]), - schema=optional_text(values["relation_schema"]), - name=str(values["relation_name"]), - ), - origin_column=str(values["origin_column"]), - destination_column=str(values["destination_column"]), - discovery=MigrationDiscovery(str(values["discovery"])), - decision=ColumnMigrationDecision(str(values["decision"])), - run_id=str(values["run_id"]), - created_at=decode_event_timestamp(values["created_at"]), - ) - - -def _event_table( - *, event: ColumnMigrationEvent, render_qualified_name: Callable[..., str | None] -) -> str: - if event.relation.schema is None: - raise MigrationStateError("column migration events require a relation schema") - return qualified_column_migration_table( - database=event.relation.database, - schema=event.relation.schema, - render_qualified_name=render_qualified_name, - ) diff --git a/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py b/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py deleted file mode 100644 index 0f9e9b4887..0000000000 --- a/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py +++ /dev/null @@ -1,204 +0,0 @@ -"""Portable SQL for append-only old-name view facts.""" - -from __future__ import annotations - -import json -from collections.abc import Callable, Iterable -from typing import Any - -from sqlbuild._native import render_event_exists_sql, render_event_insert_sql -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( - render_state_table_create_sql, -) -from sqlbuild.compiler.migrations._helpers.sql import ( - decode_event_timestamp, - optional_text, -) -from sqlbuild.compiler.migrations.constants import ( - OLD_NAME_VIEW_COLUMN_TYPES, - OLD_NAME_VIEW_COLUMNS, - OLD_NAME_VIEW_TABLE_NAME, -) -from sqlbuild.compiler.migrations.exceptions import MigrationStateError -from sqlbuild.compiler.migrations.models import MigrationRelation, OldNameViewEvent -from sqlbuild.compiler.migrations.types import OldNameViewDropReason, OldNameViewEventType - - -def qualified_old_name_view_table( - *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] -) -> str: - table: str | None = render_qualified_name( - database=database, schema=schema, name=OLD_NAME_VIEW_TABLE_NAME - ) - if table is None: - raise MigrationStateError("old-name view state requires a target schema") - return table - - -def build_old_name_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - return render_state_table_create_sql( - qualified_name=qualified_old_name_view_table( - database=database, schema=schema, render_qualified_name=render_qualified_name - ), - columns=OLD_NAME_VIEW_COLUMNS, - column_types=OLD_NAME_VIEW_COLUMN_TYPES, - required_columns=frozenset(), - render_framework_type=render_framework_type, - ) - - -def build_old_name_existing_event_sql( - *, event: OldNameViewEvent, render_qualified_name: Callable[..., str | None] -) -> str: - return render_event_exists_sql( - table=_event_table(event=event, render_qualified_name=render_qualified_name), - event_id=event.event_id, - ) - - -def build_old_name_insert_sql( - *, event: OldNameViewEvent, render_qualified_name: Callable[..., str | None] -) -> str: - return render_event_insert_sql( - table=_event_table(event=event, render_qualified_name=render_qualified_name), - columns=OLD_NAME_VIEW_COLUMNS, - column_types=OLD_NAME_VIEW_COLUMN_TYPES, - values=( - event.event_id, - event.target_name, - event.event_type.value, - event.migration_event_id, - event.destination_model, - event.old.database, - event.old.schema, - event.old.name, - event.new.database, - event.new.schema, - event.new.name, - event.view_retention, - event.archive_name, - _encode_aliases(event.column_aliases) if event.column_aliases else None, - None if event.grants_copied is None else json.dumps(list(event.grants_copied)), - event.view_sql, - event.expires_at, - None if event.drop_reason is None else event.drop_reason.value, - event.run_id, - event.created_at, - ), - ) - - -def readable_old_name_columns(stored_columns: Iterable[str]) -> tuple[str, ...]: - """Return the known columns a stored table has; columns added later may be absent.""" - - stored: frozenset[str] = frozenset(column.lower() for column in stored_columns) - return tuple(column for column in OLD_NAME_VIEW_COLUMNS if column in stored) - - -def build_old_name_read_sql( - *, - database: str | None, - schema: str, - columns: tuple[str, ...], - render_qualified_name: Callable[..., str | None], -) -> str: - table: str = qualified_old_name_view_table( - database=database, schema=schema, render_qualified_name=render_qualified_name - ) - return f"SELECT {', '.join(columns)} FROM {table} ORDER BY created_at, event_id" - - -def decode_old_name_event_row( - *, row: tuple[Any, ...], columns: tuple[str, ...] = OLD_NAME_VIEW_COLUMNS -) -> OldNameViewEvent | None: - """Decode one row of ``columns``, reading absent ones as NULL; skip unknown event types.""" - - values: dict[str, Any] = dict(zip(columns, row, strict=False)) - try: - event_type: OldNameViewEventType = OldNameViewEventType(str(values.get("event_type"))) - except ValueError: - return None - raw_reason: str | None = optional_text(values.get("drop_reason")) - raw_expires: object | None = values.get("expires_at") - return OldNameViewEvent( - event_id=str(values["event_id"]), - target_name=optional_text(values.get("target_name")), - event_type=event_type, - migration_event_id=str(values["migration_event_id"]), - destination_model=str(values["destination_model"]), - old=MigrationRelation( - database=optional_text(values.get("old_database")), - schema=optional_text(values.get("old_schema")), - name=str(values["old_name"]), - ), - new=MigrationRelation( - database=optional_text(values.get("new_database")), - schema=optional_text(values.get("new_schema")), - name=str(values["new_name"]), - ), - run_id=str(values["run_id"]), - created_at=decode_event_timestamp(values["created_at"]), - view_retention=optional_text(values.get("view_retention")), - archive_name=optional_text(values.get("archive_name")), - column_aliases=_decode_aliases(optional_text(values.get("column_aliases"))), - expires_at=None if raw_expires is None else decode_event_timestamp(raw_expires), - drop_reason=_decode_reason(raw_reason), - grants_copied=_decode_grants(optional_text(values.get("grants_copied"))), - view_sql=optional_text(values.get("view_sql")), - ) - - -def _decode_reason(raw: str | None) -> OldNameViewDropReason | None: - if raw is None: - return None - try: - return OldNameViewDropReason(raw) - except ValueError: - return None - - -def _decode_grants(raw: str | None) -> tuple[str, ...] | None: - if raw is None: - return None - try: - payload: object = json.loads(raw) - except json.JSONDecodeError: - return None - if not isinstance(payload, list): - return None - return tuple(str(statement) for statement in payload) - - -def _encode_aliases(aliases: tuple[tuple[str, str], ...]) -> str: - return json.dumps(dict(aliases), sort_keys=True, separators=(",", ":")) - - -def _decode_aliases(raw: str | None) -> tuple[tuple[str, str], ...]: - if not raw: - return () - try: - payload: object = json.loads(raw) - except json.JSONDecodeError: - return () - if not isinstance(payload, dict): - return () - return tuple(sorted((str(old), str(new)) for old, new in payload.items())) - - -def _event_table( - *, event: OldNameViewEvent, render_qualified_name: Callable[..., str | None] -) -> str: - if event.new.schema is None: - raise MigrationStateError("old-name view facts require a destination schema") - return qualified_old_name_view_table( - database=event.new.database, - schema=event.new.schema, - render_qualified_name=render_qualified_name, - ) diff --git a/src/sqlbuild/compiler/migrations/_helpers/sql.py b/src/sqlbuild/compiler/migrations/_helpers/sql.py deleted file mode 100644 index 0cf1616b6d..0000000000 --- a/src/sqlbuild/compiler/migrations/_helpers/sql.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Portable SQL for append-only model migration events.""" - -from __future__ import annotations - -from collections.abc import Callable -from datetime import UTC, datetime -from typing import Any - -from sqlbuild._native import render_event_exists_sql, render_event_insert_sql -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( - render_state_table_create_sql, -) -from sqlbuild.compiler.migrations.constants import ( - MIGRATION_COLUMN_TYPES, - MIGRATION_COLUMNS, - MIGRATION_TABLE_NAME, -) -from sqlbuild.compiler.migrations.exceptions import MigrationStateError -from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation -from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery - - -def qualified_migration_table( - *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] -) -> str: - table: str | None = render_qualified_name( - database=database, schema=schema, name=MIGRATION_TABLE_NAME - ) - if table is None: - raise MigrationStateError("model migration state requires a target schema") - return table - - -def build_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - return render_state_table_create_sql( - qualified_name=qualified_migration_table( - database=database, schema=schema, render_qualified_name=render_qualified_name - ), - columns=MIGRATION_COLUMNS, - column_types=MIGRATION_COLUMN_TYPES, - required_columns=frozenset(), - render_framework_type=render_framework_type, - ) - - -def build_existing_event_sql( - *, event: MigrationEvent, render_qualified_name: Callable[..., str | None] -) -> str: - table: str = _event_table(event=event, render_qualified_name=render_qualified_name) - return render_event_exists_sql(table=table, event_id=event.event_id) - - -def build_insert_sql( - *, event: MigrationEvent, render_qualified_name: Callable[..., str | None] -) -> str: - table: str = _event_table(event=event, render_qualified_name=render_qualified_name) - return render_event_insert_sql( - table=table, - columns=MIGRATION_COLUMNS, - column_types=MIGRATION_COLUMN_TYPES, - values=_event_values(event), - ) - - -def _event_table(*, event: MigrationEvent, render_qualified_name: Callable[..., str | None]) -> str: - if event.destination.schema is None: - raise MigrationStateError("model migration events require a destination schema") - return qualified_migration_table( - database=event.destination.database, - schema=event.destination.schema, - render_qualified_name=render_qualified_name, - ) - - -def build_read_sql( - *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] -) -> str: - table: str = qualified_migration_table( - database=database, schema=schema, render_qualified_name=render_qualified_name - ) - return f"SELECT {', '.join(MIGRATION_COLUMNS)} FROM {table} ORDER BY created_at, event_id" - - -def decode_event_row(row: tuple[Any, ...]) -> MigrationEvent: - values: dict[str, Any] = dict(zip(MIGRATION_COLUMNS, row, strict=False)) - created_at: datetime = decode_event_timestamp(values["created_at"]) - return MigrationEvent( - event_id=str(values["event_id"]), - target_name=optional_text(values["target_name"]), - origin_model=optional_text(values["origin_model"]), - origin=MigrationRelation( - database=optional_text(values["origin_database"]), - schema=optional_text(values["origin_schema"]), - name=str(values["origin_name"]), - ), - destination_model=str(values["destination_model"]), - destination=MigrationRelation( - database=optional_text(values["destination_database"]), - schema=optional_text(values["destination_schema"]), - name=str(values["destination_name"]), - ), - origin_version_hash=str(values["origin_version_hash"] or ""), - discovery=MigrationDiscovery(str(values["discovery"])), - decision=MigrationDecision(str(values["decision"])), - run_id=str(values["run_id"]), - created_at=created_at, - ) - - -def _event_values(event: MigrationEvent) -> tuple[object | None, ...]: - return ( - event.event_id, - event.target_name, - event.origin_model, - event.origin.database, - event.origin.schema, - event.origin.name, - event.destination_model, - event.destination.database, - event.destination.schema, - event.destination.name, - event.origin_version_hash, - event.discovery.value, - event.decision.value, - event.run_id, - event.created_at, - ) - - -def decode_event_timestamp(raw: object) -> datetime: - """Decode a stored event timestamp as an aware UTC datetime.""" - - created_at: datetime = raw if isinstance(raw, datetime) else datetime.fromisoformat(str(raw)) - if created_at.tzinfo is None: - created_at = created_at.replace(tzinfo=UTC) - return created_at - - -def optional_text(value: object | None) -> str | None: - """Return a stored nullable text value as a string or None.""" - - return None if value is None else str(value) diff --git a/src/sqlbuild/compiler/migrations/_helpers/writes.py b/src/sqlbuild/compiler/migrations/_helpers/writes.py deleted file mode 100644 index e01b7c8b72..0000000000 --- a/src/sqlbuild/compiler/migrations/_helpers/writes.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Idempotent append of one migration event row with bounded retries.""" - -from __future__ import annotations - -import logging -import time -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_RETRY_BASE_SECONDS - - -def append_event_row( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - existing_sql: str, - insert_sql: str, - create_table_sql: str | None, - attempts: int, - subject: str, -) -> None: - """Create the state table when given, then insert the row unless its event ID exists.""" - - attempt: int - for attempt in range(attempts): - try: - if create_table_sql is not None: - _ = execute(connection=connection, sql=create_table_sql) - if not execute(connection=connection, sql=existing_sql).fetchall(): - _ = execute(connection=connection, sql=insert_sql) - return - except Exception as error: - if attempt + 1 == attempts: - raise - logging.getLogger("sqlbuild.migrations").warning( - "migration event write attempt %s/%s failed for '%s'; retrying: %s", - attempt + 1, - attempts, - subject, - error, - ) - time.sleep(MIGRATION_WRITE_RETRY_BASE_SECONDS * (attempt + 1)) diff --git a/src/sqlbuild/compiler/migrations/constants.py b/src/sqlbuild/compiler/migrations/constants.py index 07c1e30dcb..07187e2a3c 100644 --- a/src/sqlbuild/compiler/migrations/constants.py +++ b/src/sqlbuild/compiler/migrations/constants.py @@ -2,85 +2,9 @@ from __future__ import annotations -from sqlbuild.sql_values.types import StateSqlValueType - MIGRATION_TABLE_NAME: str = "_sqlbuild_migrations" MIGRATION_WRITE_ATTEMPTS: int = 3 -MIGRATION_WRITE_RETRY_BASE_SECONDS: float = 0.05 - -MIGRATION_COLUMNS: tuple[str, ...] = ( - "event_id", - "target_name", - "origin_model", - "origin_database", - "origin_schema", - "origin_name", - "destination_model", - "destination_database", - "destination_schema", - "destination_name", - "origin_version_hash", - "discovery", - "decision", - "run_id", - "created_at", -) - -MIGRATION_COLUMN_TYPES: dict[str, StateSqlValueType] = { - **{column: StateSqlValueType.STRING for column in MIGRATION_COLUMNS}, - "created_at": StateSqlValueType.TEXT_TIMESTAMP, -} COLUMN_MIGRATION_TABLE_NAME: str = "_sqlbuild_column_migrations" -COLUMN_MIGRATION_COLUMNS: tuple[str, ...] = ( - "event_id", - "target_name", - "model_name", - "relation_database", - "relation_schema", - "relation_name", - "origin_column", - "destination_column", - "discovery", - "decision", - "run_id", - "created_at", -) - -COLUMN_MIGRATION_COLUMN_TYPES: dict[str, StateSqlValueType] = { - **{column: StateSqlValueType.STRING for column in COLUMN_MIGRATION_COLUMNS}, - "created_at": StateSqlValueType.TEXT_TIMESTAMP, -} - OLD_NAME_VIEW_TABLE_NAME: str = "_sqlbuild_old_name_views" -OLD_NAME_VIEW_CONFIG_KEY: str = "old_name_view" - -OLD_NAME_VIEW_COLUMNS: tuple[str, ...] = ( - "event_id", - "target_name", - "event_type", - "migration_event_id", - "destination_model", - "old_database", - "old_schema", - "old_name", - "new_database", - "new_schema", - "new_name", - "view_retention", - "archive_name", - "column_aliases", - "grants_copied", - "view_sql", - "expires_at", - "drop_reason", - "run_id", - "created_at", -) - -OLD_NAME_VIEW_COLUMN_TYPES: dict[str, StateSqlValueType] = { - **{column: StateSqlValueType.STRING for column in OLD_NAME_VIEW_COLUMNS}, - "expires_at": StateSqlValueType.TEXT_TIMESTAMP, - "created_at": StateSqlValueType.TEXT_TIMESTAMP, -} diff --git a/src/sqlbuild/compiler/migrations/main/_newest_column_event.py b/src/sqlbuild/compiler/migrations/main/_newest_column_event.py deleted file mode 100644 index 6158873f46..0000000000 --- a/src/sqlbuild/compiler/migrations/main/_newest_column_event.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Project the newest column migration event that mentions a column pair.""" - -from __future__ import annotations - -from collections.abc import Iterable - -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation - - -def newest_column_migration_event_mentioning( - *, - events: Iterable[ColumnMigrationEvent], - relation: MigrationRelation, - origin_column: str, - destination_column: str, -) -> ColumnMigrationEvent | None: - """Return the latest event on the relation that renamed either column.""" - - columns: frozenset[str] = frozenset({origin_column.lower(), destination_column.lower()}) - newest: ColumnMigrationEvent | None = None - event: ColumnMigrationEvent - for event in events: - if not event.mentions(relation=relation, columns=columns): - continue - if newest is None or (event.created_at, event.event_id) > ( - newest.created_at, - newest.event_id, - ): - newest = event - return newest diff --git a/src/sqlbuild/compiler/migrations/main/_newest_event.py b/src/sqlbuild/compiler/migrations/main/_newest_event.py deleted file mode 100644 index 888f0d2a9e..0000000000 --- a/src/sqlbuild/compiler/migrations/main/_newest_event.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Project the newest migration event that mentions one relation.""" - -from __future__ import annotations - -from collections.abc import Iterable - -from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation - - -def newest_migration_event_mentioning( - *, events: Iterable[MigrationEvent], relation: MigrationRelation -) -> MigrationEvent | None: - """Return the latest event naming the relation as origin or destination.""" - - newest: MigrationEvent | None = None - event: MigrationEvent - for event in events: - if not event.mentions(relation): - continue - if newest is None or (event.created_at, event.event_id) > ( - newest.created_at, - newest.event_id, - ): - newest = event - return newest diff --git a/src/sqlbuild/compiler/migrations/main/_project_old_name_views.py b/src/sqlbuild/compiler/migrations/main/_project_old_name_views.py deleted file mode 100644 index c77a46db97..0000000000 --- a/src/sqlbuild/compiler/migrations/main/_project_old_name_views.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Project old-name view histories from recorded moves and old-name facts.""" - -from __future__ import annotations - -from collections.abc import Iterable - -from sqlbuild.compiler.migrations.models import ( - MigrationEvent, - OldNameViewEvent, - OldNameViewHistory, -) -from sqlbuild.compiler.migrations.types import OldNameViewEventType - - -def project_old_name_views( - *, moves: Iterable[MigrationEvent], facts: Iterable[OldNameViewEvent] -) -> tuple[OldNameViewHistory, ...]: - """Return one history per recorded move whose old name needs a compatibility view.""" - - by_move: dict[str, dict[OldNameViewEventType, OldNameViewEvent]] = {} - fact: OldNameViewEvent - for fact in sorted(facts, key=lambda item: (item.created_at, item.event_id)): - _ = by_move.setdefault(fact.migration_event_id, {}).setdefault(fact.event_type, fact) - histories: list[OldNameViewHistory] = [] - move: MigrationEvent - for move in sorted(moves, key=lambda item: (item.created_at, item.event_id)): - recorded: dict[OldNameViewEventType, OldNameViewEvent] = by_move.get(move.event_id, {}) - required: OldNameViewEvent | None = recorded.get(OldNameViewEventType.REQUIRED) - if required is None: - continue - histories.append( - OldNameViewHistory( - move=move, - required=required, - archived=recorded.get(OldNameViewEventType.ORIGIN_ARCHIVED), - created=recorded.get(OldNameViewEventType.VIEW_CREATED), - dropped=recorded.get(OldNameViewEventType.VIEW_DROPPED), - ) - ) - return tuple(histories) diff --git a/src/sqlbuild/compiler/migrations/main/_read_column_events.py b/src/sqlbuild/compiler/migrations/main/_read_column_events.py deleted file mode 100644 index a8b0b045f8..0000000000 --- a/src/sqlbuild/compiler/migrations/main/_read_column_events.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Read append-only column migration events from one schema.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations._helpers.column_sql import ( - build_column_read_sql, - decode_column_event_row, -) -from sqlbuild.compiler.migrations.exceptions import MigrationStateError -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent - - -def read_column_migration_events( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], -) -> tuple[ColumnMigrationEvent, ...]: - """Read every column migration event this version understands from one schema.""" - - sql: str = build_column_read_sql( - database=database, schema=schema, render_qualified_name=render_qualified_name - ) - try: - rows: list[tuple[Any, ...]] = execute(connection=connection, sql=sql).fetchall() - except Exception as error: - raise MigrationStateError( - f"Unable to read column migration events from schema '{schema}': {error}" - ) from error - events: list[ColumnMigrationEvent] = [] - row: tuple[Any, ...] - for row in rows: - try: - events.append(decode_column_event_row(tuple(row))) - except ValueError: - continue - return tuple(events) diff --git a/src/sqlbuild/compiler/migrations/main/_read_events.py b/src/sqlbuild/compiler/migrations/main/_read_events.py deleted file mode 100644 index af847b5aff..0000000000 --- a/src/sqlbuild/compiler/migrations/main/_read_events.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Read append-only model migration events from one schema.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations._helpers.sql import build_read_sql, decode_event_row -from sqlbuild.compiler.migrations.exceptions import MigrationStateError -from sqlbuild.compiler.migrations.models import MigrationEvent - - -def read_migration_events( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], -) -> tuple[MigrationEvent, ...]: - """Read every migration event recorded in one schema's state table.""" - - sql: str = build_read_sql( - database=database, schema=schema, render_qualified_name=render_qualified_name - ) - try: - rows: list[tuple[Any, ...]] = execute(connection=connection, sql=sql).fetchall() - except Exception as error: - raise MigrationStateError( - f"Unable to read model migration events from schema '{schema}': {error}" - ) from error - return tuple(decode_event_row(tuple(row)) for row in rows) diff --git a/src/sqlbuild/compiler/migrations/main/_resolve_old_name_view_retention.py b/src/sqlbuild/compiler/migrations/main/_resolve_old_name_view_retention.py deleted file mode 100644 index f65d17b304..0000000000 --- a/src/sqlbuild/compiler/migrations/main/_resolve_old_name_view_retention.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Resolve how long a migrated model keeps a compatibility view at its old name.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.compiler.migrations.constants import OLD_NAME_VIEW_CONFIG_KEY - - -def resolve_old_name_view_retention( - *, config_values: Mapping[str, object], project_retention: str | None -) -> str | None: - """Return the model's retention text, the project's when unset, or None when disabled.""" - - if OLD_NAME_VIEW_CONFIG_KEY not in config_values: - return project_retention - value: object = config_values[OLD_NAME_VIEW_CONFIG_KEY] - if value is False: - return None - return str(value).strip() diff --git a/src/sqlbuild/compiler/migrations/main/column_create_table_sql.py b/src/sqlbuild/compiler/migrations/main/column_create_table_sql.py deleted file mode 100644 index 37a22d0e4d..0000000000 --- a/src/sqlbuild/compiler/migrations/main/column_create_table_sql.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Column migration state table SQL entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.compiler.migrations._helpers.column_sql import build_column_create_table_sql - - -def build_column_migration_state_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - """Build DDL that creates the column migration event table when it is missing.""" - - return build_column_create_table_sql( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - render_framework_type=render_framework_type, - ) diff --git a/src/sqlbuild/compiler/migrations/main/create_table_sql.py b/src/sqlbuild/compiler/migrations/main/create_table_sql.py deleted file mode 100644 index 75f9c3eea5..0000000000 --- a/src/sqlbuild/compiler/migrations/main/create_table_sql.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Model migration state table SQL entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.compiler.migrations._helpers.sql import build_create_table_sql - - -def build_migration_state_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - """Build DDL that creates the model migration event table when it is missing.""" - - return build_create_table_sql( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - render_framework_type=render_framework_type, - ) diff --git a/src/sqlbuild/compiler/migrations/main/old_name_view_create_table_sql.py b/src/sqlbuild/compiler/migrations/main/old_name_view_create_table_sql.py deleted file mode 100644 index c489c71eee..0000000000 --- a/src/sqlbuild/compiler/migrations/main/old_name_view_create_table_sql.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Old-name view state table SQL entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.compiler.migrations._helpers.old_name_sql import build_old_name_create_table_sql - - -def build_old_name_view_state_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - """Build DDL that creates the old-name view fact table when it is missing.""" - - return build_old_name_create_table_sql( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - render_framework_type=render_framework_type, - ) diff --git a/src/sqlbuild/compiler/migrations/main/read_old_name_view_events.py b/src/sqlbuild/compiler/migrations/main/read_old_name_view_events.py deleted file mode 100644 index a08d199876..0000000000 --- a/src/sqlbuild/compiler/migrations/main/read_old_name_view_events.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Read append-only old-name view facts from one schema.""" - -from __future__ import annotations - -from collections.abc import Callable, Iterable -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations._helpers.old_name_sql import ( - build_old_name_read_sql, - decode_old_name_event_row, - readable_old_name_columns, -) -from sqlbuild.compiler.migrations.exceptions import MigrationStateError -from sqlbuild.compiler.migrations.models import OldNameViewEvent - - -def read_old_name_view_events( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - database: str | None, - schema: str, - stored_columns: Iterable[str], - render_qualified_name: Callable[..., str | None], -) -> tuple[OldNameViewEvent, ...]: - """Read every old-name fact this version understands from the columns the table has.""" - - columns: tuple[str, ...] = readable_old_name_columns(stored_columns) - sql: str = build_old_name_read_sql( - database=database, - schema=schema, - columns=columns, - render_qualified_name=render_qualified_name, - ) - try: - rows: list[tuple[Any, ...]] = execute(connection=connection, sql=sql).fetchall() - except Exception as error: - raise MigrationStateError( - f"Unable to read old-name view facts from schema '{schema}': {error}" - ) from error - events: list[OldNameViewEvent] = [] - row: tuple[Any, ...] - for row in rows: - event: OldNameViewEvent | None = decode_old_name_event_row(row=tuple(row), columns=columns) - if event is not None: - events.append(event) - return tuple(events) diff --git a/src/sqlbuild/compiler/migrations/main/read_old_name_view_histories.py b/src/sqlbuild/compiler/migrations/main/read_old_name_view_histories.py deleted file mode 100644 index da9607a15c..0000000000 --- a/src/sqlbuild/compiler/migrations/main/read_old_name_view_histories.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Read the old-name view history of every recorded move in one schema.""" - -from __future__ import annotations - -from collections.abc import Callable, Iterable -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations.main._project_old_name_views import project_old_name_views -from sqlbuild.compiler.migrations.main._read_events import read_migration_events -from sqlbuild.compiler.migrations.main.read_old_name_view_events import ( - read_old_name_view_events, -) -from sqlbuild.compiler.migrations.models import OldNameViewHistory - - -def read_old_name_view_histories( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - database: str | None, - schema: str, - stored_columns: Iterable[str], - render_qualified_name: Callable[..., str | None], -) -> tuple[OldNameViewHistory, ...]: - """Read moves and old-name facts from a schema that holds both state tables.""" - - return project_old_name_views( - moves=read_migration_events( - connection=connection, - execute=execute, - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - ), - facts=read_old_name_view_events( - connection=connection, - execute=execute, - database=database, - schema=schema, - stored_columns=stored_columns, - render_qualified_name=render_qualified_name, - ), - ) diff --git a/src/sqlbuild/compiler/migrations/main/read_recorded_column_renames.py b/src/sqlbuild/compiler/migrations/main/read_recorded_column_renames.py deleted file mode 100644 index 74f4de2165..0000000000 --- a/src/sqlbuild/compiler/migrations/main/read_recorded_column_renames.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Read the column renames recorded on one relation since a point in time.""" - -from __future__ import annotations - -from collections.abc import Callable -from datetime import datetime -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations.main._read_column_events import read_column_migration_events -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation - - -def read_recorded_column_renames( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - relation: MigrationRelation, - since: datetime, - render_qualified_name: Callable[..., str | None], -) -> tuple[ColumnMigrationEvent, ...]: - """Return renames recorded on the relation at or after ``since``, oldest first.""" - - return tuple( - sorted( - ( - event - for event in read_column_migration_events( - connection=connection, - execute=execute, - database=relation.database, - schema=relation.schema or "", - render_qualified_name=render_qualified_name, - ) - if event.relation.matches(relation) - and event.decision.records_event - and event.created_at >= since - ), - key=lambda event: (event.created_at, event.event_id), - ) - ) diff --git a/src/sqlbuild/compiler/migrations/main/stored_old_name_view_columns.py b/src/sqlbuild/compiler/migrations/main/stored_old_name_view_columns.py deleted file mode 100644 index 3fd7053a88..0000000000 --- a/src/sqlbuild/compiler/migrations/main/stored_old_name_view_columns.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Inspect which columns a stored old-name view state table has.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.migrations.constants import OLD_NAME_VIEW_TABLE_NAME - - -def stored_old_name_view_columns( - *, adapter: BaseAdapter, connection: Any, database: str | None, schema: str -) -> tuple[str, ...]: - """Return the column names of one schema's old-name view state table.""" - - return tuple( - column.name - for column in adapter.get_columns( - connection=connection, database=database, schema=schema, name=OLD_NAME_VIEW_TABLE_NAME - ) - ) diff --git a/src/sqlbuild/compiler/migrations/main/write_column_event.py b/src/sqlbuild/compiler/migrations/main/write_column_event.py deleted file mode 100644 index 2847fd7cf7..0000000000 --- a/src/sqlbuild/compiler/migrations/main/write_column_event.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Append one column migration event idempotently.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations._helpers.column_sql import ( - build_column_existing_event_sql, - build_column_insert_sql, -) -from sqlbuild.compiler.migrations._helpers.writes import append_event_row -from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent - - -def write_column_migration_event( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - event: ColumnMigrationEvent, - render_qualified_name: Callable[..., str | None], - create_table_sql: str | None, - attempts: int = MIGRATION_WRITE_ATTEMPTS, -) -> None: - """Create the state table from adapter DDL when given, then insert the event if absent.""" - - _ = append_event_row( - connection=connection, - execute=execute, - existing_sql=build_column_existing_event_sql( - event=event, render_qualified_name=render_qualified_name - ), - insert_sql=build_column_insert_sql( - event=event, render_qualified_name=render_qualified_name - ), - create_table_sql=create_table_sql, - attempts=attempts, - subject=f"{event.model_name}.{event.destination_column}", - ) diff --git a/src/sqlbuild/compiler/migrations/main/write_event.py b/src/sqlbuild/compiler/migrations/main/write_event.py deleted file mode 100644 index a34abb2ce8..0000000000 --- a/src/sqlbuild/compiler/migrations/main/write_event.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Append one model migration event idempotently.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations._helpers.sql import build_existing_event_sql, build_insert_sql -from sqlbuild.compiler.migrations._helpers.writes import append_event_row -from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.models import MigrationEvent - - -def write_migration_event( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - event: MigrationEvent, - render_qualified_name: Callable[..., str | None], - create_table_sql: str | None, - attempts: int = MIGRATION_WRITE_ATTEMPTS, -) -> None: - """Create the state table from adapter DDL when given, then insert the event if absent.""" - - _ = append_event_row( - connection=connection, - execute=execute, - existing_sql=build_existing_event_sql( - event=event, render_qualified_name=render_qualified_name - ), - insert_sql=build_insert_sql(event=event, render_qualified_name=render_qualified_name), - create_table_sql=create_table_sql, - attempts=attempts, - subject=event.destination_model, - ) diff --git a/src/sqlbuild/compiler/migrations/main/write_old_name_view_event.py b/src/sqlbuild/compiler/migrations/main/write_old_name_view_event.py deleted file mode 100644 index 8c143f2e1f..0000000000 --- a/src/sqlbuild/compiler/migrations/main/write_old_name_view_event.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Append one old-name view fact idempotently.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.migrations._helpers.old_name_sql import ( - build_old_name_existing_event_sql, - build_old_name_insert_sql, -) -from sqlbuild.compiler.migrations._helpers.writes import append_event_row -from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.models import OldNameViewEvent - - -def write_old_name_view_event( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - event: OldNameViewEvent, - render_qualified_name: Callable[..., str | None], - create_table_sql: str | None, - attempts: int = MIGRATION_WRITE_ATTEMPTS, -) -> None: - """Create the state table from adapter DDL when given, then insert the fact if absent.""" - - _ = append_event_row( - connection=connection, - execute=execute, - existing_sql=build_old_name_existing_event_sql( - event=event, render_qualified_name=render_qualified_name - ), - insert_sql=build_old_name_insert_sql( - event=event, render_qualified_name=render_qualified_name - ), - create_table_sql=create_table_sql, - attempts=attempts, - subject=f"{event.destination_model} old name {event.old.name}", - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py index 642a82b8af..3b0738dc89 100644 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py +++ b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py @@ -5,18 +5,16 @@ from sqlbuild._native import ( build_compiled_function_fingerprint_sql, build_seed_identity, + build_selected_audit_entries, + extract_seed_columns, get_materialization_type, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.output.plan_entry import ( - build_planner_relations_context, - extract_seed_columns, -) +from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context from sqlbuild.compiler.planner._helpers.output.plan_output import ( - build_selected_audit_entries, build_selected_test_entries, plan_function, sql_test_is_selected, diff --git a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py b/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py deleted file mode 100644 index 68711fc1df..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py +++ /dev/null @@ -1,264 +0,0 @@ -"""Schema-aware completion for test-only relation fixtures.""" - -from __future__ import annotations - -import re -from collections.abc import Sequence -from functools import lru_cache -from typing import Any - -from sqlbuild._native import ( - analyze_resolved_column_reads, -) -from sqlbuild._native import ( - build_relation_fixture_completion as _native_fixture_completion, -) -from sqlbuild._native import ( - build_relation_fixture_context as build_relation_fixture_context, -) -from sqlbuild._native import ( - complete_empty_fixture_sql as complete_empty_fixture_sql, -) -from sqlbuild._native import ( - complete_typed_null_columns as complete_typed_null_columns, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( - CompiledLineageColumnFact, - CompiledLineageSourceFact, - CompileSqlReference, - DynamicColumnContractProof, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.lineage.types import ( - ColumnLineageConfidence, -) -from sqlbuild.compiler.planner.models import ( - FixtureRelationMetadata, - RelationFixtureCompletion, - RelationFixturePlanningContext, -) -from sqlbuild.compiler.planner.types import FixtureGroups, FixtureKey -from sqlbuild.compiler.references.constants import REF_PATTERN, SEED_PATTERN, SOURCE_PATTERN -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql - -_DBT_REF_PATTERN: re.Pattern[str] = re.compile(r'__dbt_ref\("([^"]+)"\s*,\s*"([^"]+)"\)') - - -def build_relation_fixture_completion( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - ordered_model_names: tuple[str, ...], - fixture_groups: FixtureGroups, - planning_context: RelationFixturePlanningContext | None = None, -) -> RelationFixtureCompletion: - """Complete required omitted columns with typed nulls unless explicitly non-nullable.""" - - return _native_fixture_completion( - project=project, - adapter=adapter, - request=(ordered_model_names, fixture_groups), - planning_context=planning_context, - ) - - -def _required_fixture_columns( - *, - ordered_model_names: tuple[str, ...], - model_map: dict[str, CompiledModelView], - fixture_keys: frozenset[FixtureKey], - relations: dict[FixtureKey, FixtureRelationMetadata], - authoritative_columns: dict[FixtureKey, frozenset[str]], - adapter: BaseAdapter, -) -> tuple[dict[FixtureKey, dict[str, set[str]]], frozenset[FixtureKey]]: - required: dict[FixtureKey, dict[str, set[str]]] = {} - star_fixture_keys: set[FixtureKey] = set() - dialect: str | None = adapter.expression_inference_profile().sql_analysis_dialect - for model_name in ordered_model_names: - model: CompiledModelView | None = model_map.get(model_name) - if model is None: - continue - for upstream in _resolved_fixture_reads( - query_sql=model.query_sql, - references=model.references, - fast_lineage=model.fast_lineage_columns or (), - fixture_keys=fixture_keys, - dialect=dialect, - ): - key: FixtureKey = ( - CompiledResourceType(upstream.resource_type), - upstream.resource_name, - ) - if key not in fixture_keys: - continue - if ( - key in authoritative_columns - and upstream.column_name.casefold() not in authoritative_columns[key] - ): - continue - required.setdefault(key, {}).setdefault(upstream.column_name, set()).add(model_name) - if model.fast_lineage_has_star: - model_star_fixture_keys: frozenset[FixtureKey] = _star_fixture_keys( - model=model, - fixture_keys=fixture_keys, - relations=relations, - adapter=adapter, - ) - star_fixture_keys.update(model_star_fixture_keys) - for key in model_star_fixture_keys: - relation: FixtureRelationMetadata | None = relations.get(key) - if relation is None or not relation.authoritative_names: - continue - for column in relation.columns: - required.setdefault(key, {}).setdefault(column.name, set()).add(model_name) - return required, frozenset(star_fixture_keys) - - -def _resolved_fixture_reads( - *, - query_sql: str, - references: tuple[CompileSqlReference, ...], - fast_lineage: Sequence[CompiledLineageColumnFact], - fixture_keys: frozenset[FixtureKey], - dialect: str | None, -) -> tuple[CompiledLineageSourceFact, ...]: - """Return external model reads without treating fallback lineage as authoritative.""" - - high_confidence_reads: list[CompiledLineageSourceFact] = [] - for fact in fast_lineage: - if fact.confidence == ColumnLineageConfidence.HIGH: - high_confidence_reads.extend(fact.upstream_columns) - needs_resolution: bool = any( - _reference_fixture_key(reference) in fixture_keys for reference in references - ) - if not needs_resolution: - return tuple(high_confidence_reads) - resolved: tuple[CompiledLineageSourceFact, ...] = _cached_resolved_column_reads( - query_sql=query_sql, - references=references, - dialect=dialect, - ) - return tuple(dict.fromkeys([*high_confidence_reads, *resolved])) - - -def _reference_fixture_key(reference: CompileSqlReference) -> FixtureKey | None: - resource_type: CompiledResourceType | None = { - SqlReferenceKind.REF: CompiledResourceType.MODEL, - SqlReferenceKind.SOURCE: CompiledResourceType.SOURCE, - SqlReferenceKind.SEED: CompiledResourceType.SEED, - }.get(SqlReferenceKind(reference.ref_kind)) - return (resource_type, reference.ref_name) if resource_type is not None else None - - -@lru_cache(maxsize=4096) -def _cached_resolved_column_reads( - *, - query_sql: str, - references: tuple[CompileSqlReference, ...], - dialect: str | None, -) -> tuple[CompiledLineageSourceFact, ...]: - return analyze_resolved_column_reads( - query_sql=query_sql, - references=references, - dialect=dialect, - ) - - -def _star_fixture_keys( - *, - model: CompiledModelView, - fixture_keys: frozenset[FixtureKey], - relations: dict[FixtureKey, FixtureRelationMetadata], - adapter: BaseAdapter, -) -> frozenset[FixtureKey]: - dependency_keys: frozenset[FixtureKey] = frozenset( - (CompiledResourceType(dependency.resource_type), dependency.name) - for dependency in model.deps - if (CompiledResourceType(dependency.resource_type), dependency.name) in fixture_keys - ) - dynamic_contract: DynamicColumnContractProof | None = model.dynamic_column_contract - if dynamic_contract is not None and dynamic_contract.output_proven: - input_relations: frozenset[str] = frozenset( - name.casefold() for name in dynamic_contract.input_relations - ) - return frozenset(key for key in dependency_keys if key[1].casefold() in input_relations) - schema_tables: list[dict[str, object]] = [] - for key in dependency_keys: - relation: FixtureRelationMetadata | None = relations.get(key) - if relation is None or not relation.columns: - continue - schema_tables.append( - { - "name": key[1], - "columns": [ - {"name": column.name, "type": column.type or "UNKNOWN"} - for column in relation.columns - ], - } - ) - if not schema_tables: - return frozenset() - polyglot_module: Any = import_polyglot_sql() - try: - analysis: object = polyglot_module.analyze_query( - _replace_relation_markers_with_stubs(model.query_sql), - { - "dialect": adapter.expression_inference_profile().sql_analysis_dialect or "generic", - "schema": {"tables": schema_tables}, - }, - ) - except polyglot_module.PolyglotError: - return frozenset() - if not isinstance(analysis, dict): - return frozenset() - star_projections: object = analysis.get("starProjections") - query_relations: object = analysis.get("relations") - if not isinstance(star_projections, list) or not isinstance(query_relations, list): - return frozenset() - relation_names_by_alias: dict[str, str] = {} - for query_relation in query_relations: - if not isinstance(query_relation, dict): - continue - name: object = query_relation.get("name") - alias: object = query_relation.get("alias") - if not isinstance(name, str): - continue - relation_names_by_alias[name.casefold()] = name.casefold() - if isinstance(alias, str): - relation_names_by_alias[alias.casefold()] = name.casefold() - star_relation_names: set[str] = set() - for projection in star_projections: - if not isinstance(projection, dict): - continue - table: object = projection.get("table") - if table is None: - expanded_columns: object = projection.get("expandedColumns") - if not isinstance(expanded_columns, list): - continue - expanded_names: set[str] = { - column.casefold() for column in expanded_columns if isinstance(column, str) - } - for key in dependency_keys: - relation: FixtureRelationMetadata | None = relations.get(key) - if relation is not None and any( - column.name.casefold() in expanded_names for column in relation.columns - ): - star_relation_names.add(key[1].casefold()) - elif isinstance(table, str): - relation_name: str | None = relation_names_by_alias.get(table.casefold()) - if relation_name is not None: - star_relation_names.add(relation_name) - return frozenset(key for key in dependency_keys if key[1].casefold() in star_relation_names) - - -def _replace_relation_markers_with_stubs(query_sql: str) -> str: - result: str = REF_PATTERN.sub(r"\1", query_sql) - result = SEED_PATTERN.sub(r"\1", result) - result = SOURCE_PATTERN.sub(r"\1", result) - return _DBT_REF_PATTERN.sub(r"\2", result) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/display.py b/src/sqlbuild/compiler/planner/_helpers/migrations/display.py deleted file mode 100644 index da617d68bc..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/display.py +++ /dev/null @@ -1,294 +0,0 @@ -"""Plan text for model migrations and identity handovers.""" - -from __future__ import annotations - -from collections.abc import Callable, Sequence - -from sqlbuild._native import append_overflow_line, count_header_style, visible_entries -from sqlbuild.compiler.migrations.types import ( - ColumnMigrationDecision, - MigrationCompatibility, - MigrationDecision, - MigrationDiscovery, - OldNameViewAction, -) -from sqlbuild.compiler.planner.constants import MANUAL_RENAME_HINT -from sqlbuild.compiler.planner.models import ( - ColumnMigrationPlanEntry, - ModelMigrationPlanEntry, - OldNameViewPlanEntry, - PlanOutput, -) -from sqlbuild.compiler.planner.types import MaterializationType -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.tree_connector import tree_connector -from sqlbuild.presentation.models import DisplayOptions - - -def format_model_migrations( - *, lines: list[str], plan: PlanOutput, display_options: DisplayOptions -) -> list[str]: - """Format declared and discovered model migrations, then identity handovers.""" - - result: list[str] = list(lines) - style: CliStyle = CliStyle(use_color=True) - header_style: Callable[[str], str] = count_header_style( - style=style, title_style=style.plan_section - ) - entries: tuple[ModelMigrationPlanEntry, ...] = plan.migration_entries - old_names: dict[str, OldNameViewPlanEntry] = { - entry.model_name: entry for entry in plan.old_name_view_entries - } - materializations: dict[str, str] = { - entry.name: str(entry.materialization_type) for entry in plan.model_entries - } - if entries: - result.append(header_style(f"Migrations ({len(entries)})")) - visible: Sequence[ModelMigrationPlanEntry] = visible_entries( - entries=entries, options=display_options - ) - entry: ModelMigrationPlanEntry - for entry in visible: - result.extend( - _migration_lines( - entry=entry, - style=style, - old_name=old_names.get(entry.model_name), - materialization=materializations.get(entry.model_name), - ) - ) - result = append_overflow_line( - lines=result, - total_count=len(entries), - visible_count=len(visible), - indent=" ", - options=display_options, - ) - result = _format_resumed_old_names( - lines=result, - plan=plan, - style=style, - header_style=header_style, - ) - return _format_column_migrations( - lines=result, plan=plan, style=style, header_style=header_style - ) - - -def _format_resumed_old_names( - *, - lines: list[str], - plan: PlanOutput, - style: CliStyle, - header_style: Callable[[str], str], -) -> list[str]: - """Format old-name steps that resume a move no longer listed as a migration.""" - - migrated: frozenset[str] = frozenset(entry.model_name for entry in plan.migration_entries) - resumed: tuple[OldNameViewPlanEntry, ...] = tuple( - entry for entry in plan.old_name_view_entries if entry.model_name not in migrated - ) - if not resumed: - return lines - result: list[str] = list(lines) - result.append(header_style(f"Old names ({len(resumed)})")) - entry: OldNameViewPlanEntry - for entry in resumed: - origin: str = entry.origin.qualified_name or entry.origin.name - destination: str = entry.destination.qualified_name or entry.destination.name - result.append( - f" {style.object_name(entry.model_name)} {style.muted(f'{origin} ->')} {destination}" - ) - result.extend(old_name_rows(entry=entry, style=style)) - return result - - -def old_name_rows(*, entry: OldNameViewPlanEntry, style: CliStyle) -> list[str]: - """Render the old name of a migration as a leaf with one fact per nested row.""" - - origin: str = entry.origin.qualified_name or entry.origin.name - facts: list[tuple[str, str]] = old_name_facts(entry) - rows: list[str] = [_property_row(label="old name", value=origin, style=style)] - index: int - label: str - value: str - for index, (label, value) in enumerate(facts): - connector: str = tree_connector(style=style, last=index == len(facts) - 1) - rows.append(f" {connector} {style.muted(label)} {value}") - return rows - - -def old_name_facts(entry: OldNameViewPlanEntry) -> list[tuple[str, str]]: - """Return what happens at a migrated model's old name, one labelled fact per row.""" - - if entry.action == OldNameViewAction.NONE: - return [("left for janitor", entry.reason or "no compatibility view")] - until: str = "" if entry.expires_at is None else f"until {entry.expires_at:%Y-%m-%d}" - view: str = f"live {until}".strip() if entry.action == OldNameViewAction.LIVE else until - facts: list[tuple[str, str]] = [("view", view)] - if entry.column_aliases: - facts.append(("columns", ", ".join(f"{old} <- {new}" for old, new in entry.column_aliases))) - if entry.grants_supported: - facts.append(("grants", _grants_text(entry))) - archived: str = ( - "already archived" if entry.action == OldNameViewAction.VIEW_ONLY else "archived" - ) - facts.append(("old table", archived)) - return facts - - -def _grants_text(entry: OldNameViewPlanEntry) -> str: - if entry.grants_copied is None: - return "copied from the old table" - return f"{entry.grants_copied} copied from the old table" - - -def _format_column_migrations( - *, - lines: list[str], - plan: PlanOutput, - style: CliStyle, - header_style: Callable[[str], str], -) -> list[str]: - """Format in-place column renames grouped under the model whose table they change.""" - - entries: tuple[ColumnMigrationPlanEntry, ...] = plan.column_migration_entries - if not entries: - return lines - result: list[str] = list(lines) - result.append(header_style(f"Column migrations ({len(entries)})")) - model_names: tuple[str, ...] = tuple(dict.fromkeys(entry.model_name for entry in entries)) - model_name: str - for model_name in model_names: - result.append(f" {style.object_name(model_name)} {style.muted('migrate columns')}") - entry: ColumnMigrationPlanEntry - for entry in entries: - if entry.model_name == model_name: - result.append(f" {_column_migration_text(entry=entry, style=style)}") - return result - - -def _column_migration_text(*, entry: ColumnMigrationPlanEntry, style: CliStyle) -> str: - rename: str = f"{entry.origin_column} -> {entry.destination_column}" - decision: str - if entry.blocks_build: - decision = style.error_strong(entry.decision.label) - elif entry.origin_missing: - decision = style.warning_strong(entry.decision.label) - elif entry.decision == ColumnMigrationDecision.DONE: - decision = style.muted(entry.decision.label) - else: - decision = style.accent_strong(entry.decision.label) - details: list[str] = [] - if entry.discovery != MigrationDiscovery.MANUAL: - details.append(entry.discovery.value) - if entry.decision == ColumnMigrationDecision.DONE and entry.completed_at is not None: - details.append(f"completed {entry.completed_at.isoformat()}") - suffix: str = f" {style.muted(f'({", ".join(details)})')}" if details else "" - return f"{rename} {decision}{suffix}" - - -def _migration_lines( - *, - entry: ModelMigrationPlanEntry, - style: CliStyle, - old_name: OldNameViewPlanEntry | None, - materialization: str | None, -) -> list[str]: - origin: str = entry.origin.qualified_name or entry.origin.name - destination: str = entry.destination.qualified_name or entry.destination.name - relation_move: str = f"{style.muted(f'{origin} ->')} {destination}" - name: str = style.object_name(entry.model_name) - decision: str = _decision_text(entry=entry, style=style) - if entry.decision == MigrationDecision.RENAMED: - decision = ( - style.muted(MigrationDecision.DONE.label) - if entry.completed_at is not None - else style.accent_strong(MigrationDecision.MIGRATE.label) - ) - rows: list[str] = [f" {name} {decision} {relation_move}"] - if entry.decision == MigrationDecision.RENAMED and entry.completed_at is None: - rows.append( - _property_row( - label="transfer", value=identity_transfer_label(materialization), style=style - ) - ) - if entry.decision.checks_compatibility: - rows.append( - _property_row( - label="compatibility", - value=_compatibility_text(compatibility=entry.compatibility, style=style), - style=style, - ) - ) - if entry.transfer is not None: - details: tuple[str | None, ...] = ( - entry.transfer.label - + ( - "" - if entry.transfer_fallback is None - else f" ({entry.transfer_fallback.label} if refused)" - ), - entry.storage_transition, - None if entry.promotion is None else f"promote by {entry.promotion.label}", - ) - rows.append( - _property_row( - label="transfer", - value=", ".join(detail for detail in details if detail), - style=style, - ) - ) - if entry.discovery != MigrationDiscovery.MANUAL: - rows.append(_property_row(label="discovery", value=entry.discovery.value, style=style)) - if entry.discovery == MigrationDiscovery.AUTOMATIC and entry.completed_at is None: - rows.append(_property_row(label="hint", value=MANUAL_RENAME_HINT, style=style)) - if entry.completed_at is not None: - rows.append( - _property_row( - label="completed", - value=f"{entry.completed_at.isoformat()} on target '{entry.target_label}'", - style=style, - ) - ) - if old_name is not None: - rows.extend(old_name_rows(entry=old_name, style=style)) - blocking: bool = entry.blocks_build or ( - entry.compatibility == MigrationCompatibility.INCOMPATIBLE - ) - finding_style: Callable[[str], str] = style.error if blocking else style.warning - rows.extend(f" {finding_style(f'! {finding}')}" for finding in entry.compatibility_findings) - return rows - - -def identity_transfer_label(materialization: str | None) -> str: - """Describe how a renamed table or view reaches its new name.""" - - if materialization == MaterializationType.VIEW: - return "recreate (view)" - return "rebuild (table)" - - -def _property_row(*, label: str, value: str, style: CliStyle) -> str: - return f" {style.muted(label)} {value}" - - -def _decision_text(*, entry: ModelMigrationPlanEntry, style: CliStyle) -> str: - decision: MigrationDecision = entry.decision - if entry.blocks_build and decision.blocks_build: - return style.error_strong(decision.label) - if entry.origin_missing: - return style.warning_strong(decision.label) - if decision in (MigrationDecision.SUPERSEDED_REPLACE, MigrationDecision.FORCED_REPLACE): - return style.warning_strong(decision.label) - if decision == MigrationDecision.DONE: - return style.muted(decision.label) - return style.accent_strong(decision.label) - - -def _compatibility_text(*, compatibility: MigrationCompatibility, style: CliStyle) -> str: - if compatibility == MigrationCompatibility.COMPATIBLE: - return style.success(compatibility.value) - if compatibility == MigrationCompatibility.INCOMPATIBLE: - return style.error(compatibility.value) - return compatibility.value diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py b/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py deleted file mode 100644 index 81fda8457b..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py +++ /dev/null @@ -1,507 +0,0 @@ -"""Plan what a build does at migrated models' old names.""" - -from __future__ import annotations - -from collections.abc import Iterable -from datetime import datetime - -from sqlbuild._native import ( - declared_column_aliases, - get_config_str, - hard_coded_relation_remedy, - match_project_relation, - old_name_claim_errors, - old_name_reference_message, - old_name_schemas_to_read, - old_name_view_skip_reason, -) -from sqlbuild._native import ( - migration_planning_database as planning_database, -) -from sqlbuild._native import ( - migration_project_schemas as project_schemas, -) -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.adapter.contract.types import RelationType -from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.migrations.main._resolve_old_name_view_retention import ( - resolve_old_name_view_retention, -) -from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location -from sqlbuild.compiler.migrations.models import MigrationRelation, OldNameViewHistory -from sqlbuild.compiler.migrations.types import ( - ColumnMigrationDecision, - MigrationDecision, - OldNameViewAction, - OldNameViewStatus, -) -from sqlbuild.compiler.planner.classes.migration_state_inspection import ( - MigrationStateInspection, -) -from sqlbuild.compiler.planner.models import ( - ColumnMigrationPlanEntry, - ModelMigrationPlanEntry, - OldNameView, - OldNameViewPlanEntry, - OldNameViewPlanning, - PlannerRuntime, - PlannerScope, - PlanWarning, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import MaterializationType, WarningSeverity -from sqlbuild.compiler.references.models import ( - LiteralSqlRelation, - ProjectRelation, - ProjectRelationIndex, - RelationName, -) -from sqlbuild.cursor_algebra.models import Duration -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.spec.contracts.models import MigrationsConfig - -_HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( - {MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT} -) -_ALIAS_DECISIONS: frozenset[ColumnMigrationDecision] = frozenset( - {ColumnMigrationDecision.RENAME, ColumnMigrationDecision.RECORD} -) -_RESUMED_STATUSES: frozenset[OldNameViewStatus] = frozenset( - {OldNameViewStatus.PENDING_ARCHIVE, OldNameViewStatus.PENDING_VIEW} -) -_PENDING_ACTIONS: dict[OldNameViewStatus, OldNameViewAction] = { - OldNameViewStatus.PENDING_ARCHIVE: OldNameViewAction.ARCHIVE_AND_VIEW, - OldNameViewStatus.PENDING_VIEW: OldNameViewAction.VIEW_ONLY, - OldNameViewStatus.LIVE: OldNameViewAction.LIVE, - OldNameViewStatus.EXPIRED: OldNameViewAction.LIVE, -} - - -def plan_old_name_views( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - migration_entries: tuple[ModelMigrationPlanEntry, ...], - column_entries: tuple[ColumnMigrationPlanEntry, ...], -) -> OldNameViewPlanning: - """Decide old-name steps for selected models and collect recorded compatibility views.""" - - models_by_name: dict[str, CompiledModelView] = { - model.name: model for model in runtime.project.models - } - selected: tuple[CompiledModelView, ...] = tuple( - models_by_name[key.name] - for key in scope.execution_order - if key in scope.selected_keys - and key.resource_type == CompiledResourceType.MODEL - and key.name in models_by_name - ) - state: MigrationStateInspection = MigrationStateInspection( - adapter=runtime.adapter, - connection=runtime.connection, - database=planning_database(runtime=runtime), - ) - schemas: set[str] = old_name_schemas_to_read( - snapshot=snapshot, - needed=_needed_schemas( - runtime=runtime, - selected=selected, - snapshot=snapshot, - migration_entries=migration_entries, - ), - ) - if schemas: - state.inspect_old_name_views(schemas=schemas) - target_name: str | None = runtime.project.effective_target_name - histories: tuple[OldNameViewHistory, ...] = tuple( - history - for history in state.old_name_histories - if history.move.target_name in (None, target_name) - ) - views: tuple[OldNameView, ...] = tuple( - _recorded_view(runtime=runtime, history=history, models=runtime.project.models) - for history in histories - if history.created is not None and history.dropped is None - ) - planner: _OldNamePlanner = _OldNamePlanner( - runtime=runtime, - models_by_name=models_by_name, - histories=histories, - views=views, - column_entries=column_entries, - ) - entries: list[OldNameViewPlanEntry] = [] - covered: set[str] = set() - entry: ModelMigrationPlanEntry - for entry in migration_entries: - planned: OldNameViewPlanEntry | None = planner.for_migration(entry) - if planned is not None: - entries.append(planned) - covered.add(entry.model_name) - model: CompiledModelView - for model in selected: - if model.name in covered: - continue - pending: OldNameViewPlanEntry | None = planner.pending_for(model) - if pending is not None: - entries.append(pending) - return OldNameViewPlanning( - entries=tuple(entries), - views=views, - warnings=( - *old_name_claim_errors( - selected=selected, snapshot=snapshot, views=views, now=runtime.invocation_time - ), - *_old_name_reference_errors(runtime=runtime, views=views, entries=tuple(entries)), - ), - ) - - -class _OldNamePlanner: - """Per-plan inputs shared by every old-name decision.""" - - def __init__( - self, - *, - runtime: PlannerRuntime, - models_by_name: dict[str, CompiledModelView], - histories: tuple[OldNameViewHistory, ...], - views: tuple[OldNameView, ...], - column_entries: tuple[ColumnMigrationPlanEntry, ...], - ) -> None: - self._runtime: PlannerRuntime = runtime - self._models_by_name: dict[str, CompiledModelView] = models_by_name - self._histories: tuple[OldNameViewHistory, ...] = histories - self._views: tuple[OldNameView, ...] = views - self._column_entries: tuple[ColumnMigrationPlanEntry, ...] = column_entries - config: MigrationsConfig = ( - runtime.project_config.migrations - if runtime.project_config is not None - else MigrationsConfig() - ) - self._project_retention: str | None = config.old_name_views - self._now: datetime = runtime.invocation_time - - def for_migration(self, entry: ModelMigrationPlanEntry) -> OldNameViewPlanEntry | None: - model: CompiledModelView | None = self._models_by_name.get(entry.model_name) - if model is None or entry.blocks_build: - return None - if _starts_move(entry): - return self._new_move(entry=entry, model=model) - if entry.decision == MigrationDecision.DONE or ( - entry.decision == MigrationDecision.RENAMED and entry.completed_at is not None - ): - history: OldNameViewHistory | None = self._newest_history( - destination=migration_relation_for_location(entry.destination), - origin=migration_relation_for_location(entry.origin), - ) - return None if history is None else self._from_history(history=history, model=model) - return None - - def pending_for(self, model: CompiledModelView) -> OldNameViewPlanEntry | None: - history: OldNameViewHistory | None = self._newest_history( - destination=migration_relation_for_location(model.destination), origin=None - ) - if history is None or history.move.destination_model != model.name: - return None - planned: OldNameViewPlanEntry | None = self._from_history(history=history, model=model) - if planned is None or not (planned.runs_steps or _skips_pending(planned)): - return None - return planned - - def _new_move( - self, *, entry: ModelMigrationPlanEntry, model: CompiledModelView - ) -> OldNameViewPlanEntry: - retention: str | None = resolve_old_name_view_retention( - config_values=model.config.values, project_retention=self._project_retention - ) - reason: str | None = self._skip_reason( - origin=entry.origin, origin_tracked=entry.origin_tracked, retention=retention - ) - if reason is not None or retention is None: - return _skipped_entry(entry=entry, reason=reason or "old_name_view false") - duration: Duration | None = Duration.parse(retention) - return OldNameViewPlanEntry( - model_name=entry.model_name, - origin=entry.origin, - destination=entry.destination, - action=OldNameViewAction.ARCHIVE_AND_VIEW, - target_name=entry.target_name, - retention=retention, - expires_at=None if duration is None else duration.add_to(self._now), - column_aliases=self._planned_aliases(model), - stores_history=_stores_history(model), - records_requirement=True, - grants_supported=self._runtime.adapter.relation_grants_supported, - ) - - def _skip_reason( - self, - *, - origin: CompiledRelationLocation, - origin_tracked: bool | None, - retention: str | None, - ) -> str | None: - """Return why the old name gets no steps; ``origin_tracked`` None means unknown.""" - - return old_name_view_skip_reason( - origin=origin, - origin_tracked=origin_tracked, - retention=retention, - models=self._models_by_name.values(), - views=self._views, - ) - - def _from_history( - self, *, history: OldNameViewHistory, model: CompiledModelView - ) -> OldNameViewPlanEntry | None: - """Resume a recorded move; its record proves SQLBuild owned the origin when it moved.""" - - status: OldNameViewStatus = history.status(now=self._now) - action: OldNameViewAction | None = _PENDING_ACTIONS.get(status) - if action is None: - return None - if status in _RESUMED_STATUSES: - reason: str | None = self._skip_reason( - origin=_location(runtime=self._runtime, relation=history.old), - origin_tracked=None, - retention=resolve_old_name_view_retention( - config_values=model.config.values, project_retention=self._project_retention - ), - ) - if reason is not None: - return _skipped_resume(runtime=self._runtime, history=history, reason=reason) - duration: Duration | None = Duration.parse(history.required.view_retention or "") - expires_at: datetime | None = ( - history.created.expires_at - if history.created is not None - else (None if duration is None else duration.add_to(self._now)) - ) - return OldNameViewPlanEntry( - model_name=model.name, - origin=_location(runtime=self._runtime, relation=history.old), - destination=_location(runtime=self._runtime, relation=history.new), - action=action, - target_name=history.move.target_name, - retention=history.required.view_retention, - expires_at=expires_at, - column_aliases=( - history.created.column_aliases - if history.created is not None - else self._planned_aliases(model) - ), - stores_history=_stores_history(model), - migration_event_id=history.move.event_id, - grants_copied=( - None - if history.created is None or history.created.grants_copied is None - else len(history.created.grants_copied) - ), - grants_supported=self._runtime.adapter.relation_grants_supported, - archived=history.archived is not None, - ) - - def _newest_history( - self, *, destination: MigrationRelation, origin: MigrationRelation | None - ) -> OldNameViewHistory | None: - matching: tuple[OldNameViewHistory, ...] = tuple( - history - for history in self._histories - if history.move.destination.matches(destination) - and (origin is None or history.move.origin.matches(origin)) - ) - if not matching: - return None - return max(matching, key=lambda item: (item.move.created_at, item.move.event_id)) - - def _planned_aliases(self, model: CompiledModelView) -> tuple[tuple[str, str], ...]: - if _stores_history(model): - return tuple( - sorted( - (entry.origin_column, entry.destination_column) - for entry in self._column_entries - if entry.model_name == model.name and entry.decision in _ALIAS_DECISIONS - ) - ) - return declared_column_aliases(model) - - -def _starts_move(entry: ModelMigrationPlanEntry) -> bool: - if entry.blocks_build: - return False - return entry.decision.moves_data or ( - entry.decision == MigrationDecision.RENAMED and entry.completed_at is None - ) - - -def _skips_pending(entry: OldNameViewPlanEntry) -> bool: - return entry.action == OldNameViewAction.NONE and entry.migration_event_id is not None - - -def _skipped_resume( - *, runtime: PlannerRuntime, history: OldNameViewHistory, reason: str -) -> OldNameViewPlanEntry: - return OldNameViewPlanEntry( - model_name=history.move.destination_model, - origin=_location(runtime=runtime, relation=history.old), - destination=_location(runtime=runtime, relation=history.new), - action=OldNameViewAction.NONE, - target_name=history.move.target_name, - migration_event_id=history.move.event_id, - reason=reason, - ) - - -def _stores_history(model: CompiledModelView) -> bool: - return ( - get_config_str(values=model.config.values, key="materialized") in _HISTORY_MATERIALIZATIONS - ) - - -def _skipped_entry(*, entry: ModelMigrationPlanEntry, reason: str) -> OldNameViewPlanEntry: - return OldNameViewPlanEntry( - model_name=entry.model_name, - origin=entry.origin, - destination=entry.destination, - action=OldNameViewAction.NONE, - target_name=entry.target_name, - reason=reason, - ) - - -def _needed_schemas( - *, - runtime: PlannerRuntime, - selected: tuple[CompiledModelView, ...], - snapshot: WarehouseSnapshot, - migration_entries: tuple[ModelMigrationPlanEntry, ...], -) -> set[str]: - """Return schemas whose old-name facts this plan needs.""" - - schemas: set[str] = { - model.destination.schema for model in selected if model.destination.schema is not None - } - schemas.update( - entry.origin.schema for entry in migration_entries if entry.origin.schema is not None - ) - if runtime.project.unmatched_literal_sql_relations or any( - _may_claim_view(model=model, snapshot=snapshot) for model in selected - ): - schemas.update(project_schemas(runtime=runtime)) - return schemas - - -def _may_claim_view(*, model: CompiledModelView, snapshot: WarehouseSnapshot) -> bool: - relation: RelationInfo | None = snapshot.existing_relations.get(model.name) - return relation is not None and _is_view(relation) - - -def _is_view(relation: RelationInfo) -> bool: - return normalize_relation_type(relation.relation_type) == RelationType.VIEW - - -def _old_name_reference_errors( - *, - runtime: PlannerRuntime, - views: tuple[OldNameView, ...], - entries: tuple[OldNameViewPlanEntry, ...], -) -> tuple[PlanWarning, ...]: - """Refuse literal Python SQL that reads a name kept only as a compatibility view.""" - - literals: tuple[LiteralSqlRelation, ...] = runtime.project.unmatched_literal_sql_relations - if not literals: - return () - index: ProjectRelationIndex = ProjectRelationIndex( - relations=tuple( - dict.fromkeys( - ( - *( - _old_name_relation(old=view.old, model_name=view.destination_model) - for view in views - ), - *( - _old_name_relation(old=entry.origin, model_name=entry.model_name) - for entry in entries - if entry.runs_steps - ), - ) - ) - ) - ) - errors: list[PlanWarning] = [] - literal: LiteralSqlRelation - for literal in literals: - match: ProjectRelation | None = match_project_relation( - index=index, relation=literal.relation - ) - if match is None or match.compatibility_for is None: - continue - written: str = ".".join( - part - for part in (literal.relation.database, literal.relation.schema, literal.relation.name) - if part - ) - errors.append( - PlanWarning( - model_name=None, - severity=WarningSeverity.ERROR, - message=( - f"{literal.relative_path}:{literal.line}: " - + old_name_reference_message( - owner_label=literal.owner_label, - written=written, - method=literal.method, - model_name=match.compatibility_for, - ) - + "; " - + hard_coded_relation_remedy(owner_kind=literal.owner_kind, ref=match.ref) - ), - code="P008", - ) - ) - return tuple(errors) - - -def _old_name_relation(*, old: CompiledRelationLocation, model_name: str) -> ProjectRelation: - return ProjectRelation( - ref=SqlResourceRef(kind=SqlResourceRefKind.MODEL, name=model_name), - relation=RelationName(name=old.name, schema=old.schema, database=old.database), - compatibility_for=model_name, - ) - - -def _recorded_view( - *, runtime: PlannerRuntime, history: OldNameViewHistory, models: Iterable[CompiledModelView] -) -> OldNameView: - return OldNameView( - destination_model=history.move.destination_model, - old=_location(runtime=runtime, relation=history.old), - new=_location(runtime=runtime, relation=history.new), - expires_at=None if history.created is None else history.created.expires_at, - column_aliases=() if history.created is None else history.created.column_aliases, - migration_event_id=history.move.event_id, - target_name=history.move.target_name, - name_reused_by=next( - ( - model.name - for model in models - if migration_relation_for_location(model.destination).matches(history.old) - ), - None, - ), - ) - - -def _location(*, runtime: PlannerRuntime, relation: MigrationRelation) -> CompiledRelationLocation: - return CompiledRelationLocation( - database=relation.database, - schema=relation.schema, - name=relation.name, - qualified_name=runtime.adapter.render_qualified_name( - database=relation.database, schema=relation.schema, name=relation.name - ), - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py deleted file mode 100644 index 93ec534f32..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Audit plan entry construction.""" - -from __future__ import annotations - -from sqlbuild._native import ( - assert_no_unresolved_sql_markers, - reads_outside_target_lineage, - render_audit_sql, - resolve_attachment_kind, - resolve_effective_run_scope, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.auditing.types import ( - AuditAttachmentKind, - AuditRunScope, - AuditSeverity, -) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledAuditView -from sqlbuild.compiler.planner.models import AuditPlanEntry -from sqlbuild.spec.contracts.models import SourceEntry - - -def plan_audit( - *, - audit: CompiledAuditView, - model_locations: dict[str, CompiledRelationLocation], - seed_locations: dict[str, CompiledRelationLocation], - source_map: dict[str, SourceEntry], - adapter: BaseAdapter, - lineage_upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - lineage_downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - model_materializations: dict[str, str], -) -> AuditPlanEntry: - """Build an audit plan entry with refs and sources resolved.""" - - resolved_sql: str = render_audit_sql( - unresolved_sql=audit.sql_body, - model_locations=model_locations, - seed_locations=seed_locations, - source_map=source_map, - adapter=adapter, - ) - assert_no_unresolved_sql_markers( - sql=resolved_sql, - context=f"audit '{audit.name}' planned SQL", - ) - evidence_resolved_sql: str | None = None - if audit.evidence_sql is not None: - evidence_resolved_sql = render_audit_sql( - unresolved_sql=audit.evidence_sql, - model_locations=model_locations, - seed_locations=seed_locations, - source_map=source_map, - adapter=adapter, - ) - assert_no_unresolved_sql_markers( - sql=evidence_resolved_sql, - context=f"audit '{audit.name}' planned evidence SQL", - ) - - attachment_kind: AuditAttachmentKind - attached_target_name: str | None - attachment_kind, attached_target_name = resolve_attachment_kind( - audit=audit, - downstream_deps=lineage_downstream_deps, - ) - - attached_materialization: str | None = None - if attachment_kind == AuditAttachmentKind.MODEL and attached_target_name is not None: - attached_materialization = model_materializations.get(attached_target_name) - - requested_run_scope: AuditRunScope = AuditRunScope( - audit.run_scope if audit.run_scope is not None else AuditRunScope.FINAL - ) - severity: AuditSeverity = AuditSeverity( - audit.severity if audit.severity is not None else AuditSeverity.WARN - ) - - effective_run_scope: AuditRunScope = resolve_effective_run_scope( - requested_run_scope=requested_run_scope, - attached_model_materialization=attached_materialization, - ) - attached_target_kind: AttachedAuditTargetKind | None = audit.attached_target_kind - if attached_target_kind is None and attachment_kind == AuditAttachmentKind.MODEL: - attached_target_kind = AttachedAuditTargetKind.MODEL - elif attached_target_kind is None and attachment_kind == AuditAttachmentKind.SOURCE: - attached_target_kind = AttachedAuditTargetKind.SOURCE - - return AuditPlanEntry( - key=audit.key, - name=audit.name, - definition_name=audit.definition_name, - resolved_sql=resolved_sql, - unresolved_sql=audit.sql_body, - attachment_kind=attachment_kind, - severity=severity, - requested_run_scope=requested_run_scope, - effective_run_scope=effective_run_scope, - evaluation_mode=audit.evaluation_mode, - value_column=( - None if audit.measurement_contract is None else audit.measurement_contract.value_column - ), - sample_count_column=( - None - if audit.measurement_contract is None - else audit.measurement_contract.sample_count_column - ), - sample_unit=( - None if audit.measurement_contract is None else audit.measurement_contract.sample_unit - ), - thresholds=audit.thresholds, - minimum_samples=audit.minimum_samples, - evidence_resolved_sql=evidence_resolved_sql, - evidence_unresolved_sql=audit.evidence_sql, - evidence_limit=audit.evidence_limit, - scope_deps=audit.scope_deps, - attached_target_kind=attached_target_kind, - attached_target_name=attached_target_name, - attached_column_name=audit.attached_column_name, - always_run=audit.always_run, - description=audit.description, - reads_outside_target_lineage=reads_outside_target_lineage( - audit=audit, upstream_deps=lineage_upstream_deps - ), - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index 1d7271da0a..c398c92029 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -8,56 +8,41 @@ from typing import Any from sqlbuild._native import ( - build_model_warnings, + apply_configured_microbatch_limits, + assemble_model_plan_entry, + audit_is_selected, + build_cursor_input_relations, build_source_load_map, - cap_microbatch_bounds, - check_cursor_type_consistency, - count_microbatches, detect_model_changes, - get_config_bool, get_config_cursor_bound, - get_config_int, get_config_str, - get_config_string_tuple, get_materialization_type, microbatch_guidance, - microbatch_limit_warning, - render_source_relation, replaces_incremental_relation, resolve_bounded_cursor_override, - resolve_cursor_input_roles, - resolve_effective_microbatch_batch_size, - resolve_lineage_reference, - resolve_microbatch_limit_config, + resolve_microbatch_lookback, resolve_model_plan_action, - resolve_schema_actions, + runtime_cursor_producer_names, source_binding_diagnostics, validate_capped_watermark_inputs, + validate_source_cursor_input_columns, ) -from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, CompilerDiagnostic, - CompileSqlReference, - CursorInputRoles, ) from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, CompiledAuditView, - CompiledFunctionView, CompiledModelView, CompiledProjectView, CompiledResourceType, - CompiledSeedView, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( check_full_rebuild_allowed, describe_full_rebuild_cause, @@ -119,41 +104,29 @@ PlannerScope, PlanWarning, ResolvedModelAction, - SchemaAction, WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import ( BackfillAction, - ContractPolicy, CursorGrain, CursorType, - GraphResourceKind, IncrementalMode, - IncrementalStrategy, MaterializationType, MicrobatchStrategy, - OnSchemaChange, PlanAction, PlanReason, WarningSeverity, ) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind +from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.cursor_algebra.models import TimestampValue from sqlbuild.cursor_algebra.types import BoundSentinel from sqlbuild.spec.contracts.models import ( FutureCursorsConfig, LocalConfig, ProjectConfig, - SchemaColumn, - SchemaDynamicColumnFamily, SourceEntry, StartCursorsConfig, ) -from sqlbuild.spec.contracts.types import MicrobatchLimitAction, TableType - -_IDEMPOTENT_MICROBATCH_STRATEGIES: frozenset[IncrementalStrategy] = frozenset( - (IncrementalStrategy.DELETE_INSERT, IncrementalStrategy.MERGE) -) @dataclass(frozen=True) @@ -171,33 +144,6 @@ class _MicrobatchRangeInputs: replaces_relation: bool = False -@dataclass(frozen=True) -class _ModelPlanConfig: - incremental_strategy: str | None - incremental_mode: str | None - microbatch_strategy: str | None - cursor_watermark_mode: str | None - batch_size: str | None - snapshot_strategy: str | None - updated_at_column: str | None - check_columns: tuple[str, ...] - observed_at_column: str | None - historical_input: str | None - valid_from_column: str | None - valid_to_column: str | None - initial_valid_from: str | None - invalidate_hard_deletes: bool - snapshot_full_refresh: str | None - snapshot_schema_change: str | None - - -@dataclass(frozen=True) -class _CustomMaterializationConfig: - name: str | None - config: dict[str, object] - placeholders: dict[str, str] - - def build_planner_relations_context( *, project: CompiledProjectView, @@ -405,7 +351,7 @@ def build_plan_entries( end_cursor_override: str | None = inputs.end_cursor_override entries: list[ModelPlanEntry] = [] warnings: list[PlanWarning] = [] - runtime_cursor_producer_names: frozenset[str] = _runtime_cursor_producer_names( + runtime_producer_names: frozenset[str] = runtime_cursor_producer_names( scope=scope, source_map=relations.source_map, ) @@ -444,7 +390,7 @@ def build_plan_entries( source_map=relations.source_read_map, source_warehouse_columns=relations.source_warehouse_columns, star_exclude_keyword=relations.star_exclude_keyword, - runtime_cursor_producer_names=runtime_cursor_producer_names, + runtime_cursor_producer_names=runtime_producer_names, future_cursor_config=inputs.future_cursor_config, start_cursor_config=inputs.start_cursor_config, invocation_time=inputs.invocation_time, @@ -481,7 +427,7 @@ def build_plan_entries( ), ) if entry.action != PlanAction.SKIP: - entry, limit_warnings = _apply_configured_microbatch_limits( + entry, limit_warnings = apply_configured_microbatch_limits( entry=entry, inputs=inputs ) warnings.extend(limit_warnings) @@ -527,143 +473,6 @@ def build_plan_entries( return PlannerModelEntryResults(entries=tuple(entries), warnings=tuple(warnings)) -def _apply_configured_microbatch_limits( - *, entry: ModelPlanEntry, inputs: PlanEntryBuildInputs -) -> tuple[ModelPlanEntry, tuple[PlanWarning, ...]]: - """Apply CLI, project, and model policies in their documented precedence order.""" - - if inputs.max_microbatches_is_override: - limited_entry, warning = _apply_microbatch_limit( - entry=entry, - max_batches=inputs.max_microbatches, - action=MicrobatchLimitAction.ERROR, - ) - return limited_entry, (() if warning is None else (warning,)) - - if entry.declared_microbatch_limit_action is not None: - warnings: list[PlanWarning] = [] - declared_action: MicrobatchLimitAction = entry.declared_microbatch_limit_action - if inputs.max_microbatches is not None: - _, safety_warning = _apply_microbatch_limit( - entry=entry, - max_batches=inputs.max_microbatches, - action=inputs.microbatch_limit_action, - ) - entry = replace( - entry, - microbatch_safety_limit=inputs.max_microbatches, - microbatch_safety_limit_action=inputs.microbatch_limit_action, - ) - if safety_warning is not None: - warnings.append(safety_warning) - entry, model_warning = _apply_microbatch_limit( - entry=entry, - max_batches=entry.microbatch_limit, - action=declared_action, - ) - if model_warning is not None: - warnings.append(model_warning) - return entry, tuple(warnings) - - max_batches: int | None = ( - None - if entry.microbatch_strategy == MicrobatchStrategy.ROLLING_WINDOW - and entry.reason != PlanReason.FULL_REFRESH - and entry.start_cursor_override is None - and entry.end_cursor_override is None - else entry.microbatch_limit or inputs.max_microbatches - ) - limited_entry, warning = _apply_microbatch_limit( - entry=entry, - max_batches=max_batches, - action=inputs.microbatch_limit_action, - ) - return limited_entry, (() if warning is None else (warning,)) - - -def _apply_microbatch_limit( - *, entry: ModelPlanEntry, max_batches: int | None, action: MicrobatchLimitAction -) -> tuple[ModelPlanEntry, PlanWarning | None]: - if entry.action == PlanAction.SKIP: - return entry, None - batch_count: int | None = None - effective_batch_size: str | None = None - if ( - entry.microbatch_range is not None - and entry.batch_size is not None - and entry.cursor_type is not None - ): - effective_grain: str | None = resolve_effective_timestamp_grain( - cursor_type=entry.cursor_type, - downstream_grain=entry.cursor_grain, - microbatch_strategy=entry.microbatch_strategy, - cursor_input_grains=tuple( - relation.cursor_grain for relation in entry.cursor_input_relations - ), - ) - effective_batch_size = ( - resolve_effective_microbatch_batch_size( - batch_size=entry.batch_size, - effective_grain=effective_grain, - ) - if effective_grain is not None - else entry.batch_size - ) - batch_count = count_microbatches( - bounds=entry.microbatch_range, - batch_size=effective_batch_size, - cursor_type=entry.cursor_type, - equal_bounds_are_batch=entry.action == PlanAction.CREATE_TABLE, - ) - warning: str | None = ( - None - if batch_count is None - else microbatch_limit_warning( - model_name=entry.name, - max_batches=max_batches, - batch_count=batch_count, - action=action, - ) - ) - limited_range: CursorBounds | None = entry.microbatch_range - if ( - warning is not None - and effective_batch_size is not None - and entry.microbatch_range is not None - and max_batches is not None - and action - in { - MicrobatchLimitAction.CAP_FROM_END, - MicrobatchLimitAction.CAP_FROM_START, - } - ): - limited_range = cap_microbatch_bounds( - bounds=entry.microbatch_range, - batch_size=effective_batch_size, - cursor_type=entry.cursor_type or "", - max_batches=max_batches, - action=action, - ) - limited_entry: ModelPlanEntry = replace( - entry, - microbatch_range=limited_range, - microbatch_limit=max_batches, - microbatch_limit_count=batch_count, - microbatch_limit_action=action if max_batches is not None else None, - microbatch_limit_warning=warning, - ) - if warning is None: - return limited_entry, None - if action == MicrobatchLimitAction.ERROR: - raise CompileInputError(warning) - return limited_entry, PlanWarning( - model_name=entry.name, - severity=WarningSeverity.WARNING, - code="MICROBATCH_LIMIT_EXCEEDED", - message=warning, - ) - - def plan_model_from_change( *, model: CompiledModelView, @@ -728,42 +537,12 @@ def plan_model_from_change( full_rebuild_cause=full_rebuild_cause, ) - on_schema_change: OnSchemaChange | None = _get_on_schema_change(model) - schema_actions: tuple[SchemaAction, ...] = resolve_schema_actions( - schema_findings=change_result.schema_findings, - on_schema_change=on_schema_change, - ) - - type_enforcement: bool = _get_type_enforcement(model) - declared_columns: tuple[ColumnInfo, ...] = _get_declared_columns(model) - contract_enforced: bool = model.config.values.get("contract") == ContractPolicy.ENFORCED - contract_columns: tuple[ColumnInfo, ...] = _get_contract_columns(model) - contract_dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = ( - model.schema_entry.dynamic_columns if model.schema_entry is not None else () - ) - unique_key: tuple[str, ...] = _get_unique_key(model) - warehouse_columns: tuple[ColumnInfo, ...] = snapshot.existing_columns.get(model.name, ()) - - warnings: tuple[PlanWarning, ...] = build_model_warnings( - model_name=model.name, - materialization_type=materialization_type, - change_result=change_result, - schema_actions=schema_actions, - on_schema_change=on_schema_change, - type_enforcement=type_enforcement, - ) - pre_hooks, post_hooks = resolve_model_hook_entries( model=model, adapter=adapter, context=context, external_sql_reference_resolver=external_sql_reference_resolver, ) - cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") - cursor_grain: str | None = get_config_str(values=model.config.values, key="cursor_grain") - cursor_start: str | None = get_config_cursor_bound( - values=model.config.values, key="cursor_start" - ) effective_future_cursor_config, effective_start_cursor_config = _resolve_cursor_safety_configs( model=model, context=context ) @@ -772,7 +551,7 @@ def plan_model_from_change( models_by_name=context.models_by_name, functions_by_name=context.functions_by_name, ) - cursor_input_relations: tuple[CursorInputRelation, ...] = _build_cursor_input_relations( + cursor_input_relations: tuple[CursorInputRelation, ...] = build_cursor_input_relations( model=model, adapter=adapter, model_locations=context.model_locations, @@ -783,8 +562,8 @@ def plan_model_from_change( cursor_column=cursor_column, runtime_cursor_producer_names=context.runtime_cursor_producer_names, ) - runtime_owned_cursor_bounds: bool = _has_runtime_owned_cursor_watermarks( - cursor_input_relations + runtime_owned_cursor_bounds: bool = any( + relation.is_runtime_owned for relation in cursor_input_relations ) and not (start_cursor_override is not None and end_cursor_override is not None) cursor_bounds: CursorBounds | None = _compute_plan_cursor_bounds( model=model, @@ -809,18 +588,6 @@ def plan_model_from_change( cursor_overrides=cursor_overrides, ) - cursor_type_warning: PlanWarning | None = check_cursor_type_consistency( - model_name=model.name, - cursor_column=cursor_column, - cursor_type=cursor_type, - warehouse_columns=warehouse_columns, - sql_analysis_enabled=sql_analysis_enabled, - ) - if cursor_type_warning is not None: - warnings = (*warnings, cursor_type_warning) - - plan_config: _ModelPlanConfig = _resolve_model_plan_config(model=model) - microbatch_range: CursorBounds | None = _compute_microbatch_range( model=model, inputs=_MicrobatchRangeInputs( @@ -838,139 +605,39 @@ def plan_model_from_change( ), ) - warnings = _append_cursor_safety_warnings( - warnings=warnings, - model_name=model.name, - bounds=microbatch_range or cursor_bounds, - ) - - fingerprint: Fingerprint | None = snapshot.fingerprints.models.get(model.name) - previous_query_sql: str | None = fingerprint.definition if fingerprint is not None else None - - custom: _CustomMaterializationConfig = _resolve_custom_materialization_config( - model=model, materialization_type=materialization_type - ) - - ddl_cursor_bounds: CursorBounds | None = ( - _build_runtime_placeholder_bounds() - if runtime_owned_cursor_bounds and cursor_column is not None - else (microbatch_range if microbatch_range else cursor_bounds) - ) - logical_ddl: str = _render_table_type_ddl( - ddl=_build_logical_ddl_from_adapter( - adapter=adapter, - action=action, - resolved_sql=resolved_sql, - destination=model.destination, - unique_key=unique_key, - merge_exclude_columns=get_config_string_tuple( - values=model.config.values, key="merge_exclude_columns" - ) - or (), - warehouse_columns=warehouse_columns, - cursor_column=cursor_column, - cursor_bounds=ddl_cursor_bounds, - cursor_type=cursor_type, - ), - table_type=model.config.table_type.value.value, - ) - - entry: ModelPlanEntry = ModelPlanEntry( - key=model.key, - name=model.name, - relative_path=model.relative_path, - materialization_type=materialization_type, + entry: ModelPlanEntry + warnings: tuple[PlanWarning, ...] + entry, warnings = assemble_model_plan_entry( + model=model, + adapter=adapter, + snapshot=snapshot, + change_result=change_result, action=action, reason=reason, - destination=model.destination, - fingerprint_query_sql=model.query_sql, + materialization_type=materialization_type, resolved_sql=resolved_sql, - logical_ddl=logical_ddl, - incremental_strategy=plan_config.incremental_strategy, - incremental_mode=plan_config.incremental_mode, - microbatch_strategy=plan_config.microbatch_strategy, - cursor_watermark_mode=plan_config.cursor_watermark_mode, + sql_analysis_enabled=sql_analysis_enabled, + pre_hooks=pre_hooks, + post_hooks=post_hooks, cursor_column=cursor_column, - cursor_type=cursor_type, - cursor_grain=cursor_grain, - cursor_start=cursor_start, - cursor_end=get_config_cursor_bound(values=model.config.values, key="cursor_end"), - lookback=( - resolve_microbatch_lookback( - model=model, - effective_grain=resolve_effective_timestamp_grain( - cursor_type=cursor_type, - downstream_grain=cursor_grain, - microbatch_strategy=plan_config.microbatch_strategy, - cursor_input_grains=tuple( - relation.cursor_grain for relation in cursor_input_relations - ), - ), - ) - if plan_config.incremental_mode == IncrementalMode.MICROBATCH - else get_config_str(values=model.config.values, key="lookback") - ), + cursor_input_relations=cursor_input_relations, + runtime_owned_cursor_bounds=runtime_owned_cursor_bounds, cursor_bounds=cursor_bounds, + microbatch_range=microbatch_range, empty_cursor_inputs=empty_cursor_inputs, waiting_on_empty_inputs=waiting_on_empty_inputs, - cursor_input_relations=cursor_input_relations, - batch_size=plan_config.batch_size, - batch_concurrency=get_config_int(values=model.config.values, key="batch_concurrency") or 1, - unaccounted_partition_policy=get_config_str( - values=model.config.values, key="unaccounted_partition_policy" - ), - microbatch_range=microbatch_range, - microbatch_limit=resolve_microbatch_limit_config(values=model.config.values)[0], - declared_microbatch_limit_action=resolve_microbatch_limit_config( - values=model.config.values - )[1], + backfill=backfill, start_cursor_override=start_cursor_override, end_cursor_override=end_cursor_override, future_cursor_config=effective_future_cursor_config, start_cursor_config=effective_start_cursor_config, invocation_time=context.invocation_time, - unique_key=unique_key, - merge_exclude_columns=get_config_string_tuple( - values=model.config.values, key="merge_exclude_columns" - ) - or (), - snapshot_strategy=plan_config.snapshot_strategy, - updated_at_column=plan_config.updated_at_column, - check_columns=plan_config.check_columns, - observed_at_column=plan_config.observed_at_column, - historical_input=plan_config.historical_input, - valid_from_column=plan_config.valid_from_column, - valid_to_column=plan_config.valid_to_column, - initial_valid_from=plan_config.initial_valid_from, - invalidate_hard_deletes=plan_config.invalidate_hard_deletes, - snapshot_full_refresh=plan_config.snapshot_full_refresh, - snapshot_schema_change=plan_config.snapshot_schema_change, - table_type=model.config.table_type.value.value, - on_schema_change=on_schema_change, - type_enforcement=type_enforcement, - declared_columns=declared_columns, - contract_enforced=contract_enforced, - contract_columns=contract_columns, - contract_dynamic_columns=contract_dynamic_columns, - pre_hooks=pre_hooks, - post_hooks=post_hooks, - previous_query_sql=previous_query_sql, - fingerprint_metadata_json=change_result.fingerprint_metadata_json, - previous_metadata_json=change_result.previous_metadata_json, - fingerprint_version_hash=change_result.fingerprint_version_hash, - previous_version_hash=change_result.previous_version_hash, - query_changed=change_result.query_changed, - config_changed=change_result.config_changed, - schema_actions=schema_actions, - schema_findings=change_result.schema_findings, - backfill=backfill, - changed_functions=change_result.changed_functions, - custom_materialization_name=custom.name, - custom_config=custom.config, - custom_placeholders=custom.placeholders, ) - - return entry, warnings + return entry, _append_cursor_safety_warnings( + warnings=warnings, + model_name=model.name, + bounds=microbatch_range or cursor_bounds, + ) def resolve_cursor_overrides( @@ -1029,79 +696,6 @@ def resolve_cursor_overrides( return resolved_start, resolved_end -def _get_on_schema_change(model: CompiledModelView) -> OnSchemaChange | None: - """Extract on_schema_change from model config.""" - - raw: object | None = model.config.values.get("on_schema_change") - if raw is None: - return None - if isinstance(raw, str) and raw in set(OnSchemaChange): - return OnSchemaChange(raw) - raise PlannerInputError(f"model '{model.name}': unknown on_schema_change '{raw}'") - - -def extract_seed_columns(seed: CompiledSeedView) -> tuple[ColumnInfo, ...]: - """Extract declared columns from seed schema entry.""" - - columns: list[ColumnInfo] = [] - col: SchemaColumn - for col in seed.schema_entry.columns: - if col.type is not None: - columns.append(ColumnInfo(name=col.name, type=col.type)) - return tuple(columns) - - -def _get_type_enforcement(model: CompiledModelView) -> bool: - """Resolve whether type enforcement is active for a model.""" - - if model.schema_entry is not None and model.schema_entry.type_enforcement is not None: - return model.schema_entry.type_enforcement - return False - - -def _get_declared_columns(model: CompiledModelView) -> tuple[ColumnInfo, ...]: - """Extract declared columns with types from schema entry.""" - - if model.schema_entry is None: - return () - columns: list[ColumnInfo] = [] - col: SchemaColumn - for col in model.schema_entry.columns: - if col.type is not None: - columns.append(ColumnInfo(name=col.name, type=col.type)) - return tuple(columns) - - -def _get_contract_columns(model: CompiledModelView) -> tuple[ColumnInfo, ...]: - if model.schema_entry is None: - return () - return tuple( - ColumnInfo(name=col.name, type=col.type or "") for col in model.schema_entry.columns - ) - - -def _get_unique_key(model: CompiledModelView) -> tuple[str, ...]: - """Extract unique_key from model config as a normalized tuple.""" - - raw: object | None = model.config.values.get("unique_key") - if isinstance(raw, str): - return (raw,) - if isinstance(raw, list): - return tuple(k for k in raw if isinstance(k, str)) - return () - - -def _get_check_columns(model: CompiledModelView) -> tuple[str, ...]: - """Extract check_columns from model config as a normalized tuple.""" - - raw: object | None = model.config.values.get("check_columns") - if isinstance(raw, str): - return (raw,) - if isinstance(raw, list): - return tuple(column for column in raw if isinstance(column, str)) - return () - - def _resolve_cursor_safety_configs( *, model: CompiledModelView, context: ModelPlanContext ) -> tuple[FutureCursorsConfig | None, StartCursorsConfig | None]: @@ -1111,59 +705,6 @@ def _resolve_cursor_safety_configs( ) -def _resolve_model_plan_config(*, model: CompiledModelView) -> _ModelPlanConfig: - return _ModelPlanConfig( - incremental_strategy=get_config_str(values=model.config.values, key="incremental_strategy"), - incremental_mode=get_config_str(values=model.config.values, key="incremental_mode"), - microbatch_strategy=get_config_str(values=model.config.values, key="microbatch_strategy"), - cursor_watermark_mode=get_config_str( - values=model.config.values, key="cursor_watermark_mode" - ), - batch_size=get_config_str(values=model.config.values, key="batch_size"), - snapshot_strategy=get_config_str(values=model.config.values, key="snapshot_strategy"), - updated_at_column=get_config_str(values=model.config.values, key="updated_at"), - check_columns=_get_check_columns(model), - observed_at_column=get_config_str(values=model.config.values, key="observed_at"), - historical_input=get_config_str(values=model.config.values, key="historical_input"), - valid_from_column=get_config_str(values=model.config.values, key="valid_from_column"), - valid_to_column=get_config_str(values=model.config.values, key="valid_to_column"), - initial_valid_from=get_config_str(values=model.config.values, key="initial_valid_from"), - invalidate_hard_deletes=get_config_bool( - values=model.config.values, key="invalidate_hard_deletes" - ) - or False, - snapshot_full_refresh=get_config_str( - values=model.config.values, key="snapshot_full_refresh" - ), - snapshot_schema_change=get_config_str( - values=model.config.values, key="snapshot_schema_change" - ), - ) - - -def _resolve_custom_materialization_config( - *, model: CompiledModelView, materialization_type: MaterializationType -) -> _CustomMaterializationConfig: - if materialization_type != MaterializationType.CUSTOM: - return _CustomMaterializationConfig(name=None, config={}, placeholders={}) - raw_name: object | None = model.config.values.get("materialized") - raw_config: object | None = model.config.values.get("config") - raw_placeholders: object | None = model.config.values.get("placeholders") - return _CustomMaterializationConfig( - name=raw_name if isinstance(raw_name, str) else None, - config=( - {str(key): value for key, value in raw_config.items()} - if isinstance(raw_config, dict) - else {} - ), - placeholders=( - {str(key): str(value) for key, value in raw_placeholders.items()} - if isinstance(raw_placeholders, dict) - else {} - ), - ) - - def _compute_microbatch_range( *, model: CompiledModelView, inputs: _MicrobatchRangeInputs ) -> CursorBounds | None: @@ -1269,36 +810,6 @@ def _compute_microbatch_range( ) -def resolve_microbatch_lookback( - *, model: CompiledModelView, effective_grain: str | None = None -) -> str | None: - """Resolve explicit or default lookback for one microbatch model.""" - - lookback: str | None = get_config_str(values=model.config.values, key="lookback") - if lookback is not None: - return lookback - raw_strategy: str | None = get_config_str( - values=model.config.values, key="incremental_strategy" - ) - strategy: IncrementalStrategy | None = ( - IncrementalStrategy(raw_strategy) if raw_strategy is not None else None - ) - if strategy not in _IDEMPOTENT_MICROBATCH_STRATEGIES: - return None - batch_size: str | None = get_config_str(values=model.config.values, key="batch_size") - if batch_size is None: - return None - resolved_grain: str | None = effective_grain or get_config_str( - values=model.config.values, key="cursor_grain" - ) - if resolved_grain is None: - return batch_size - return resolve_effective_microbatch_batch_size( - batch_size=batch_size, - effective_grain=resolved_grain, - ) - - def _compute_plan_cursor_bounds( *, model: CompiledModelView, @@ -1488,352 +999,6 @@ def _append_maximum_start_warning( ) -def _render_table_type_ddl(*, ddl: str, table_type: str) -> str: - """Render permanent Snowflake table DDL from the adapter's transient default.""" - - return ddl.replace("TRANSIENT TABLE", "TABLE", 1) if table_type == TableType.PERMANENT else ddl - - -def _build_logical_ddl_from_adapter( - *, - adapter: BaseAdapter, - action: PlanAction, - resolved_sql: str, - destination: CompiledRelationLocation, - unique_key: tuple[str, ...], - merge_exclude_columns: tuple[str, ...], - warehouse_columns: tuple[ColumnInfo, ...], - cursor_column: str | None = None, - cursor_bounds: CursorBounds | None = None, - cursor_type: str | None = None, -) -> str: - """Generate logical DDL using adapter render methods.""" - - qualified_name: str = destination.qualified_name or destination.name - - if action == PlanAction.CREATE_VIEW: - return ";\n\n".join( - adapter.render_create_view_as(destination=qualified_name, sql=resolved_sql) - ) - - if action == PlanAction.CREATE_TABLE: - return ";\n\n".join( - adapter.render_create_table_as(destination=qualified_name, sql=resolved_sql) - ) - - if action == PlanAction.INCREMENTAL_APPEND: - return ";\n\n".join(adapter.render_append(destination=qualified_name, sql=resolved_sql)) - - if action == PlanAction.INCREMENTAL_DELETE_INSERT: - if cursor_column is not None and cursor_bounds is not None: - return ";\n\n".join( - adapter.render_delete_insert_cursor( - destination=qualified_name, - sql=resolved_sql, - cursor_column=cursor_column, - cursor_start=sentinel_to_token(sentinel=cursor_bounds.start), - cursor_end=sentinel_to_token(sentinel=cursor_bounds.end), - cursor_type=cursor_type, - ) - ) - return ";\n\n".join( - adapter.render_delete_insert( - destination=qualified_name, - sql=resolved_sql, - unique_key=unique_key, - ) - ) - - if action == PlanAction.INCREMENTAL_MERGE: - source_columns: tuple[str, ...] = tuple(col.name for col in warehouse_columns) - return ";\n\n".join( - adapter.render_merge( - destination=qualified_name, - sql=resolved_sql, - unique_key=unique_key, - source_columns=source_columns, - exclude_columns=merge_exclude_columns, - ) - ) - - return "" - - -def _build_cursor_input_relations( - *, - model: CompiledModelView, - adapter: BaseAdapter, - model_locations: dict[str, CompiledRelationLocation], - models_by_name: dict[str, CompiledModelView], - functions_by_name: dict[str, CompiledFunctionView] | None = None, - seed_locations: dict[str, CompiledRelationLocation], - source_map: dict[str, SourceEntry], - cursor_column: str | None, - runtime_cursor_producer_names: frozenset[str], -) -> tuple[CursorInputRelation, ...]: - """Build cursor-bearing input relation metadata for runtime range discovery.""" - - materialized: str | None = get_config_str(values=model.config.values, key="materialized") - if materialized != MaterializationType.INCREMENTAL or cursor_column is None: - return () - - cursor_watermark_inputs: dict[str, str] = resolve_cursor_input_roles( - model=model - ).watermark_inputs - relations: list[CursorInputRelation] = [] - input_name: str - input_cursor_column: str - for input_name, input_cursor_column in cursor_watermark_inputs.items(): - ref: CompileSqlReference = resolve_lineage_reference( - model=model, - input_name=input_name, - models_by_name=models_by_name, - functions_by_name=functions_by_name or {}, - ) - producer_model: CompiledModelView | None = ( - models_by_name.get(ref.ref_name) if ref.ref_kind == SqlReferenceKind.REF else None - ) - relation: str | None = _resolve_cursor_input_relation( - ref=ref, - adapter=adapter, - model_locations=model_locations, - seed_locations=seed_locations, - source_map=source_map, - ) - if relation is not None: - relations.append( - CursorInputRelation( - relation=relation, - cursor_column=input_cursor_column, - cursor_grain=( - CursorGrain(input_grain) - if ( - input_grain := _resolve_cursor_input_grain( - ref=ref, models_by_name=models_by_name - ) - ) - is not None - else None - ), - is_model_backed=( - ref.ref_kind == SqlReferenceKind.REF and ref.ref_name in model_locations - ), - is_runtime_produced=ref.ref_name in runtime_cursor_producer_names, - input_name=ref.ref_name, - input_kind=_cursor_input_kind(ref=ref, model_locations=model_locations), - terminal_cursor_start=( - parse( - raw=get_config_cursor_bound( - values=producer_model.config.values, key="cursor_start" - ), - cursor_type=get_config_str( - values=model.config.values, key="cursor_type" - ) - or CursorType.TIMESTAMP, - ) - if producer_model is not None - and get_config_cursor_bound( - values=producer_model.config.values, key="cursor_start" - ) - is not None - else None - ), - terminal_cursor_end=( - parse( - raw=get_config_cursor_bound( - values=producer_model.config.values, key="cursor_end" - ), - cursor_type=get_config_str( - values=model.config.values, key="cursor_type" - ) - or CursorType.TIMESTAMP, - ) - if producer_model is not None - and get_config_cursor_bound( - values=producer_model.config.values, key="cursor_end" - ) - is not None - else None - ), - ) - ) - return tuple(relations) - - -def validate_source_cursor_input_columns( - *, - model: CompiledModelView, - cursor_column: str | None, - models_by_name: dict[str, CompiledModelView] | None = None, - functions_by_name: dict[str, CompiledFunctionView] | None = None, - source_map: dict[str, SourceEntry] | None = None, - source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]], -) -> None: - """Validate cursor input columns using contracts before source warehouse metadata.""" - - materialized: str | None = get_config_str(values=model.config.values, key="materialized") - if materialized != MaterializationType.INCREMENTAL or cursor_column is None: - return - - cursor_roles: CursorInputRoles = resolve_cursor_input_roles(model=model) - effective_models_by_name: dict[str, CompiledModelView] = models_by_name or {} - effective_source_map: dict[str, SourceEntry] = source_map or {} - authored_inputs: tuple[tuple[str, str], ...] = tuple( - cursor_roles.filter_inputs.items() - ) + tuple(cursor_roles.watermark_inputs.items()) - input_name: str - input_cursor_column: str - for input_name, input_cursor_column in authored_inputs: - ref: CompileSqlReference = resolve_lineage_reference( - model=model, - input_name=input_name, - models_by_name=effective_models_by_name, - functions_by_name=functions_by_name or {}, - ) - if ref.ref_kind == SqlReferenceKind.REF: - upstream_model: CompiledModelView | None = effective_models_by_name.get(ref.ref_name) - if upstream_model is None: - continue - has_enforced_contract: bool = ( - upstream_model.config.values.get("contract") == ContractPolicy.ENFORCED - ) - if not has_enforced_contract: - continue - declared_names: tuple[str, ...] = _model_declared_column_names(upstream_model) - if input_cursor_column.lower() in {name.lower() for name in declared_names}: - continue - declared_display: str = ", ".join(declared_names) or "none" - raise PlannerInputError( - f"model '{model.name}': {CURSOR_INPUTS_CONFIG_KEY} " - f"references model '{ref.ref_name}' " - f"column '{input_cursor_column}', but that model contract does not expose " - f"the column. Known output columns: {declared_display}", - code="S302", - ) - if ref.ref_kind != SqlReferenceKind.SOURCE: - continue - source_entry: SourceEntry | None = effective_source_map.get(ref.ref_name) - if source_entry is not None and source_entry.contract == ContractPolicy.ENFORCED: - declared_names = tuple(column.name for column in source_entry.columns) - if input_cursor_column.lower() in {name.lower() for name in declared_names}: - continue - declared_display = ", ".join(declared_names) or "none" - raise PlannerInputError( - f"model '{model.name}': {CURSOR_INPUTS_CONFIG_KEY} " - f"references source '{ref.ref_name}' " - f"column '{input_cursor_column}', but that source contract does not expose " - f"the column. Declared contract columns: {declared_display}", - code="S302", - ) - known_columns: tuple[ColumnInfo, ...] | None = source_warehouse_columns.get(ref.ref_name) - if known_columns is None: - continue - known_column_names: frozenset[str] = frozenset(col.name.lower() for col in known_columns) - if input_cursor_column.lower() in known_column_names: - continue - known_display: str = ", ".join(col.name for col in known_columns) or "none" - raise PlannerInputError( - f"model '{model.name}': {CURSOR_INPUTS_CONFIG_KEY} references source '{ref.ref_name}' " - f"column '{input_cursor_column}', but that source does not expose the column. " - f"Known source columns: {known_display}", - code="S302", - ) - - -def _model_declared_column_names(model: CompiledModelView) -> tuple[str, ...]: - if model.schema_entry is None: - return () - return tuple(column.name for column in model.schema_entry.columns) - - -def _has_runtime_owned_cursor_watermarks( - cursor_input_relations: tuple[CursorInputRelation, ...], -) -> bool: - """Return whether any cursor input is produced by this invocation.""" - - return any(relation.is_runtime_owned for relation in cursor_input_relations) - - -def _runtime_cursor_producer_names( - *, scope: PlannerScope, source_map: dict[str, SourceEntry] -) -> frozenset[str]: - """Return selected relations whose contents are produced during this invocation.""" - - names: set[str] = set() - key: CompiledObjectKey - for key in scope.selected_keys: - if key.resource_type in {CompiledResourceType.MODEL, CompiledResourceType.SEED}: - names.add(key.name) - elif key.resource_type == CompiledResourceType.SOURCE: - source: SourceEntry | None = source_map.get(key.name) - if source is not None and source.loader is not None: - names.add(key.name) - return frozenset(names) - - -def _build_runtime_placeholder_bounds() -> CursorBounds: - """Return placeholder cursor bounds for runtime-owned models.""" - - return CursorBounds(start=BoundSentinel.START, end=BoundSentinel.END) - - -def _resolve_cursor_input_relation( - *, - ref: CompileSqlReference, - adapter: BaseAdapter, - model_locations: dict[str, CompiledRelationLocation], - seed_locations: dict[str, CompiledRelationLocation], - source_map: dict[str, SourceEntry], -) -> str | None: - """Resolve one cursor input reference to a qualified relation name.""" - - if ref.ref_kind == SqlReferenceKind.REF: - target: CompiledRelationLocation | None = model_locations.get(ref.ref_name) - if target is None: - target = seed_locations.get(ref.ref_name) - return target.qualified_name if target is not None else None - if ref.ref_kind == SqlReferenceKind.SEED: - seed_target: CompiledRelationLocation | None = seed_locations.get(ref.ref_name) - return seed_target.qualified_name if seed_target is not None else None - if ref.ref_kind == SqlReferenceKind.SOURCE: - source: SourceEntry | None = source_map.get(ref.ref_name) - if source is None: - return None - return render_source_relation(entry=source, adapter=adapter) - return None - - -def _cursor_input_kind( - *, ref: CompileSqlReference, model_locations: dict[str, CompiledRelationLocation] -) -> GraphResourceKind | None: - """Return the resource kind a resolved cursor input reference reads.""" - - if ref.ref_kind == SqlReferenceKind.REF: - return ( - GraphResourceKind.MODEL if ref.ref_name in model_locations else GraphResourceKind.SEED - ) - if ref.ref_kind == SqlReferenceKind.SEED: - return GraphResourceKind.SEED - if ref.ref_kind == SqlReferenceKind.SOURCE: - return GraphResourceKind.SOURCE - return None - - -def _resolve_cursor_input_grain( - *, - ref: CompileSqlReference, - models_by_name: dict[str, CompiledModelView], -) -> str | None: - """Resolve timestamp grain metadata for a model-backed cursor input relation.""" - - if ref.ref_kind != SqlReferenceKind.REF: - return None - upstream_model: CompiledModelView | None = models_by_name.get(ref.ref_name) - if upstream_model is None: - return None - return get_config_str(values=upstream_model.config.values, key="cursor_grain") - - def scope_overlaps( *, scope_deps: tuple[CompiledObjectKey, ...], @@ -1848,22 +1013,6 @@ def scope_overlaps( return False -def audit_is_selected( - *, - audit: CompiledAuditView, - selected_keys: frozenset[CompiledObjectKey], -) -> bool: - """Return whether an audit belongs to the selected logical target scope.""" - - if audit.attached_target_kind is not None and audit.attached_target_name is not None: - target_key: CompiledObjectKey = CompiledObjectKey( - resource_type=AttachedAuditTargetKind(audit.attached_target_kind).resource_type, - name=audit.attached_target_name, - ) - return target_key in selected_keys - return scope_overlaps(scope_deps=audit.scope_deps, selected_keys=selected_keys) - - def _selected_audit_relation_keys( *, project: CompiledProjectView, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py index 61c803dd83..850550e264 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py @@ -3,8 +3,11 @@ from __future__ import annotations from sqlbuild._native import ( + build_relation_fixture_context, + build_selected_audit_entries, build_source_load_entries, - project_lineage_views, + extract_seed_columns, + plan_sql_tests, upstream_loader_dependency_names, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -13,7 +16,6 @@ CompiledSqlTest, ) from sqlbuild.compiler.compile.types import ( - CompiledAuditView, CompiledFunctionView, CompiledProjectView, ) @@ -25,19 +27,11 @@ PythonHookEntry, ) from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.graph.models import LineageGraphViews -from sqlbuild.compiler.planner._helpers.fixtures.completion import ( - build_relation_fixture_context, -) -from sqlbuild.compiler.planner._helpers.output.audit_entry import plan_audit from sqlbuild.compiler.planner._helpers.output.plan_entry import ( - audit_is_selected, build_model_materializations, - extract_seed_columns, scope_overlaps, ) from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_function_sql -from sqlbuild.compiler.planner._helpers.sql_tests.assembly import plan_sql_tests from sqlbuild.compiler.planner.exceptions import ( PlannerInputError, ) @@ -420,46 +414,6 @@ def plan_function( ) -def build_selected_audit_entries( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - relations: PlannerRelationsContext, - model_materializations: dict[str, str], -) -> list[AuditPlanEntry]: - entries: list[AuditPlanEntry] = [] - selected_audits: tuple[CompiledAuditView, ...] = tuple( - audit - for audit in project.audits - if audit_is_selected(audit=audit, selected_keys=scope.selected_keys) - ) - if not selected_audits: - return entries - lineage: LineageGraphViews = project_lineage_views(project) - lineage_upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - lineage.upstream_deps - ) - lineage_downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - lineage.downstream_deps - ) - audit: CompiledAuditView - for audit in selected_audits: - entries.append( - plan_audit( - audit=audit, - model_locations=relations.model_locations, - seed_locations=relations.seed_locations, - source_map=relations.source_read_map, - adapter=adapter, - lineage_upstream_deps=lineage_upstream_deps, - lineage_downstream_deps=lineage_downstream_deps, - model_materializations=model_materializations, - ) - ) - return entries - - def plan_selected_sql_tests( *, project: CompiledProjectView, diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py index 51064cf363..8e1dc9dff7 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py @@ -4,8 +4,7 @@ import time -from sqlbuild._native import plan_column_migrations, plan_model_migrations -from sqlbuild.compiler.planner._helpers.migrations.old_names import plan_old_name_views +from sqlbuild._native import plan_column_migrations, plan_model_migrations, plan_old_name_views from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( effectively_full_refreshed_model_names, diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/artifact_names.py b/src/sqlbuild/compiler/planner/_helpers/scenario/artifact_names.py deleted file mode 100644 index 1de215dfe6..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/artifact_names.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Scenario artifact physical-name helper implementations.""" - -from __future__ import annotations - -import re - -from sqlbuild.adapter.relations.main.fit_artifact_logical_name import fit_artifact_logical_name -from sqlbuild.compiler.planner.constants import ( - SCENARIO_ARTIFACT_KINDS, - SCENARIO_ARTIFACT_PREFIX, - SCENARIO_HASH_PREFIX_LENGTH, -) -from sqlbuild.compiler.planner.models import ParsedScenarioArtifactName - -_SCENARIO_ARTIFACT_NAME_RE: re.Pattern[str] = re.compile( - rf"^{re.escape(SCENARIO_ARTIFACT_PREFIX)}" - rf"(?P[0-9a-f]{{{SCENARIO_HASH_PREFIX_LENGTH}}})__" - rf"(?P{'|'.join(SCENARIO_ARTIFACT_KINDS)})__" - r"(?P.+)$" -) - - -def build_scenario_artifact_physical_name( - *, - hash_prefix: str, - kind: str, - logical_name: str, - identifier_limit: int, -) -> str: - fixed_prefix: str = f"{SCENARIO_ARTIFACT_PREFIX}{hash_prefix}__{kind}__" - logical_part: str = fit_scenario_artifact_logical_name( - logical_name=logical_name, - fixed_prefix=fixed_prefix, - identifier_limit=identifier_limit, - ) - return f"{fixed_prefix}{logical_part}" - - -def parse_scenario_artifact_physical_name(name: str) -> ParsedScenarioArtifactName | None: - match: re.Match[str] | None = _SCENARIO_ARTIFACT_NAME_RE.fullmatch(name) - if match is None: - return None - return ParsedScenarioArtifactName( - hash_prefix=match.group("hash_prefix"), - kind=match.group("kind"), - logical_name=match.group("logical_name"), - ) - - -def is_scenario_artifact_physical_name(name: str) -> bool: - return parse_scenario_artifact_physical_name(name) is not None - - -def fit_scenario_artifact_logical_name( - *, logical_name: str, fixed_prefix: str, identifier_limit: int -) -> str: - return fit_artifact_logical_name( - logical_name=logical_name, - fixed_prefix=fixed_prefix, - identifier_limit=identifier_limit, - artifact_label="Scenario artifact", - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/artifacts.py b/src/sqlbuild/compiler/planner/_helpers/scenario/artifacts.py deleted file mode 100644 index f55697843f..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/artifacts.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Deterministic physical artifact naming for SQL scenarios.""" - -from __future__ import annotations - -import hashlib -from collections.abc import Callable - -from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.planner.constants import ( - SCENARIO_DEFAULT_IDENTIFIER_LIMIT, - SCENARIO_HASH_PREFIX_LENGTH, - SCENARIO_PLAN_HASH_COLLISION, - SCENARIO_PLAN_INVALID_HASH_PREFIX, - SCENARIO_PLAN_RELATION_COLLISION, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.scenarios._build_scenario_artifact_physical_name import ( - build_scenario_artifact_physical_name, -) -from sqlbuild.compiler.planner.models import ( - ScenarioArtifactIdentity, - ScenarioArtifactName, - ScenarioRelationMap, -) -from sqlbuild.compiler.planner.types import ScenarioArtifactKind - - -def compute_scenario_hash_prefix( - *, - project_name: str, - scenario_name: str, - run_namespace: str | None = None, - prefix_length: int = SCENARIO_HASH_PREFIX_LENGTH, -) -> str: - """Return a stable scenario artifact hash prefix.""" - - if prefix_length < 1: - raise PlannerInputError( - "Scenario hash prefix length must be at least 1", - code=SCENARIO_PLAN_INVALID_HASH_PREFIX, - ) - hash_input: str = f"{project_name}:{scenario_name}" - if run_namespace is not None: - hash_input = f"{project_name}:{run_namespace}:{scenario_name}" - return hashlib.sha256(hash_input.encode("utf-8")).hexdigest()[:prefix_length] - - -def build_scenario_hash_index( - *, - project_name: str, - scenarios: tuple[CompiledSqlScenario, ...], - run_namespace: str | None = None, - prefix_length: int = SCENARIO_HASH_PREFIX_LENGTH, -) -> dict[str, str]: - """Return scenario name to hash prefix, failing clearly on prefix collisions.""" - - by_prefix: dict[str, CompiledSqlScenario] = {} - result: dict[str, str] = {} - scenario: CompiledSqlScenario - for scenario in scenarios: - prefix: str = compute_scenario_hash_prefix( - project_name=project_name, - scenario_name=scenario.name, - run_namespace=run_namespace, - prefix_length=prefix_length, - ) - existing: CompiledSqlScenario | None = by_prefix.get(prefix) - if existing is not None: - raise PlannerInputError( - "Scenario artifact hash collision: scenarios " - f"'{existing.name}' and '{scenario.name}' both map to hash prefix " - f"'{prefix}'. Rename one scenario file so SQLBuild can generate " - "distinct warehouse artifact prefixes.", - code=SCENARIO_PLAN_HASH_COLLISION, - help=( - "Rename one scenario file so SQLBuild can generate distinct artifact prefixes." - ), - ) - by_prefix[prefix] = scenario - result[scenario.name] = prefix - return result - - -def build_scenario_artifact_name( - *, - hash_prefix: str, - kind: ScenarioArtifactKind | str, - logical_name: str, - identifier_limit: int = SCENARIO_DEFAULT_IDENTIFIER_LIMIT, -) -> str: - """Build one deterministic scenario physical relation name.""" - - normalized_kind: ScenarioArtifactKind = ScenarioArtifactKind(kind) - return build_scenario_artifact_physical_name( - hash_prefix=hash_prefix, - kind=normalized_kind.value, - logical_name=logical_name, - identifier_limit=identifier_limit, - ) - - -def build_scenario_relation_map( - *, - scenario_name: str, - hash_prefix: str, - artifacts: tuple[ScenarioArtifactIdentity, ...], - identifier_limit: int = SCENARIO_DEFAULT_IDENTIFIER_LIMIT, - normalize_identifier: Callable[[str], str] | None = None, -) -> ScenarioRelationMap: - """Build and collision-check physical relation names for one scenario.""" - - normalizer: Callable[[str], str] = normalize_identifier or (lambda value: value) - resolved_artifacts: list[ScenarioArtifactName] = [] - seen: dict[str, ScenarioArtifactIdentity] = {} - artifact: ScenarioArtifactIdentity - for artifact in artifacts: - physical_name: str = build_scenario_artifact_name( - hash_prefix=hash_prefix, - kind=artifact.kind, - logical_name=artifact.logical_name, - identifier_limit=identifier_limit, - ) - normalized_physical_name: str = normalizer(physical_name) - existing: ScenarioArtifactIdentity | None = seen.get(normalized_physical_name) - if existing is not None: - raise PlannerInputError( - f"Scenario relation name collision in scenario '{scenario_name}'. " - f"Both scenario artifacts map to '{physical_name}': " - f"{ScenarioArtifactKind(existing.kind).value} {existing.logical_name} and " - f"{ScenarioArtifactKind(artifact.kind).value} {artifact.logical_name}. " - "Rename one artifact " - "or shorten one model/source name.", - code=SCENARIO_PLAN_RELATION_COLLISION, - help="Rename or shorten one of the colliding scenario artifacts.", - ) - seen[normalized_physical_name] = artifact - resolved_artifacts.append( - ScenarioArtifactName(identity=artifact, physical_name=physical_name) - ) - - return ScenarioRelationMap( - scenario_name=scenario_name, - hash_prefix=hash_prefix, - artifacts=tuple(resolved_artifacts), - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py deleted file mode 100644 index da8e69b766..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py +++ /dev/null @@ -1,744 +0,0 @@ -"""Scenario relation override planning helpers.""" - -from __future__ import annotations - -import re -from collections.abc import Callable -from dataclasses import replace - -from sqlbuild._native import ( - build_compiled_function_fingerprint_sql, - build_execution_upstream_deps, - compiled_model_record, - render_source_relation, - topologically_order_keys, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.constants import ( - DBT_REF_TEST_CTE_PREFIX, - REF_TEST_CTE_PREFIX, - SEED_TEST_CTE_PREFIX, - SOURCE_TEST_CTE_PREFIX, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, - CompiledSqlScenario, - CompileSqlScenarioCte, -) -from sqlbuild.compiler.compile.types import ( - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - CompiledSeedView, -) -from sqlbuild.compiler.planner._helpers.fixtures.completion import ( - build_relation_fixture_completion, -) -from sqlbuild.compiler.planner._helpers.output.plan_entry import extract_seed_columns, plan_model -from sqlbuild.compiler.planner._helpers.resolve.refs import ( - build_function_locations, - replace_executable_matches, -) -from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_function_sql -from sqlbuild.compiler.planner.constants import ( - SCENARIO_PLAN_INVALID_FIXTURE, - SCENARIO_PLAN_MISSING_FIXTURE_SQL, - SCENARIO_PLAN_MISSING_RELATION_TARGET, - SCENARIO_PLAN_UNKNOWN_SEED, - SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - CursorOverridePair, - FunctionPlanEntry, - ModelPlanContext, - ModelPlanEntry, - PlanWarning, - RelationFixtureCompletion, - ScenarioAssertionExpectationPlan, - ScenarioExecutionPlan, - ScenarioExpectedExpectationPlan, - ScenarioFixturePlan, - ScenarioGraphPlan, - ScenarioRelationPlan, - SeedPlanEntry, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import ( - FixtureKey, - ScenarioArtifactKind, -) -from sqlbuild.compiler.references.constants import ( - DBT_REF_CALL_PREFIX, - REF_CALL_PREFIX, - SEED_CALL_PREFIX, - SOURCE_CALL_PREFIX, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import SourceEntry - -_NAME_ARGUMENT_PATTERN_TEXT: str = r'"(?P[^"]+)"\)' -_REF_PATTERN: re.Pattern[str] = re.compile(REF_CALL_PREFIX + _NAME_ARGUMENT_PATTERN_TEXT) -_SEED_PATTERN: re.Pattern[str] = re.compile(SEED_CALL_PREFIX + _NAME_ARGUMENT_PATTERN_TEXT) -_SOURCE_PATTERN: re.Pattern[str] = re.compile(SOURCE_CALL_PREFIX + _NAME_ARGUMENT_PATTERN_TEXT) -_DBT_REF_PATTERN: re.Pattern[str] = re.compile( - DBT_REF_CALL_PREFIX + r'"(?P[^"]+)"(?:\s*,\s*"(?P[^"]+)")?\)' -) - - -def resolve_scenario_check_sql( - *, - sql: str, - relation_plan: ScenarioRelationPlan, - adapter: BaseAdapter, - source_label: str, - source_lexical_syntax: SqlLexicalSyntax, -) -> str: - """Resolve refs, seeds, and sources in authored scenario expected/assertion SQL.""" - - def _ref_target(match: re.Match[str]) -> str | None: - return _location_name(relation_plan.model_locations.get(match.group("name"))) - - def _seed_target(match: re.Match[str]) -> str | None: - return _location_name(relation_plan.seed_locations.get(match.group("name"))) - - def _dbt_ref_target(match: re.Match[str]) -> str | None: - return _location_name( - relation_plan.dbt_ref_fixture_locations.get(_dbt_ref_fixture_name(match)) - ) - - return _replace_relation_markers( - sql=sql, - resolvers=( - (_REF_PATTERN, _ref_target), - (_SEED_PATTERN, _seed_target), - ( - _SOURCE_PATTERN, - _source_resolver(source_map=relation_plan.source_map, adapter=adapter), - ), - (_DBT_REF_PATTERN, _dbt_ref_target), - ), - source_label=source_label, - source_lexical_syntax=source_lexical_syntax, - ) - - -def _source_resolver( - *, source_map: dict[str, SourceEntry], adapter: BaseAdapter -) -> Callable[[re.Match[str]], str | None]: - def _source_target(match: re.Match[str]) -> str | None: - source: SourceEntry | None = source_map.get(match.group("name")) - return None if source is None else render_source_relation(entry=source, adapter=adapter) - - return _source_target - - -def _location_name(location: CompiledRelationLocation | None) -> str | None: - return None if location is None else location.qualified_name - - -def _marker_replacer( - *, resolve_target: Callable[[re.Match[str]], str | None], source_label: str -) -> Callable[[re.Match[str]], str]: - def _replace(match: re.Match[str]) -> str: - target: str | None = resolve_target(match) - if target is None: - raise PlannerInputError( - f"Scenario SQL in {source_label} references {match.group(0)!r}, which is " - "not a relation in this scenario", - code=SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER, - help="Reference a model, seed, source, or dbt ref the scenario builds or " - "provides as a fixture.", - ) - return target - - return _replace - - -def _replace_relation_markers( - *, - sql: str, - resolvers: tuple[tuple[re.Pattern[str], Callable[[re.Match[str]], str | None]], ...], - source_label: str, - source_lexical_syntax: SqlLexicalSyntax, -) -> str: - result: str = sql - pattern: re.Pattern[str] - resolve_target: Callable[[re.Match[str]], str | None] - for pattern, resolve_target in resolvers: - try: - result = replace_executable_matches( - sql=result, - pattern=pattern, - replace=_marker_replacer(resolve_target=resolve_target, source_label=source_label), - lexical_syntax=source_lexical_syntax, - ) - except CompileInputError as error: - raise PlannerInputError( - f"Scenario SQL in {source_label} could not be scanned for relation markers: " - f"{error}", - code=SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER, - help="Close every quoted string and block comment using the project dialect's " - "quoting and comment rules.", - ) from None - return result - - -def build_scenario_execution_plan( - *, - scenario: CompiledSqlScenario, - project: CompiledProjectView, - adapter: BaseAdapter, - graph_plan: ScenarioGraphPlan, - relation_plan: ScenarioRelationPlan, - source_lexical_syntax: SqlLexicalSyntax, - snapshot: WarehouseSnapshot | None = None, - source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]] | None = None, - sql_analysis_enabled: bool = True, -) -> tuple[ScenarioExecutionPlan, tuple[PlanWarning, ...]]: - """Build a dry-run execution plan for one SQL scenario.""" - - effective_snapshot: WarehouseSnapshot = snapshot or WarehouseSnapshot() - effective_source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]] = ( - source_warehouse_columns or {} - ) - models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} - functions_by_key: dict[CompiledObjectKey, CompiledFunctionView] = { - function.key: function for function in project.functions - } - function_locations: dict[str, CompiledRelationLocation] = build_function_locations( - project.functions - ) - scenario_model_names: frozenset[str] = frozenset(graph_plan.model_names) - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_execution_upstream_deps(project) - ) - ordered_model_keys: tuple[CompiledObjectKey, ...] = tuple( - key - for key in topologically_order_keys(upstream=upstream_deps) - if key.resource_type == CompiledResourceType.MODEL and key.name in scenario_model_names - ) - scenario_fixture_groups: tuple[tuple[CompiledResourceType, dict[str, str]], ...] = ( - ( - CompiledResourceType.SOURCE, - _extract_fixture_ctes(scenario=scenario, prefix=SOURCE_TEST_CTE_PREFIX), - ), - ( - CompiledResourceType.MODEL, - _extract_fixture_ctes(scenario=scenario, prefix=REF_TEST_CTE_PREFIX), - ), - ( - CompiledResourceType.SEED, - _extract_fixture_ctes(scenario=scenario, prefix=SEED_TEST_CTE_PREFIX), - ), - ) - fixture_sql_overrides: dict[FixtureKey, str] | None = None - if sql_analysis_enabled: - fixture_completion: RelationFixtureCompletion = build_relation_fixture_completion( - project=project, - adapter=adapter, - ordered_model_names=tuple(key.name for key in ordered_model_keys), - fixture_groups=scenario_fixture_groups, - ) - if fixture_completion.diagnostics: - source_path: str = _scenario_source_label(scenario) - details: str = "\n".join( - f"- {diagnostic.message}" for diagnostic in fixture_completion.diagnostics - ) - raise PlannerInputError( - f"Scenario '{scenario.name}' has invalid relation fixtures in " - f"{source_path}:\n{details}", - code=SCENARIO_PLAN_INVALID_FIXTURE, - ) - fixture_sql_overrides = fixture_completion.fixture_sql_by_key - fixture_plans: tuple[ScenarioFixturePlan, ...] = build_scenario_fixture_plans( - scenario=scenario, - graph_plan=graph_plan, - relation_plan=relation_plan, - adapter=adapter, - source_lexical_syntax=source_lexical_syntax, - fixture_sql_overrides=fixture_sql_overrides, - ) - seed_entries: tuple[SeedPlanEntry, ...] = build_scenario_seed_entries( - project=project, - graph_plan=graph_plan, - relation_plan=relation_plan, - ) - function_entries: tuple[FunctionPlanEntry, ...] = tuple( - _build_scenario_function_entry( - function=functions_by_key[key], - adapter=adapter, - relation_plan=relation_plan, - function_locations=function_locations, - source_warehouse_columns=effective_source_warehouse_columns, - ) - for key in topologically_order_keys(upstream=upstream_deps) - if key in graph_plan.function_deps and key in functions_by_key - ) - - model_entries: list[ModelPlanEntry] = [] - warnings: list[PlanWarning] = [] - key: CompiledObjectKey - for key in ordered_model_keys: - model: CompiledModelView = models_by_name[key.name] - scenario_target: CompiledRelationLocation = _required_target( - targets=relation_plan.model_locations, - name=model.name, - kind=ScenarioArtifactKind.MODEL, - ) - scenario_query_sql: str = _resolve_model_dbt_ref_fixtures( - query_sql=model.query_sql, - relation_plan=relation_plan, - ) - entry: ModelPlanEntry - model_warnings: tuple[PlanWarning, ...] - entry, model_warnings = plan_model( - model=replace( - compiled_model_record(model), - destination=scenario_target, - query_sql=scenario_query_sql, - ), - snapshot=effective_snapshot, - adapter=adapter, - context=ModelPlanContext( - model_locations=relation_plan.model_locations, - models_by_name=models_by_name, - functions_by_name={function.name: function for function in project.functions}, - seed_locations=relation_plan.seed_locations, - function_locations=function_locations, - source_map=relation_plan.source_map, - source_warehouse_columns=effective_source_warehouse_columns, - star_exclude_keyword=adapter.star_exclude_keyword(), - ), - sql_analysis_enabled=sql_analysis_enabled, - query_change_tracking=False, - full_refresh=True, - cursor_overrides=CursorOverridePair(), - ) - model_entries.append(entry) - warnings.extend(model_warnings) - - expected_expectations: tuple[ScenarioExpectedExpectationPlan, ...] = tuple( - _build_expected_check_plan( - expected_cte=expected_cte, - relation_plan=relation_plan, - adapter=adapter, - source_label=_scenario_source_label(scenario), - source_lexical_syntax=source_lexical_syntax, - ) - for expected_cte in scenario.expected_ctes - ) - assertion_expectations: tuple[ScenarioAssertionExpectationPlan, ...] = tuple( - ScenarioAssertionExpectationPlan( - name=assertion_cte.name.removeprefix("__assert__"), - sql=resolve_scenario_check_sql( - sql=assertion_cte.sql_body, - relation_plan=relation_plan, - adapter=adapter, - source_label=_scenario_source_label(scenario), - source_lexical_syntax=source_lexical_syntax, - ), - ) - for assertion_cte in scenario.assertion_ctes - ) - - return ( - ScenarioExecutionPlan( - key=scenario.key, - name=scenario.name, - graph_plan=graph_plan, - relation_plan=relation_plan, - fixture_plans=fixture_plans, - seed_entries=seed_entries, - function_entries=function_entries, - model_entries=tuple(model_entries), - hook_functions=project.hook_functions, - expected_expectations=expected_expectations, - assertion_expectations=assertion_expectations, - ), - tuple(warnings), - ) - - -def _build_scenario_function_entry( - *, - function: CompiledFunctionView, - adapter: BaseAdapter, - relation_plan: ScenarioRelationPlan, - function_locations: dict[str, CompiledRelationLocation], - source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]], -) -> FunctionPlanEntry: - return FunctionPlanEntry( - key=function.key, - name=function.name, - relative_path=function.relative_path, - destination=function.destination, - arguments=function.arguments, - returns=function.returns, - body_sql=resolve_function_sql( - adapter=adapter, - function=function, - model_locations=relation_plan.model_locations, - seed_locations=relation_plan.seed_locations, - function_locations=function_locations, - source_map=relation_plan.source_map, - source_warehouse_columns=source_warehouse_columns, - star_exclude_keyword=adapter.star_exclude_keyword(), - ), - fingerprint_query_sql=build_compiled_function_fingerprint_sql(function), - fingerprint_destination=function.fingerprint_destination, - return_columns=function.return_columns, - language=function.language, - source_file_path=function.source_file_path, - runtime_version=function.runtime_version, - entry_point=function.entry_point, - packages=function.packages, - ) - - -def build_scenario_fixture_plans( - *, - scenario: CompiledSqlScenario, - graph_plan: ScenarioGraphPlan, - relation_plan: ScenarioRelationPlan, - adapter: BaseAdapter, - source_lexical_syntax: SqlLexicalSyntax, - fixture_sql_overrides: dict[FixtureKey, str] | None = None, -) -> tuple[ScenarioFixturePlan, ...]: - """Build self-contained fixture SQL plans, including shared helper CTEs.""" - - source_label: str = _scenario_source_label(scenario) - helper_ctes: tuple[CompileSqlScenarioCte, ...] = _extract_helper_ctes(scenario) - resolved_helper_ctes: tuple[CompileSqlScenarioCte, ...] = tuple( - replace( - helper_cte, - sql_body=_resolve_project_source_refs( - sql=helper_cte.sql_body, - source_map=relation_plan.project_source_map, - adapter=adapter, - source_label=source_label, - source_lexical_syntax=source_lexical_syntax, - ), - ) - for helper_cte in helper_ctes - ) - source_ctes: dict[str, str] = _extract_fixture_ctes( - scenario=scenario, - prefix=SOURCE_TEST_CTE_PREFIX, - ) - ref_ctes: dict[str, str] = _extract_fixture_ctes( - scenario=scenario, - prefix=REF_TEST_CTE_PREFIX, - ) - seed_ctes: dict[str, str] = _extract_fixture_ctes( - scenario=scenario, - prefix=SEED_TEST_CTE_PREFIX, - ) - dbt_ref_ctes: dict[str, str] = _extract_fixture_ctes( - scenario=scenario, - prefix=DBT_REF_TEST_CTE_PREFIX, - ) - overrides: dict[FixtureKey, str] = fixture_sql_overrides or {} - source_ctes = _apply_fixture_overrides( - resource_type=CompiledResourceType.SOURCE, - fixture_sql=source_ctes, - overrides=overrides, - ) - ref_ctes = _apply_fixture_overrides( - resource_type=CompiledResourceType.MODEL, - fixture_sql=ref_ctes, - overrides=overrides, - ) - seed_ctes = _apply_fixture_overrides( - resource_type=CompiledResourceType.SEED, - fixture_sql=seed_ctes, - overrides=overrides, - ) - - plans: list[ScenarioFixturePlan] = [] - source_name: str - for source_name in graph_plan.source_fixture_names: - plans.append( - ScenarioFixturePlan( - kind=ScenarioArtifactKind.SOURCE, - logical_name=source_name, - destination=_required_target( - targets=relation_plan.source_fixture_locations, - name=source_name, - kind=ScenarioArtifactKind.SOURCE, - ), - sql=_wrap_sql_with_helpers( - sql=_resolve_project_source_refs( - sql=_required_fixture_sql( - fixture_sql=source_ctes, logical_name=source_name, kind="source" - ), - source_map=relation_plan.project_source_map, - adapter=adapter, - source_label=source_label, - source_lexical_syntax=source_lexical_syntax, - ), - helper_ctes=resolved_helper_ctes, - ), - ) - ) - - ref_name: str - for ref_name in graph_plan.ref_fixture_names: - plans.append( - ScenarioFixturePlan( - kind=ScenarioArtifactKind.REF, - logical_name=ref_name, - destination=_required_target( - targets=relation_plan.ref_fixture_locations, - name=ref_name, - kind=ScenarioArtifactKind.REF, - ), - sql=_wrap_sql_with_helpers( - sql=_resolve_project_source_refs( - sql=_required_fixture_sql( - fixture_sql=ref_ctes, logical_name=ref_name, kind="ref" - ), - source_map=relation_plan.project_source_map, - adapter=adapter, - source_label=source_label, - source_lexical_syntax=source_lexical_syntax, - ), - helper_ctes=resolved_helper_ctes, - ), - ) - ) - - seed_name: str - for seed_name in graph_plan.seed_fixture_names: - plans.append( - ScenarioFixturePlan( - kind=ScenarioArtifactKind.SEED, - logical_name=seed_name, - destination=_required_target( - targets=relation_plan.seed_fixture_locations, - name=seed_name, - kind=ScenarioArtifactKind.SEED, - ), - sql=_wrap_sql_with_helpers( - sql=_resolve_project_source_refs( - sql=_required_fixture_sql( - fixture_sql=seed_ctes, logical_name=seed_name, kind="seed" - ), - source_map=relation_plan.project_source_map, - adapter=adapter, - source_label=source_label, - source_lexical_syntax=source_lexical_syntax, - ), - helper_ctes=resolved_helper_ctes, - ), - ) - ) - - dbt_ref_name: str - for dbt_ref_name in graph_plan.dbt_ref_fixture_names: - plans.append( - ScenarioFixturePlan( - kind=ScenarioArtifactKind.DBT_REF, - logical_name=dbt_ref_name, - destination=_required_target( - targets=relation_plan.dbt_ref_fixture_locations, - name=dbt_ref_name, - kind=ScenarioArtifactKind.DBT_REF, - ), - sql=_wrap_sql_with_helpers( - sql=_resolve_project_source_refs( - sql=_required_fixture_sql( - fixture_sql=dbt_ref_ctes, - logical_name=dbt_ref_name, - kind="dbt_ref", - ), - source_map=relation_plan.project_source_map, - adapter=adapter, - source_label=source_label, - source_lexical_syntax=source_lexical_syntax, - ), - helper_ctes=resolved_helper_ctes, - ), - ) - ) - - return tuple(plans) - - -def _apply_fixture_overrides( - *, - resource_type: CompiledResourceType, - fixture_sql: dict[str, str], - overrides: dict[FixtureKey, str], -) -> dict[str, str]: - return {name: overrides.get((resource_type, name), sql) for name, sql in fixture_sql.items()} - - -def build_scenario_seed_entries( - *, - project: CompiledProjectView, - graph_plan: ScenarioGraphPlan, - relation_plan: ScenarioRelationPlan, -) -> tuple[SeedPlanEntry, ...]: - """Build project seed load entries for required seeds not overridden by fixtures.""" - - seeds_by_name: dict[str, CompiledSeedView] = {seed.name: seed for seed in project.seeds} - seed_fixture_names: frozenset[str] = frozenset(graph_plan.seed_fixture_names) - seed_entries: list[SeedPlanEntry] = [] - seed_name: str - for seed_name in graph_plan.seed_names: - if seed_name in seed_fixture_names: - continue - seed: CompiledSeedView | None = seeds_by_name.get(seed_name) - if seed is None: - raise PlannerInputError( - f"Scenario '{graph_plan.name}' requires unknown seed '{seed_name}'", - code=SCENARIO_PLAN_UNKNOWN_SEED, - ) - seed_entries.append( - SeedPlanEntry( - key=seed.key, - name=seed.name, - destination=_required_target( - targets=relation_plan.seed_locations, - name=seed_name, - kind=ScenarioArtifactKind.SEED, - ), - file_path=seed.seed_file.file_path, - columns=extract_seed_columns(seed), - csv_settings=seed.schema_entry.csv_settings, - ) - ) - return tuple(seed_entries) - - -def _build_expected_check_plan( - *, - expected_cte: CompileSqlScenarioCte, - relation_plan: ScenarioRelationPlan, - adapter: BaseAdapter, - source_label: str, - source_lexical_syntax: SqlLexicalSyntax, -) -> ScenarioExpectedExpectationPlan: - model_name: str = expected_cte.name.removeprefix("__expected__") - actual_destination: CompiledRelationLocation = _required_target( - targets=relation_plan.model_locations, - name=model_name, - kind=ScenarioArtifactKind.MODEL, - ) - return ScenarioExpectedExpectationPlan( - model_name=model_name, - actual_destination=actual_destination, - expected_sql=resolve_scenario_check_sql( - sql=expected_cte.sql_body, - relation_plan=relation_plan, - adapter=adapter, - source_label=source_label, - source_lexical_syntax=source_lexical_syntax, - ), - ) - - -def _extract_helper_ctes(scenario: CompiledSqlScenario) -> tuple[CompileSqlScenarioCte, ...]: - helpers: list[CompileSqlScenarioCte] = [] - cte: CompileSqlScenarioCte - for cte in scenario.authored_ctes: - if cte.name.startswith(SOURCE_TEST_CTE_PREFIX): - continue - if cte.name.startswith(REF_TEST_CTE_PREFIX): - continue - if cte.name.startswith(SEED_TEST_CTE_PREFIX): - continue - if cte.name.startswith(DBT_REF_TEST_CTE_PREFIX): - continue - helpers.append(cte) - return tuple(helpers) - - -def _extract_fixture_ctes(*, scenario: CompiledSqlScenario, prefix: str) -> dict[str, str]: - result: dict[str, str] = {} - cte: CompileSqlScenarioCte - for cte in scenario.authored_ctes: - if cte.name.startswith(prefix): - result[cte.name.removeprefix(prefix)] = cte.sql_body - return result - - -def _wrap_sql_with_helpers(*, sql: str, helper_ctes: tuple[CompileSqlScenarioCte, ...]) -> str: - if not helper_ctes: - return sql - helper_parts: list[str] = [] - helper_cte: CompileSqlScenarioCte - for helper_cte in helper_ctes: - helper_parts.append(f"{helper_cte.name} AS ({helper_cte.sql_body})") - return f"WITH {', '.join(helper_parts)} {sql}" - - -def _resolve_project_source_refs( - *, - sql: str, - source_map: dict[str, SourceEntry], - adapter: BaseAdapter, - source_label: str, - source_lexical_syntax: SqlLexicalSyntax, -) -> str: - return _replace_relation_markers( - sql=sql, - resolvers=((_SOURCE_PATTERN, _source_resolver(source_map=source_map, adapter=adapter)),), - source_label=source_label, - source_lexical_syntax=source_lexical_syntax, - ) - - -def _scenario_source_label(scenario: CompiledSqlScenario) -> str: - return str(scenario.source_path or scenario.name) - - -def _required_fixture_sql(*, fixture_sql: dict[str, str], logical_name: str, kind: str) -> str: - sql: str | None = fixture_sql.get(logical_name) - if sql is None: - raise PlannerInputError( - f"Scenario is missing {kind} fixture SQL '{logical_name}'", - code=SCENARIO_PLAN_MISSING_FIXTURE_SQL, - ) - return sql - - -def _required_target( - *, - targets: dict[str, CompiledRelationLocation], - name: str, - kind: ScenarioArtifactKind, -) -> CompiledRelationLocation: - target: CompiledRelationLocation | None = targets.get(name) - if target is None: - raise PlannerInputError( - f"Scenario relation plan is missing {kind.value} target '{name}'", - code=SCENARIO_PLAN_MISSING_RELATION_TARGET, - help="This is likely a SQLBuild bug. Please file an issue with the scenario name.", - ) - return target - - -def _resolve_model_dbt_ref_fixtures(*, query_sql: str, relation_plan: ScenarioRelationPlan) -> str: - def _replace_dbt_ref(match: re.Match[str]) -> str: - target: CompiledRelationLocation | None = relation_plan.dbt_ref_fixture_locations.get( - _dbt_ref_fixture_name(match) - ) - if target is None or target.qualified_name is None: - return match.group(0) - return target.qualified_name - - return _DBT_REF_PATTERN.sub(_replace_dbt_ref, query_sql) - - -def _dbt_ref_fixture_name(match: re.Match[str]) -> str: - first: str = match.group("first") - second: str | None = match.group("second") - if second is None: - return first - return f"{first}__{second}" diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py deleted file mode 100644 index 3776ff41f7..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py +++ /dev/null @@ -1,356 +0,0 @@ -"""SQL-native unit-test plan entries built from native chain planning.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.constants import ( - DBT_REF_TEST_CTE_PREFIX, - EXPECTED_TEST_CTE_PREFIX, - REF_TEST_CTE_PREFIX, - SEED_TEST_CTE_PREFIX, - SOURCE_TEST_CTE_PREFIX, - TABLE_FN_TEST_CTE_PREFIX, -) -from sqlbuild.compiler.compile.models import ( - CompiledDirectLogicSqlTestPayload, - CompiledModelSqlTestPayload, - CompiledObjectKey, - CompiledSqlTest, - CompileSqlTestCte, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - SqlTestMode, -) -from sqlbuild.compiler.planner._helpers.fixtures.completion import build_relation_fixture_context -from sqlbuild.compiler.planner._helpers.sql_tests.fixture_validation import ( - build_validated_test_fixtures, - fixture_location, -) -from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( - plan_sql_tests_natively, - resolve_sql_test_model_chains, -) -from sqlbuild.compiler.planner.exceptions import SqlTestFixtureValidationError -from sqlbuild.compiler.planner.models import ( - NativeSqlTestPlan, - RelationFixturePlanningContext, - SqlTestPlanEntry, - SqlTestPlanResult, -) -from sqlbuild.executor.testing.main.missing_expected_columns import ( - describe_missing_expected_columns, -) - -_FUNCTION_RESOURCE_TYPES: frozenset[CompiledResourceType] = frozenset( - {CompiledResourceType.UDF, CompiledResourceType.TABLE_FN} -) - - -def plan_sql_tests( - *, - tests: tuple[CompiledSqlTest, ...], - project: CompiledProjectView, - adapter: BaseAdapter, - sql_analysis_enabled: bool = False, - validate_fixtures: bool = False, - fixture_planning_context: RelationFixturePlanningContext | None = None, -) -> tuple[SqlTestPlanResult, ...]: - """Plan SQL tests in one native batch, reporting fixture diagnostics per test.""" - - planned_tests: tuple[CompiledSqlTest, ...] = tests - diagnostics_by_index: dict[int, tuple[str, ...]] = {} - validation_context: RelationFixturePlanningContext | None = None - if sql_analysis_enabled and validate_fixtures: - validation_context = fixture_planning_context or build_relation_fixture_context( - project=project - ) - planned_tests, diagnostics_by_index = _validate_fixtures( - tests=tests, - project=project, - adapter=adapter, - fixture_planning_context=validation_context, - ) - plannable_indexes: tuple[int, ...] = tuple( - index for index in range(len(tests)) if index not in diagnostics_by_index - ) - plans: tuple[NativeSqlTestPlan, ...] = plan_sql_tests_natively( - project=project, - tests=tuple(planned_tests[index] for index in plannable_indexes), - adapter=adapter, - sql_analysis_enabled=sql_analysis_enabled, - render_sql=False, - ) - plans_by_index: dict[int, NativeSqlTestPlan] = dict(zip(plannable_indexes, plans, strict=True)) - models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} - results: list[SqlTestPlanResult] = [] - for index, test in enumerate(tests): - plan: NativeSqlTestPlan | None = plans_by_index.get(index) - if plan is None: - results.append( - SqlTestPlanResult(entry=None, fixture_diagnostics=diagnostics_by_index[index]) - ) - continue - plan_diagnostics: tuple[str, ...] = ( - *plan.error_messages, - *_expected_column_diagnostics( - test=test, plan=plan, fixture_planning_context=validation_context - ), - ) - if plan_diagnostics: - results.append(SqlTestPlanResult(entry=None, fixture_diagnostics=plan_diagnostics)) - continue - results.append( - SqlTestPlanResult( - entry=_plan_entry( - test=test, - plan=plan, - project=project, - models_by_name=models_by_name, - sql_analysis_enabled=sql_analysis_enabled, - ), - warnings=plan.warnings, - ) - ) - return tuple(results) - - -def _expected_column_diagnostics( - *, - test: CompiledSqlTest, - plan: NativeSqlTestPlan, - fixture_planning_context: RelationFixturePlanningContext | None, -) -> tuple[str, ...]: - """Report expected columns that authoritative model output metadata proves absent.""" - - if fixture_planning_context is None: - return () - diagnostics: list[str] = [] - for step in plan.chain: - available_columns: frozenset[str] | None = ( - fixture_planning_context.authoritative_columns.get( - (CompiledResourceType.MODEL, step.model_name) - ) - ) - if step.expected_columns is None or available_columns is None: - continue - message: str | None = describe_missing_expected_columns( - model_name=step.model_name, - expected_columns=step.expected_columns, - available_columns=available_columns, - ) - if message is None: - continue - location: str = fixture_location( - test=test, resource_type=CompiledResourceType.SQL_TEST, name=step.model_name - ) - diagnostics.append(f"SQL test '{test.name}': {location}: {message}") - return tuple(diagnostics) - - -def _validate_fixtures( - *, - tests: tuple[CompiledSqlTest, ...], - project: CompiledProjectView, - adapter: BaseAdapter, - fixture_planning_context: RelationFixturePlanningContext, -) -> tuple[tuple[CompiledSqlTest, ...], dict[int, tuple[str, ...]]]: - """Return tests with validated fixture completions plus diagnostics by test index.""" - - model_indexes: tuple[int, ...] = tuple( - index - for index, test in enumerate(tests) - if isinstance(test.payload, CompiledModelSqlTestPayload) - ) - if not model_indexes: - return tests, {} - chains: tuple[tuple[str, ...], ...] = resolve_sql_test_model_chains( - project=project, - tests=tuple(tests[index] for index in model_indexes), - ) - chains_by_index: dict[int, tuple[str, ...]] = dict(zip(model_indexes, chains, strict=True)) - context: RelationFixturePlanningContext = fixture_planning_context - validated_tests: list[CompiledSqlTest] = [] - diagnostics_by_index: dict[int, tuple[str, ...]] = {} - for index, test in enumerate(tests): - chain: tuple[str, ...] | None = chains_by_index.get(index) - if chain is None: - validated_tests.append(test) - continue - try: - validated_tests.append( - _with_validated_fixtures( - test=test, - project=project, - adapter=adapter, - ordered_model_names=chain, - fixture_planning_context=context, - ) - ) - except SqlTestFixtureValidationError as error: - validated_tests.append(test) - diagnostics_by_index[index] = error.diagnostics - return tuple(validated_tests), diagnostics_by_index - - -def _with_validated_fixtures( - *, - test: CompiledSqlTest, - project: CompiledProjectView, - adapter: BaseAdapter, - ordered_model_names: tuple[str, ...], - fixture_planning_context: RelationFixturePlanningContext, -) -> CompiledSqlTest: - payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload = test.payload - if not isinstance(payload, CompiledModelSqlTestPayload): - return test - mock_refs, mock_sources, mock_seeds, expected_outputs = build_validated_test_fixtures( - test=test, - project=project, - adapter=adapter, - ordered_model_names=ordered_model_names, - mock_refs=_prefixed_ctes(ctes=payload.authored_ctes, prefix=REF_TEST_CTE_PREFIX), - mock_sources=_prefixed_ctes(ctes=payload.authored_ctes, prefix=SOURCE_TEST_CTE_PREFIX), - mock_seeds=_prefixed_ctes(ctes=payload.authored_ctes, prefix=SEED_TEST_CTE_PREFIX), - expected_outputs=_expected_outputs(payload=payload), - planning_context=fixture_planning_context, - ) - completed_bodies: tuple[tuple[str, dict[str, str]], ...] = ( - (REF_TEST_CTE_PREFIX, mock_refs), - (SOURCE_TEST_CTE_PREFIX, mock_sources), - (SEED_TEST_CTE_PREFIX, mock_seeds), - ) - authored_ctes: tuple[CompileSqlTestCte, ...] = tuple( - _completed_cte(cte=cte, completed_bodies=completed_bodies) for cte in payload.authored_ctes - ) - expected_ctes: tuple[CompileSqlTestCte, ...] = tuple( - replace( - cte, - sql_body=expected_outputs.get( - cte.name.removeprefix(EXPECTED_TEST_CTE_PREFIX), cte.sql_body - ), - ) - for cte in payload.expected_ctes - ) - return replace( - test, - payload=replace(payload, authored_ctes=authored_ctes, expected_ctes=expected_ctes), - ) - - -def _completed_cte( - *, - cte: CompileSqlTestCte, - completed_bodies: tuple[tuple[str, dict[str, str]], ...], -) -> CompileSqlTestCte: - for prefix, bodies in completed_bodies: - if cte.name.startswith(prefix): - return replace(cte, sql_body=bodies.get(cte.name.removeprefix(prefix), cte.sql_body)) - return cte - - -def _expected_outputs(*, payload: CompiledModelSqlTestPayload) -> dict[str, str]: - return { - cte.name.removeprefix(EXPECTED_TEST_CTE_PREFIX): cte.sql_body - for cte in payload.expected_ctes - } - - -def _prefixed_ctes(*, ctes: tuple[CompileSqlTestCte, ...], prefix: str) -> dict[str, str]: - return { - cte.name.removeprefix(prefix): cte.sql_body for cte in ctes if cte.name.startswith(prefix) - } - - -def _prefixed_names(*, test: CompiledSqlTest, prefix: str) -> tuple[str, ...]: - if not isinstance(test.payload, CompiledModelSqlTestPayload): - return () - return tuple(sorted(_prefixed_ctes(ctes=test.payload.authored_ctes, prefix=prefix))) - - -def _plan_entry( - *, - test: CompiledSqlTest, - plan: NativeSqlTestPlan, - project: CompiledProjectView, - models_by_name: dict[str, CompiledModelView], - sql_analysis_enabled: bool, -) -> SqlTestPlanEntry: - mock_table_function_names: tuple[str, ...] = _prefixed_names( - test=test, prefix=TABLE_FN_TEST_CTE_PREFIX - ) - return SqlTestPlanEntry( - key=test.key, - name=test.name, - source_path=test.source_path, - block_index=test.block_index, - parent_name=test.parent_name, - case_name=test.case_name, - case_index=test.case_index, - case_fingerprint=test.case_fingerprint, - parameter_schema=test.parameter_schema, - parameter_values=test.parameter_values, - mock_ref_names=_prefixed_names(test=test, prefix=REF_TEST_CTE_PREFIX), - mock_source_names=_prefixed_names(test=test, prefix=SOURCE_TEST_CTE_PREFIX), - mock_seed_names=_prefixed_names(test=test, prefix=SEED_TEST_CTE_PREFIX), - mock_dbt_ref_names=_prefixed_names(test=test, prefix=DBT_REF_TEST_CTE_PREFIX), - mock_table_function_names=mock_table_function_names, - chain=plan.chain, - assertions=plan.assertions, - scope_deps=test.scope_deps, - function_deps=( - _direct_function_deps(test=test, project=project) - if isinstance(test.payload, CompiledDirectLogicSqlTestPayload) - else _chain_function_deps( - model_names=plan.model_names, - models_by_name=models_by_name, - mocked_table_functions=frozenset(mock_table_function_names), - ) - ), - sql_analysis_enabled=sql_analysis_enabled, - ) - - -def _chain_function_deps( - *, - model_names: tuple[str, ...], - models_by_name: dict[str, CompiledModelView], - mocked_table_functions: frozenset[str], -) -> tuple[CompiledObjectKey, ...]: - deps: dict[CompiledObjectKey, None] = {} - for model_name in model_names: - model: CompiledModelView | None = models_by_name.get(model_name) - if model is None: - continue - for dep in model.deps: - if dep.resource_type not in _FUNCTION_RESOURCE_TYPES: - continue - if dep.resource_type == CompiledResourceType.TABLE_FN and ( - dep.name in mocked_table_functions - ): - continue - deps[dep] = None - return tuple(deps) - - -def _direct_function_deps( - *, test: CompiledSqlTest, project: CompiledProjectView -) -> tuple[CompiledObjectKey, ...]: - if not isinstance(test.payload, CompiledDirectLogicSqlTestPayload): - return () - resource_type: CompiledResourceType | None = { - SqlTestMode.UDF: CompiledResourceType.UDF, - SqlTestMode.TABLE_FN: CompiledResourceType.TABLE_FN, - }.get(test.payload.mode) - if resource_type is None: - return () - tested_names: frozenset[str] = frozenset(test.payload.tested_resource_names) - return tuple( - function.key - for function in project.functions - if function.key.resource_type == resource_type and function.name in tested_names - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/comments.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/comments.py deleted file mode 100644 index 14418ad9da..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/comments.py +++ /dev/null @@ -1,48 +0,0 @@ -"""SQL comment handling for textual unit-test fallback resolution.""" - -from __future__ import annotations - -import re - -_SQL_NON_CODE_PATTERN: re.Pattern[str] = re.compile( - r"'(?:''|[^'])*'|\"(?:\"\"|[^\"])*\"|`(?:``|[^`])*`|" - r"\$\$.*?\$\$|--[^\n]*|/\*.*?\*/", - re.DOTALL, -) - - -def uncommented_pattern_matches(*, pattern: re.Pattern[str], sql: str) -> tuple[re.Match[str], ...]: - """Find pattern matches outside SQL comments and quoted values.""" - - return uncommented_matches_by_pattern(patterns=(pattern,), sql=sql)[0] - - -def uncommented_matches_by_pattern( - *, patterns: tuple[re.Pattern[str], ...], sql: str -) -> tuple[tuple[re.Match[str], ...], ...]: - """Find matches for multiple patterns with one protected-region scan.""" - - if all(pattern.search(sql) is None for pattern in patterns): - return tuple(() for _pattern in patterns) - protected_ranges: tuple[tuple[int, int], ...] = tuple( - (match.start(), match.end()) for match in _SQL_NON_CODE_PATTERN.finditer(sql) - ) - - def _matches(pattern: re.Pattern[str]) -> tuple[re.Match[str], ...]: - protected_index: int = 0 - matches: list[re.Match[str]] = [] - match: re.Match[str] - for match in pattern.finditer(sql): - while ( - protected_index < len(protected_ranges) - and protected_ranges[protected_index][1] <= match.start() - ): - protected_index += 1 - if ( - protected_index == len(protected_ranges) - or match.start() < protected_ranges[protected_index][0] - ): - matches.append(match) - return tuple(matches) - - return tuple(_matches(pattern) for pattern in patterns) diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py deleted file mode 100644 index 31bf70ada0..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Cursor windows rendered for models that SQL tests evaluate from their real SQL.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild._native import cursor_compare as compare -from sqlbuild._native import cursor_floor_to_grain as floor_to_grain -from sqlbuild._native import cursor_parse as parse -from sqlbuild._native import cursor_render as render -from sqlbuild._native import get_config_str, resolve_cursor_intrinsics -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompiledSqlTest -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner._helpers.resolve.cursor_intrinsics import ( - render_cursor_intrinsic_bounds, -) -from sqlbuild.compiler.planner.models import CursorBounds -from sqlbuild.compiler.planner.types import CursorGrain, CursorType -from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.types import CursorScalar - -_DEFAULT_TIMESTAMP_START: str = "1900-01-01 00:00:00" -_DEFAULT_TIMESTAMP_END: str = "2999-12-31 00:00:00" -_DEFAULT_INTEGER_START: str = "-1000000000000000" -_DEFAULT_INTEGER_END: str = "1000000000000000" -_CURSOR_START_KEY: str = "cursor_start" -_CURSOR_END_KEY: str = "cursor_end" - - -@dataclass(frozen=True) -class _ModelCursor: - name: str - cursor_type: CursorType - cursor_grain: CursorGrain | None - - -def uses_cursor_intrinsics(*, sql: str) -> bool: - """Return whether SQL calls `__cursor_start()` or `__cursor_end()`.""" - - _, found = resolve_cursor_intrinsics(sql=sql) - return found - - -def declares_cursor_window(*, test: CompiledSqlTest) -> bool: - """Return whether a SQL test header declares its own cursor window.""" - - return test.test_block.cursor_start is not None or test.test_block.cursor_end is not None - - -def render_test_cursor_intrinsics( - *, - sql: str, - model: CompiledModelView, - adapter: BaseAdapter, - test: CompiledSqlTest | None, -) -> str: - """Render cursor intrinsics with the test's declared window, or the wide default window.""" - - if not uses_cursor_intrinsics(sql=sql): - return sql - cursor: _ModelCursor = _model_cursor(model=model) - start, end = _test_window(cursor=cursor, test=test) - return render_cursor_intrinsic_bounds( - sql=sql, - bounds=CursorBounds(start=start, end=end), - cursor_type=cursor.cursor_type, - adapter=adapter, - ) - - -def declared_window_cursor_models( - *, test: CompiledSqlTest, chain_models: tuple[CompiledModelView, ...] -) -> tuple[CompiledModelView, ...]: - """Validate a declared window against every chain model using intrinsics and return them.""" - - cursor_models: tuple[CompiledModelView, ...] = tuple( - model for model in chain_models if uses_cursor_intrinsics(sql=model.query_sql) - ) - if not cursor_models: - raise CompileInputError( - f"{_test_label(test=test)} declares cursor_start or cursor_end, but no model it " - "evaluates uses __cursor_start() or __cursor_end()" - ) - for model in cursor_models: - _test_window(cursor=_model_cursor(model=model), test=test) - return cursor_models - - -def _test_window( - *, cursor: _ModelCursor, test: CompiledSqlTest | None -) -> tuple[CursorScalar, CursorScalar]: - declared_start: str | None = test.test_block.cursor_start if test is not None else None - declared_end: str | None = test.test_block.cursor_end if test is not None else None - integer: bool = cursor.cursor_type == CursorType.INTEGER - start: CursorScalar = _bound( - raw=declared_start, - default=_DEFAULT_INTEGER_START if integer else _DEFAULT_TIMESTAMP_START, - key=_CURSOR_START_KEY, - cursor=cursor, - test=test, - ) - end: CursorScalar = _bound( - raw=declared_end, - default=_DEFAULT_INTEGER_END if integer else _DEFAULT_TIMESTAMP_END, - key=_CURSOR_END_KEY, - cursor=cursor, - test=test, - ) - if test is not None and compare(left=start, right=end) >= 0: - raise CompileInputError( - f"{_test_label(test=test)}: cursor_start '{render(value=start)}' must be before " - f"the exclusive cursor_end '{render(value=end)}' for model '{cursor.name}'" - ) - return start, end - - -def _bound( - *, - raw: str | None, - default: str, - key: str, - cursor: _ModelCursor, - test: CompiledSqlTest | None, -) -> CursorScalar: - if raw is None or test is None: - return parse(raw=default, cursor_type=cursor.cursor_type) - try: - value: CursorScalar = parse(raw=raw, cursor_type=cursor.cursor_type) - except CursorAlgebraError: - expected: str = ( - "an integer" if cursor.cursor_type == CursorType.INTEGER else "an ISO timestamp or date" - ) - raise CompileInputError( - f"{_test_label(test=test)}: {key} '{raw}' must be {expected} because model " - f"'{cursor.name}' uses cursor_type {cursor.cursor_type.value}" - ) from None - if cursor.cursor_grain is not None and compare( - left=floor_to_grain(value=value, grain=cursor.cursor_grain), right=value - ): - raise CompileInputError( - f"{_test_label(test=test)}: {key} '{raw}' is not aligned to cursor_grain " - f"{cursor.cursor_grain.value} of model '{cursor.name}'" - ) - return value - - -def _model_cursor(*, model: CompiledModelView) -> _ModelCursor: - cursor_type: CursorType = CursorType( - get_config_str(values=model.config.values, key="cursor_type") or CursorType.TIMESTAMP - ) - grain: str | None = get_config_str(values=model.config.values, key="cursor_grain") - return _ModelCursor( - name=model.name, - cursor_type=cursor_type, - cursor_grain=( - CursorGrain(grain) - if grain is not None and cursor_type == CursorType.TIMESTAMP - else None - ), - ) - - -def _test_label(*, test: CompiledSqlTest) -> str: - return f"SQL test '{test.name}' in '{test.test_file.relative_path}'" diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py deleted file mode 100644 index 7de7a40e3c..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py +++ /dev/null @@ -1,192 +0,0 @@ -"""Static SQL-test fixture shape validation.""" - -from __future__ import annotations - -import re - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( - CompiledSqlTest, - FixtureColumnInference, - InferredColumn, -) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.planner._helpers.fixtures.completion import ( - build_relation_fixture_completion, - build_relation_fixture_context, - complete_empty_fixture_sql, - complete_typed_null_columns, -) -from sqlbuild.compiler.planner.exceptions import SqlTestFixtureValidationError -from sqlbuild.compiler.planner.models import ( - RelationFixtureCompletion, - RelationFixtureDiagnostic, - RelationFixturePlanningContext, -) - -_COLLECTION_TYPES: frozenset[str] = frozenset( - {"ARRAY", "LIST", "MAP", "OBJECT", "STRUCT", "VARIANT"} -) - - -def build_validated_test_fixtures( - *, - test: CompiledSqlTest, - project: CompiledProjectView, - adapter: BaseAdapter, - ordered_model_names: tuple[str, ...], - mock_refs: dict[str, str], - mock_sources: dict[str, str], - mock_seeds: dict[str, str], - expected_outputs: dict[str, str], - planning_context: RelationFixturePlanningContext | None = None, -) -> tuple[dict[str, str], dict[str, str], dict[str, str], dict[str, str]]: - """Report all statically provable missing columns and collection-type mismatches.""" - - fixture_groups: tuple[tuple[CompiledResourceType, dict[str, str]], ...] = ( - (CompiledResourceType.MODEL, mock_refs), - (CompiledResourceType.SOURCE, mock_sources), - (CompiledResourceType.SEED, mock_seeds), - ) - fixture_context: RelationFixturePlanningContext = ( - planning_context or build_relation_fixture_context(project=project) - ) - completion: RelationFixtureCompletion = build_relation_fixture_completion( - project=project, - adapter=adapter, - ordered_model_names=ordered_model_names, - fixture_groups=fixture_groups, - planning_context=fixture_context, - ) - errors: list[str] = [ - _located_diagnostic(test=test, diagnostic=diagnostic) - for diagnostic in completion.diagnostics - ] - completed_expected_outputs: dict[str, str] = dict(expected_outputs) - type_fixture_groups: tuple[tuple[CompiledResourceType, dict[str, str]], ...] = ( - *fixture_groups, - (CompiledResourceType.SQL_TEST, expected_outputs), - ) - for resource_type, fixtures in type_fixture_groups: - for name, sql in fixtures.items(): - inferred_columns: tuple[InferredColumn, ...] | None - if resource_type == CompiledResourceType.SQL_TEST: - completed_sql, empty_error = complete_empty_fixture_sql( - sql=sql, - relation=fixture_context.relations.get((CompiledResourceType.MODEL, name)), - adapter=adapter, - ) - if empty_error is not None: - errors.append( - f"{fixture_location(test=test, resource_type=resource_type, name=name)}: " - f"expected output '{name}' {empty_error}" - ) - completed_expected_outputs[name] = completed_sql - inference: FixtureColumnInference | None = fixture_context.fixture_inferences.infer( - query_sql=completed_sql, - inference_profile=adapter.expression_inference_profile(), - ) - inferred_columns = inference.columns if inference is not None else None - if inference is not None: - completed_expected_outputs[name] = complete_typed_null_columns( - sql=completed_sql, - inference=inference, - expected_types=completion.expected_types.get( - (CompiledResourceType.MODEL, name), {} - ), - adapter=adapter, - ) - else: - inferred_columns = completion.inferred_by_fixture.get((resource_type, name)) - if inferred_columns is None: - continue - type_key: tuple[CompiledResourceType, str] = ( - CompiledResourceType.MODEL - if resource_type == CompiledResourceType.SQL_TEST - else resource_type, - name, - ) - for column in inferred_columns: - expected_type: str | None = completion.expected_types.get(type_key, {}).get( - column.name.casefold() - ) - if not _types_are_incompatible(expected=expected_type, inferred=column.type): - continue - fixture_label: str = ( - "expected output" - if resource_type == CompiledResourceType.SQL_TEST - else f"mock {resource_type.value}" - ) - errors.append( - f"{fixture_location(test=test, resource_type=resource_type, name=name)}: " - f"{fixture_label} '{name}' column '{column.name}' has incompatible type " - f"{column.type}; resource type is {expected_type}. Use an explicit CAST, " - "ARRAY_CONSTRUCT, or PARSE_JSON as appropriate" - ) - if errors: - raise SqlTestFixtureValidationError( - tuple(f"SQL test '{test.name}': {diagnostic}" for diagnostic in sorted(set(errors))) - ) - return ( - _completed_fixture_group( - resource_type=CompiledResourceType.MODEL, - fixtures=mock_refs, - completion=completion, - ), - _completed_fixture_group( - resource_type=CompiledResourceType.SOURCE, - fixtures=mock_sources, - completion=completion, - ), - _completed_fixture_group( - resource_type=CompiledResourceType.SEED, - fixtures=mock_seeds, - completion=completion, - ), - completed_expected_outputs, - ) - - -def _located_diagnostic(*, test: CompiledSqlTest, diagnostic: RelationFixtureDiagnostic) -> str: - return ( - f"{fixture_location(test=test, resource_type=diagnostic.key[0], name=diagnostic.key[1])}: " - f"{diagnostic.message}" - ) - - -def _completed_fixture_group( - *, - resource_type: CompiledResourceType, - fixtures: dict[str, str], - completion: RelationFixtureCompletion, -) -> dict[str, str]: - return { - name: completion.fixture_sql_by_key.get((resource_type, name), sql) - for name, sql in fixtures.items() - } - - -def fixture_location( - *, test: CompiledSqlTest, resource_type: CompiledResourceType, name: str -) -> str: - """Return the test file and line that defines one fixture or expected-output CTE.""" - - prefix: str = { - CompiledResourceType.MODEL: "__ref__", - CompiledResourceType.SOURCE: "__source__", - CompiledResourceType.SEED: "__seed__", - CompiledResourceType.SQL_TEST: "__expected__", - }[resource_type] - match: re.Match[str] | None = re.search( - rf"(?im)^\s*{re.escape(prefix + name)}\s+AS\s*\(", test.test_file.contents - ) - line: int = 1 if match is None else test.test_file.contents.count("\n", 0, match.start()) + 1 - return f"{test.test_file.relative_path.as_posix()}:{line}" - - -def _types_are_incompatible(*, expected: str | None, inferred: str | None) -> bool: - if expected is None or inferred is None: - return False - expected_base: str = expected.upper().split("(", maxsplit=1)[0].strip() - inferred_base: str = inferred.upper().split("(", maxsplit=1)[0].strip() - return (expected_base in _COLLECTION_TYPES) != (inferred_base in _COLLECTION_TYPES) diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py deleted file mode 100644 index 9a8f915454..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py +++ /dev/null @@ -1,396 +0,0 @@ -"""Native SQL-test chain planning and comparison rendering boundary.""" - -from __future__ import annotations - -import time - -import orjson - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.classes.sql_test_cte_locator import SqlTestCteLocator -from sqlbuild.compiler.compile.constants import SQL_TEST_HELPER_REFERENCE_CODE -from sqlbuild.compiler.compile.exceptions import CompileInputError, SqlTestReferenceError -from sqlbuild.compiler.compile.models import ( - CompiledDirectLogicSqlTestPayload, - CompiledModelSqlTestPayload, - CompiledSqlTest, -) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.planner._helpers.sql_tests.cursor_window import ( - declared_window_cursor_models, - declares_cursor_window, - render_test_cursor_intrinsics, -) -from sqlbuild.compiler.planner.exceptions import NativeSqlTestPlanningError, PlannerInputError -from sqlbuild.compiler.planner.models import ( - ChainStep, - NativeSqlTestArtifact, - NativeSqlTestPlan, - PlanWarning, - SqlTestAssertionStep, -) -from sqlbuild.compiler.planner.types import WarningSeverity -from sqlbuild.compiler.profiling.classes.context import CompileTimingContext -from sqlbuild.compiler.profiling.models import CompileTimingCollector -from sqlbuild.compiler.sql_test_glue.constants import CURSOR_INTRINSIC_NAMES -from sqlbuild.compiler.sql_test_glue.models import ( - NativeSqlTestChainRequest, - NativeSqlTestPlanningRequest, -) -from sqlbuild.compiler.sql_test_glue.types import ( - NativeAssertionStepRow, - NativeChainStepRow, - NativeSqlTestPlanRow, -) - -_UNRESOLVED_REFERENCE_PREFIX: str = "sql_test_reference:" - -_CALL_SUFFIX_SENTINEL: str = "__SQLBUILD_CALL_SUFFIX__" -_NATIVE_WORKERS: int = 4 - - -def plan_compiled_sql_test_artifacts( - *, - project: CompiledProjectView, - tests: tuple[CompiledSqlTest, ...], - adapter: BaseAdapter, - sql_analysis_enabled: bool, -) -> tuple[NativeSqlTestArtifact, ...]: - """Plan and render SQL tests from the compiled objects, with natively projected errors.""" - - artifacts: list[NativeSqlTestArtifact] = [] - for plan in _plan_compiled_sql_tests( - project=project, - tests=tests, - adapter=adapter, - sql_analysis_enabled=sql_analysis_enabled, - render_sql=True, - include_plan=False, - ): - if plan.sql is None: - raise NativeSqlTestPlanningError("native SQL-test planning omitted rendered SQL") - artifacts.append( - NativeSqlTestArtifact( - sql=plan.sql, model_names=plan.model_names, error_messages=plan.error_messages - ) - ) - return tuple(artifacts) - - -def sql_test_plan_error_messages(*, warnings: tuple[PlanWarning, ...]) -> tuple[str, ...]: - """Return each distinct ERROR-severity planning message for one SQL test, in order.""" - - return tuple( - dict.fromkeys( - warning.message for warning in warnings if warning.severity is WarningSeverity.ERROR - ) - ) - - -def plan_sql_tests_natively( - *, - project: CompiledProjectView, - tests: tuple[CompiledSqlTest, ...], - adapter: BaseAdapter, - sql_analysis_enabled: bool, - render_sql: bool, - include_plan: bool = True, -) -> tuple[NativeSqlTestPlan, ...]: - """Plan SQL-test chains and assertions in one native batch, optionally rendering SQL.""" - - return _plan_compiled_sql_tests( - project=project, - tests=tests, - adapter=adapter, - sql_analysis_enabled=sql_analysis_enabled, - render_sql=render_sql, - include_plan=include_plan, - ) - - -def _plan_compiled_sql_tests( - *, - project: CompiledProjectView, - tests: tuple[CompiledSqlTest, ...], - adapter: BaseAdapter, - sql_analysis_enabled: bool, - render_sql: bool, - include_plan: bool, -) -> tuple[NativeSqlTestPlan, ...]: - """Plan from the compiled objects natively; Python supplies only the adapter's renderings.""" - - if not tests: - return () - start_ns: int = time.perf_counter_ns() - rendered_model_sql: dict[int, str] = { - index: render_test_cursor_intrinsics( - sql=model.query_sql, model=model, adapter=adapter, test=None - ) - for index, model in enumerate(project.models) - if _mentions_cursor_intrinsic(model.query_sql) - } - functions: tuple[tuple[str, str, str, str, str], ...] = _function_templates( - project=project, adapter=adapter - ) - rendered_test_overrides: dict[int, dict[str, str]] = _rendered_test_overrides( - project=project, tests=tests, adapter=adapter - ) - request: NativeSqlTestPlanningRequest = NativeSqlTestPlanningRequest( - models=project.models, - rendered_model_sql=rendered_model_sql, - functions=functions, - tests=tests, - rendered_test_overrides=rendered_test_overrides, - sql_analysis_enabled=sql_analysis_enabled, - sql_analysis_dialect=adapter.sql_analysis_dialect(), - set_difference_operator=adapter.render_set_difference_operator(), - requires_derived_table_aliases=adapter.requires_derived_table_aliases(), - lexical_syntax=project.sql_lexical_syntax.native_mapping, - render_sql=render_sql, - include_plan=include_plan, - ) - try: - rows, planning_ns, rendering_ns = _native.plan_compiled_sql_tests(request) - except ValueError as error: - raise _planning_error(error=error, tests=tests) from None - _record_planning_timing(start_ns=start_ns, planning_ns=planning_ns, rendering_ns=rendering_ns) - return tuple(_plan_from_row(row=row) for row in rows) - - -def _plan_from_row(*, row: NativeSqlTestPlanRow) -> NativeSqlTestPlan: - sql, chain, assertions, model_names, warnings, error_messages = row - return NativeSqlTestPlan( - chain=tuple(_chain_step_from_row(row=step) for step in chain), - assertions=tuple(_assertion_step_from_row(row=step) for step in assertions), - model_names=tuple(model_names), - warnings=tuple( - PlanWarning(model_name=model_name, severity=WarningSeverity(severity), message=message) - for model_name, severity, message in warnings - ), - error_messages=tuple(error_messages), - sql=sql, - ) - - -def _assertion_step_from_row(*, row: NativeAssertionStepRow) -> SqlTestAssertionStep: - name, resolved_sql, lifted_ctes, comparison_body_sql = row - return SqlTestAssertionStep( - name=name, - resolved_sql=resolved_sql, - lifted_ctes=tuple(lifted_ctes), - comparison_body_sql=comparison_body_sql, - ) - - -def _chain_step_from_row(*, row: NativeChainStepRow) -> ChainStep: - ( - model_name, - resolved_sql, - expected_cte_sql, - lifted_ctes, - comparison_body_sql, - expected_columns, - expected_lifted_ctes, - ) = row - return ChainStep( - model_name=model_name, - resolved_sql=resolved_sql, - expected_cte_sql=expected_cte_sql, - lifted_ctes=tuple(lifted_ctes), - comparison_body_sql=comparison_body_sql, - expected_columns=None if expected_columns is None else tuple(expected_columns), - expected_lifted_ctes=tuple(expected_lifted_ctes), - ) - - -def _mentions_cursor_intrinsic(sql: str) -> bool: - """Whether SQL names a cursor intrinsic; SQL that does not is never rendered differently.""" - - return any(name in sql for name in CURSOR_INTRINSIC_NAMES) - - -def _rendered_test_overrides( - *, project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...], adapter: BaseAdapter -) -> dict[int, dict[str, str]]: - """Cursor-rendered model overrides for the tests whose overrides cursor windows change.""" - - models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} - windowed_indexes: tuple[int, ...] = tuple( - index - for index, test in enumerate(tests) - if isinstance(test.payload, CompiledModelSqlTestPayload) - and declares_cursor_window(test=test) - ) - chains: tuple[tuple[str, ...], ...] = ( - resolve_sql_test_model_chains( - project=project, tests=tuple(tests[index] for index in windowed_indexes) - ) - if windowed_indexes - else () - ) - chains_by_index: dict[int, tuple[str, ...]] = dict(zip(windowed_indexes, chains, strict=True)) - rendered: dict[int, dict[str, str]] = {} - for index, test in enumerate(tests): - payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload = test.payload - if not isinstance(payload, CompiledModelSqlTestPayload): - continue - chain: tuple[str, ...] | None = chains_by_index.get(index) - if chain is None and not any( - _mentions_cursor_intrinsic(sql) for sql in payload.model_query_overrides.values() - ): - continue - overrides: dict[str, str] | None = _windowed_model_query_overrides( - test=test, models_by_name=models_by_name, adapter=adapter, chain=chain - ) - if overrides is not None: - rendered[index] = overrides - return rendered - - -def _record_planning_timing(*, start_ns: int, planning_ns: int, rendering_ns: int) -> None: - elapsed_ns: int = time.perf_counter_ns() - start_ns - boundary_overhead_ns: int = max(0, elapsed_ns - planning_ns - rendering_ns) - collector: CompileTimingCollector | None = CompileTimingContext.active.get() - if collector is not None: - collector.add( - phase="test_planning_ms", - elapsed_ns=planning_ns + boundary_overhead_ns, - ) - collector.add(phase="comparison_render_ms", elapsed_ns=rendering_ns) - - -def resolve_sql_test_model_chains( - *, project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...] -) -> tuple[tuple[str, ...], ...]: - """Return each test's ordered unmocked model chain; direct-logic tests have none.""" - - if not tests: - return () - try: - native_chains: list[list[str]] = _native.resolve_compiled_sql_test_chains( - NativeSqlTestChainRequest( - models=project.models, - tests=tests, - lexical_syntax=project.sql_lexical_syntax.native_mapping, - ) - ) - except ValueError as error: - raise _planning_error(error=error, tests=tests) from None - return tuple(tuple(chain) for chain in native_chains) - - -def _planning_error(*, error: ValueError, tests: tuple[CompiledSqlTest, ...]) -> Exception: - message: str = str(error) - if message.startswith(_UNRESOLVED_REFERENCE_PREFIX): - return _unresolved_reference_error( - payload=orjson.loads(message.removeprefix(_UNRESOLVED_REFERENCE_PREFIX)), tests=tests - ) - if message.startswith("compile_input:"): - return CompileInputError(message.removeprefix("compile_input:")) - if message.startswith("planner_input:"): - return PlannerInputError(message.removeprefix("planner_input:")) - return NativeSqlTestPlanningError(f"native SQL-test planning failed: {message}") - - -def _unresolved_reference_error( - *, payload: dict[str, str], tests: tuple[CompiledSqlTest, ...] -) -> Exception: - """Locate a reference call the native planner could not resolve in its test file.""" - - test: CompiledSqlTest | None = next( - ( - test - for test in tests - if test.name == payload["testName"] - and str(test.test_file.relative_path) == payload["fileLabel"] - ), - None, - ) - cte_name: str = payload["cteName"] - call: str = payload["call"] - message: str = f"SQL test CTE '{cte_name}' calls {call}, which the test query cannot resolve" - help_text: str = ( - f"Mock it in the test, for example {payload['mockCte']} AS (SELECT ...), or call " - '__ref("") on a model the test runs.' - ) - if test is None: - return CompileInputError(message, code=SQL_TEST_HELPER_REFERENCE_CODE, help=help_text) - return SqlTestReferenceError( - message, - location=SqlTestCteLocator.locate( - test_file=test.test_file, test_block=test.test_block, cte_name=cte_name, call=call - ), - help=help_text, - ) - - -def _function_templates( - *, project: CompiledProjectView, adapter: BaseAdapter -) -> tuple[tuple[str, str, str, str, str], ...]: - """Each function's name and the adapter's UDF and table-function call templates.""" - - functions: list[tuple[str, str, str, str, str]] = [] - for function in project.functions: - target: str | None = function.destination.qualified_name - if target is None: - continue - udf_prefix, udf_suffix = _render_call_template( - rendered=adapter.render_udf_call( - target=target, - call_suffix_sql=_CALL_SUFFIX_SENTINEL, - ) - ) - table_function_prefix, table_function_suffix = _render_call_template( - rendered=adapter.render_table_function_call( - target=target, - call_suffix_sql=_CALL_SUFFIX_SENTINEL, - ) - ) - functions.append( - (function.name, udf_prefix, udf_suffix, table_function_prefix, table_function_suffix) - ) - return tuple(functions) - - -def _windowed_model_query_overrides( - *, - test: CompiledSqlTest, - models_by_name: dict[str, CompiledModelView], - adapter: BaseAdapter, - chain: tuple[str, ...] | None, -) -> dict[str, str] | None: - payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload = test.payload - if not isinstance(payload, CompiledModelSqlTestPayload): - return None - chain_names: frozenset[str] = frozenset(chain or ()) - overrides: dict[str, str] = {} - for model_name, sql in payload.model_query_overrides.items(): - model: CompiledModelView | None = models_by_name.get(model_name) - overrides[model_name] = ( - sql - if model is None - else render_test_cursor_intrinsics( - sql=sql, - model=model, - adapter=adapter, - test=test if model_name in chain_names else None, - ) - ) - if chain is not None: - chain_models: tuple[CompiledModelView, ...] = tuple( - models_by_name[name] for name in chain if name in models_by_name - ) - for model in declared_window_cursor_models(test=test, chain_models=chain_models): - if model.name not in overrides: - overrides[model.name] = render_test_cursor_intrinsics( - sql=model.query_sql, model=model, adapter=adapter, test=test - ) - return overrides - - -def _render_call_template(*, rendered: str) -> tuple[str, str]: - prefix, separator, suffix = rendered.partition(_CALL_SUFFIX_SENTINEL) - if not separator or _CALL_SUFFIX_SENTINEL in suffix: - raise NativeSqlTestPlanningError("adapter SQL function call template is not deterministic") - return prefix, suffix diff --git a/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py b/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py deleted file mode 100644 index 25ce1f4011..0000000000 --- a/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py +++ /dev/null @@ -1,358 +0,0 @@ -"""Read-only warehouse evidence gathered once for all model migrations in one plan.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild._native import read_latest_fingerprints -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo -from sqlbuild.adapter.relations.main.get_columns_for_inspection import get_columns_for_inspection -from sqlbuild.adapter.relations.main.list_relations_for_inspection import ( - list_relations_for_inspection, -) -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME, NODE_TYPE_MODEL -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet -from sqlbuild.compiler.migrations.constants import ( - COLUMN_MIGRATION_TABLE_NAME, - MIGRATION_TABLE_NAME, - OLD_NAME_VIEW_TABLE_NAME, -) -from sqlbuild.compiler.migrations.main._project_old_name_views import project_old_name_views -from sqlbuild.compiler.migrations.main._read_column_events import read_column_migration_events -from sqlbuild.compiler.migrations.main._read_events import read_migration_events -from sqlbuild.compiler.migrations.main.read_old_name_view_events import ( - read_old_name_view_events, -) -from sqlbuild.compiler.migrations.main.stored_old_name_view_columns import ( - stored_old_name_view_columns, -) -from sqlbuild.compiler.migrations.models import ( - ColumnMigrationEvent, - MigrationEvent, - MigrationRelation, - OldNameViewEvent, - OldNameViewHistory, -) -from sqlbuild.compiler.planner.models import WarehouseFingerprints - - -class MigrationStateInspection: - """Relations, fingerprints, and migration events for one planning database.""" - - def __init__( - self, - *, - adapter: BaseAdapter, - connection: Any, - database: str | None, - known_fingerprints: WarehouseFingerprints | None = None, - ) -> None: - self._adapter: BaseAdapter = adapter - self._connection: Any = connection - self._database: str | None = database - self._known_fingerprints: dict[str, tuple[Fingerprint, ...]] = ( - known_fingerprints.unfiltered_schemas - if known_fingerprints is not None and known_fingerprints.unfiltered_database == database - else {} - ) - self._read_schemas: set[str] = set() - self._fingerprints: dict[str, tuple[Fingerprint, ...]] = {} - self._events: list[MigrationEvent] = [] - self._event_schemas: set[str] = set() - self._relations: dict[tuple[str, str], RelationInfo] = {} - self._columns: dict[tuple[str, str], tuple[ColumnInfo, ...]] = {} - self._column_event_schemas: set[str] = set() - self._column_events: list[ColumnMigrationEvent] = [] - self._old_name_schemas: set[str] = set() - self._old_name_moves: list[MigrationEvent] = [] - self._old_name_facts: list[OldNameViewEvent] = [] - - @property - def database(self) -> str | None: - """Return the database whose namespaces this inspection reads.""" - - return self._database - - @property - def events(self) -> tuple[MigrationEvent, ...]: - """Return every migration event read so far.""" - - return tuple(self._events) - - @property - def column_events(self) -> tuple[ColumnMigrationEvent, ...]: - """Return every column migration event read so far.""" - - return tuple(self._column_events) - - def inspect_column_events(self, *, schemas: set[str]) -> None: - """Read the column migration state table once for every schema not inspected yet.""" - - pending: tuple[str, ...] = tuple( - sorted(schema for schema in schemas if schema.lower() not in self._column_event_schemas) - ) - if not pending: - return - self._column_event_schemas.update(schema.lower() for schema in pending) - listed: tuple[RelationInfo, ...] = list_relations_for_inspection( - adapter=self._adapter, - connection=self._connection, - database=self._database, - schemas=pending, - names=(COLUMN_MIGRATION_TABLE_NAME,), - ) - present: set[str] = { - (relation.schema or "").lower() - for relation in listed - if relation.name.lower() == COLUMN_MIGRATION_TABLE_NAME - } - schema: str - for schema in pending: - if schema.lower() in present: - self._column_events.extend( - read_column_migration_events( - connection=self._connection, - execute=self._adapter.execute, - database=self._database, - schema=schema, - render_qualified_name=self._adapter.render_qualified_name, - ) - ) - - @property - def old_name_histories(self) -> tuple[OldNameViewHistory, ...]: - """Return the old-name history of every move read by ``inspect_old_name_views``.""" - - return project_old_name_views(moves=self._old_name_moves, facts=self._old_name_facts) - - def inspect_old_name_views(self, *, schemas: set[str]) -> None: - """Read moves and old-name facts once for every schema not inspected yet.""" - - pending: tuple[str, ...] = tuple( - sorted(schema for schema in schemas if schema.lower() not in self._old_name_schemas) - ) - if not pending: - return - self._old_name_schemas.update(schema.lower() for schema in pending) - listed: tuple[RelationInfo, ...] = list_relations_for_inspection( - adapter=self._adapter, - connection=self._connection, - database=self._database, - schemas=pending, - names=(MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME), - ) - state_tables: set[tuple[str, str]] = { - ((relation.schema or "").lower(), relation.name.lower()) for relation in listed - } - schema: str - for schema in pending: - if (schema.lower(), OLD_NAME_VIEW_TABLE_NAME) not in state_tables or ( - schema.lower(), - MIGRATION_TABLE_NAME, - ) not in state_tables: - continue - self._old_name_facts.extend( - read_old_name_view_events( - connection=self._connection, - execute=self._adapter.execute, - database=self._database, - schema=schema, - stored_columns=stored_old_name_view_columns( - adapter=self._adapter, - connection=self._connection, - database=self._database, - schema=schema, - ), - render_qualified_name=self._adapter.render_qualified_name, - ) - ) - self._old_name_moves.extend( - read_migration_events( - connection=self._connection, - execute=self._adapter.execute, - database=self._database, - schema=schema, - render_qualified_name=self._adapter.render_qualified_name, - ) - ) - - def inspect_migration_events(self, *, schemas: set[str]) -> None: - """Read migration events once from schemas known to hold the migration table.""" - - schema: str - for schema in sorted(schemas): - if schema.lower() in self._event_schemas: - continue - self._event_schemas.add(schema.lower()) - self._events.extend( - read_migration_events( - connection=self._connection, - execute=self._adapter.execute, - database=self._database, - schema=schema, - render_qualified_name=self._adapter.render_qualified_name, - ) - ) - - def inspect_schemas(self, *, schemas: set[str]) -> None: - """Read state tables once for every schema not inspected yet.""" - - pending: tuple[str, ...] = tuple( - sorted(schema for schema in schemas if schema.lower() not in self._read_schemas) - ) - if not pending: - return - self._read_schemas.update(schema.lower() for schema in pending) - listed: tuple[RelationInfo, ...] = list_relations_for_inspection( - adapter=self._adapter, - connection=self._connection, - database=self._database, - schemas=pending, - names=(MIGRATION_TABLE_NAME, FINGERPRINT_TABLE_NAME), - ) - state_tables: set[tuple[str, str]] = { - ((relation.schema or "").lower(), relation.name.lower()) for relation in listed - } - schema: str - for schema in pending: - if (schema.lower(), MIGRATION_TABLE_NAME) in state_tables: - self.inspect_migration_events(schemas={schema}) - if (schema.lower(), FINGERPRINT_TABLE_NAME) in state_tables: - self._fingerprints[schema.lower()] = self._read_model_fingerprints(schema=schema) - - def _read_model_fingerprints(self, *, schema: str) -> tuple[Fingerprint, ...]: - latest: tuple[Fingerprint, ...] | None = self._known_fingerprints.get(schema.lower()) - if latest is None: - fingerprint_set: FingerprintSet = read_latest_fingerprints( - connection=self._connection, - execute=self._adapter.execute, - table_exists=True, - database=self._database, - schema=schema, - render_qualified_name=self._adapter.render_qualified_name, - render_read_latest_sql=self._adapter.render_read_latest_fingerprints_sql, - ) - latest = tuple((fingerprint_set.fingerprints_by_identity or {}).values()) - return tuple( - fingerprint for fingerprint in latest if fingerprint.node_type == NODE_TYPE_MODEL - ) - - def inspect_relations(self, *, locations: tuple[CompiledRelationLocation, ...]) -> None: - """List candidate relations and their columns with one metadata read.""" - - pending: tuple[CompiledRelationLocation, ...] = tuple( - location - for location in locations - if location.schema is not None and _relation_key(location) not in self._relations - ) - if not pending: - return - listed: tuple[RelationInfo, ...] = list_relations_for_inspection( - adapter=self._adapter, - connection=self._connection, - database=self._database, - schemas=tuple(sorted({location.schema or "" for location in pending})), - names=tuple(sorted({location.name for location in pending})), - ) - wanted: set[tuple[str, str]] = {_relation_key(location) for location in pending} - matched: tuple[RelationInfo, ...] = tuple( - relation for relation in listed if _listed_key(relation) in wanted - ) - columns: dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]] = ( - get_columns_for_inspection( - adapter=self._adapter, connection=self._connection, relations=matched - ) - if matched - else {} - ) - relation: RelationInfo - for relation in matched: - self._relations[_listed_key(relation)] = relation - self._columns[_listed_key(relation)] = columns.get(relation.identity, ()) - - def relation(self, location: CompiledRelationLocation) -> RelationInfo | None: - """Return the listed relation for a location.""" - - return self._relations.get(_relation_key(location)) - - def relation_columns(self, location: CompiledRelationLocation) -> tuple[ColumnInfo, ...]: - """Return listed columns for a location.""" - - return self._columns.get(_relation_key(location), ()) - - def model_fingerprint( - self, *, location: CompiledRelationLocation, model_name: str | None - ) -> Fingerprint | None: - """Return the latest model fingerprint that built the location or named model.""" - - candidates: tuple[Fingerprint, ...] = self._fingerprints.get( - (location.schema or "").lower(), () - ) - target: MigrationRelation = MigrationRelation( - database=location.database, schema=location.schema, name=location.name - ) - by_relation: tuple[Fingerprint, ...] = tuple( - fingerprint - for fingerprint in candidates - if _fingerprint_built(fingerprint=fingerprint, target=target) - ) - if by_relation: - return max(by_relation, key=_fingerprint_order) - if model_name is None: - return None - named: tuple[Fingerprint, ...] = tuple( - fingerprint for fingerprint in candidates if fingerprint.node_name == model_name - ) - return max(named, key=_fingerprint_order) if named else None - - def named_fingerprint(self, *, model_name: str) -> Fingerprint | None: - """Return the newest model fingerprint with this node name in any inspected schema.""" - - matches: list[Fingerprint] = [] - fingerprints: tuple[Fingerprint, ...] - for fingerprints in self._fingerprints.values(): - matches.extend( - fingerprint for fingerprint in fingerprints if fingerprint.node_name == model_name - ) - return max(matches, key=_fingerprint_order) if matches else None - - def orphan_fingerprints( - self, *, project_model_names: frozenset[str] - ) -> tuple[Fingerprint, ...]: - """Return latest model fingerprints whose names are no longer in the project.""" - - orphans: list[Fingerprint] = [] - fingerprints: tuple[Fingerprint, ...] - for fingerprints in self._fingerprints.values(): - orphans.extend( - fingerprint - for fingerprint in fingerprints - if fingerprint.node_name not in project_model_names - ) - return tuple(orphans) - - -def _relation_key(location: CompiledRelationLocation) -> tuple[str, str]: - return ((location.schema or "").lower(), location.name.lower()) - - -def _listed_key(relation: RelationInfo) -> tuple[str, str]: - return ((relation.schema or "").lower(), relation.name.lower()) - - -def _fingerprint_built(*, fingerprint: Fingerprint, target: MigrationRelation) -> bool: - if fingerprint.target_name is None: - return False - return target.matches( - MigrationRelation( - database=fingerprint.target_database, - schema=fingerprint.target_schema, - name=fingerprint.target_name, - ) - ) - - -def _fingerprint_order(fingerprint: Fingerprint) -> tuple[object, str]: - return (fingerprint.ts.isoformat(), fingerprint.run_id) diff --git a/src/sqlbuild/compiler/planner/exceptions.py b/src/sqlbuild/compiler/planner/exceptions.py index 9b3763810f..fb72dc94a5 100644 --- a/src/sqlbuild/compiler/planner/exceptions.py +++ b/src/sqlbuild/compiler/planner/exceptions.py @@ -23,14 +23,6 @@ class NativeSqlTestPlanningError(PlannerInputError): """Raised when the native SQL-test planning boundary returns invalid data.""" -class SqlTestFixtureValidationError(PlannerInputError): - """Raised with every invalid fixture diagnostic for one SQL test.""" - - def __init__(self, diagnostics: tuple[str, ...]) -> None: - self.diagnostics = diagnostics - super().__init__("\n- ".join(diagnostics)) - - class FutureCursorSafetyError(PlannerInputError): """Raised when an effective cursor exceeds the configured future limit.""" diff --git a/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py b/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py index 65b71f266b..62e4997dad 100644 --- a/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py +++ b/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py @@ -1,11 +1,11 @@ """Planner entrypoint for SQL-test artifacts planned natively from the compiled objects.""" +from sqlbuild._native import ( + plan_compiled_sql_test_artifacts as _plan_compiled_sql_test_artifacts, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( - plan_compiled_sql_test_artifacts as _plan_compiled_sql_test_artifacts, -) from sqlbuild.compiler.planner.models import NativeSqlTestArtifact diff --git a/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py b/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py index 94695b3960..4412ed1b5c 100644 --- a/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py +++ b/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py @@ -1,10 +1,10 @@ """Public planner entrypoint for batched SQL-test model closure discovery.""" -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlTest -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( +from sqlbuild._native import ( resolve_sql_test_model_chains, ) +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlTest +from sqlbuild.compiler.compile.types import CompiledProjectView def sql_test_model_chain_names_by_key( diff --git a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py index 5caddd6e1f..7915fda271 100644 --- a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py +++ b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py @@ -2,8 +2,7 @@ from __future__ import annotations -from sqlbuild._native import append_overflow_line, visible_entries -from sqlbuild.compiler.planner._helpers.migrations.display import format_model_migrations +from sqlbuild._native import append_overflow_line, format_model_migrations, visible_entries from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.presentation.models import DisplayOptions diff --git a/src/sqlbuild/compiler/planner/main/scenarios/_build_scenario_artifact_physical_name.py b/src/sqlbuild/compiler/planner/main/scenarios/_build_scenario_artifact_physical_name.py index 8bcc581641..f1e0241e81 100644 --- a/src/sqlbuild/compiler/planner/main/scenarios/_build_scenario_artifact_physical_name.py +++ b/src/sqlbuild/compiler/planner/main/scenarios/_build_scenario_artifact_physical_name.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.planner._helpers.scenario.artifact_names import ( +from sqlbuild._native import ( build_scenario_artifact_physical_name as _build_scenario_artifact_physical_name, ) diff --git a/src/sqlbuild/compiler/planner/main/scenarios/is_scenario_artifact_physical_name.py b/src/sqlbuild/compiler/planner/main/scenarios/is_scenario_artifact_physical_name.py index d5c430bc29..bced73f581 100644 --- a/src/sqlbuild/compiler/planner/main/scenarios/is_scenario_artifact_physical_name.py +++ b/src/sqlbuild/compiler/planner/main/scenarios/is_scenario_artifact_physical_name.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.planner._helpers.scenario.artifact_names import ( +from sqlbuild._native import ( is_scenario_artifact_physical_name as _is_scenario_artifact_physical_name, ) diff --git a/src/sqlbuild/executor/migrations/_helpers/columns.py b/src/sqlbuild/executor/migrations/_helpers/columns.py index 31231a63b4..195558d9c2 100644 --- a/src/sqlbuild/executor/migrations/_helpers/columns.py +++ b/src/sqlbuild/executor/migrations/_helpers/columns.py @@ -5,13 +5,12 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import deterministic_column_migration_event_id +from sqlbuild._native import deterministic_column_migration_event_id, write_migration_state_event from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location -from sqlbuild.compiler.migrations.main.write_column_event import write_column_migration_event from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation from sqlbuild.compiler.migrations.types import ColumnMigrationDecision from sqlbuild.compiler.planner.models import ColumnMigrationPlanEntry @@ -90,7 +89,8 @@ def _rename_and_record( event: ColumnMigrationEvent = column_migration_event( entry=entry, decision=decision, run_id=run_id ) - write_column_migration_event( + write_migration_state_event( + kind="column", connection=connection, execute=adapter.execute, event=event, diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py b/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py index 2ddafcdb9f..1ad02e77bf 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py @@ -5,17 +5,15 @@ from datetime import datetime from typing import Any -from sqlbuild._native import deterministic_old_name_view_event_id +from sqlbuild._native import ( + deterministic_old_name_view_event_id, + read_migration_state_events, + stored_old_name_view_columns, + write_migration_state_event, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.main.read_old_name_view_events import ( - read_old_name_view_events, -) from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location -from sqlbuild.compiler.migrations.main.stored_old_name_view_columns import ( - stored_old_name_view_columns, -) -from sqlbuild.compiler.migrations.main.write_old_name_view_event import write_old_name_view_event from sqlbuild.compiler.migrations.models import MigrationRelation, OldNameViewEvent from sqlbuild.compiler.migrations.types import OldNameViewEventType from sqlbuild.compiler.planner.models import OldNameViewPlanEntry @@ -67,7 +65,8 @@ def record_old_name_fact( ) -> None: """Append one old-name fact through the adapter's state-table rendering.""" - write_old_name_view_event( + write_migration_state_event( + kind="old_name", connection=connection, execute=adapter.execute, event=event, @@ -98,7 +97,8 @@ def recorded_old_name_facts( ) facts: dict[OldNameViewEventType, OldNameViewEvent] = {} event: OldNameViewEvent - for event in read_old_name_view_events( + for event in read_migration_state_events( + kind="old_name", connection=connection, execute=adapter.execute, database=relation.database, diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py index 45656d83b5..e8495b0dd5 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py @@ -8,7 +8,7 @@ from datetime import UTC, datetime, timedelta from typing import Any -from sqlbuild._native import deterministic_migration_event_id +from sqlbuild._native import deterministic_migration_event_id, read_recorded_column_renames from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.main.grants_for_columns import grants_for_columns @@ -20,9 +20,6 @@ from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.migrations.constants import COLUMN_MIGRATION_TABLE_NAME -from sqlbuild.compiler.migrations.main.read_recorded_column_renames import ( - read_recorded_column_renames, -) from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location from sqlbuild.compiler.migrations.models import ( ColumnMigrationEvent, diff --git a/src/sqlbuild/executor/migrations/_helpers/promotion.py b/src/sqlbuild/executor/migrations/_helpers/promotion.py index e3cd213f9c..42d2f1be39 100644 --- a/src/sqlbuild/executor/migrations/_helpers/promotion.py +++ b/src/sqlbuild/executor/migrations/_helpers/promotion.py @@ -5,12 +5,11 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import deterministic_migration_event_id +from sqlbuild._native import deterministic_migration_event_id, write_migration_state_event from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location -from sqlbuild.compiler.migrations.main.write_event import write_migration_event from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation, OldNameViewEvent from sqlbuild.compiler.migrations.types import MigrationDecision, OldNameViewEventType from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, OldNameViewPlanEntry @@ -89,7 +88,8 @@ def record_event( ) -> None: """Append one migration event through the adapter's state-table rendering.""" - write_migration_event( + write_migration_state_event( + kind="model", connection=connection, execute=adapter.execute, event=event, diff --git a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py index e33c45671b..3abcd272a3 100644 --- a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py +++ b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py @@ -6,7 +6,12 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import deterministic_old_name_view_event_id +from sqlbuild._native import ( + deterministic_old_name_view_event_id, + read_old_name_view_histories, + stored_old_name_view_columns, + write_migration_state_event, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import RelationInfo @@ -20,13 +25,6 @@ MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, ) -from sqlbuild.compiler.migrations.main.read_old_name_view_histories import ( - read_old_name_view_histories, -) -from sqlbuild.compiler.migrations.main.stored_old_name_view_columns import ( - stored_old_name_view_columns, -) -from sqlbuild.compiler.migrations.main.write_old_name_view_event import write_old_name_view_event from sqlbuild.compiler.migrations.models import OldNameViewEvent, OldNameViewHistory from sqlbuild.compiler.migrations.types import ( OldNameViewDropReason, @@ -165,7 +163,8 @@ def _record_drop( *, adapter: BaseAdapter, connection: Any, view: JanitorOldNameView, run_id: str ) -> None: required: OldNameViewEvent = view.history.required - write_old_name_view_event( + write_migration_state_event( + kind="old_name", connection=connection, execute=adapter.execute, event=OldNameViewEvent( diff --git a/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py b/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py index 62b071b5a9..9c193493fe 100644 --- a/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py +++ b/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py @@ -13,7 +13,7 @@ from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.executor.testing._helpers.native_requests import comparison_request -from sqlbuild.executor.testing.main.missing_expected_columns import ( +from sqlbuild.executor.testing.main._missing_expected_columns import ( describe_missing_expected_columns, ) from sqlbuild.executor.testing.types import NativeSqlTestRenderingModule diff --git a/src/sqlbuild/executor/testing/main/missing_expected_columns.py b/src/sqlbuild/executor/testing/main/_missing_expected_columns.py similarity index 100% rename from src/sqlbuild/executor/testing/main/missing_expected_columns.py rename to src/sqlbuild/executor/testing/main/_missing_expected_columns.py diff --git a/src/sqlbuild/presentation/main/tree_connector.py b/src/sqlbuild/presentation/main/tree_connector.py deleted file mode 100644 index 549f493cda..0000000000 --- a/src/sqlbuild/presentation/main/tree_connector.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public tree connector rendering entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.structure import tree_connector as _tree_connector -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def tree_connector(*, style: CliStyle, last: bool) -> str: - """Render a dim tree connector for a group entry.""" - - return _tree_connector(style=style, last=last) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py index 523bbcde73..eb59249a10 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py @@ -200,6 +200,31 @@ def build_scenario_python_hooks_project_files() -> dict[str, str]: return repo_files +def build_scenario_dependency_chain_project_files() -> dict[str, str]: + """Build a three-model scenario with partial source fixtures and final checks.""" + + repo_files: dict[str, str] = build_partial_fixture_scenario_project_files() + repo_files["models/orders_summary.sql"] = ( + 'MODEL (description "Summarize fixture orders.", materialized table);\n' + 'SELECT order_id, status FROM __ref("orders")\n' + ) + repo_files["models/orders_final.sql"] = ( + 'MODEL (description "Final scenario output.", materialized table);\n' + 'SELECT order_id, status FROM __ref("orders_summary")\n' + ) + repo_files["tests/scenarios/partial_orders.sql"] = ( + "SCENARIO (description 'Check a dependency chain.');\n" + "WITH helper AS (SELECT 1 AS order_id),\n" + "__source__raw_orders AS (SELECT order_id FROM helper),\n" + "__expected__orders_final AS (\n" + " SELECT 1 AS order_id, CAST(NULL AS VARCHAR) AS status\n" + "),\n" + '__assert__positive_ids AS (SELECT * FROM __ref("orders_final") WHERE order_id < 1)\n' + "SELECT 1\n" + ) + return repo_files + + def build_capture_safety_project_files(*, use_project_row_limit: bool) -> dict[str, str]: """Build scenario e2e files with optional project snapshot limit config.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_namespace.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_namespace.py index d6c89a3c98..2ce20bfabd 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_namespace.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_namespace.py @@ -11,8 +11,8 @@ import pytest +from sqlbuild._native import compute_scenario_hash_prefix from sqlbuild.cli.entry.main.entry import main -from sqlbuild.compiler.planner._helpers.scenario.artifacts import compute_scenario_hash_prefix from sqlbuild.executor.scenario._helpers.execution import run as scenario_execution from tests.e2e.src.sqlbuild.cli.commands.main.scenario._test_types import ( ScenarioNamespaceE2ETestCase, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py index 70e7a62f47..5b87573170 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py @@ -23,6 +23,7 @@ assert_local_duckdb_state, assert_runtime_artifact_contains, build_partial_fixture_scenario_project_files, + build_scenario_dependency_chain_project_files, build_scenario_project_files, build_scenario_python_hooks_project_files, list_scenario_relation_names, @@ -86,6 +87,15 @@ def test_given_unclosed_scenario_comment_when_running_cli_then_error_names_scena "__sqlbuild_partial_fixture", ), ), + ScenarioPartialFixtureE2ETestCase( + description="three-model dependency chain preserves helper fixtures and final checks", + repo_files=build_scenario_dependency_chain_project_files(), + expected_stdout_fragment="PASS=1 FAIL=0 TOTAL=1", + expected_artifact_fragments=( + 'CAST(NULL AS VARCHAR) AS "status"', + "WITH helper AS (SELECT 1 AS order_id)", + ), + ), ), ids=lambda case: case.description, ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py index 47bee99870..7500593196 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py @@ -378,6 +378,6 @@ def fail_non_transactional_record(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(DuckDbAdapter, "supports_transactional_ddl", lambda self: False) monkeypatch.setattr( - "sqlbuild.executor.migrations._helpers.columns.write_column_migration_event", + "sqlbuild.executor.migrations._helpers.columns.write_migration_state_event", _raise_interruption, ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 4c8eb6891d..2286abfaee 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -513,7 +513,7 @@ def fail_record(monkeypatch: pytest.MonkeyPatch) -> None: """Crash after the clone but before the migration event is recorded.""" monkeypatch.setattr( - "sqlbuild.executor.migrations._helpers.promotion.write_migration_event", + "sqlbuild.executor.migrations._helpers.promotion.write_migration_state_event", _raise_interruption, ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py index 5b95c8f51f..0b236fa5de 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py @@ -197,7 +197,7 @@ def fail_janitor_drop_fact(monkeypatch: pytest.MonkeyPatch) -> None: """Crash after the janitor drops a view but before the drop is recorded.""" monkeypatch.setattr( - "sqlbuild.executor.old_name_views._helpers.janitor.write_old_name_view_event", + "sqlbuild.executor.old_name_views._helpers.janitor.write_migration_state_event", _raise_interruption, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py index c7e1899610..5a0cdf10b5 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -11,6 +11,7 @@ class FixturePlanningIntegrationTestCase: description: str expected_sql_fragment: str + query_sql: str = 'SELECT order_id, status FROM __source("raw_orders")' @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index ef7dab0c6b..04f67da526 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -2,7 +2,6 @@ from __future__ import annotations -import re from pathlib import Path from sqlbuild.compiler.compile.constants import ( @@ -29,10 +28,13 @@ DiscoveredSqlTestBlock, DiscoveredSqlTestFile, ) -from sqlbuild.compiler.planner._helpers.sql_tests.comments import uncommented_pattern_matches +from sqlbuild.compiler.references.types import SqlReferenceKind from tests.integration.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( ExecuteChainTestCase, ) +from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly.helpers import ( + reference_names_by_kind, +) _STUB_TEST_FILE: DiscoveredSqlTestFile = DiscoveredSqlTestFile( file_path=Path("tests/unit/test_chain.sql"), @@ -109,12 +111,9 @@ def build_test_and_project( model_name: str query_sql: str for model_name, query_sql in test_case.model_queries.items(): - dependency_names: tuple[str, ...] = tuple( - match.group(1) - for match in uncommented_pattern_matches( - pattern=re.compile(r'__ref\("([^"]+)"\)'), sql=query_sql - ) - ) + dependency_names: tuple[str, ...] = reference_names_by_kind(query_sql=query_sql)[ + SqlReferenceKind.REF + ] dependencies: tuple[CompiledObjectKey, ...] = tuple( CompiledObjectKey( resource_type=CompiledResourceType.MODEL, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index e02a9b154d..f1b2b4c26d 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -5,6 +5,7 @@ import pytest +from sqlbuild._native import plan_audit from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledAudit, @@ -16,7 +17,6 @@ DiscoveredAuditBlock, DiscoveredAuditFile, ) -from sqlbuild.compiler.planner._helpers.output.audit_entry import plan_audit from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.spec.contracts.models import SourceEntry from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index 14ea786712..3a910007b9 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -5,14 +5,15 @@ import pytest -from sqlbuild._native import build_compiled_project, discover_project_inputs -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner._helpers.fixtures.completion import ( +from sqlbuild._native import ( + build_compiled_project, build_relation_fixture_completion, build_relation_fixture_context, + discover_project_inputs, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner._helpers.output.plan_output import build_selected_test_entries from sqlbuild.compiler.planner.models import ( RelationFixtureCompletion, @@ -33,6 +34,21 @@ description="partial source fixture", expected_sql_fragment='CAST(NULL AS VARCHAR) AS "status"', ), + FixturePlanningIntegrationTestCase( + description="qualified star through alias", + query_sql='SELECT o.* FROM __source("raw_orders") AS o', + expected_sql_fragment='CAST(NULL AS VARCHAR) AS "status"', + ), + FixturePlanningIntegrationTestCase( + description="CTE and quoted column reads", + query_sql='WITH selected AS (SELECT "order_id", "status" FROM __source("raw_orders")) SELECT * FROM selected', + expected_sql_fragment='CAST(NULL AS VARCHAR) AS "status"', + ), + FixturePlanningIntegrationTestCase( + description="marker-looking strings and comments", + query_sql='SELECT order_id, status, \'__ref("unused")\' AS note FROM __source("raw_orders") /* __source("missing") */', + expected_sql_fragment='CAST(NULL AS VARCHAR) AS "status"', + ), ), ids=lambda case: case.description, ) @@ -51,7 +67,7 @@ def test_given_partial_fixture_when_compiling_and_planning_then_completes_requir 'database = ":memory:"\n' ), "models/orders.sql": ( - 'MODEL (description "Test model orders.");\n\nSELECT order_id, status FROM __source("raw_orders")\n' + f'MODEL (description "Test model orders.");\n\n{test_case.query_sql}\n' ), "sources/raw_orders.yml": ( "sources:\n" @@ -206,9 +222,9 @@ def test_given_source_contract_when_completing_fixture_then_respects_authoritati completed: RelationFixtureCompletion = build_relation_fixture_completion( project=project, adapter=adapter, - ordered_model_names=("orders",), - fixture_groups=( - (CompiledResourceType.SOURCE, {"raw_orders": "SELECT 1 AS ORDER_ID, 2 AS extra"}), + request=( + ("orders",), + ((CompiledResourceType.SOURCE, {"raw_orders": "SELECT 1 AS ORDER_ID, 2 AS extra"}),), ), planning_context=context, ) diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py index 57fb768f0a..fe0ec8fa83 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -117,6 +117,7 @@ def test_given_native_source_when_checking_module_paths_then_reports_errors( "sqlbuild.executor.runtime", "sqlbuild.fingerprints", "sqlbuild.lineage", + "sqlbuild.migrations", "sqlbuild.planner", "sqlbuild.planner.changes", ), diff --git a/tests/unit/scripts/planner_differential/_helpers/capturing/test_canonical.py b/tests/unit/scripts/planner_differential/_helpers/capturing/test_canonical.py index b0431ca765..129358d1b0 100644 --- a/tests/unit/scripts/planner_differential/_helpers/capturing/test_canonical.py +++ b/tests/unit/scripts/planner_differential/_helpers/capturing/test_canonical.py @@ -58,6 +58,21 @@ raw="_sqb_archive__20261014t004426z__migration_origin__orders", expected_text="_sqb_archive____migration_origin__orders", ), + MaskTextTestCase( + description="replay_range_date_relative_to_the_run_date", + raw="range 2026-10-11 → 2026-10-14", + expected_text="range → ", + ), + MaskTextTestCase( + description="retention_date_relative_to_the_run_date", + raw="live until 2026-11-13", + expected_text="live until ", + ), + MaskTextTestCase( + description="fixed_data_date_is_kept", + raw="partition 2026-04-02", + expected_text="partition 2026-04-02", + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py b/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py index 302b428eac..f9fda0f342 100644 --- a/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py +++ b/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py @@ -6,9 +6,8 @@ import pytest +from sqlbuild._native import read_migration_state_events, write_migration_state_event from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.migrations.main._read_events import read_migration_events -from sqlbuild.compiler.migrations.main.write_event import write_migration_event from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery from tests.unit.src.sqlbuild.adapters.sqlserver._test_types import ( @@ -92,7 +91,8 @@ def test_given_sqlserver_when_writing_migration_event_then_statements_are_tsql( rows=test_case.existing_rows ) - write_migration_event( + write_migration_state_event( + kind="model", connection=connection, execute=adapter.execute, event=_EVENT, @@ -151,7 +151,8 @@ def test_given_sqlserver_datetime2_rows_when_reading_events_then_decodes_utc_eve ) ) - events: tuple[MigrationEvent, ...] = read_migration_events( + events: tuple[MigrationEvent, ...] = read_migration_state_events( + kind="model", connection=connection, execute=adapter.execute, database=None, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py index e7d28eb393..28e1acf3db 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from sqlbuild.compiler.migrations.models import MigrationRelation -from sqlbuild.compiler.migrations.types import OldNameViewEventType +from sqlbuild.compiler.migrations.types import OldNameViewDropReason, OldNameViewEventType @dataclass(frozen=True) @@ -100,3 +100,21 @@ class PinnedOldNameEventIdTestCase: event_type: OldNameViewEventType migration_event_id: str expected_event_id: str + + +@dataclass(frozen=True) +class StoredOldNameRowTestCase: + description: str + column_aliases_sql: str + grants_copied_sql: str + drop_reason_sql: str + expected_aliases: tuple[tuple[str, str], ...] + expected_grants: tuple[str, ...] | None + expected_drop_reason: OldNameViewDropReason | None + + +@dataclass(frozen=True) +class StoredColumnRowTestCase: + description: str + decisions: tuple[str, ...] + expected_decisions: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py index cd49f9f219..916b3c6873 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py @@ -5,12 +5,12 @@ import duckdb import pytest -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.main._newest_column_event import ( +from sqlbuild._native import ( newest_column_migration_event_mentioning, + read_migration_state_events, + write_migration_state_event, ) -from sqlbuild.compiler.migrations.main._read_column_events import read_column_migration_events -from sqlbuild.compiler.migrations.main.write_column_event import write_column_migration_event +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.migrations.models import ColumnMigrationEvent from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( ColumnEventStorageTestCase, @@ -50,7 +50,8 @@ def test_given_column_events_when_writing_and_reading_then_rows_are_idempotent( ) for _ in range(test_case.write_count): - write_column_migration_event( + write_migration_state_event( + kind="column", connection=connection, execute=adapter.execute, event=event, @@ -66,7 +67,8 @@ def test_given_column_events_when_writing_and_reading_then_rows_are_idempotent( f"('future', 'fct_orders', 'fct_orders', 'tax', 'levy', 'manual', " f"'{foreign_decision}', 'run-9', TIMESTAMP '2026-01-09')" ) - stored: tuple[ColumnMigrationEvent, ...] = read_column_migration_events( + stored: tuple[ColumnMigrationEvent, ...] = read_migration_state_events( + kind="column", connection=connection, execute=adapter.execute, database=None, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py index 87d21f00ee..7c1e77718d 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py @@ -5,10 +5,12 @@ import duckdb import pytest +from sqlbuild._native import ( + newest_migration_event_mentioning, + read_migration_state_events, + write_migration_state_event, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.main._newest_event import newest_migration_event_mentioning -from sqlbuild.compiler.migrations.main._read_events import read_migration_events -from sqlbuild.compiler.migrations.main.write_event import write_migration_event from sqlbuild.compiler.migrations.models import MigrationEvent from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( NewestEventTestCase, @@ -87,7 +89,8 @@ def test_given_repeated_event_write_when_reading_then_event_is_stored_once( ) for _ in range(test_case.write_count): - write_migration_event( + write_migration_state_event( + kind="model", connection=connection, execute=adapter.execute, event=event, @@ -96,7 +99,8 @@ def test_given_repeated_event_write_when_reading_then_event_is_stored_once( database=None, schema="main" ), ) - stored: tuple[MigrationEvent, ...] = read_migration_events( + stored: tuple[MigrationEvent, ...] = read_migration_state_events( + kind="model", connection=connection, execute=adapter.execute, database=None, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py index 4e7638cbf7..eb25c85b74 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py @@ -7,23 +7,20 @@ import duckdb import pytest -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.constants import OLD_NAME_VIEW_COLUMNS -from sqlbuild.compiler.migrations.main._project_old_name_views import project_old_name_views -from sqlbuild.compiler.migrations.main.read_old_name_view_events import ( - read_old_name_view_events, +from sqlbuild._native import ( + MigrationStateInspection, + project_old_name_views, + read_migration_state_events, + stored_old_name_view_columns, + write_migration_state_event, ) -from sqlbuild.compiler.migrations.main.write_event import write_migration_event -from sqlbuild.compiler.migrations.main.write_old_name_view_event import write_old_name_view_event +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.migrations.models import ( MigrationEvent, OldNameViewEvent, OldNameViewHistory, ) from sqlbuild.compiler.migrations.types import OldNameViewEventType -from sqlbuild.compiler.planner.classes.migration_state_inspection import ( - MigrationStateInspection, -) from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( OldNameViewAbsentColumnsTestCase, OldNameViewProjectionTestCase, @@ -69,7 +66,8 @@ def test_given_old_name_facts_when_writing_and_reading_then_rows_are_idempotent( ) for _ in range(test_case.write_count): - write_old_name_view_event( + write_migration_state_event( + kind="old_name", connection=connection, execute=adapter.execute, event=fact, @@ -85,12 +83,15 @@ def test_given_old_name_facts_when_writing_and_reading_then_rows_are_idempotent( f"VALUES ('future', '{foreign}', 'm', 'daily_revenue', 'revenue', 'daily_revenue', " "'run-9', TIMESTAMP '2026-01-09')" ) - stored: tuple[OldNameViewEvent, ...] = read_old_name_view_events( + stored: tuple[OldNameViewEvent, ...] = read_migration_state_events( + kind="old_name", connection=connection, execute=adapter.execute, database=None, schema="main", - stored_columns=OLD_NAME_VIEW_COLUMNS, + stored_columns=stored_old_name_view_columns( + adapter=adapter, connection=connection, database=None, schema="main" + ), render_qualified_name=adapter.render_qualified_name, ) @@ -199,7 +200,8 @@ def test_given_state_table_without_optional_columns_when_planning_then_absent_co adapter: DuckDbAdapter = DuckDbAdapter() connection: duckdb.DuckDBPyConnection = duckdb.connect(":memory:") move: MigrationEvent = migration_event(origin="revenue", destination="daily_revenue", day=1) - write_migration_event( + write_migration_state_event( + kind="model", connection=connection, execute=adapter.execute, event=move, @@ -214,7 +216,8 @@ def test_given_state_table_without_optional_columns_when_planning_then_absent_co OldNameViewEventType.ORIGIN_ARCHIVED, OldNameViewEventType.VIEW_CREATED, ): - write_old_name_view_event( + write_migration_state_event( + kind="old_name", connection=connection, execute=adapter.execute, event=old_name_fact( diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py new file mode 100644 index 0000000000..0b18656d46 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py @@ -0,0 +1,131 @@ +"""Unit coverage for decoding hand-written persisted migration state rows.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import duckdb +import pytest + +from sqlbuild._native import read_migration_state_events, stored_old_name_view_columns +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, OldNameViewEvent +from sqlbuild.compiler.migrations.types import OldNameViewDropReason +from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( + StoredColumnRowTestCase, + StoredOldNameRowTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + StoredOldNameRowTestCase( + description="stored aliases are sorted and grants keep their order", + column_aliases_sql="""'{"b":"total","a":"amount"}'""", + grants_copied_sql="""'["GRANT SELECT ON main.orders TO reporting"]'""", + drop_reason_sql="'early'", + expected_aliases=(("a", "amount"), ("b", "total")), + expected_grants=("GRANT SELECT ON main.orders TO reporting",), + expected_drop_reason=OldNameViewDropReason.EARLY, + ), + StoredOldNameRowTestCase( + description="unreadable JSON and an unknown drop reason read as absent", + column_aliases_sql="'{not json'", + grants_copied_sql="""'{"a": 1}'""", + drop_reason_sql="'from_a_future_version'", + expected_aliases=(), + expected_grants=None, + expected_drop_reason=None, + ), + StoredOldNameRowTestCase( + description="null columns read as no aliases and unknown grants", + column_aliases_sql="NULL", + grants_copied_sql="NULL", + drop_reason_sql="NULL", + expected_aliases=(), + expected_grants=None, + expected_drop_reason=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_persisted_old_name_row_when_reading_then_codecs_decode_stored_text( + test_case: StoredOldNameRowTestCase, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: duckdb.DuckDBPyConnection = duckdb.connect(":memory:") + _ = connection.execute( + adapter.render_create_old_name_view_state_table_sql(database=None, schema="main") + ) + _ = connection.execute( + "INSERT INTO main._sqlbuild_old_name_views (event_id, event_type, migration_event_id, " + "destination_model, old_schema, old_name, new_schema, new_name, column_aliases, " + "grants_copied, drop_reason, run_id, created_at) VALUES ('fact', 'view_dropped', 'move', " + "'daily_orders', 'main', 'orders', 'main', 'daily_orders', " + f"{test_case.column_aliases_sql}, {test_case.grants_copied_sql}, " + f"{test_case.drop_reason_sql}, 'run-1', '2026-01-02T03:04:05+00:00')" + ) + + stored: tuple[OldNameViewEvent, ...] = read_migration_state_events( + kind="old_name", + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + stored_columns=stored_old_name_view_columns( + adapter=adapter, connection=connection, database=None, schema="main" + ), + render_qualified_name=adapter.render_qualified_name, + ) + + assert len(stored) == 1 + assert stored[0].column_aliases == test_case.expected_aliases + assert stored[0].grants_copied == test_case.expected_grants + assert stored[0].drop_reason == test_case.expected_drop_reason + assert stored[0].created_at == datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC) + + +@pytest.mark.parametrize( + "test_case", + [ + StoredColumnRowTestCase( + description="rows with a decision from a newer version are skipped", + decisions=("rename", "decided_by_a_future_version", "record"), + expected_decisions=("rename", "record"), + ), + ], + ids=lambda case: case.description, +) +def test_given_persisted_column_rows_when_reading_then_unknown_rows_are_skipped_in_order( + test_case: StoredColumnRowTestCase, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: duckdb.DuckDBPyConnection = duckdb.connect(":memory:") + _ = connection.execute( + adapter.render_create_column_migration_state_table_sql(database=None, schema="main") + ) + index: int + decision: str + for index, decision in enumerate(test_case.decisions): + _ = connection.execute( + "INSERT INTO main._sqlbuild_column_migrations (event_id, model_name, relation_schema, " + "relation_name, origin_column, destination_column, discovery, decision, run_id, " + f"created_at) VALUES ('event-{index}', 'orders', 'main', 'orders', 'amount', " + f"'total', 'manual', '{decision}', 'run-{index}', '2026-01-0{index + 1}')" + ) + + stored: tuple[ColumnMigrationEvent, ...] = read_migration_state_events( + kind="column", + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + ) + + assert tuple(event.decision.value for event in stored) == test_case.expected_decisions + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 18f2365be1..bf57006e39 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -14,6 +14,7 @@ import sqlbuild._native as _native from sqlbuild._native import ( build_downstream_deps, + build_scenario_relation_map, compiled_model_record, compute_query_hash, detect_changes, @@ -66,7 +67,6 @@ ) from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner._helpers.scenario.artifacts import build_scenario_relation_map from sqlbuild.compiler.planner.models import ( BackfillResult, ChangeDetectionResult, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_old_name_rows.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_old_name_rows.py index 929f6a6529..443abdb3a2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_old_name_rows.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_old_name_rows.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.planner._helpers.migrations.display import old_name_rows +from sqlbuild._native import old_name_rows from sqlbuild.presentation.classes.cli_style import CliStyle from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( OldNameRowsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 8106ea1ef6..c17745d1c6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -2,13 +2,19 @@ from __future__ import annotations -import re from collections.abc import Callable from pathlib import Path from types import MappingProxyType from typing import Any -from sqlbuild._native import expand_sql_macros +from sqlbuild._native import ( + expand_sql_macros, + extract_sql_references, + plan_compiled_sql_test_artifacts, + plan_sql_tests, + plan_sql_tests_natively, + sql_test_plan_error_messages, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter @@ -44,20 +50,15 @@ DiscoveredSqlTestBlock, DiscoveredSqlTestFile, ) -from sqlbuild.compiler.planner._helpers.sql_tests.assembly import plan_sql_tests -from sqlbuild.compiler.planner._helpers.sql_tests.comments import uncommented_pattern_matches -from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( - plan_compiled_sql_test_artifacts, - plan_sql_tests_natively, - sql_test_plan_error_messages, -) from sqlbuild.compiler.planner.models import ( NativeSqlTestPlan, PlanWarning, SqlTestPlanEntry, SqlTestPlanResult, ) +from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( PlanTestChainTestCase, @@ -380,25 +381,24 @@ def build_test_and_project( def _model_dependencies(*, query_sql: str) -> tuple[str, ...]: - return tuple( - dict.fromkeys( - match.group(1) - for match in uncommented_pattern_matches( - pattern=re.compile(r'__ref\("([^"]+)"\)'), sql=query_sql - ) - ) - ) + return _reference_names(query_sql=query_sql, kind=SqlReferenceKind.REF) def _table_function_dependencies(*, query_sql: str) -> tuple[str, ...]: - return tuple( - dict.fromkeys( - match.group(1) - for match in uncommented_pattern_matches( - pattern=re.compile(r'__table_fn\("([^"]+)"\)'), sql=query_sql - ) - ) - ) + return _reference_names(query_sql=query_sql, kind=SqlReferenceKind.TABLE_FUNCTION) + + +def _reference_names(*, query_sql: str, kind: SqlReferenceKind) -> tuple[str, ...]: + return reference_names_by_kind(query_sql=query_sql)[kind] + + +def reference_names_by_kind(*, query_sql: str) -> dict[SqlReferenceKind, tuple[str, ...]]: + """Return distinct logical reference names in authored order, grouped by reference kind.""" + + names: dict[SqlReferenceKind, dict[str, None]] = {kind: {} for kind in SqlReferenceKind} + for reference in extract_sql_references(sql=query_sql, syntax=SqlLexicalSyntax()): + names[SqlReferenceKind(reference.ref_kind)][reference.ref_name] = None + return {kind: tuple(kind_names) for kind, kind_names in names.items()} def _build_model_query_overrides( @@ -541,7 +541,9 @@ def build_windowed_sql_test(*, cursor_start: str | None, cursor_end: str | None) return CompiledSqlTest( key=CompiledObjectKey(resource_type=CompiledResourceType.SQL_TEST, name="orders_case"), - scope_deps=(), + scope_deps=( + CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="daily_orders"), + ), name="orders_case", test_file=DiscoveredSqlTestFile( file_path=Path("tests/unit/orders_case.sql"), @@ -556,5 +558,34 @@ def build_windowed_sql_test(*, cursor_start: str | None, cursor_end: str | None) cursor_start=cursor_start, cursor_end=cursor_end, ), - sql_body="", + sql_body=( + f"WITH {EXPECTED_TEST_CTE_PREFIX}daily_orders AS " + "(SELECT TIMESTAMP '2026-02-01' AS order_date) SELECT 1" + ), + payload=CompiledModelSqlTestPayload( + expected_model_names=("daily_orders",), + expected_ctes=( + CompileSqlTestCte( + name=f"{EXPECTED_TEST_CTE_PREFIX}daily_orders", + sql_body="SELECT TIMESTAMP '2026-02-01' AS order_date", + ), + ), + ), + ) + + +def plan_cursor_window_model_sql( + *, model: CompiledModelView, test: CompiledSqlTest, adapter: BaseAdapter +) -> str: + """Plan one windowed SQL test and return the rendered SQL of its cursor model step.""" + + project: CompiledProjectView = CompiledProject( + run_id="test_run", + effective_target_name=None, + effective_connection={}, + effective_vars={}, + models=(model,), ) + entry, _ = plan_single_test(test=test, project=project, adapter=adapter) + (step,) = entry.chain + return step.resolved_sql diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_cursor_window.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_cursor_window.py index 6532187bc8..933806dbd0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_cursor_window.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_cursor_window.py @@ -4,17 +4,14 @@ import pytest -from sqlbuild.compiler.planner._helpers.sql_tests.cursor_window import ( - render_test_cursor_intrinsics, -) from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( CursorWindowRenderingTestCase, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly.helpers import ( CURSOR_WINDOW_ADAPTERS, - CURSOR_WINDOW_MODEL_SQL, build_cursor_model, build_windowed_sql_test, + plan_cursor_window_model_sql, ) @@ -63,13 +60,12 @@ def test_given_model_using_cursor_intrinsics_when_rendering_default_window_then_bounds_are_typed( test_case: CursorWindowRenderingTestCase, ) -> None: - rendered: str = render_test_cursor_intrinsics( - sql=CURSOR_WINDOW_MODEL_SQL, + rendered: str = plan_cursor_window_model_sql( model=build_cursor_model( cursor_type=test_case.cursor_type, cursor_grain=test_case.cursor_grain ), + test=build_windowed_sql_test(cursor_start=None, cursor_end=None), adapter=CURSOR_WINDOW_ADAPTERS[test_case.adapter_name], - test=None, ) assert rendered == test_case.expected_sql @@ -96,15 +92,14 @@ def test_given_model_using_cursor_intrinsics_when_rendering_default_window_then_ def test_given_declared_test_window_when_rendering_then_declared_bounds_replace_defaults( test_case: CursorWindowRenderingTestCase, ) -> None: - rendered: str = render_test_cursor_intrinsics( - sql=CURSOR_WINDOW_MODEL_SQL, + rendered: str = plan_cursor_window_model_sql( model=build_cursor_model( cursor_type=test_case.cursor_type, cursor_grain=test_case.cursor_grain ), - adapter=CURSOR_WINDOW_ADAPTERS[test_case.adapter_name], test=build_windowed_sql_test( cursor_start=test_case.cursor_start, cursor_end=test_case.cursor_end ), + adapter=CURSOR_WINDOW_ADAPTERS[test_case.adapter_name], ) assert rendered == test_case.expected_sql diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index 77a9ebd7ed..5761e52fc5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -6,7 +6,9 @@ import pytest from sqlbuild._native import ( + plan_compiled_sql_test_artifacts, project_with, + sql_test_plan_error_messages, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -24,10 +26,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile -from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( - plan_compiled_sql_test_artifacts, - sql_test_plan_error_messages, -) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.main.commands._relations import resolve_static_relation_context from sqlbuild.compiler.planner.main.commands._scope import resolve_static_command_scope diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index a88912efab..a371b8a5d6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -2,6 +2,7 @@ import pytest +from sqlbuild._native import plan_audit from sqlbuild.compiler.auditing.types import AuditAttachmentKind, AuditRunScope from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -9,7 +10,6 @@ CompileSqlReference, ) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledAuditView -from sqlbuild.compiler.planner._helpers.output.audit_entry import plan_audit from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import SourceEntry diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index 8616d21334..76098d4a9a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -5,6 +5,8 @@ import pytest +import sqlbuild._native as sql_test_assembly +from sqlbuild import _native as fixture_completion from sqlbuild._native import ( compiled_model_record, project_with, @@ -19,10 +21,8 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner._helpers.fixtures import completion as fixture_completion from sqlbuild.compiler.planner._helpers.output import plan_output from sqlbuild.compiler.planner._helpers.output.plan_output import build_selected_test_entries -from sqlbuild.compiler.planner._helpers.sql_tests import assembly as sql_test_assembly from sqlbuild.compiler.planner.classes import fixture_column_inferences as fixture_inferences from sqlbuild.compiler.planner.models import ( FixtureColumnMetadata, @@ -102,13 +102,7 @@ def test_given_many_tests_when_building_entries_then_project_fixture_metadata_is ) metadata_builder: Mock = Mock(wraps=fixture_completion.build_relation_fixture_context) monkeypatch.setattr(plan_output, "build_relation_fixture_context", metadata_builder) - fixture_completion._cached_resolved_column_reads.cache_clear() - fallback_analyzer: Mock = Mock(wraps=fixture_completion.analyze_resolved_column_reads) - monkeypatch.setattr( - fixture_completion, - "analyze_resolved_column_reads", - fallback_analyzer, - ) + fixture_completion.fixture_read_cache_info(reset=True) topology_builder: Mock = Mock(wraps=sql_test_assembly.resolve_sql_test_model_chains) monkeypatch.setattr(sql_test_assembly, "resolve_sql_test_model_chains", topology_builder) @@ -124,7 +118,7 @@ def test_given_many_tests_when_building_entries_then_project_fixture_metadata_is assert len(entries) == test_case.test_count assert warnings == [] assert metadata_builder.call_count == test_case.expected_metadata_builds - assert fallback_analyzer.call_count == test_case.expected_analysis_calls + assert fixture_completion.fixture_read_cache_info()[0] == test_case.expected_analysis_calls assert topology_builder.call_count == test_case.expected_topology_builds @@ -239,14 +233,7 @@ def test_given_complete_authoritative_fixture_when_completing_then_skips_column_ (CompiledResourceType.MODEL, model.name): {}, }, ) - fallback_analyzer: Mock = Mock( - side_effect=AssertionError("complete fixtures must not trigger model analysis") - ) - monkeypatch.setattr( - fixture_completion, - "analyze_resolved_column_reads", - fallback_analyzer, - ) + fixture_completion.fixture_read_cache_info(reset=True) fixture_inference: Mock = Mock(wraps=fixture_inferences.infer_fixture_column_facts) monkeypatch.setattr( fixture_inferences, @@ -257,14 +244,16 @@ def test_given_complete_authoritative_fixture_when_completing_then_skips_column_ completed: RelationFixtureCompletion = fixture_completion.build_relation_fixture_completion( project=project_with(project=project, models=(model,)), adapter=PlannerTestAdapter(), - ordered_model_names=(model.name,), - fixture_groups=((CompiledResourceType.SOURCE, {"raw_orders": test_case.fixture_sql}),), + request=( + (model.name,), + ((CompiledResourceType.SOURCE, {"raw_orders": test_case.fixture_sql}),), + ), planning_context=context, ) assert completed.diagnostics == () assert completed.fixture_sql_by_key[source_key] == test_case.expected_fixture_sql - assert fallback_analyzer.call_count == test_case.expected_analysis_calls + assert fixture_completion.fixture_read_cache_info()[0] == test_case.expected_analysis_calls assert fixture_inference.call_count == test_case.expected_inference_calls diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py index 64b45550b2..437ceae8ce 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py @@ -4,6 +4,10 @@ import pytest +from sqlbuild._native import ( + apply_configured_microbatch_limits, + apply_microbatch_limit, +) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( @@ -11,10 +15,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner._helpers.output.plan_entry import ( - _apply_configured_microbatch_limits, - _apply_microbatch_limit, -) from sqlbuild.compiler.planner.models import ( CursorBounds, CursorInputRelation, @@ -87,7 +87,7 @@ def test_given_nested_model_policy_when_applying_outer_limit_then_precedence_is_ declared_microbatch_limit_action=MicrobatchLimitAction.CAP_FROM_END, ) - limited_entry, warnings = _apply_configured_microbatch_limits( + limited_entry, warnings = apply_configured_microbatch_limits( entry=entry, inputs=PlanEntryBuildInputs( max_microbatches=test_case.configured_limit, @@ -153,7 +153,7 @@ def test_given_coarse_effective_grain_when_applying_limit_then_authored_batch_se microbatch_range=CursorBounds(start="2026-01-01", end="2026-04-01"), ) - limited_entry, warning = _apply_microbatch_limit( + limited_entry, warning = apply_microbatch_limit( entry=entry, max_batches=3, action=MicrobatchLimitAction.WARN, @@ -272,7 +272,7 @@ def test_given_planner_owned_range_when_applying_limit_then_expected_decision_is ) action: MicrobatchLimitAction = MicrobatchLimitAction(test_case.action) - limited_entry, warning = _apply_microbatch_limit( + limited_entry, warning = apply_microbatch_limit( entry=entry, max_batches=test_case.max_batches, action=action ) @@ -323,7 +323,7 @@ def test_given_excess_planner_owned_range_when_action_is_error_then_planning_fai ) with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - _apply_microbatch_limit( + apply_microbatch_limit( entry=entry, max_batches=test_case.max_batches, action=MicrobatchLimitAction(test_case.action), diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py index 351663804b..1a38e56bda 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import resolve_microbatch_lookback from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner._helpers.output.plan_entry import resolve_microbatch_lookback from sqlbuild.compiler.planner.types import IncrementalStrategy from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( MicrobatchLookbackTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py index 13545647f5..f158cefe5b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py @@ -2,14 +2,12 @@ import pytest -from sqlbuild.compiler.planner._helpers.scenario.artifact_names import ( - parse_scenario_artifact_physical_name, -) -from sqlbuild.compiler.planner._helpers.scenario.artifacts import ( +from sqlbuild._native import ( build_scenario_artifact_name, build_scenario_hash_index, build_scenario_relation_map, compute_scenario_hash_prefix, + parse_scenario_artifact_physical_name, ) from sqlbuild.compiler.planner.constants import ( SCENARIO_PLAN_HASH_COLLISION, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py index e307c9bdea..b1e7474eb1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py @@ -7,8 +7,11 @@ import pytest from sqlbuild._native import ( + build_scenario_execution_plan, + build_scenario_fixture_plans, build_scenario_relation_plan, project_with, + resolve_scenario_check_sql, ) from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter @@ -22,11 +25,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner._helpers.scenario.relations import ( - build_scenario_execution_plan, - build_scenario_fixture_plans, - resolve_scenario_check_sql, -) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( ScenarioExecutionPlan, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py index 2137d7e8b7..f89f17f262 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py @@ -6,6 +6,11 @@ import pytest +from sqlbuild._native import ( + build_cursor_input_relations, + runtime_cursor_producer_names, + validate_source_cursor_input_columns, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -19,12 +24,7 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner._helpers.output.plan_entry import ( - _build_cursor_input_relations, - _runtime_cursor_producer_names, - build_planner_relations_context, - validate_source_cursor_input_columns, -) +from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context from sqlbuild.compiler.planner._helpers.warehouse.source_columns import gather_source_columns from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( @@ -447,7 +447,7 @@ def test_given_view_filter_and_transitive_watermark_when_planning_then_relation_ ) view_model: CompiledModelView = models_by_name["orders_view"] - relations: tuple[CursorInputRelation, ...] = _build_cursor_input_relations( + relations: tuple[CursorInputRelation, ...] = build_cursor_input_relations( model=model, adapter=DuckDbAdapter(), model_locations={view_model.name: view_model.destination}, @@ -608,7 +608,7 @@ def test_given_table_function_and_transitive_watermark_when_planning_then_source source_map={"raw_orders": SourceEntry(name="raw_orders", schema="raw")}, source_warehouse_columns={"raw_orders": (ColumnInfo(name="loaded_at", type="TIMESTAMP"),)}, ) - relations: tuple[CursorInputRelation, ...] = _build_cursor_input_relations( + relations: tuple[CursorInputRelation, ...] = build_cursor_input_relations( model=model, adapter=DuckDbAdapter(), model_locations={name: upstream.destination for name, upstream in models_by_name.items()}, @@ -716,7 +716,7 @@ def test_given_selected_relations_when_classifying_runtime_watermarks_then_only_ execution_order=(), ) - names: frozenset[str] = _runtime_cursor_producer_names( + names: frozenset[str] = runtime_cursor_producer_names( scope=scope, source_map=test_case.source_map, ) diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py index 8deb2b0c05..cfbbbf145c 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py @@ -4,11 +4,11 @@ import pytest -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.planner._helpers.scenario.artifacts import ( +from sqlbuild._native import ( build_scenario_artifact_name, compute_scenario_hash_prefix, ) +from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.executor.janitor.main.execute import execute_janitor_plan from sqlbuild.executor.janitor.main.plan import build_janitor_plan diff --git a/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py b/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py index f1bdb26f8f..5df4303783 100644 --- a/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py +++ b/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py @@ -5,8 +5,8 @@ import duckdb import pytest +from sqlbuild._native import read_migration_state_events from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.main._read_column_events import read_column_migration_events from sqlbuild.compiler.migrations.models import ColumnMigrationEvent from sqlbuild.executor.migrations._helpers.columns import apply_model_column_renames from tests.unit.src.sqlbuild.executor.migrations._helpers._test_types import ( @@ -57,7 +57,8 @@ def test_given_overlapping_build_between_renames_when_applying_then_reconciles_f ), run_id="run-1", ) - events: tuple[ColumnMigrationEvent, ...] = read_column_migration_events( + events: tuple[ColumnMigrationEvent, ...] = read_migration_state_events( + kind="column", connection=connection, execute=adapter.execute, database=None,