From 2d40315db5a5fdb04d1e38bbac733365fc81963b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 03:34:15 +0100 Subject: [PATCH 01/11] refactor: add native warehouse inspection planning decisions Add pure planner decisions for warehouse inspection: physical cursor read grouping and statements, retention directions and change phases, table type copy names, missing source table messages, unknown freshness warnings, full rebuild causes, managed source call scanning, deferral target templates, and the migration fingerprint cache key and entry format, byte-compatible with the stored cache rows. --- crates/sqlbuild-analysis/src/lib.rs | 1 + .../_helpers/cache_entries.rs | 12 + .../src/planner_warehouse/_helpers/mod.rs | 3 + .../src/planner_warehouse/constants.rs | 29 + .../planner_warehouse/main/cursor_bounds.rs | 13 + .../main/cursor_query_groups.rs | 41 + .../main/cursor_query_identity.rs | 17 + .../main/cursor_query_sql.rs | 21 + .../main/fingerprint_cache_entry.rs | 36 + .../main/fingerprint_cache_key.rs | 36 + .../main/fingerprint_cache_lookup.rs | 39 + .../main/fingerprint_cache_shared_digest.rs | 40 + .../main/freshness_unknown_message.rs | 31 + .../main/full_rebuild_cause.rs | 26 + .../main/missing_source_tables_help.rs | 32 + .../main/missing_source_tables_message.rs | 50 + .../src/planner_warehouse/main/mod.rs | 19 + .../main/retention_change.rs | 19 + .../main/retention_direction.rs | 25 + .../main/source_call_names.rs | 21 + .../main/source_deferral_field.rs | 63 + .../main/table_type_copy_name.rs | 14 + .../src/planner_warehouse/mod.rs | 8 + .../src/planner_warehouse/models.rs | 52 + .../src/planner_warehouse/tests.rs | 14 + .../src/planner_warehouse/tests/helpers.rs | 26 + .../tests/test_cursor_queries.rs | 63 + .../tests/test_fingerprint_cache.rs | 102 + .../planner_warehouse/tests/test_messages.rs | 110 ++ .../planner_warehouse/tests/test_retention.rs | 124 ++ .../tests/test_source_texts.rs | 70 + .../src/planner_warehouse/tests/test_types.rs | 79 + ...tion.py => _get_columns_for_inspection.py} | 0 ...n.py => _list_relations_for_inspection.py} | 0 .../planner/_helpers/planning/__init__.py | 0 .../planner/_helpers/planning/buildability.py | 60 - .../planner/_helpers/planning/entries.py | 94 - .../planner/_helpers/planning/full_refresh.py | 67 - .../planner/_helpers/planning/identities.py | 117 -- .../_helpers/planning/output_assembly.py | 270 --- .../_helpers/planning/reconciliation.py | 35 - .../planner/_helpers/planning/retention.py | 455 ----- .../_helpers/planning/scope_pruning.py | 28 - .../planner/_helpers/planning/scopes.py | 66 - .../_helpers/planning/warehouse_state.py | 125 -- .../_helpers/warehouse/semantic_sources.py | 139 -- .../planner/_helpers/warehouse/snapshot.py | 1696 ----------------- .../_helpers/warehouse/source_columns.py | 133 -- .../_helpers/warehouse/source_deferral.py | 261 --- .../_helpers/warehouse/source_freshness.py | 104 - .../warehouse/source_freshness_warnings.py | 53 - .../_helpers/warehouse/source_tables.py | 180 -- .../classes/migration_fingerprint_cache.py | 234 --- .../_helpers/test_warehouse_cursor_queries.py | 532 ------ .../test_warehouse_cursor_resolution.py | 89 - .../test_warehouse_cursor_concurrency.py | 139 -- 56 files changed, 1236 insertions(+), 4877 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/_helpers/cache_entries.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/constants.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_bounds.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_groups.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_identity.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_sql.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/fingerprint_cache_entry.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/fingerprint_cache_key.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/fingerprint_cache_lookup.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/fingerprint_cache_shared_digest.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/freshness_unknown_message.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/full_rebuild_cause.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/missing_source_tables_help.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/missing_source_tables_message.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/retention_change.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/retention_direction.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/source_call_names.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/source_deferral_field.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/main/table_type_copy_name.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/models.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/tests.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/tests/test_fingerprint_cache.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/tests/test_messages.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/tests/test_retention.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/tests/test_source_texts.rs create mode 100644 crates/sqlbuild-analysis/src/planner_warehouse/tests/test_types.rs rename src/sqlbuild/adapter/relations/main/{get_columns_for_inspection.py => _get_columns_for_inspection.py} (100%) rename src/sqlbuild/adapter/relations/main/{list_relations_for_inspection.py => _list_relations_for_inspection.py} (100%) delete mode 100644 src/sqlbuild/compiler/planner/_helpers/planning/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/planning/buildability.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/planning/entries.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/planning/full_refresh.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/planning/identities.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/planning/reconciliation.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/planning/retention.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/planning/scope_pruning.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/planning/scopes.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/warehouse/semantic_sources.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/warehouse/source_columns.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/warehouse/source_freshness.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/warehouse/source_freshness_warnings.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/warehouse/source_tables.py delete mode 100644 src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py delete mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_queries.py delete mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_resolution.py delete mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_warehouse_cursor_concurrency.py diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 4d8753e629..79b7b528c6 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -27,6 +27,7 @@ pub mod planner_scenario; pub mod planner_scope; pub mod planner_selection; pub mod planner_strategy; +pub mod planner_warehouse; pub mod python_plan; pub mod python_selection; pub mod python_values; diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/_helpers/cache_entries.rs b/crates/sqlbuild-analysis/src/planner_warehouse/_helpers/cache_entries.rs new file mode 100644 index 0000000000..33d50e26bc --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/_helpers/cache_entries.rs @@ -0,0 +1,12 @@ +//! Digests binding stored fingerprint cache payloads to their keys. + +use sha2::{Digest, Sha256}; + +/// SHA-256 of the cache key, a NUL byte and the serialized payload. +pub(crate) fn entry_digest(cache_key: &str, serialized: &str) -> String { + let mut digest = Sha256::new(); + digest.update(cache_key.as_bytes()); + digest.update(b"\0"); + digest.update(serialized.as_bytes()); + format!("{:x}", digest.finalize()) +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_warehouse/_helpers/mod.rs new file mode 100644 index 0000000000..dbc16fe136 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/_helpers/mod.rs @@ -0,0 +1,3 @@ +//! Shared helpers of warehouse-inspection planning. + +pub(crate) mod cache_entries; diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/constants.rs b/crates/sqlbuild-analysis/src/planner_warehouse/constants.rs new file mode 100644 index 0000000000..110d65ed20 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/constants.rs @@ -0,0 +1,29 @@ +//! Fixed texts and limits of warehouse-inspection planning. + +pub(crate) const TIMESTAMP_CURSOR_TYPE: &str = "timestamp"; +pub(crate) const RETENTION_INCREASE: &str = "increase"; +pub(crate) const RETENTION_DECREASE: &str = "decrease"; +pub(crate) const RETENTION_MIXED: &str = "mixed"; +pub(crate) const RETENTION_MATCH: &str = "match"; +pub(crate) const RETENTION_PHASE_PRE: &str = "pre"; +pub(crate) const RETENTION_PHASE_POST: &str = "post"; +pub(crate) const CHANGE_PHASE_PREPARE: &str = "prepare"; +pub(crate) const CHANGE_PHASE_FINALIZE: &str = "finalize"; +pub(crate) const TABLE_TYPE_COPY_PREFIX: &str = "__sqb_type_swap__"; +pub(crate) const MESSAGE_ITEM_LIMIT: usize = 10; +pub(crate) const FRESHNESS_NAME_LIMIT: usize = 10; +pub(crate) const SOURCE_FRESHNESS_UNKNOWN_WARNING_TITLE: &str = "source freshness unknown"; +pub(crate) const FRESHNESS_UNAVAILABLE_EXPLANATION: &str = "adapter freshness metadata is unavailable, for example for a view, an external table, or a table without a last-modified time"; +pub(crate) const FRESHNESS_ERROR_EXPLANATION: &str = "the freshness observation failed"; +pub(crate) const FRESHNESS_UNAVAILABLE: &str = "unavailable"; +pub(crate) const FRESHNESS_ERROR: &str = "error"; +pub(crate) const SOURCE_CALL_OPEN: &str = "__source(\""; +pub(crate) const CONTEXT_PLACEHOLDER_OPEN: &str = "${CTX:"; +pub(crate) const VARIABLE_PLACEHOLDER_OPEN: &str = "${"; +pub(crate) const SOURCE_DEFERRAL_CONTEXT_FIELDS: [&str; 2] = ["schema", "database"]; +pub(crate) const FINGERPRINT_CACHE_VERSION: i64 = 1; +pub(crate) const FINGERPRINT_CACHE_ENTRY_SEPARATOR: char = '\n'; +pub(crate) const MAX_FINGERPRINT_CACHE_ENTRY_CHARACTERS: usize = 4_096; +pub(crate) const MIGRATION_FINGERPRINT_ALGORITHM: &str = "migration-fingerprint-v1"; +pub(crate) const CHANGE_KIND_FIRST_RUN: &str = "first_run"; +pub(crate) const CHANGE_KIND_RENAMED: &str = "renamed"; diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_bounds.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_bounds.rs new file mode 100644 index 0000000000..856b5c5a2b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_bounds.rs @@ -0,0 +1,13 @@ +//! Which bounds one physical cursor read selects. + +use crate::planner_warehouse::models::PhysicalCursorQuery; + +/// `min`, `max` or `min,max`, as shown in progress and debug events. +pub fn cursor_bounds(query: &PhysicalCursorQuery) -> String { + match (query.min_tags.is_empty(), query.max_tags.is_empty()) { + (false, false) => "min,max".to_owned(), + (false, true) => "min".to_owned(), + (true, false) => "max".to_owned(), + (true, true) => String::new(), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_groups.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_groups.rs new file mode 100644 index 0000000000..8a88abb6fc --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_groups.rs @@ -0,0 +1,41 @@ +//! Group logical cursor bounds by the physical relation column that answers them. + +use crate::planner_warehouse::constants::TIMESTAMP_CURSOR_TYPE; +use crate::planner_warehouse::models::{CursorTagRequest, PhysicalCursorQuery}; + +/// Merge requests into one query per relation, column and type, in first-request order. +pub fn cursor_query_groups(requests: &[CursorTagRequest]) -> Vec { + let mut groups: Vec = Vec::new(); + for request in requests { + let cursor_type = request + .cursor_type + .clone() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| TIMESTAMP_CURSOR_TYPE.to_owned()); + let position = groups.iter().position(|group| { + group.relation == request.relation + && group.cursor_column == request.cursor_column + && group.cursor_type == cursor_type + }); + let index = position.unwrap_or_else(|| { + groups.push(PhysicalCursorQuery { + relation: request.relation.clone(), + cursor_column: request.cursor_column.clone(), + cursor_type, + min_tags: Vec::new(), + max_tags: Vec::new(), + }); + groups.len() - 1 + }); + let group = &mut groups[index]; + if let Some(min_tag) = &request.min_tag + && !group.min_tags.contains(min_tag) + { + group.min_tags.push(min_tag.clone()); + } + if !group.max_tags.contains(&request.max_tag) { + group.max_tags.push(request.max_tag.clone()); + } + } + groups +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_identity.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_identity.rs new file mode 100644 index 0000000000..562cf0b0f7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_identity.rs @@ -0,0 +1,17 @@ +//! The progress label of one physical cursor read. + +use crate::planner_warehouse::main::cursor_bounds::cursor_bounds; +use crate::planner_warehouse::models::PhysicalCursorQuery; + +/// `(index/total): relation.column [bounds]`. +pub fn cursor_query_identity( + query: &PhysicalCursorQuery, + (index, total): (usize, usize), +) -> String { + format!( + "({index}/{total}): {}.{} [{}]", + query.relation, + query.cursor_column, + cursor_bounds(query) + ) +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_sql.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_sql.rs new file mode 100644 index 0000000000..b86da84c9f --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_sql.rs @@ -0,0 +1,21 @@ +//! The unmerged MIN/MAX statement of one physical cursor read. + +use crate::planner_warehouse::models::PhysicalCursorQuery; + +/// Select the requested bounds of the cursor column as text. +pub fn cursor_query_sql(query: &PhysicalCursorQuery) -> String { + let mut parts: Vec = Vec::new(); + if !query.min_tags.is_empty() { + parts.push(format!( + "CAST(MIN({}) AS VARCHAR) AS _min", + query.cursor_column + )); + } + if !query.max_tags.is_empty() { + parts.push(format!( + "CAST(MAX({}) AS VARCHAR) AS _max", + query.cursor_column + )); + } + format!("SELECT {} FROM {}", parts.join(", "), query.relation) +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/fingerprint_cache_entry.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/fingerprint_cache_entry.rs new file mode 100644 index 0000000000..12a3db99c9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/fingerprint_cache_entry.rs @@ -0,0 +1,36 @@ +//! The stored payload of one migration fingerprint cache entry. + +use crate::planner_warehouse::_helpers::cache_entries::entry_digest; +use crate::planner_warehouse::constants::{ + FINGERPRINT_CACHE_ENTRY_SEPARATOR, FINGERPRINT_CACHE_VERSION, +}; +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; + +/// `digest` newline `{"f":...,"k":...,"v":1}`, the digest binding the payload to its key. +pub fn fingerprint_cache_entry( + cache_key: &str, + fingerprint: Option<&str>, +) -> Result { + let value = JsonValue::Object(vec![ + ( + "v".to_owned(), + JsonValue::Integer(JsonInteger::from(FINGERPRINT_CACHE_VERSION)), + ), + ("k".to_owned(), JsonValue::String(cache_key.to_owned())), + ( + "f".to_owned(), + fingerprint.map_or(JsonValue::Null, |text| JsonValue::String(text.to_owned())), + ), + ]); + let options = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }; + let serialized = dumps(&value, &JsonDialect::Stdlib(options))?; + Ok(format!( + "{}{FINGERPRINT_CACHE_ENTRY_SEPARATOR}{serialized}", + entry_digest(cache_key, &serialized) + )) +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/fingerprint_cache_key.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/fingerprint_cache_key.rs new file mode 100644 index 0000000000..54ba1c8825 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/fingerprint_cache_key.rs @@ -0,0 +1,36 @@ +//! The persistent key of one migration fingerprint cache entry. + +use crate::planner_warehouse::models::FingerprintCacheKey; +use sha2::{Digest, Sha256}; +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +/// SHA-256 of the compact, non-ASCII-escaped JSON array of the key parts. +pub fn fingerprint_cache_key(key: &FingerprintCacheKey) -> Result { + let renames: Vec = key + .visible_renames + .iter() + .map(|(name, identity)| { + JsonValue::Array(vec![ + JsonValue::String(name.clone()), + JsonValue::String(identity.clone()), + ]) + }) + .collect(); + let value = JsonValue::Array(vec![ + JsonValue::String(key.shared_digest.clone()), + JsonValue::String(key.query_sql.clone()), + JsonValue::String(key.metadata_json.clone()), + key.dialect + .clone() + .map_or(JsonValue::Null, JsonValue::String), + JsonValue::Array(renames), + ]); + let options = StdlibJsonOptions { + ensure_ascii: false, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }; + let text = dumps(&value, &JsonDialect::Stdlib(options))?; + Ok(format!("{:x}", Sha256::digest(text.as_bytes()))) +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/fingerprint_cache_lookup.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/fingerprint_cache_lookup.rs new file mode 100644 index 0000000000..a3a8fbd28b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/fingerprint_cache_lookup.rs @@ -0,0 +1,39 @@ +//! Validate one stored migration fingerprint cache entry. + +use crate::planner_warehouse::_helpers::cache_entries::entry_digest; +use crate::planner_warehouse::constants::{ + FINGERPRINT_CACHE_ENTRY_SEPARATOR, FINGERPRINT_CACHE_VERSION, + MAX_FINGERPRINT_CACHE_ENTRY_CHARACTERS, +}; +use serde_json::Value; + +/// The stored fingerprint (possibly `None`), or `None` when the entry is not valid for the key. +pub fn fingerprint_cache_lookup( + contents: &str, + expected_cache_key: &str, +) -> Option> { + if contents.chars().count() > MAX_FINGERPRINT_CACHE_ENTRY_CHARACTERS { + return None; + } + let (stored_digest, serialized) = contents.split_once(FINGERPRINT_CACHE_ENTRY_SEPARATOR)?; + if !stored_digest.is_ascii() || stored_digest != entry_digest(expected_cache_key, serialized) { + return None; + } + let Ok(Value::Object(values)) = serde_json::from_str::(serialized) else { + return None; + }; + let version_matches = match values.get("v") { + Some(Value::Number(number)) => number.as_f64() == Some(FINGERPRINT_CACHE_VERSION as f64), + Some(Value::Bool(flag)) => *flag, + _ => false, + }; + let key_matches = values.get("k").and_then(Value::as_str) == Some(expected_cache_key); + if !version_matches || !key_matches { + return None; + } + match values.get("f") { + None | Some(Value::Null) => Some(None), + Some(Value::String(fingerprint)) => Some(Some(fingerprint.clone())), + Some(_) => None, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/fingerprint_cache_shared_digest.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/fingerprint_cache_shared_digest.rs new file mode 100644 index 0000000000..bd69559fd0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/fingerprint_cache_shared_digest.rs @@ -0,0 +1,40 @@ +//! The digest shared by every migration fingerprint cache key of one build. + +use crate::planner_warehouse::constants::{ + FINGERPRINT_CACHE_VERSION, MIGRATION_FINGERPRINT_ALGORITHM, +}; +use sha2::{Digest, Sha256}; +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; + +/// SHA-256 of the sorted JSON of the algorithm, cache version and package versions. +pub fn fingerprint_cache_shared_digest( + sqlbuild_version: &str, + polyglot_version: &str, +) -> Result { + let value = JsonValue::Object(vec![ + ( + "algorithm".to_owned(), + JsonValue::String(MIGRATION_FINGERPRINT_ALGORITHM.to_owned()), + ), + ( + "cache_version".to_owned(), + JsonValue::Integer(JsonInteger::from(FINGERPRINT_CACHE_VERSION)), + ), + ( + "sqlbuild_version".to_owned(), + JsonValue::String(sqlbuild_version.to_owned()), + ), + ( + "polyglot_version".to_owned(), + JsonValue::String(polyglot_version.to_owned()), + ), + ]); + let options = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None) + }; + let text = dumps(&value, &JsonDialect::Stdlib(options))?; + Ok(format!("{:x}", Sha256::digest(text.as_bytes()))) +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/freshness_unknown_message.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/freshness_unknown_message.rs new file mode 100644 index 0000000000..a9858fd01c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/freshness_unknown_message.rs @@ -0,0 +1,31 @@ +//! The plan warning for sources whose freshness could not be observed. + +use crate::planner_warehouse::constants::{ + FRESHNESS_ERROR, FRESHNESS_ERROR_EXPLANATION, FRESHNESS_NAME_LIMIT, FRESHNESS_UNAVAILABLE, + FRESHNESS_UNAVAILABLE_EXPLANATION, SOURCE_FRESHNESS_UNKNOWN_WARNING_TITLE, +}; + +/// The warning for one explained reason, or `None` for reasons that need no warning. +pub fn freshness_unknown_message(reason: &str, names: &[String]) -> Option { + let explanation = if reason == FRESHNESS_UNAVAILABLE { + FRESHNESS_UNAVAILABLE_EXPLANATION + } else if reason == FRESHNESS_ERROR { + FRESHNESS_ERROR_EXPLANATION + } else { + return None; + }; + let mut ordered: Vec<&String> = names.iter().collect(); + ordered.sort(); + let mut shown = ordered + .iter() + .take(FRESHNESS_NAME_LIMIT) + .map(|name| name.as_str()) + .collect::>() + .join(", "); + if ordered.len() > FRESHNESS_NAME_LIMIT { + shown = format!("{shown} and {} more", ordered.len() - FRESHNESS_NAME_LIMIT); + } + Some(format!( + "{SOURCE_FRESHNESS_UNKNOWN_WARNING_TITLE} ({reason}) for {shown}: {explanation}; run sqb freshness for each source's details" + )) +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/full_rebuild_cause.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/full_rebuild_cause.rs new file mode 100644 index 0000000000..adfea2a43b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/full_rebuild_cause.rs @@ -0,0 +1,26 @@ +//! Why an incremental model is planned as a full rebuild. + +use crate::planner_warehouse::constants::{CHANGE_KIND_FIRST_RUN, CHANGE_KIND_RENAMED}; +use crate::planner_warehouse::models::FullRebuildFacts; + +/// The cause text shown in plans and full-refresh refusals. +pub fn full_rebuild_cause(facts: &FullRebuildFacts) -> String { + if facts.full_refresh { + "--full-refresh".to_owned() + } else if facts.change_kind == CHANGE_KIND_FIRST_RUN { + "first run".to_owned() + } else if facts.change_kind == CHANGE_KIND_RENAMED { + "rename".to_owned() + } else if facts.query_changed { + "query changed with replay_on_change full".to_owned() + } else if !facts.changed_functions.is_empty() { + format!( + "function {} changed with replay_on_change full", + facts.changed_functions.join(", ") + ) + } else if facts.has_schema_findings { + "schema changed with replay_on_change full".to_owned() + } else { + "full backfill".to_owned() + } +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/missing_source_tables_help.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/missing_source_tables_help.rs new file mode 100644 index 0000000000..804e334189 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/missing_source_tables_help.rs @@ -0,0 +1,32 @@ +//! The remedy for selected source tables the warehouse cannot read. + +use crate::planner_warehouse::constants::MESSAGE_ITEM_LIMIT; +use crate::planner_warehouse::models::MissingSourceTable; +use std::collections::BTreeSet; + +/// Suggest creating, granting or correcting the sources, with their declarations. +pub fn missing_source_tables_help(missing: &[MissingSourceTable]) -> String { + let locations: Vec = missing + .iter() + .take(MESSAGE_ITEM_LIMIT) + .map(|item| format!("'{}' at {}", item.source_name, item.location)) + .collect(); + let roles: BTreeSet<&str> = missing + .iter() + .filter_map(|item| item.role.as_deref()) + .collect(); + let role_text = if roles.is_empty() { + String::new() + } else { + format!(" ({})", roles.into_iter().collect::>().join(", ")) + }; + let grant = if missing.iter().any(|item| item.unreadable) { + format!("grant SELECT on it to the role in use{role_text}, ") + } else { + String::new() + }; + format!( + "create or load the table, {grant}correct its database, schema, or table in the source declaration, or stop reading it from the selected models; declared {}", + locations.join(", ") + ) +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/missing_source_tables_message.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/missing_source_tables_message.rs new file mode 100644 index 0000000000..e13b2cfcf7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/missing_source_tables_message.rs @@ -0,0 +1,50 @@ +//! The error message listing selected source tables the warehouse cannot read. + +use crate::planner_warehouse::constants::MESSAGE_ITEM_LIMIT; +use crate::planner_warehouse::models::MissingSourceTable; + +/// Summarise missing tables, at most ten listed with their readers. +pub fn missing_source_tables_message(missing: &[MissingSourceTable]) -> String { + let single = missing.len() == 1; + let noun = if single { + "source table" + } else { + "source tables" + }; + let verb = if single { "does" } else { "do" }; + let readability = if !missing.iter().any(|item| item.unreadable) { + "" + } else if single { + " or is not readable by the current role" + } else { + " or are not readable by the current role" + }; + let mut lines: Vec = vec![format!( + "cannot build selected scope: {} {noun} read by selected resources {verb} not exist in the warehouse{readability}:", + missing.len() + )]; + for item in missing.iter().take(MESSAGE_ITEM_LIMIT) { + let suffix = if item.unreadable { + let role = item.role.as_ref().map_or_else( + || "the current role".to_owned(), + |role| format!("role {role}"), + ); + format!("; the warehouse reports it does not exist or is not readable by {role}") + } else { + String::new() + }; + lines.push(format!( + " - source '{}' ({}), read by {}{suffix}", + item.source_name, + item.relation, + item.read_by.join(", ") + )); + } + if missing.len() > MESSAGE_ITEM_LIMIT { + lines.push(format!( + " - ... and {} more", + missing.len() - MESSAGE_ITEM_LIMIT + )); + } + lines.join("\n") +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/mod.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/mod.rs new file mode 100644 index 0000000000..4cd8559b89 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/mod.rs @@ -0,0 +1,19 @@ +//! Entry points of warehouse-inspection planning. + +pub mod cursor_bounds; +pub mod cursor_query_groups; +pub mod cursor_query_identity; +pub mod cursor_query_sql; +pub mod fingerprint_cache_entry; +pub mod fingerprint_cache_key; +pub mod fingerprint_cache_lookup; +pub mod fingerprint_cache_shared_digest; +pub mod freshness_unknown_message; +pub mod full_rebuild_cause; +pub mod missing_source_tables_help; +pub mod missing_source_tables_message; +pub mod retention_change; +pub mod retention_direction; +pub mod source_call_names; +pub mod source_deferral_field; +pub mod table_type_copy_name; diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/retention_change.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/retention_change.rs new file mode 100644 index 0000000000..85c8dcf9d7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/retention_change.rs @@ -0,0 +1,19 @@ +//! The plan direction and phase of one rendered retention change. + +use crate::planner_warehouse::constants::{ + CHANGE_PHASE_FINALIZE, CHANGE_PHASE_PREPARE, RETENTION_DECREASE, RETENTION_INCREASE, + RETENTION_PHASE_POST, RETENTION_PHASE_PRE, +}; + +/// `(direction, phase)` for a change phase, falling back to the state direction. +pub fn retention_change<'a>(change_phase: &str, fallback: &'a str) -> (&'a str, &'static str) { + if change_phase == CHANGE_PHASE_PREPARE { + (RETENTION_INCREASE, RETENTION_PHASE_PRE) + } else if change_phase == CHANGE_PHASE_FINALIZE { + (RETENTION_DECREASE, RETENTION_PHASE_POST) + } else if fallback == RETENTION_INCREASE { + (fallback, RETENTION_PHASE_PRE) + } else { + (fallback, RETENTION_PHASE_POST) + } +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/retention_direction.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/retention_direction.rs new file mode 100644 index 0000000000..538897f077 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/retention_direction.rs @@ -0,0 +1,25 @@ +//! Compare inspected retention with the desired days. + +use crate::planner_warehouse::constants::{ + RETENTION_DECREASE, RETENTION_INCREASE, RETENTION_MATCH, RETENTION_MIXED, +}; + +/// Direction from the Delta retention values, or the effective days when none are set. +pub fn retention_direction( + delta_values: [Option; 2], + effective_days: i64, + desired_days: i64, +) -> &'static str { + let mut values: Vec = delta_values.into_iter().flatten().collect(); + if values.is_empty() { + values.push(effective_days); + } + let increase = values.iter().any(|value| *value < desired_days); + let decrease = values.iter().any(|value| *value > desired_days); + match (increase, decrease) { + (true, true) => RETENTION_MIXED, + (true, false) => RETENTION_INCREASE, + (false, true) => RETENTION_DECREASE, + (false, false) => RETENTION_MATCH, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/source_call_names.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/source_call_names.rs new file mode 100644 index 0000000000..9031665579 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/source_call_names.rs @@ -0,0 +1,21 @@ +//! Source names called as `__source("name")` in SQL text. + +use crate::planner_warehouse::constants::SOURCE_CALL_OPEN; + +/// Every non-overlapping `__source("name")` name, in text order. +pub fn source_call_names(sql: &str) -> Vec { + let mut names: Vec = Vec::new(); + let mut position = 0; + while let Some(found) = sql[position..].find(SOURCE_CALL_OPEN) { + let start = position + found + SOURCE_CALL_OPEN.len(); + let rest = &sql[start..]; + match rest.find('"') { + Some(end) if end > 0 && rest[end + 1..].starts_with(')') => { + names.push(rest[..end].to_owned()); + position = start + end + 2; + } + _ => position += found + 1, + } + } + names +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/source_deferral_field.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/source_deferral_field.rs new file mode 100644 index 0000000000..c2d65329f8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/source_deferral_field.rs @@ -0,0 +1,63 @@ +//! Resolve a deferral target's database or schema template for one source. + +use crate::planner_warehouse::constants::{ + CONTEXT_PLACEHOLDER_OPEN, SOURCE_DEFERRAL_CONTEXT_FIELDS, VARIABLE_PLACEHOLDER_OPEN, +}; + +/// Substitute `${CTX:field}` with the logical value, then `${var}` through `render_variable`. +pub fn source_deferral_field( + (target_value, logical_value): (Option<&str>, Option<&str>), + render_variable: impl FnMut(&str) -> Result, E>, +) -> Result, E> { + let Some(target_value) = target_value else { + return Ok(logical_value.map(str::to_owned)); + }; + let contexts = substitute( + (target_value, CONTEXT_PLACEHOLDER_OPEN, |_| true), + |key: &str| { + Ok::<_, E>( + SOURCE_DEFERRAL_CONTEXT_FIELDS + .contains(&key) + .then(|| logical_value.unwrap_or_default().to_owned()), + ) + }, + )?; + let variables = substitute( + (&contexts, VARIABLE_PLACEHOLDER_OPEN, |name| { + !name.contains(':') + }), + render_variable, + )?; + Ok(Some(variables)) +} + +fn substitute( + (text, open, accepts): (&str, &str, fn(&str) -> bool), + mut replace: impl FnMut(&str) -> Result, E>, +) -> Result { + let mut output = String::new(); + let mut position = 0; + while let Some(found) = text[position..].find(open) { + let start = position + found; + let body_start = start + open.len(); + match text[body_start..].find('}') { + Some(length) if length > 0 && accepts(&text[body_start..body_start + length]) => { + let key = &text[body_start..body_start + length]; + output.push_str(&text[position..start]); + let end = body_start + length + 1; + match replace(key)? { + Some(value) => output.push_str(&value), + None => output.push_str(&text[start..end]), + } + position = end; + } + _ => { + let next = start + open.chars().next().map_or(1, char::len_utf8); + output.push_str(&text[position..next]); + position = next; + } + } + } + output.push_str(&text[position..]); + Ok(output) +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/table_type_copy_name.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/table_type_copy_name.rs new file mode 100644 index 0000000000..0c96d8c374 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/table_type_copy_name.rs @@ -0,0 +1,14 @@ +//! The deterministic conversion copy name of a table-type change. + +use crate::planner_warehouse::constants::TABLE_TYPE_COPY_PREFIX; +use sha2::{Digest, Sha256}; + +/// Prefix, the target name fitted to the identifier limit, and a short digest suffix. +pub fn table_type_copy_name(target_name: &str, identifier_limit: i64) -> String { + let digest = format!("{:x}", Sha256::digest(target_name.as_bytes())); + let suffix = format!("__{}", &digest[..12]); + let reserved = (TABLE_TYPE_COPY_PREFIX.chars().count() + suffix.chars().count()) as i64; + let available = usize::try_from((identifier_limit - reserved).max(0)).unwrap_or(0); + let fitted: String = target_name.chars().take(available).collect(); + format!("{TABLE_TYPE_COPY_PREFIX}{fitted}{suffix}") +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/mod.rs b/crates/sqlbuild-analysis/src/planner_warehouse/mod.rs new file mode 100644 index 0000000000..ec3c487e2e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/mod.rs @@ -0,0 +1,8 @@ +//! Pure warehouse-inspection planning decisions: cursor reads, retention, sources and caches. + +mod _helpers; +mod constants; +pub mod main; +pub mod models; +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/models.rs b/crates/sqlbuild-analysis/src/planner_warehouse/models.rs new file mode 100644 index 0000000000..7fa64309a2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/models.rs @@ -0,0 +1,52 @@ +//! Values exchanged with warehouse-inspection planning. + +/// One logical cursor bound wanted from a physical relation column. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CursorTagRequest { + pub relation: String, + pub cursor_column: String, + pub cursor_type: Option, + pub min_tag: Option, + pub max_tag: String, +} + +/// One physical relation and cursor column read once for every logical bound it serves. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PhysicalCursorQuery { + pub relation: String, + pub cursor_column: String, + pub cursor_type: String, + pub min_tags: Vec, + pub max_tags: Vec, +} + +/// A selected external source table the warehouse did not list and could not read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MissingSourceTable { + pub source_name: String, + pub relation: String, + pub read_by: Vec, + pub unreadable: bool, + pub role: Option, + pub location: String, +} + +/// The change facts that explain an incremental full rebuild. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FullRebuildFacts { + pub full_refresh: bool, + pub change_kind: String, + pub query_changed: bool, + pub changed_functions: Vec, + pub has_schema_findings: bool, +} + +/// The key parts of one migration fingerprint cache entry. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FingerprintCacheKey { + pub shared_digest: String, + pub query_sql: String, + pub metadata_json: String, + pub dialect: Option, + pub visible_renames: Vec<(String, String)>, +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/tests.rs b/crates/sqlbuild-analysis/src/planner_warehouse/tests.rs new file mode 100644 index 0000000000..9069837606 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/tests.rs @@ -0,0 +1,14 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_cursor_queries.rs"] +mod test_cursor_queries; +#[path = "tests/test_fingerprint_cache.rs"] +mod test_fingerprint_cache; +#[path = "tests/test_messages.rs"] +mod test_messages; +#[path = "tests/test_retention.rs"] +mod test_retention; +#[path = "tests/test_source_texts.rs"] +mod test_source_texts; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/tests/helpers.rs b/crates/sqlbuild-analysis/src/planner_warehouse/tests/helpers.rs new file mode 100644 index 0000000000..3687bf4c31 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/tests/helpers.rs @@ -0,0 +1,26 @@ +//! Shared builders for warehouse-inspection planning tests. + +use crate::planner_warehouse::models::{CursorTagRequest, MissingSourceTable}; + +/// A cursor tag request on the `updated_at` column. +pub(super) fn request(relation: &str, min_tag: Option<&str>, max_tag: &str) -> CursorTagRequest { + CursorTagRequest { + relation: relation.to_owned(), + cursor_column: "updated_at".to_owned(), + cursor_type: None, + min_tag: min_tag.map(str::to_owned), + max_tag: max_tag.to_owned(), + } +} + +/// A missing source table read by `read_by`, unreadable when a role is given. +pub(super) fn missing(name: &str, read_by: &[&str], role: Option<&str>) -> MissingSourceTable { + MissingSourceTable { + source_name: name.to_owned(), + relation: format!("raw.{name}"), + read_by: read_by.iter().map(|reader| (*reader).to_owned()).collect(), + unreadable: role.is_some(), + role: role.map(str::to_owned), + location: format!("sources/{name}.yml:2"), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs new file mode 100644 index 0000000000..10b026f12e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs @@ -0,0 +1,63 @@ +//! Logical cursor bounds share one physical read per relation column and type. + +use crate::planner_warehouse::main::cursor_query_groups::cursor_query_groups; +use crate::planner_warehouse::main::cursor_query_identity::cursor_query_identity; +use crate::planner_warehouse::main::cursor_query_sql::cursor_query_sql; +use crate::planner_warehouse::tests::helpers::request; +use crate::planner_warehouse::tests::test_types::CursorGroupTestCase; + +#[test] +fn given_logical_bounds_when_grouping_then_reads_each_relation_once_in_first_order() { + let test_cases = [ + CursorGroupTestCase { + description: "target max and shared upstream bounds merge", + requests: vec![ + request("analytics.orders", None, "orders__target__max"), + request( + "raw.events", + Some("orders__events__min"), + "orders__events__max", + ), + request( + "raw.events", + Some("totals__events__min"), + "totals__events__max", + ), + request( + "raw.events", + Some("orders__events__min"), + "orders__events__max", + ), + ], + expected_sql: vec![ + "SELECT CAST(MAX(updated_at) AS VARCHAR) AS _max FROM analytics.orders", + "SELECT CAST(MIN(updated_at) AS VARCHAR) AS _min, CAST(MAX(updated_at) AS VARCHAR) AS _max FROM raw.events", + ], + expected_identities: vec![ + "(1/2): analytics.orders.updated_at [max]", + "(2/2): raw.events.updated_at [min,max]", + ], + }, + CursorGroupTestCase { + description: "no requests read nothing", + requests: Vec::new(), + expected_sql: Vec::new(), + expected_identities: Vec::new(), + }, + ]; + for test_case in test_cases { + let groups = cursor_query_groups(&test_case.requests); + let sql: Vec = groups.iter().map(cursor_query_sql).collect(); + let identities: Vec = groups + .iter() + .enumerate() + .map(|(index, group)| cursor_query_identity(group, (index + 1, groups.len()))) + .collect(); + assert_eq!(sql, test_case.expected_sql, "{}", test_case.description); + assert_eq!( + identities, test_case.expected_identities, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_fingerprint_cache.rs b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_fingerprint_cache.rs new file mode 100644 index 0000000000..aacfbb7ead --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_fingerprint_cache.rs @@ -0,0 +1,102 @@ +//! Migration fingerprint cache keys and entries stay byte-identical to the stored format. + +use crate::planner_warehouse::main::fingerprint_cache_entry::fingerprint_cache_entry; +use crate::planner_warehouse::main::fingerprint_cache_key::fingerprint_cache_key; +use crate::planner_warehouse::main::fingerprint_cache_lookup::fingerprint_cache_lookup; +use crate::planner_warehouse::main::fingerprint_cache_shared_digest::fingerprint_cache_shared_digest; +use crate::planner_warehouse::models::FingerprintCacheKey; +use crate::planner_warehouse::tests::test_types::{ + FingerprintDigestTestCase, FingerprintLookupTestCase, +}; + +#[test] +fn given_key_parts_when_hashing_then_matches_stored_digests() { + let test_cases = [FingerprintDigestTestCase { + description: "python cache digests", + key: FingerprintCacheKey { + shared_digest: "shared".to_owned(), + query_sql: "SELECT 1".to_owned(), + metadata_json: "{}".to_owned(), + dialect: Some("duckdb".to_owned()), + visible_renames: vec![("orders".to_owned(), "abc".to_owned())], + }, + expected_key: "835490d6bfb10f11b9649eb56324524a30685ef55634d585940417179b4e28ad", + expected_shared: "ca6125db465a1a7f10fc6b210bb82bc6641998ca007324621992c1285325a121", + expected_entry: "391ffa0fdaf437fff6e7f6d1f0fc01e5509751580667cc37f101ba020c707eb6\n{\"f\":\"fp1\",\"k\":\"k1\",\"v\":1}", + }]; + for test_case in test_cases { + assert_eq!( + fingerprint_cache_key(&test_case.key).unwrap_or_default(), + test_case.expected_key, + "{}", + test_case.description + ); + assert_eq!( + fingerprint_cache_shared_digest("1.0", "2.0").unwrap_or_default(), + test_case.expected_shared, + "{}", + test_case.description + ); + assert_eq!( + fingerprint_cache_entry("k1", Some("fp1")).unwrap_or_default(), + test_case.expected_entry, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_stored_entry_when_looking_up_then_accepts_only_intact_entries_for_the_key() { + let test_cases = [ + FingerprintLookupTestCase { + description: "intact fingerprint", + stored_fingerprint: Some("fp1"), + tamper: None, + lookup_key: "k1", + expected_lookup: Some(Some("fp1")), + }, + FingerprintLookupTestCase { + description: "intact absent fingerprint", + stored_fingerprint: None, + tamper: None, + lookup_key: "k1", + expected_lookup: Some(None), + }, + FingerprintLookupTestCase { + description: "other key", + stored_fingerprint: Some("fp1"), + tamper: None, + lookup_key: "k2", + expected_lookup: None, + }, + FingerprintLookupTestCase { + description: "edited payload", + stored_fingerprint: Some("fp1"), + tamper: Some(("fp1", "fp2")), + lookup_key: "k1", + expected_lookup: None, + }, + FingerprintLookupTestCase { + description: "missing separator", + stored_fingerprint: Some("fp1"), + tamper: Some(("\n", "")), + lookup_key: "k1", + expected_lookup: None, + }, + ]; + for test_case in test_cases { + let entry = fingerprint_cache_entry("k1", test_case.stored_fingerprint).unwrap_or_default(); + let stored = test_case + .tamper + .map_or(entry.clone(), |(from, to)| entry.replace(from, to)); + assert_eq!( + fingerprint_cache_lookup(&stored, test_case.lookup_key), + test_case + .expected_lookup + .map(|found| found.map(str::to_owned)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_messages.rs b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_messages.rs new file mode 100644 index 0000000000..c119e89243 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_messages.rs @@ -0,0 +1,110 @@ +//! Planner messages keep their wording, limits and ordering. + +use crate::planner_warehouse::main::freshness_unknown_message::freshness_unknown_message; +use crate::planner_warehouse::main::full_rebuild_cause::full_rebuild_cause; +use crate::planner_warehouse::main::missing_source_tables_help::missing_source_tables_help; +use crate::planner_warehouse::main::missing_source_tables_message::missing_source_tables_message; +use crate::planner_warehouse::models::FullRebuildFacts; +use crate::planner_warehouse::tests::helpers::missing; +use crate::planner_warehouse::tests::test_types::{ + FreshnessMessageTestCase, MissingSourceTestCase, RebuildCauseTestCase, +}; + +#[test] +fn given_missing_source_tables_when_reporting_then_lists_readers_and_remedies() { + let test_cases = [ + MissingSourceTestCase { + description: "one missing table", + missing: vec![missing("orders", &["fact_orders"], None)], + expected_message: "cannot build selected scope: 1 source table read by selected resources does not exist in the warehouse:\n - source 'orders' (raw.orders), read by fact_orders", + expected_help: "create or load the table, correct its database, schema, or table in the source declaration, or stop reading it from the selected models; declared 'orders' at sources/orders.yml:2", + }, + MissingSourceTestCase { + description: "unreadable table names the role", + missing: vec![ + missing("orders", &["a", "b"], None), + missing("customers", &["c"], Some("ANALYST")), + ], + expected_message: "cannot build selected scope: 2 source tables read by selected resources do not exist in the warehouse or are not readable by the current role:\n - source 'orders' (raw.orders), read by a, b\n - source 'customers' (raw.customers), read by c; the warehouse reports it does not exist or is not readable by role ANALYST", + expected_help: "create or load the table, grant SELECT on it to the role in use (ANALYST), correct its database, schema, or table in the source declaration, or stop reading it from the selected models; declared 'orders' at sources/orders.yml:2, 'customers' at sources/customers.yml:2", + }, + ]; + for test_case in test_cases { + assert_eq!( + missing_source_tables_message(&test_case.missing), + test_case.expected_message, + "{}", + test_case.description + ); + assert_eq!( + missing_source_tables_help(&test_case.missing), + test_case.expected_help, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_unknown_freshness_when_warning_then_sorts_and_limits_names() { + let many: Vec = (0..12).rev().map(|index| format!("s{index:02}")).collect(); + let test_cases = [ + FreshnessMessageTestCase { + description: "error reason with many names", + reason: "error", + names: many, + expected_message: Some("source freshness unknown (error) for s00, s01, s02, s03, s04, s05, s06, s07, s08, s09 and 2 more: the freshness observation failed; run sqb freshness for each source's details".to_owned()), + }, + FreshnessMessageTestCase { + description: "unconfigured reason is silent", + reason: "no_freshness_config", + names: vec!["orders".to_owned()], + expected_message: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + freshness_unknown_message(test_case.reason, &test_case.names), + test_case.expected_message, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_change_facts_when_describing_rebuild_then_uses_first_matching_cause() { + let test_cases = [ + RebuildCauseTestCase { + description: "flag wins", + facts: FullRebuildFacts { + full_refresh: true, + change_kind: "first_run".to_owned(), + ..FullRebuildFacts::default() + }, + expected_cause: "--full-refresh", + }, + RebuildCauseTestCase { + description: "changed functions", + facts: FullRebuildFacts { + change_kind: "changed".to_owned(), + changed_functions: vec!["f".to_owned(), "g".to_owned()], + ..FullRebuildFacts::default() + }, + expected_cause: "function f, g changed with replay_on_change full", + }, + RebuildCauseTestCase { + description: "nothing specific", + facts: FullRebuildFacts::default(), + expected_cause: "full backfill", + }, + ]; + for test_case in test_cases { + assert_eq!( + full_rebuild_cause(&test_case.facts), + test_case.expected_cause, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_retention.rs b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_retention.rs new file mode 100644 index 0000000000..f36ac93554 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_retention.rs @@ -0,0 +1,124 @@ +//! Retention directions and phases follow inspected state and rendered change phases. + +use crate::planner_warehouse::main::retention_change::retention_change; +use crate::planner_warehouse::main::retention_direction::retention_direction; +use crate::planner_warehouse::main::table_type_copy_name::table_type_copy_name; +use crate::planner_warehouse::tests::test_types::{ + CopyNameTestCase, RetentionChangeTestCase, RetentionDirectionTestCase, +}; + +#[test] +fn given_inspected_retention_when_comparing_then_returns_direction() { + let test_cases = [ + RetentionDirectionTestCase { + description: "effective days below desired increase", + delta_values: [None, None], + effective_days: 1, + desired_days: 7, + expected_direction: "increase", + }, + RetentionDirectionTestCase { + description: "delta values on both sides are mixed", + delta_values: [Some(3), Some(30)], + effective_days: 7, + desired_days: 7, + expected_direction: "mixed", + }, + RetentionDirectionTestCase { + description: "delta value above desired decreases", + delta_values: [None, Some(30)], + effective_days: 1, + desired_days: 7, + expected_direction: "decrease", + }, + RetentionDirectionTestCase { + description: "equal days match", + delta_values: [None, None], + effective_days: 7, + desired_days: 7, + expected_direction: "match", + }, + ]; + for test_case in test_cases { + assert_eq!( + retention_direction( + test_case.delta_values, + test_case.effective_days, + test_case.desired_days + ), + test_case.expected_direction, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_rendered_change_when_phasing_then_prepare_and_finalize_override_fallback() { + let test_cases = [ + RetentionChangeTestCase { + description: "prepare increases first", + change_phase: "prepare", + fallback: "mixed", + expected_change: ("increase", "pre"), + }, + RetentionChangeTestCase { + description: "finalize decreases last", + change_phase: "finalize", + fallback: "mixed", + expected_change: ("decrease", "post"), + }, + RetentionChangeTestCase { + description: "alter increase runs before", + change_phase: "alter", + fallback: "increase", + expected_change: ("increase", "pre"), + }, + RetentionChangeTestCase { + description: "alter decrease runs after", + change_phase: "alter", + fallback: "decrease", + expected_change: ("decrease", "post"), + }, + ]; + for test_case in test_cases { + assert_eq!( + retention_change(test_case.change_phase, test_case.fallback), + test_case.expected_change, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_target_name_when_naming_copy_then_fits_identifier_limit() { + let test_cases = [ + CopyNameTestCase { + description: "short name keeps whole target", + target_name: "orders", + identifier_limit: 255, + expected_name: "__sqb_type_swap__orders__1c168adb00d2", + }, + CopyNameTestCase { + description: "tight limit truncates target", + target_name: "orders", + identifier_limit: 34, + expected_name: "__sqb_type_swap__ord__1c168adb00d2", + }, + CopyNameTestCase { + description: "limit below affixes drops target", + target_name: "orders", + identifier_limit: 3, + expected_name: "__sqb_type_swap____1c168adb00d2", + }, + ]; + for test_case in test_cases { + assert_eq!( + table_type_copy_name(test_case.target_name, test_case.identifier_limit), + test_case.expected_name, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_source_texts.rs b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_source_texts.rs new file mode 100644 index 0000000000..0ef4ccfaae --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_source_texts.rs @@ -0,0 +1,70 @@ +//! Source calls and deferral templates follow Python's regular expression semantics. + +use crate::planner_warehouse::main::source_call_names::source_call_names; +use crate::planner_warehouse::main::source_deferral_field::source_deferral_field; +use crate::planner_warehouse::tests::test_types::{DeferralFieldTestCase, SourceCallTestCase}; + +#[test] +fn given_sql_when_scanning_source_calls_then_returns_quoted_names_in_order() { + let test_cases = [ + SourceCallTestCase { + description: "two calls", + sql: r#"SELECT * FROM __source("orders") JOIN __source("customers")"#, + expected_names: vec!["orders", "customers"], + }, + SourceCallTestCase { + description: "unterminated and empty calls are skipped", + sql: r#"__source("") __source("a" x) __source("b")"#, + expected_names: vec!["b"], + }, + SourceCallTestCase { + description: "no calls", + sql: "SELECT 1", + expected_names: Vec::new(), + }, + ]; + for test_case in test_cases { + assert_eq!( + source_call_names(test_case.sql), + test_case.expected_names, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_target_template_when_resolving_then_substitutes_context_and_known_variables() { + let test_cases = [ + DeferralFieldTestCase { + description: "absent target keeps logical value", + target_value: None, + logical_value: Some("raw"), + expected_value: Some("raw"), + }, + DeferralFieldTestCase { + description: "context and variables substitute", + target_value: Some("${CTX:schema}_${env}_${missing}_${CTX:other}"), + logical_value: Some("raw"), + expected_value: Some("raw_prod_${missing}_${CTX:other}"), + }, + DeferralFieldTestCase { + description: "variable with colon is retried at the next character", + target_value: Some("${a:${env}}"), + logical_value: None, + expected_value: Some("${a:prod}"), + }, + ]; + for test_case in test_cases { + let resolved = source_deferral_field( + (test_case.target_value, test_case.logical_value), + |name: &str| Ok::<_, ()>((name == "env").then(|| "prod".to_owned())), + ); + assert_eq!( + resolved, + Ok(test_case.expected_value.map(str::to_owned)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_types.rs new file mode 100644 index 0000000000..f4ae048082 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_types.rs @@ -0,0 +1,79 @@ +//! Cases for warehouse-inspection planning decisions. + +pub(super) struct CursorGroupTestCase { + pub description: &'static str, + pub requests: Vec, + pub expected_sql: Vec<&'static str>, + pub expected_identities: Vec<&'static str>, +} + +pub(super) struct RetentionDirectionTestCase { + pub description: &'static str, + pub delta_values: [Option; 2], + pub effective_days: i64, + pub desired_days: i64, + pub expected_direction: &'static str, +} + +pub(super) struct RetentionChangeTestCase { + pub description: &'static str, + pub change_phase: &'static str, + pub fallback: &'static str, + pub expected_change: (&'static str, &'static str), +} + +pub(super) struct CopyNameTestCase { + pub description: &'static str, + pub target_name: &'static str, + pub identifier_limit: i64, + pub expected_name: &'static str, +} + +pub(super) struct SourceCallTestCase { + pub description: &'static str, + pub sql: &'static str, + pub expected_names: Vec<&'static str>, +} + +pub(super) struct DeferralFieldTestCase { + pub description: &'static str, + pub target_value: Option<&'static str>, + pub logical_value: Option<&'static str>, + pub expected_value: Option<&'static str>, +} + +pub(super) struct FingerprintLookupTestCase { + pub description: &'static str, + pub stored_fingerprint: Option<&'static str>, + pub tamper: Option<(&'static str, &'static str)>, + pub lookup_key: &'static str, + pub expected_lookup: Option>, +} + +pub(super) struct MissingSourceTestCase { + pub description: &'static str, + pub missing: Vec, + pub expected_message: &'static str, + pub expected_help: &'static str, +} + +pub(super) struct FreshnessMessageTestCase { + pub description: &'static str, + pub reason: &'static str, + pub names: Vec, + pub expected_message: Option, +} + +pub(super) struct RebuildCauseTestCase { + pub description: &'static str, + pub facts: crate::planner_warehouse::models::FullRebuildFacts, + pub expected_cause: &'static str, +} + +pub(super) struct FingerprintDigestTestCase { + pub description: &'static str, + pub key: crate::planner_warehouse::models::FingerprintCacheKey, + pub expected_key: &'static str, + pub expected_shared: &'static str, + pub expected_entry: &'static str, +} diff --git a/src/sqlbuild/adapter/relations/main/get_columns_for_inspection.py b/src/sqlbuild/adapter/relations/main/_get_columns_for_inspection.py similarity index 100% rename from src/sqlbuild/adapter/relations/main/get_columns_for_inspection.py rename to src/sqlbuild/adapter/relations/main/_get_columns_for_inspection.py diff --git a/src/sqlbuild/adapter/relations/main/list_relations_for_inspection.py b/src/sqlbuild/adapter/relations/main/_list_relations_for_inspection.py similarity index 100% rename from src/sqlbuild/adapter/relations/main/list_relations_for_inspection.py rename to src/sqlbuild/adapter/relations/main/_list_relations_for_inspection.py diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/__init__.py b/src/sqlbuild/compiler/planner/_helpers/planning/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/buildability.py b/src/sqlbuild/compiler/planner/_helpers/planning/buildability.py deleted file mode 100644 index 1e6d2a951d..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/planning/buildability.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Selected-scope buildability validation for execution planning.""" - -from __future__ import annotations - -from sqlbuild._native import ( - build_execution_edge_origins, - check_buildability, - missing_upstream_message, -) -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.warehouse.source_tables import ( - check_selected_source_tables_exist, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - DeferralInputs, - MissingUpstream, - PlannerRuntime, - PlannerScopeResolution, - PlannerWarehouseState, - WarehouseSnapshot, -) - - -def check_selected_scope_buildability( - *, - runtime: PlannerRuntime, - scopes: PlannerScopeResolution, - warehouse: PlannerWarehouseState, - deferral: DeferralInputs, -) -> None: - """Raise a planner input error when selected upstream inputs are missing.""" - - project: CompiledProjectView = runtime.project - snapshot: WarehouseSnapshot = warehouse.snapshot - external_seed_keys: frozenset[CompiledObjectKey] = frozenset( - seed.key for seed in project.seeds if seed.external - ) - missing: tuple[MissingUpstream, ...] = check_buildability( - selected_keys=scopes.selected_scope.selected_keys, - upstream_deps=scopes.selected_scope.upstream_deps, - snapshot=snapshot, - deferred_relations=deferral.deferred_relations, - satisfied_keys=external_seed_keys, - ) - if missing: - raise PlannerInputError( - missing_upstream_message( - missing=missing, edge_origins=build_execution_edge_origins(project) - ), - code="S301", - ) - check_selected_source_tables_exist( - project=project, - adapter=runtime.adapter, - connection=runtime.connection, - scope=scopes.selected_scope, - relations=warehouse.inspection_relations, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/entries.py b/src/sqlbuild/compiler/planner/_helpers/planning/entries.py deleted file mode 100644 index 7f65552afb..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/planning/entries.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Plan entry construction phase for execution planning.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.output.plan_entry import ( - build_plan_entries, - build_planner_relations_context, -) -from sqlbuild.compiler.planner.models import ( - DeferralInputs, - PlanEntryBuildInputs, - PlannerChangeReconciliation, - PlannerEntryResults, - PlannerIdentityContext, - PlannerModelEntryResults, - PlannerOverrides, - PlannerPolicies, - PlannerRelationsContext, - PlannerRuntime, - PlannerScopePruningResult, - PlannerWarehouseState, -) -from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult - - -def build_planner_entry_results( - *, - runtime: PlannerRuntime, - warehouse: PlannerWarehouseState, - identities: PlannerIdentityContext, - overrides: PlannerOverrides, - policies: PlannerPolicies, - deferral: DeferralInputs, - pruning: PlannerScopePruningResult, - reconciliation: PlannerChangeReconciliation, - source_freshness: DirectSourceFreshnessPlanningResult, -) -> PlannerEntryResults: - """Build execution model entries for one plan build.""" - - execution_relations: PlannerRelationsContext = build_planner_relations_context( - project=runtime.project, - adapter=runtime.adapter, - connection=runtime.connection, - scope=pruning.execution_scope, - deferral=deferral, - project_config=runtime.project_config, - local_config=runtime.local_config, - known_source_columns=warehouse.inspection_relations.source_warehouse_columns, - ) - model_entry_results: PlannerModelEntryResults = build_plan_entries( - project=runtime.project, - adapter=runtime.adapter, - scope=pruning.execution_scope, - snapshot=warehouse.snapshot, - relations=execution_relations, - resolved_actions=reconciliation.resolved_actions, - cursor_overrides=overrides.cursor_overrides, - full_refresh=overrides.full_refresh, - build_inputs=PlanEntryBuildInputs( - source_freshness_blocked_model_names=( - source_freshness.propagation.blocked_model_names - if source_freshness.propagation is not None - else frozenset() - ), - external_blocked_model_names=frozenset(overrides.external_blocked_model_names), - future_cursor_config=( - runtime.project_config.cursors.future - if runtime.project_config is not None - else None - ), - start_cursor_config=( - runtime.project_config.cursors.start if runtime.project_config is not None else None - ), - invocation_time=runtime.invocation_time, - max_microbatches=( - overrides.max_microbatches - if overrides.max_microbatches is not None - else ( - runtime.project_config.microbatches.limits.max_batches - if runtime.project_config is not None - else None - ) - ), - max_microbatches_is_override=overrides.max_microbatches is not None, - microbatch_limit_action=( - runtime.project_config.microbatches.limits.action - if runtime.project_config is not None - else PlanEntryBuildInputs().microbatch_limit_action - ), - ), - ) - return PlannerEntryResults( - model_entry_results=model_entry_results, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/full_refresh.py b/src/sqlbuild/compiler/planner/_helpers/planning/full_refresh.py deleted file mode 100644 index 88e57852f1..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/planning/full_refresh.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Per-model full-refresh resolution.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.planner.constants import FULL_REFRESH_DISABLED_REBUILD_CODE -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ChangeDetectionResult -from sqlbuild.compiler.planner.types import ChangeKind - - -def resolve_model_full_refresh(*, model: CompiledModelView, cli_full_refresh: bool) -> bool: - """Resolve dbt-compatible nullable model full-refresh semantics.""" - - configured: object | None = model.config.values.get("full_refresh") - return configured if isinstance(configured, bool) else cli_full_refresh - - -def effectively_full_refreshed_model_names( - *, project: CompiledProjectView, cli_full_refresh: bool -) -> frozenset[str]: - """Return model names that are effectively full refreshed for this invocation.""" - - return frozenset( - model.name - for model in project.models - if resolve_model_full_refresh(model=model, cli_full_refresh=cli_full_refresh) - ) - - -def describe_full_rebuild_cause(*, change_result: ChangeDetectionResult, full_refresh: bool) -> str: - """Describe why an incremental model is planned as a full rebuild.""" - - if full_refresh: - return "--full-refresh" - if change_result.change_kind == ChangeKind.FIRST_RUN: - return "first run" - if change_result.change_kind == ChangeKind.RENAMED: - return "rename" - if change_result.query_changed: - return "query changed with replay_on_change full" - if change_result.changed_functions: - names: str = ", ".join(change_result.changed_functions) - return f"function {names} changed with replay_on_change full" - if change_result.schema_findings: - return "schema changed with replay_on_change full" - return "full backfill" - - -def check_full_rebuild_allowed( - *, model: CompiledModelView, change_result: ChangeDetectionResult, full_rebuild_cause: str -) -> None: - """Refuse a full rebuild of a full_refresh false model unless its own SQL or first run asks.""" - - if model.config.values.get("full_refresh") is not False: - return - if change_result.change_kind == ChangeKind.FIRST_RUN or change_result.query_changed: - return - raise PlannerInputError( - f"model '{model.name}' sets full_refresh false, but the plan would fully rebuild it " - f"({full_rebuild_cause})", - code=FULL_REFRESH_DISABLED_REBUILD_CODE, - help=( - "set replay_on_change to forward or bounded- on this model to keep its " - "existing history, or remove full_refresh false to allow the rebuild" - ), - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/identities.py b/src/sqlbuild/compiler/planner/_helpers/planning/identities.py deleted file mode 100644 index 30bcab34c4..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/planning/identities.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Version identity and stale-warning change phases for execution planning.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild._native import build_direct_model_version_identities, detect_model_changes_in_scope -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.planner.models import ( - ChangeDetectionResult, - DirectModelVersionIdentities, - PlannerChangeResults, - PlannerIdentityContext, - PlannerScopeResolution, - WarehouseSnapshot, -) - - -def build_planner_identity_context( - *, - project: CompiledProjectView, - scopes: PlannerScopeResolution, - include_stale_warning_identities: bool = True, -) -> PlannerIdentityContext: - """Build expected version identities for the inspection and stale-warning scopes.""" - - version_identities: DirectModelVersionIdentities = build_direct_model_version_identities( - functions=project.functions, - seeds=project.seeds, - scope=scopes.inspection_scope, - hook_functions=project.hook_functions, - dialect=project.sql_analysis_dialect, - ) - return PlannerIdentityContext( - version_identities=version_identities, - stale_warning_identities=( - build_direct_model_version_identities( - functions=project.functions, - seeds=project.seeds, - scope=scopes.stale_warning_scope, - hook_functions=project.hook_functions, - dialect=project.sql_analysis_dialect, - ) - if include_stale_warning_identities - else version_identities - ), - ) - - -def detect_stale_warning_changes( - *, - project: CompiledProjectView, - scopes: PlannerScopeResolution, - snapshot: WarehouseSnapshot, - identities: PlannerIdentityContext, - execution_changes: PlannerChangeResults, -) -> PlannerChangeResults: - """Detect only stale-warning model changes not already known from execution planning.""" - - reusable_model_names: frozenset[str] = _reusable_execution_model_names( - identities=identities, - execution_changes=execution_changes, - query_change_tracking=project.settings.query_change_tracking, - ) - stale_keys: frozenset[CompiledObjectKey] = frozenset( - scopes.stale_warning_scope.all_keys.values() - ) - remaining_model_keys: frozenset[CompiledObjectKey] = frozenset( - key - for key in stale_keys - if key.resource_type == CompiledResourceType.MODEL and key.name not in reusable_model_names - ) - remaining_model_changes: dict[str, ChangeDetectionResult] = detect_model_changes_in_scope( - project=project, - scope=replace( - scopes.stale_warning_scope, - selected_keys=remaining_model_keys, - ), - snapshot=snapshot, - query_change_tracking=True, - expected_version_hashes=identities.stale_warning_identities.model_version_hashes, - expected_metadata_jsons=identities.stale_warning_identities.model_metadata_jsons, - ) - return PlannerChangeResults( - models={ - **{ - name: change - for name, change in execution_changes.models.items() - if name in reusable_model_names - }, - **remaining_model_changes, - }, - functions={}, - ) - - -def _reusable_execution_model_names( - *, - identities: PlannerIdentityContext, - execution_changes: PlannerChangeResults, - query_change_tracking: bool, -) -> frozenset[str]: - if not query_change_tracking: - return frozenset() - execution_identities: DirectModelVersionIdentities = identities.version_identities - stale_identities: DirectModelVersionIdentities = identities.stale_warning_identities - return frozenset( - model_name - for model_name in execution_changes.models - if ( - execution_identities.model_version_hashes.get(model_name) - == stale_identities.model_version_hashes.get(model_name) - and execution_identities.model_metadata_jsons.get(model_name) - == stale_identities.model_metadata_jsons.get(model_name) - ) - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py b/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py deleted file mode 100644 index 209616597e..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py +++ /dev/null @@ -1,270 +0,0 @@ -"""Plan output assembly phases for execution planning.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild._native import ( - build_stale_out_of_selection_warnings, - with_migration_fingerprints, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner._helpers.output.plan_output import build_plan_output -from sqlbuild.compiler.planner._helpers.planning.retention import plan_retention, plan_table_types -from sqlbuild.compiler.planner._helpers.warehouse.source_freshness_warnings import ( - build_source_freshness_unknown_warnings, -) -from sqlbuild.compiler.planner.models import ( - ColumnRenameHint, - ModelPlanEntry, - PlannedSqlTests, - PlannerChangeReconciliation, - PlannerChangeResults, - PlannerEntryResults, - PlannerIdentityContext, - PlannerOverrides, - PlannerPolicies, - PlannerRuntime, - PlannerScopePruningResult, - PlannerScopeResolution, - PlannerWarehouseState, - PlanOutput, - PlanOutputExtras, - PlanWarning, -) -from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult -from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus - - -def assemble_base_plan_output( - *, - runtime: PlannerRuntime, - warehouse: PlannerWarehouseState, - identities: PlannerIdentityContext, - overrides: PlannerOverrides, - pruning: PlannerScopePruningResult, - reconciliation: PlannerChangeReconciliation, - entries: PlannerEntryResults, - source_freshness: DirectSourceFreshnessPlanningResult, - planned_sql_tests: PlannedSqlTests | None = None, -) -> PlanOutput: - """Assemble the base plan output with freshness and pruning metadata attached.""" - - plan_output: PlanOutput = build_plan_output( - project=runtime.project, - adapter=runtime.adapter, - scope=pruning.execution_scope, - snapshot=warehouse.snapshot, - relations=warehouse.inspection_relations, - changes=reconciliation.changes, - model_entry_results=entries.model_entry_results, - reload_sources=overrides.reload_sources, - extras=PlanOutputExtras( - seed_version_hashes=identities.version_identities.seed_version_hashes, - seed_metadata_jsons=identities.version_identities.seed_metadata_jsons, - planned_sql_tests=planned_sql_tests, - ), - ) - plan_output = replace( - plan_output, - source_freshness=source_freshness, - cursor_overrides=overrides.cursor_overrides, - ) - if pruning.pruned_direct_model_names: - plan_output = replace( - plan_output, - metadata={ - **plan_output.metadata, - "direct_pruned_model_names": pruning.pruned_direct_model_names, - }, - ) - return plan_output - - -def with_storage_policies( - *, - plan_output: PlanOutput, - runtime: PlannerRuntime, - warehouse: PlannerWarehouseState, - scopes: PlannerScopeResolution, - policies: PlannerPolicies, -) -> PlanOutput: - """Attach migrations and selected Snowflake table-type and retention work to the plan.""" - - return replace( - plan_output, - model_entries=with_column_rename_hints( - entries=with_migration_fingerprints( - entries=plan_output.model_entries, - models_by_name={model.name: model for model in runtime.project.models}, - fingerprints=warehouse.migration_fingerprints, - dialect=runtime.adapter.sql_analysis_dialect(), - ) - if policies.record_migration_fingerprints - else plan_output.model_entries, - hints=warehouse.column_rename_hints, - ), - migration_entries=warehouse.migration_entries, - column_migration_entries=warehouse.column_migration_entries, - old_name_view_entries=warehouse.old_name_view_entries, - old_name_views=warehouse.old_name_views, - warnings=(*warehouse.migration_warnings, *plan_output.warnings), - table_type_entries=plan_table_types( - runtime=runtime, warehouse=warehouse, scope=scopes.selected_scope - ), - retention_entries=plan_retention( - runtime=runtime, warehouse=warehouse, scope=scopes.selected_scope - ), - ) - - -def with_column_rename_hints( - *, entries: tuple[ModelPlanEntry, ...], hints: tuple[ColumnRenameHint, ...] -) -> tuple[ModelPlanEntry, ...]: - """Attach near-match column rename hints to the model entries they describe.""" - - by_model: dict[str, list[ColumnRenameHint]] = {} - hint: ColumnRenameHint - for hint in hints: - by_model.setdefault(hint.model_name, []).append(hint) - return tuple( - replace(entry, column_rename_hints=tuple(by_model[entry.name])) - if entry.name in by_model - else entry - for entry in entries - ) - - -def with_plan_warnings( - *, - runtime: PlannerRuntime, - scopes: PlannerScopeResolution, - warehouse: PlannerWarehouseState, - identities: PlannerIdentityContext, - stale_warning_changes: PlannerChangeResults, - pruning: PlannerScopePruningResult, - source_freshness: DirectSourceFreshnessPlanningResult, - plan_output: PlanOutput, - policies: PlannerPolicies, -) -> PlanOutput: - """Append source freshness and stale-out-of-selection warnings to the plan.""" - - freshness_warnings: tuple[PlanWarning, ...] = build_source_freshness_unknown_warnings( - source_freshness=source_freshness - ) - if freshness_warnings: - plan_output = replace( - plan_output, - warnings=(*plan_output.warnings, *freshness_warnings), - ) - if not policies.selection_diagnostics: - return plan_output - stale_out_of_selection_warnings: tuple[PlanWarning, ...] = ( - build_stale_out_of_selection_warnings( - original_scope=scopes.stale_warning_scope, - execution_scope=pruning.execution_scope, - changes=stale_warning_changes, - snapshot=warehouse.snapshot, - version_identities=identities.stale_warning_identities, - source_freshness=source_freshness, - include_sources=False, - model_changes_complete=True, - ) - ) - if stale_out_of_selection_warnings: - plan_output = replace( - plan_output, - warnings=(*plan_output.warnings, *stale_out_of_selection_warnings), - ) - return plan_output - - -def with_plan_metadata( - *, - plan_output: PlanOutput, - pruning: PlannerScopePruningResult, - source_freshness: DirectSourceFreshnessPlanningResult, - policies: PlannerPolicies, -) -> PlanOutput: - """Attach direct source-freshness metadata to the plan output.""" - - direct_remaining_stale_model_names: tuple[str, ...] = tuple( - sorted( - pruning.direct_identity_stale_model_names - - frozenset( - key.name - for key in pruning.inspection_scope.selected_keys - if key.resource_type == CompiledResourceType.MODEL - ) - ) - ) - plan_output = replace( - plan_output, - metadata={ - **plan_output.metadata, - "direct_source_freshness": _serialize_direct_source_freshness_metadata( - source_freshness - ), - "direct_remaining_stale_model_names": direct_remaining_stale_model_names, - "selection_diagnostics": { - "mode": "direct", - "enabled": policies.selection_diagnostics, - }, - }, - ) - return plan_output - - -def _serialize_direct_source_freshness_metadata( - source_freshness: DirectSourceFreshnessPlanningResult, -) -> dict[str, object]: - changed_source_names: tuple[str, ...] = tuple( - sorted(identity.source_name for identity in source_freshness.changed_identities) - ) - unchanged_source_names: tuple[str, ...] = tuple( - sorted(identity.source_name for identity in source_freshness.unchanged_identities) - ) - stale_model_names: tuple[str, ...] = ( - tuple(sorted(source_freshness.propagation.stale_model_names)) - if source_freshness.propagation is not None - else () - ) - blocked_model_names: tuple[str, ...] = ( - tuple(sorted(source_freshness.propagation.blocked_model_names)) - if source_freshness.propagation is not None - else () - ) - age_warning_source_names: tuple[str, ...] = tuple( - sorted( - identity.source_name - for identity, status in source_freshness.age_statuses.items() - if status == SourceFreshnessAgeStatus.WARN - ) - ) - age_error_source_names: tuple[str, ...] = tuple( - sorted( - identity.source_name - for identity, status in source_freshness.age_statuses.items() - if status == SourceFreshnessAgeStatus.ERROR - ) - ) - return { - "observed_source_names": tuple( - sorted(record.source_name for record in source_freshness.observed_records) - ), - "changed_source_names": changed_source_names, - "unchanged_source_names": unchanged_source_names, - "unknown_source_names": tuple(sorted(source_freshness.unknown_source_names)), - "unknown_source_details": tuple( - { - "source_name": unknown.source_name, - "reason": unknown.reason.value, - "message": unknown.message, - } - for unknown in source_freshness.unknown_sources.values() - ), - "age_warning_source_names": age_warning_source_names, - "age_error_source_names": age_error_source_names, - "stale_model_names": stale_model_names, - "blocked_model_names": blocked_model_names, - } diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/reconciliation.py b/src/sqlbuild/compiler/planner/_helpers/planning/reconciliation.py deleted file mode 100644 index d1070fcc6b..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/planning/reconciliation.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Write-hash reconciliation phase for execution planning.""" - -from __future__ import annotations - -from sqlbuild._native import merge_recomputed_model_changes, with_honest_model_write_hashes -from sqlbuild.compiler.planner.models import ( - PlannerChangeReconciliation, - PlannerChangeResults, - PlannerIdentityContext, - PlannerResolvedActions, - PlannerScopePruningResult, - PlannerWarehouseState, -) - - -def reconcile_execution_changes( - *, - warehouse: PlannerWarehouseState, - identities: PlannerIdentityContext, - pruning: PlannerScopePruningResult, - changes: PlannerChangeResults, -) -> PlannerChangeReconciliation: - """Recompute honest write hashes and merge them into resolved actions.""" - - honest_changes: PlannerChangeResults = with_honest_model_write_hashes( - scope=pruning.execution_scope, - snapshot=warehouse.snapshot, - changes=changes, - version_identities=identities.version_identities, - ) - resolved_actions: PlannerResolvedActions = merge_recomputed_model_changes( - resolved_actions=pruning.resolved_actions, - changes=honest_changes, - ) - return PlannerChangeReconciliation(changes=honest_changes, resolved_actions=resolved_actions) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/retention.py b/src/sqlbuild/compiler/planner/_helpers/planning/retention.py deleted file mode 100644 index 2e508119f6..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/planning/retention.py +++ /dev/null @@ -1,455 +0,0 @@ -"""Warehouse-aware retention planning.""" - -from __future__ import annotations - -import hashlib -from collections import defaultdict -from dataclasses import replace - -from sqlbuild._native import resolve_target_config -from sqlbuild.adapter.contract.models import ( - RelationInfo, - RelationLookup, - RenderedRetentionChange, - RetentionRequest, - RetentionState, -) -from sqlbuild.adapter.contract.types import BuiltinAdapter, RetentionChangePhase, RetentionScope -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - PlannerRuntime, - PlannerScope, - PlannerWarehouseState, - RetentionPlanEntry, - TableTypePlanEntry, -) -from sqlbuild.compiler.planner.types import ( - MaterializationType, - RetentionDirection, - RetentionPlanPhase, -) -from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention, TargetConfig -from sqlbuild.spec.contracts.types import TableType - -_DECREASE_WARNING: str = "Retention decreases are irreversible once warehouse history expires." -_DOWNGRADE_HISTORY_DAYS: int = 90 -_TABLE_TYPE_COPY_PREFIX: str = "__sqb_type_swap__" - - -def plan_table_types( - *, runtime: PlannerRuntime, warehouse: PlannerWarehouseState, scope: PlannerScope -) -> tuple[TableTypePlanEntry, ...]: - """Produce independent entries for selected Snowflake table-type drift.""" - - selected_names: frozenset[str] = frozenset( - key.name for key in scope.selected_keys if key.resource_type == CompiledResourceType.MODEL - ) - selected_models: tuple[CompiledModelView, ...] = tuple( - model for model in runtime.project.models if model.name in selected_names - ) - declared_models: tuple[CompiledModelView, ...] = tuple( - model for model in selected_models if model.config.table_type.declared - ) - if declared_models and runtime.adapter.adapter_name != BuiltinAdapter.SNOWFLAKE: - names: str = ", ".join(model.name for model in declared_models) - raise PlannerInputError( - f"table_type is not supported on this adapter; declared by model(s): {names}" - ) - if runtime.adapter.adapter_name != BuiltinAdapter.SNOWFLAKE: - return () - target: TargetConfig = effective_target_config(runtime=runtime) - entries: list[TableTypePlanEntry] = [] - for model in selected_models: - if not MaterializationType.is_table_backed( - materialized=model.config.values.get("materialized") - ): - continue - relation: RelationInfo | None = warehouse.snapshot.existing_relations.get(model.name) - if relation is None: - continue - desired_transient: bool = model.config.table_type.value == TableType.TRANSIENT - if relation.is_transient is not None and relation.is_transient == desired_transient: - continue - downgrade: bool = relation.is_transient is False and desired_transient - entries.append( - TableTypePlanEntry( - model_name=model.name, - destination=model.destination, - copy_name=table_type_copy_name( - target_name=model.destination.name, - identifier_limit=runtime.adapter.maximum_identifier_length(), - ), - desired_type=model.config.table_type.value.value, - actual_type=( - None - if relation.is_transient is None - else TableType.TRANSIENT.value - if relation.is_transient - else TableType.PERMANENT.value - ), - source=model.config.table_type.source.value, - downgrade=downgrade, - downgrade_policy=target.table_type_downgrade.value, - irreversible_warning=( - f"Downgrading may discard up to {_DOWNGRADE_HISTORY_DAYS} days of " - "time-travel history." - if downgrade - else None - ), - ) - ) - return tuple(entries) - - -def table_type_copy_name(*, target_name: str, identifier_limit: int) -> str: - """Return a deterministic identifier-fitted conversion copy name.""" - - digest: str = hashlib.sha256(target_name.encode()).hexdigest()[:12] - suffix: str = f"__{digest}" - available: int = max(0, identifier_limit - len(_TABLE_TYPE_COPY_PREFIX) - len(suffix)) - return f"{_TABLE_TYPE_COPY_PREFIX}{target_name[:available]}{suffix}" - - -def plan_retention( - *, - runtime: PlannerRuntime, - warehouse: PlannerWarehouseState, - scope: PlannerScope, -) -> tuple[RetentionPlanEntry, ...]: - """Inspect managed selected retention scopes and produce independent plan entries.""" - - selected_names: frozenset[str] = frozenset( - key.name for key in scope.selected_keys if key.resource_type == CompiledResourceType.MODEL - ) - selected_models: tuple[CompiledModelView, ...] = tuple( - model - for model in runtime.project.models - if model.name in selected_names - and not model.config.time_travel_retention.unmanaged - and model.config.time_travel_retention.desired_days is not None - ) - if not selected_models: - return () - entries: list[RetentionPlanEntry] = [] - if runtime.adapter.adapter_name == BuiltinAdapter.BIGQUERY: - entries.extend(_plan_bigquery_retention(runtime=runtime, models=selected_models)) - else: - for model in selected_models: - entries.extend( - _plan_relation_retention(runtime=runtime, warehouse=warehouse, model=model) - ) - decrease_policy: str = effective_target_config( - runtime=runtime - ).time_travel_retention_decrease.value - return tuple(replace(entry, decrease_policy=decrease_policy) for entry in entries) - - -def _plan_relation_retention( - *, runtime: PlannerRuntime, warehouse: PlannerWarehouseState, model: CompiledModelView -) -> tuple[RetentionPlanEntry, ...]: - retention: ResolvedTimeTravelRetention = model.config.time_travel_retention - if retention.desired_days is None or retention.source is None: - raise PlannerInputError(f"model '{model.name}': managed retention is incomplete") - desired_days: int = retention.desired_days - source: str = retention.source.value - schema: str | None = model.destination.schema - if schema is None: - raise PlannerInputError(f"model '{model.name}': retention requires a destination schema") - request: RetentionRequest = RetentionRequest( - request_id=model.name, - scope=RetentionScope.RELATION, - database=model.destination.database, - schema=schema, - name=model.destination.name, - desired_days=desired_days, - ) - if model.name not in warehouse.snapshot.existing_relations: - if ( - runtime.adapter.adapter_name == BuiltinAdapter.SNOWFLAKE - and desired_days > 1 - and model.config.table_type.value == TableType.TRANSIENT - ): - raise PlannerInputError( - f"model '{model.name}': Snowflake transient tables cannot retain " - f"time travel for {desired_days} days; retention above one day requires a " - "permanent table; set table_type permanent on the model, its materialization " - "default, or the target default_table_type" - ) - changes: tuple[RenderedRetentionChange, ...] = runtime.adapter.render_retention_changes( - request=request - ) - return ( - _entry( - request=request, - model_names=(model.name,), - state=None, - source=source, - direction=RetentionDirection.APPLY_AFTER_CREATE, - phase=RetentionPlanPhase.AFTER_CREATE, - statements=_flatten_changes(changes=changes), - ), - ) - relation: RelationInfo = warehouse.snapshot.existing_relations[model.name] - relation_is_transient: bool | None = relation.is_transient - listed_state: RetentionState | None = None - if relation.identity == RelationLookup.key( - database=model.destination.database, schema=schema, name=model.destination.name - ): - listed_state = runtime.adapter.retention_state_from_relation( - request=request, relation=relation - ) - state: RetentionState = listed_state or runtime.adapter.inspect_retention( - connection=runtime.connection, request=request - ) - if ( - state.is_transient - and desired_days > 1 - and model.config.table_type.value == TableType.TRANSIENT - ): - raise PlannerInputError( - f"model '{model.name}': Snowflake transient tables cannot retain time travel for " - f"{desired_days} days; retention above one day requires a permanent table; convert " - "it by setting table_type permanent on the model, its materialization default, or " - "the target default_table_type" - ) - if ( - runtime.adapter.adapter_name == BuiltinAdapter.SNOWFLAKE - and desired_days > 1 - and relation_is_transient is True - and model.config.table_type.value == TableType.PERMANENT - ): - changes: tuple[RenderedRetentionChange, ...] = runtime.adapter.render_retention_changes( - request=request - ) - return ( - _entry( - request=request, - model_names=(model.name,), - state=state, - source=source, - direction=RetentionDirection.APPLY_AFTER_CREATE, - phase=RetentionPlanPhase.AFTER_CREATE, - statements=_flatten_changes(changes=changes), - ), - ) - direction: RetentionDirection = _state_direction(state=state, desired_days=desired_days) - if direction == RetentionDirection.MATCH: - return ( - _entry( - request=request, - model_names=(model.name,), - state=state, - source=source, - direction=direction, - phase=RetentionPlanPhase.NONE, - ), - ) - changes = runtime.adapter.render_retention_changes(request=request, state=state) - return tuple( - _entry( - request=request, - model_names=(model.name,), - state=state, - source=source, - direction=_change_direction(change=change, fallback=direction), - phase=_change_phase(change=change, fallback=direction), - statements=change.statements, - ) - for change in changes - ) - - -def _plan_bigquery_retention( - *, runtime: PlannerRuntime, models: tuple[CompiledModelView, ...] -) -> tuple[RetentionPlanEntry, ...]: - target: TargetConfig = effective_target_config(runtime=runtime) - if not target.owns_time_travel_retention_namespace: - raise PlannerInputError( - "BigQuery managed time_travel_retention requires target " - "owns_time_travel_retention_namespace = true" - ) - all_managed: dict[tuple[str | None, str], list[CompiledModelView]] = defaultdict(list) - model: CompiledModelView - for model in runtime.project.models: - retention: ResolvedTimeTravelRetention = model.config.time_travel_retention - if ( - retention.unmanaged - or retention.desired_days is None - or model.destination.schema is None - ): - continue - all_managed[(model.destination.database, model.destination.schema)].append(model) - selected_scopes: set[tuple[str | None, str]] = { - (model.destination.database, str(model.destination.schema)) for model in models - } - entries: list[RetentionPlanEntry] = [] - for database, schema in sorted(selected_scopes, key=lambda item: (item[0] or "", item[1])): - scope_models: list[CompiledModelView] = all_managed[(database, schema)] - desired_values: set[int] = { - desired - for item in scope_models - if (desired := item.config.time_travel_retention.desired_days) is not None - } - if len(desired_values) != 1: - details: str = ", ".join( - f"{item.name}={item.config.time_travel_retention.desired_days}d" - for item in scope_models - ) - raise PlannerInputError( - f"BigQuery dataset {database + '.' if database else ''}{schema} has conflicting " - f"managed retention values: {details}" - ) - desired_days: int = desired_values.pop() - sources: str = ",".join( - sorted( - { - source.value - for item in scope_models - if (source := item.config.time_travel_retention.source) is not None - } - ) - ) - request: RetentionRequest = RetentionRequest( - request_id=f"{database + '.' if database else ''}{schema}", - scope=RetentionScope.NAMESPACE, - database=database, - schema=schema, - desired_days=desired_days, - ) - state: RetentionState = runtime.adapter.inspect_retention( - connection=runtime.connection, request=request - ) - if not state.exists: - changes: tuple[RenderedRetentionChange, ...] = runtime.adapter.render_retention_changes( - request=request - ) - entries.append( - _entry( - request=request, - model_names=tuple(sorted(item.name for item in scope_models)), - state=state, - source=sources, - direction=RetentionDirection.APPLY_AFTER_CREATE, - phase=RetentionPlanPhase.AFTER_CREATE, - statements=_flatten_changes(changes=changes), - ) - ) - continue - direction: RetentionDirection = _state_direction(state=state, desired_days=desired_days) - changes: tuple[RenderedRetentionChange, ...] = ( - () - if direction == RetentionDirection.MATCH - else runtime.adapter.render_retention_changes(request=request, state=state) - ) - statements: tuple[str, ...] = _flatten_changes(changes=changes) - entries.append( - _entry( - request=request, - model_names=tuple(sorted(item.name for item in scope_models)), - state=state, - source=sources, - direction=direction, - phase=( - RetentionPlanPhase.NONE - if direction == RetentionDirection.MATCH - else RetentionPlanPhase.PRE - if direction == RetentionDirection.INCREASE - else RetentionPlanPhase.POST - ), - statements=statements, - ) - ) - return tuple(entries) - - -def effective_target_config(*, runtime: PlannerRuntime) -> TargetConfig: - """Return the resolved config of the target being planned, or defaults without one.""" - - if ( - runtime.project_config is None - or runtime.local_config is None - or runtime.project.effective_target_name is None - ): - return TargetConfig() - return resolve_target_config( - project_config=runtime.project_config, - local_config=runtime.local_config, - target_name=runtime.project.effective_target_name, - ) - - -def _flatten_changes(*, changes: tuple[RenderedRetentionChange, ...]) -> tuple[str, ...]: - statements: list[str] = [] - for change in changes: - statements.extend(change.statements) - return tuple(statements) - - -def _state_direction(*, state: RetentionState, desired_days: int) -> RetentionDirection: - values: tuple[int, ...] = tuple( - value - for value in ( - state.delta_log_retention_days, - state.delta_deleted_file_retention_days, - ) - if value is not None - ) or (state.effective_days,) - has_increase: bool = any(value < desired_days for value in values) - has_decrease: bool = any(value > desired_days for value in values) - if has_increase and has_decrease: - return RetentionDirection.MIXED - if has_increase: - return RetentionDirection.INCREASE - if has_decrease: - return RetentionDirection.DECREASE - return RetentionDirection.MATCH - - -def _change_direction( - *, change: RenderedRetentionChange, fallback: RetentionDirection -) -> RetentionDirection: - if change.phase == RetentionChangePhase.PREPARE: - return RetentionDirection.INCREASE - if change.phase == RetentionChangePhase.FINALIZE: - return RetentionDirection.DECREASE - return fallback - - -def _change_phase( - *, change: RenderedRetentionChange, fallback: RetentionDirection -) -> RetentionPlanPhase: - if change.phase == RetentionChangePhase.PREPARE: - return RetentionPlanPhase.PRE - if change.phase == RetentionChangePhase.FINALIZE: - return RetentionPlanPhase.POST - return ( - RetentionPlanPhase.PRE - if fallback == RetentionDirection.INCREASE - else RetentionPlanPhase.POST - ) - - -def _entry( - *, - request: RetentionRequest, - model_names: tuple[str, ...], - state: RetentionState | None, - source: str, - direction: RetentionDirection, - phase: RetentionPlanPhase, - statements: tuple[str, ...] = (), -) -> RetentionPlanEntry: - return RetentionPlanEntry( - request=request, - model_names=model_names, - actual_days=None if state is None else state.configured_days, - effective_days=None if state is None else state.effective_days, - source=source, - direction=direction, - phase=phase, - statements=statements, - irreversible_warning=( - _DECREASE_WARNING if direction == RetentionDirection.DECREASE else None - ), - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/scope_pruning.py b/src/sqlbuild/compiler/planner/_helpers/planning/scope_pruning.py deleted file mode 100644 index 7386ed1cd9..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/planning/scope_pruning.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Direct unchanged-scope pruning phase for execution planning.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.models import ( - PlannerResolvedActions, - PlannerScope, - PlannerScopePruningResult, - PlannerScopeResolution, -) - - -def prune_planner_execution_scope( - *, - scopes: PlannerScopeResolution, - resolved_actions: PlannerResolvedActions, -) -> PlannerScopePruningResult: - """Derive the execution scope from the resolved inspection scope.""" - - inspection_scope: PlannerScope = scopes.inspection_scope - execution_scope: PlannerScope = inspection_scope - return PlannerScopePruningResult( - inspection_scope=inspection_scope, - execution_scope=execution_scope, - resolved_actions=resolved_actions, - pruned_direct_model_names=(), - direct_identity_stale_model_names=frozenset(), - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py b/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py deleted file mode 100644 index 22bb97e84b..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Scope resolution phase for execution planning.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild._native import build_planner_scope, transitive_closure_many -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import ( - PlannerPolicies, - PlannerScope, - PlannerScopeResolution, - PlannerSelection, -) - - -def resolve_planner_scopes( - *, - project: CompiledProjectView, - selection: PlannerSelection, - policies: PlannerPolicies, -) -> PlannerScopeResolution: - """Resolve selected, stale-warning, and inspection scopes for one plan.""" - - selected_scope: PlannerScope = build_planner_scope( - project=project, - select=selection.select, - exclude=selection.exclude, - auto_load_sources=policies.auto_load_sources, - selected_keys=selection.selected_keys, - python_read_source_names=selection.python_read_source_names, - sql_test_selection=selection.sql_test_selection, - ) - full_scope: PlannerScope = build_planner_scope( - project=project, - select=(), - exclude=(), - auto_load_sources=policies.auto_load_sources, - ) - stale_warning_keys: frozenset[CompiledObjectKey] = transitive_closure_many( - starts=selected_scope.selected_keys, - edges=full_scope.upstream_deps, - include_starts=True, - ) - stale_warning_scope: PlannerScope = replace( - full_scope, - all_keys={ - name: key for name, key in full_scope.all_keys.items() if key in stale_warning_keys - }, - models_by_name={ - name: model - for name, model in full_scope.models_by_name.items() - if model.key in stale_warning_keys - }, - selected_keys=selected_scope.selected_keys, - execution_order=tuple( - key for key in full_scope.execution_order if key in stale_warning_keys - ), - user_selected_keys=selected_scope.user_selected_keys, - ) - return PlannerScopeResolution( - selected_scope=selected_scope, - stale_warning_scope=stale_warning_scope, - inspection_scope=selected_scope, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py deleted file mode 100644 index 8e1dc9dff7..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Warehouse inspection phase for execution planning.""" - -from __future__ import annotations - -import time - -from sqlbuild._native import plan_column_migrations, plan_model_migrations, plan_old_name_views -from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context -from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( - effectively_full_refreshed_model_names, -) -from sqlbuild.compiler.planner._helpers.warehouse.snapshot import gather_warehouse_snapshot -from sqlbuild.compiler.planner.classes.migration_fingerprint_cache import ( - MigrationFingerprintCache, -) -from sqlbuild.compiler.planner.models import ( - ColumnMigrationPlanning, - CursorSnapshotScope, - DeferralInputs, - ModelMigrationPlanning, - OldNameViewPlanning, - PlannerOverrides, - PlannerRelationsContext, - PlannerRuntime, - PlannerScopeResolution, - PlannerWarehouseState, - WarehouseSnapshot, -) - - -def gather_planner_warehouse_state( - *, - runtime: PlannerRuntime, - scopes: PlannerScopeResolution, - overrides: PlannerOverrides, - deferral: DeferralInputs, -) -> PlannerWarehouseState: - """Gather the warehouse snapshot and inspection relations in one pass.""" - - warehouse_start: float = time.monotonic() - fingerprints: MigrationFingerprintCache = MigrationFingerprintCache( - root=runtime.project.compile_cache_dir - ) - if runtime.on_progress is not None: - runtime.on_progress("Inspecting warehouse state...") - snapshot: WarehouseSnapshot = gather_warehouse_snapshot( - project=runtime.project, - adapter=runtime.adapter, - connection=runtime.connection, - execute=runtime.adapter.execute, - selected_keys=frozenset(scopes.stale_warning_scope.all_keys.values()), - full_refresh_model_names=effectively_full_refreshed_model_names( - project=runtime.project, - cli_full_refresh=overrides.full_refresh, - ), - on_progress=runtime.on_progress, - deferred_locations=deferral.deferred_locations, - cursor_scope=CursorSnapshotScope( - model_keys=scopes.selected_scope.selected_keys, - runtime_producer_keys=scopes.selected_scope.selected_keys, - invocation_time=runtime.invocation_time, - start_cursor_config=( - runtime.project_config.cursors.start if runtime.project_config is not None else None - ), - cursor_overrides=overrides.cursor_overrides, - ), - ) - migrations: ModelMigrationPlanning = plan_model_migrations( - runtime=runtime, - scope=scopes.selected_scope, - snapshot=snapshot, - overrides=overrides, - deferral=deferral, - fingerprints=fingerprints, - ) - fingerprints.persist() - inspection_relations: PlannerRelationsContext = build_planner_relations_context( - project=runtime.project, - adapter=runtime.adapter, - connection=runtime.connection, - scope=scopes.inspection_scope, - deferral=deferral, - project_config=runtime.project_config, - local_config=runtime.local_config, - ) - columns: ColumnMigrationPlanning = plan_column_migrations( - runtime=runtime, - scope=scopes.selected_scope, - snapshot=migrations.snapshot, - full_refresh_model_names=effectively_full_refreshed_model_names( - project=runtime.project, - cli_full_refresh=overrides.full_refresh, - ), - physical_relations={ - entry.model_name: entry.origin.qualified_name or entry.origin.name - for entry in migrations.entries - if entry.decision.moves_data - }, - overrides=overrides, - deferral=deferral, - source_columns=inspection_relations.source_warehouse_columns, - ) - old_names: OldNameViewPlanning = plan_old_name_views( - runtime=runtime, - scope=scopes.selected_scope, - snapshot=snapshot, - migration_entries=migrations.entries, - column_entries=columns.entries, - ) - if runtime.on_progress is not None: - runtime.on_progress( - f"Inspected warehouse state. ({time.monotonic() - warehouse_start:.2f}s)" - ) - runtime.on_progress("Generating plan...") - return PlannerWarehouseState( - snapshot=columns.snapshot, - inspection_relations=inspection_relations, - migration_entries=migrations.entries, - migration_warnings=(*migrations.warnings, *columns.warnings, *old_names.warnings), - column_migration_entries=columns.entries, - column_rename_hints=columns.hints, - old_name_view_entries=old_names.entries, - old_name_views=old_names.views, - migration_fingerprints=fingerprints, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/semantic_sources.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/semantic_sources.py deleted file mode 100644 index 9c0e7e5e19..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/semantic_sources.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Complete missing source metadata in bounded, database-grouped batches.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo -from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog -from sqlbuild.adapter.relations.main.active_inspection_catalog import active_inspection_catalog -from sqlbuild.adapter.relations.main.get_columns_for_inspection import get_columns_for_inspection -from sqlbuild.adapter.relations.main.list_relations_for_inspection import ( - list_relations_for_inspection, -) -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.types import ContractPolicy -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.spec.contracts.models import SourceEntry - - -def get_semantic_source_columns( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - source_read_map: dict[str, SourceEntry], - columns: dict[str, tuple[ColumnInfo, ...]], - selected_keys: frozenset[CompiledObjectKey], -) -> dict[str, tuple[ColumnInfo, ...]]: - """Reuse collected columns and inspect only selected, unresolved table sources.""" - if not project.settings.sql_analysis or connection is None: - return columns - required: set[str] = set() - for model in project.models: - if model.key in selected_keys and model.config.values.get("sql_analysis") is not False: - required.update( - reference.ref_name - for reference in model.references - if reference.ref_kind == SqlReferenceKind.SOURCE - ) - by_database: dict[str | None, list[SourceEntry]] = {} - for name in sorted(required - columns.keys()): - entry: SourceEntry | None = source_read_map.get(name) - if ( - entry is None - or entry.expression is not None - or entry.contract == ContractPolicy.ENFORCED - ): - continue - by_database.setdefault(entry.database, []).append(entry) - result: dict[str, tuple[ColumnInfo, ...]] = dict(columns) - for database, entries in by_database.items(): - relations: tuple[RelationInfo, ...] = _list_source_candidates( - adapter=adapter, connection=connection, database=database, entries=tuple(entries) - ) - chosen: dict[str, RelationInfo] = {} - for entry in entries: - candidates: tuple[RelationInfo, ...] = tuple( - relation - for relation in relations - if relation.name.casefold() == (entry.table or entry.name).casefold() - and ( - entry.schema is None - or (relation.schema or "").casefold() == entry.schema.casefold() - ) - ) - if len(candidates) == 1: - chosen[entry.name] = candidates[0] - inspected: dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]] = ( - get_columns_for_inspection( - adapter=adapter, - connection=connection, - relations=tuple(dict.fromkeys(chosen.values())), - ) - if chosen - else {} - ) - name: str - relation: RelationInfo - for name, relation in chosen.items(): - identity: tuple[str | None, str | None, str] = ( - relation.database.lower() if relation.database else None, - relation.schema.lower() if relation.schema else None, - relation.name.lower(), - ) - if identity in inspected: - result[name] = inspected[identity] - return result - - -def _list_source_candidates( - *, adapter: BaseAdapter, connection: Any, database: str | None, entries: tuple[SourceEntry, ...] -) -> tuple[RelationInfo, ...]: - """List candidates by name in any schema; schema listings first answer scoped catalogs.""" - - pending: tuple[SourceEntry, ...] = entries - relations: list[RelationInfo] = [] - catalog: InspectionCatalog | None = active_inspection_catalog( - adapter=adapter, connection=connection - ) - scoped: tuple[SourceEntry, ...] = tuple(entry for entry in entries if entry.schema is not None) - if catalog is not None and catalog.schema_scoped and scoped: - relations.extend( - list_relations_for_inspection( - adapter=adapter, - connection=connection, - database=database, - schemas=tuple(sorted({entry.schema or "" for entry in scoped})), - names=tuple(sorted({entry.table or entry.name for entry in scoped})), - ) - ) - pending = tuple( - entry for entry in entries if not _has_candidate(entry=entry, relations=relations) - ) - if pending: - relations.extend( - list_relations_for_inspection( - adapter=adapter, - connection=connection, - database=database, - schemas=None, - names=tuple(sorted({entry.table or entry.name for entry in pending})), - ) - ) - unique: dict[tuple[str | None, str | None, str], RelationInfo] = {} - relation: RelationInfo - for relation in relations: - _ = unique.setdefault(relation.identity, relation) - return tuple(unique.values()) - - -def _has_candidate(*, entry: SourceEntry, relations: list[RelationInfo]) -> bool: - name: str = (entry.table or entry.name).casefold() - return any( - relation.name.casefold() == name - and (entry.schema is None or (relation.schema or "").casefold() == entry.schema.casefold()) - for relation in relations - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py deleted file mode 100644 index b18690f71c..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py +++ /dev/null @@ -1,1696 +0,0 @@ -"""Gather a frozen point-in-time warehouse snapshot for planning.""" - -from __future__ import annotations - -import logging -import time -from collections.abc import Callable -from dataclasses import dataclass, replace -from datetime import datetime -from typing import Any - -from sqlbuild._native import ( - build_execution_upstream_deps, - build_upstream_intermediate_source_map, - classify_cursor_sql_type, - get_config_cursor_bound, - get_config_str, - read_latest_fingerprints, - render_source_relation, - resolve_cursor_input_roles, - resolve_lineage_reference, - select_latest_fingerprints, - transitive_closure_many, -) -from sqlbuild._native import cursor_compare as compare -from sqlbuild._native import cursor_exclusive_to_inclusive as exclusive_to_inclusive -from sqlbuild._native import cursor_observed_partition as observed_partition -from sqlbuild._native import cursor_parse as parse -from sqlbuild._native import cursor_render as render -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog -from sqlbuild.adapter.relations.main.active_inspection_catalog import active_inspection_catalog -from sqlbuild.adapter.relations.main.get_columns_for_inspection import get_columns_for_inspection -from sqlbuild.adapter.relations.main.list_relations_for_inspection import ( - list_relations_for_inspection, -) -from sqlbuild.adapter.relations.main.record_inspection_query import record_inspection_query -from sqlbuild.adapter.relations.main.run_bounded_inspections import run_bounded_inspections -from sqlbuild.adapter.relations.main.run_recorded_inspection_query import ( - run_recorded_inspection_query, -) -from sqlbuild.adapter.relations.models import InspectionQueryRecord -from sqlbuild.compiler.compile.constants import MIGRATE_FROM_CONFIG_KEY -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, - CompileSqlReference, -) -from sqlbuild.compiler.compile.types import ( - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - CompiledSeedView, - CompiledSourceView, -) -from sqlbuild.compiler.fingerprints.constants import ( - FINGERPRINT_TABLE_NAME, - FUNCTION_NODE_TYPES, - NODE_TYPE_MODEL, - NODE_TYPE_SEED, -) -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet -from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME -from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( - effectively_full_refreshed_model_names, -) -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - normalize_cursor_snapshot_grain, - resolve_effective_timestamp_grain, -) -from sqlbuild.compiler.planner._helpers.resolve.cursor_policies import resolve_start_cursor_config -from sqlbuild.compiler.planner._helpers.resolve.maximum_start import maximum_allowed_start -from sqlbuild.compiler.planner.constants import METADATA_NAME_FILTER_LIMIT -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - CursorInputEvidence, - CursorOverrides, - CursorSnapshotScope, - ModelCursorSnapshot, - WarehouseFingerprints, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import ( - ContractPolicy, - CursorGrain, - CursorType, - CursorWatermarkMode, - MaterializationType, - MicrobatchStrategy, -) -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME -from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.models import DateValue -from sqlbuild.cursor_algebra.types import CursorScalar -from sqlbuild.diagnostics.main.log_debug_event import log_debug_event -from sqlbuild.spec.contracts.models import SourceEntry, StartCursorsConfig - -_DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.planner") - - -@dataclass(frozen=True) -class _PhysicalCursorQuery: - """One physical relation and cursor column to inspect.""" - - relation: str - cursor_column: str - min_tags: tuple[str, ...] - max_tags: tuple[str, ...] - cursor_type: str = CursorType.TIMESTAMP - - -@dataclass(frozen=True) -class _CursorQueryOutcome: - """Result of one physical cursor statement; ``row`` is None when nothing was returned.""" - - row: Any - elapsed_seconds: float - error: Exception | None = None - - -@dataclass(frozen=True) -class _CursorQueryResults: - """Merged cursor values plus the tags of physical reads that failed.""" - - values: dict[str, CursorScalar] - failed_tags: frozenset[str] = frozenset() - - -@dataclass(frozen=True) -class _UpstreamCursorInfo: - """Pre-resolved upstream cursor metadata for one ref.""" - - tag_min: str - tag_max: str - relation: str - cursor_column: str - input_name: str = "" - cursor_grain: CursorGrain | None = None - terminal_cursor_start: CursorScalar | None = None - terminal_cursor_end: CursorScalar | None = None - - -@dataclass(frozen=True) -class _CursorModelInfo: - """Pre-resolved cursor metadata for one incremental model.""" - - model_name: str - target_tag: str | None - target_relation: str | None - cursor_column: str - upstreams: tuple[_UpstreamCursorInfo, ...] - cursor_type: str | None = None - cursor_grain: str | None = None - effective_cursor_grain: str | None = None - start_cursor_config: StartCursorsConfig | None = None - has_start_override: bool = False - cursor_watermark_mode: CursorWatermarkMode = CursorWatermarkMode.ALL - microbatch_strategy: str | None = None - origin_cursor_column: str | None = None - - @property - def physical_cursor_column(self) -> str: - """Return the target column holding the cursor now, before any pending rename.""" - - return self.origin_cursor_column or self.cursor_column - - -@dataclass(frozen=True) -class _CursorGatherInputs: - """Selection and policy inputs for cursor warehouse inspection.""" - - selected_keys: frozenset[CompiledObjectKey] | None - full_refresh_model_names: frozenset[str] - deferred_locations: dict[str, CompiledRelationLocation] | None - runtime_producer_keys: frozenset[CompiledObjectKey] | None - invocation_time: datetime | None - start_cursor_config: StartCursorsConfig | None - cursor_overrides: CursorOverrides | None - target_relation_overrides: dict[str, str] | None = None - origin_cursor_column_overrides: dict[str, str] | None = None - existing_columns: dict[str, tuple[ColumnInfo, ...]] | None = None - - -@dataclass(frozen=True) -class _StateTableSchemas: - """Lower-cased schemas where each planner-read state table exists.""" - - fingerprints: frozenset[str] - source_freshness: frozenset[str] - old_name_views: frozenset[str] - migrations: frozenset[str] - - -def gather_warehouse_snapshot( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - execute: AdapterExecute[Any, Any], - selected_keys: frozenset[CompiledObjectKey] | None = None, - full_refresh: bool = False, - full_refresh_model_names: frozenset[str] | None = None, - on_progress: Callable[[str], None] | None = None, - deferred_locations: dict[str, CompiledRelationLocation] | None = None, - cursor_scope: CursorSnapshotScope | None = None, -) -> WarehouseSnapshot: - """Gather relations, columns, and fingerprints for all target schemas.""" - - relevant_keys: frozenset[CompiledObjectKey] | None = _relevant_state_keys( - project=project, - selected_keys=selected_keys, - ) - database: str | None = _resolve_database(project=project, selected_keys=relevant_keys) - schemas: tuple[str, ...] = _collect_target_schemas(project=project, selected_keys=relevant_keys) - metadata_names: tuple[str, ...] | None = _build_metadata_name_filter( - project=project, - selected_keys=relevant_keys, - ) - if not schemas and metadata_names is None: - return WarehouseSnapshot() - query_schemas: tuple[str, ...] | None = schemas or None - _prefetch_inspection_schemas( - project=project, - adapter=adapter, - connection=connection, - database=database, - schemas=schemas, - relevant_keys=relevant_keys, - ) - - relations: dict[str, RelationInfo] - state_schemas: _StateTableSchemas - relations, state_schemas = _gather_relations( - project=project, - adapter=adapter, - connection=connection, - database=database, - schemas=query_schemas, - names=metadata_names, - ) - fingerprint_state_schemas: frozenset[str] = state_schemas.fingerprints - freshness_state_schemas: frozenset[str] = state_schemas.source_freshness - columns: dict[str, tuple[ColumnInfo, ...]] = _gather_columns( - project=project, - adapter=adapter, - connection=connection, - relations=relations, - ) - fingerprints: WarehouseFingerprints = _gather_fingerprints( - adapter=adapter, - connection=connection, - execute=execute, - database=database, - schemas=query_schemas, - fingerprint_state_schemas=fingerprint_state_schemas, - node_names=_selected_node_names(relevant_keys), - read_unfiltered=_shares_unfiltered_fingerprints( - project=project, selected_keys=relevant_keys - ), - ) - - effective_full_refresh_names: frozenset[str] = ( - full_refresh_model_names - if full_refresh_model_names is not None - else effectively_full_refreshed_model_names( - project=project, - cli_full_refresh=full_refresh, - ) - ) - cursor_snapshots: dict[str, ModelCursorSnapshot] = _gather_cursor_snapshots( - project=project, - adapter=adapter, - connection=connection, - execute=execute, - existing_relations=relations, - on_progress=on_progress, - inputs=_CursorGatherInputs( - selected_keys=cursor_scope.model_keys if cursor_scope is not None else selected_keys, - full_refresh_model_names=effective_full_refresh_names, - deferred_locations=deferred_locations, - runtime_producer_keys=( - cursor_scope.runtime_producer_keys if cursor_scope is not None else None - ), - invocation_time=(cursor_scope.invocation_time if cursor_scope is not None else None), - start_cursor_config=( - cursor_scope.start_cursor_config if cursor_scope is not None else None - ), - cursor_overrides=(cursor_scope.cursor_overrides if cursor_scope is not None else None), - existing_columns=columns, - ), - ) - - return WarehouseSnapshot( - existing_relations=relations, - existing_columns=columns, - fingerprints=fingerprints, - cursor_snapshots=cursor_snapshots, - source_freshness_state_schemas=freshness_state_schemas, - column_dialect=adapter.sql_analysis_dialect(), - old_name_view_state_schemas=state_schemas.old_name_views, - migration_state_schemas=state_schemas.migrations, - listed_state_schemas=( - frozenset(schema.lower() for schema in query_schemas) - if query_schemas is not None - else None - ), - ) - - -def gather_redirected_cursor_snapshots( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - existing_relations: dict[str, RelationInfo], - target_relations: dict[str, str], - cursor_scope: CursorSnapshotScope, - full_refresh_model_names: frozenset[str], - deferred_locations: dict[str, CompiledRelationLocation] | None = None, - on_progress: Callable[[str], None] | None = None, - origin_cursor_columns: dict[str, str] | None = None, -) -> dict[str, ModelCursorSnapshot]: - """Gather cursor snapshots for models whose history lives in another relation or column.""" - - if not target_relations: - return {} - model_keys: frozenset[CompiledObjectKey] = frozenset( - CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=name) - for name in target_relations - ) - snapshots: dict[str, ModelCursorSnapshot] = _gather_cursor_snapshots( - project=project, - adapter=adapter, - connection=connection, - execute=adapter.execute, - existing_relations=existing_relations, - on_progress=on_progress, - inputs=_CursorGatherInputs( - selected_keys=model_keys, - full_refresh_model_names=full_refresh_model_names, - deferred_locations=deferred_locations, - runtime_producer_keys=cursor_scope.runtime_producer_keys, - invocation_time=cursor_scope.invocation_time, - start_cursor_config=cursor_scope.start_cursor_config, - cursor_overrides=cursor_scope.cursor_overrides, - target_relation_overrides=target_relations, - origin_cursor_column_overrides=origin_cursor_columns, - ), - ) - model_map: dict[str, CompiledModelView] = {model.name: model for model in project.models} - return { - name: replace(snapshot, target_relation=model_map[name].destination.qualified_name) - for name, snapshot in snapshots.items() - if name in model_map - } - - -def _prefetch_inspection_schemas( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - database: str | None, - schemas: tuple[str, ...], - relevant_keys: frozenset[CompiledObjectKey] | None, -) -> None: - """List target and source schemas together, concurrently; later reads report failures.""" - - catalog: InspectionCatalog | None = active_inspection_catalog( - adapter=adapter, connection=connection - ) - if catalog is None: - return - scopes: list[tuple[str | None, str]] = [(database, schema) for schema in schemas] - source: CompiledSourceView - for source in project.sources: - entry: SourceEntry = source.source_entry - if ( - (relevant_keys is not None and source.key not in relevant_keys) - or entry.expression is not None - or entry.schema is None - ): - continue - scopes.append((entry.database, entry.schema)) - catalog.prefetch_schema_listings(scopes=tuple(scopes), best_effort=True) - - -def _relevant_state_keys( - *, - project: CompiledProjectView, - selected_keys: frozenset[CompiledObjectKey] | None, -) -> frozenset[CompiledObjectKey] | None: - """Retain state identities for the complete upstream closure of the selection.""" - - if selected_keys is None: - return None - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_execution_upstream_deps(project) - ) - return transitive_closure_many(starts=selected_keys, edges=upstream_deps, include_starts=True) - - -def _resolve_database( - *, - project: CompiledProjectView, - selected_keys: frozenset[CompiledObjectKey] | None, -) -> str | None: - """Extract the database from the first model target that declares one.""" - - model: CompiledModelView - for model in project.models: - if selected_keys is not None and model.key not in selected_keys: - continue - if model.destination.database is not None: - return model.destination.database - seed: CompiledSeedView - for seed in project.seeds: - if selected_keys is not None and seed.key not in selected_keys: - continue - if seed.destination.database is not None: - return seed.destination.database - function: CompiledFunctionView - for function in project.functions: - if selected_keys is not None and function.key not in selected_keys: - continue - if function.destination.database is not None: - return function.destination.database - return project.effective_target_database - - -def _collect_target_schemas( - *, - project: CompiledProjectView, - selected_keys: frozenset[CompiledObjectKey] | None, -) -> tuple[str, ...]: - """Collect distinct non-null target schemas from models, seeds, and functions.""" - - schemas: set[str] = set() - model: CompiledModelView - for model in project.models: - if selected_keys is not None and model.key not in selected_keys: - continue - if model.destination.schema is not None: - schemas.add(model.destination.schema) - seed: CompiledSeedView - for seed in project.seeds: - if selected_keys is not None and seed.key not in selected_keys: - continue - if seed.destination.schema is not None: - schemas.add(seed.destination.schema) - function: CompiledFunctionView - for function in project.functions: - if selected_keys is not None and function.key not in selected_keys: - continue - if function.destination.schema is not None: - schemas.add(function.destination.schema) - if not schemas and project.effective_target_schema is not None: - schemas.add(project.effective_target_schema) - return tuple(sorted(schemas)) - - -def _build_metadata_name_filter( - *, - project: CompiledProjectView, - selected_keys: frozenset[CompiledObjectKey] | None, -) -> tuple[str, ...] | None: - names: set[str] = set() - if selected_keys is None: - model: CompiledModelView - for model in project.models: - names.add(model.destination.name) - seed: CompiledSeedView - for seed in project.seeds: - names.add(seed.destination.name) - else: - selected_names: frozenset[str] = frozenset(key.name for key in selected_keys) - model_map: dict[str, CompiledModelView] = {model.name: model for model in project.models} - seed_map: dict[str, CompiledSeedView] = {seed.name: seed for seed in project.seeds} - source_map: dict[str, SourceEntry] = { - source.source_entry.name: source.source_entry for source in project.sources - } - key: CompiledObjectKey - for key in selected_keys: - selected_model: CompiledModelView | None = model_map.get(key.name) - if selected_model is not None: - names.add(selected_model.destination.name) - names.update( - _model_upstream_names( - model=selected_model, - model_map=model_map, - seed_map=seed_map, - selected_names=selected_names, - ) - ) - continue - selected_seed: CompiledSeedView | None = seed_map.get(key.name) - if selected_seed is not None: - names.add(selected_seed.destination.name) - continue - selected_source: SourceEntry | None = source_map.get(key.name) - if selected_source is not None: - names.add(selected_source.table or selected_source.name) - upstream_intermediate_source: SourceEntry - for upstream_intermediate_source in build_upstream_intermediate_source_map( - project=project, - selected_keys=selected_keys, - ).values(): - names.add(upstream_intermediate_source.table or upstream_intermediate_source.name) - if not names or len(names) > METADATA_NAME_FILTER_LIMIT: - return None - names.add(FINGERPRINT_TABLE_NAME) - names.add(SOURCE_FRESHNESS_TABLE_NAME) - names.add(OLD_NAME_VIEW_TABLE_NAME) - names.add(MIGRATION_TABLE_NAME) - return tuple(sorted(names)) - - -def _model_upstream_names( - *, - model: CompiledModelView, - model_map: dict[str, CompiledModelView], - seed_map: dict[str, CompiledSeedView], - selected_names: frozenset[str], -) -> frozenset[str]: - names: set[str] = set() - reference: CompileSqlReference - for reference in model.references: - if ( - reference.ref_kind - not in { - SqlReferenceKind.REF, - SqlReferenceKind.SEED, - } - or reference.ref_name in selected_names - ): - continue - if reference.ref_kind == SqlReferenceKind.REF: - upstream_model: CompiledModelView | None = model_map.get(reference.ref_name) - if upstream_model is not None: - names.add(upstream_model.destination.name) - continue - if reference.ref_kind == SqlReferenceKind.SEED: - upstream_seed: CompiledSeedView | None = seed_map.get(reference.ref_name) - if upstream_seed is not None: - names.add(upstream_seed.destination.name) - continue - return frozenset(names) - - -def _gather_relations( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - database: str | None, - schemas: tuple[str, ...] | None, - names: tuple[str, ...] | None, -) -> tuple[dict[str, RelationInfo], _StateTableSchemas]: - """Fetch relations and the schemas where fingerprint, freshness and old-name state live.""" - - relations: tuple[RelationInfo, ...] = list_relations_for_inspection( - adapter=adapter, connection=connection, database=database, schemas=schemas, names=names - ) - result: dict[str, RelationInfo] = {} - logical_names_by_identity: dict[tuple[str | None, str | None, str], str] = {} - model: CompiledModelView - for model in project.models: - logical_names_by_identity[_location_identity(model.destination)] = model.name - seed: CompiledSeedView - for seed in project.seeds: - logical_names_by_identity[_location_identity(seed.destination)] = seed.name - function: CompiledFunctionView - for function in project.functions: - logical_names_by_identity[_location_identity(function.destination)] = function.name - fingerprint_schemas: set[str] = set() - freshness_schemas: set[str] = set() - old_name_schemas: set[str] = set() - migration_schemas: set[str] = set() - relation: RelationInfo - for relation in relations: - if relation.name == MIGRATION_TABLE_NAME and relation.schema is not None: - migration_schemas.add(relation.schema.lower()) - if relation.name == OLD_NAME_VIEW_TABLE_NAME: - if relation.schema is not None: - old_name_schemas.add(relation.schema.lower()) - continue - if relation.name == FINGERPRINT_TABLE_NAME: - if relation.schema is not None: - fingerprint_schemas.add(relation.schema.lower()) - continue - if relation.name == SOURCE_FRESHNESS_TABLE_NAME: - if relation.schema is not None: - freshness_schemas.add(relation.schema.lower()) - continue - result[logical_names_by_identity.get(relation.identity, relation.name)] = relation - return result, _StateTableSchemas( - fingerprints=frozenset(fingerprint_schemas), - source_freshness=frozenset(freshness_schemas), - old_name_views=frozenset(old_name_schemas), - migrations=frozenset(migration_schemas), - ) - - -def _location_identity( - location: CompiledRelationLocation, -) -> tuple[str | None, str | None, str]: - return ( - None if location.database is None else location.database.lower(), - None if location.schema is None else location.schema.lower(), - location.name.lower(), - ) - - -def _gather_columns( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - relations: dict[str, RelationInfo], -) -> dict[str, tuple[ColumnInfo, ...]]: - """Fetch columns only for listed relations that planning looks up by node name.""" - - node_names: frozenset[str] = frozenset( - ( - *(model.name for model in project.models), - *(seed.name for seed in project.seeds), - *(function.name for function in project.functions), - ) - ) - needed: dict[str, RelationInfo] = { - logical_name: relation - for logical_name, relation in relations.items() - if logical_name in node_names - } - all_columns: dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]] = ( - get_columns_for_inspection( - adapter=adapter, connection=connection, relations=tuple(needed.values()) - ) - if needed - else {} - ) - return { - logical_name: all_columns[relation.identity] - for logical_name, relation in needed.items() - if relation.identity in all_columns - } - - -def _gather_fingerprints( - *, - adapter: BaseAdapter, - connection: Any, - execute: AdapterExecute[Any, Any], - database: str | None, - schemas: tuple[str, ...] | None, - fingerprint_state_schemas: frozenset[str], - node_names: tuple[str, ...] | None, - read_unfiltered: bool, -) -> WarehouseFingerprints: - """Read latest fingerprints per target schema, keeping unfiltered reads for migrations.""" - - if schemas is None: - return WarehouseFingerprints() - model_fingerprints: dict[str, Fingerprint] = {} - function_fingerprints: dict[str, Fingerprint] = {} - seed_fingerprints: dict[str, Fingerprint] = {} - python_fingerprints: dict[tuple[str, str], Fingerprint] = {} - unfiltered_schemas: dict[str, tuple[Fingerprint, ...]] = {} - filtered_node_types: tuple[str, ...] = (NODE_TYPE_MODEL, *FUNCTION_NODE_TYPES, NODE_TYPE_SEED) - schema: str - for schema in schemas: - fingerprint_set: FingerprintSet = read_latest_fingerprints( - connection=connection, - execute=execute, - table_exists=schema.lower() in fingerprint_state_schemas, - database=database, - schema=schema, - render_qualified_name=adapter.render_qualified_name, - render_read_latest_sql=adapter.render_read_latest_fingerprints_sql, - node_names=None if read_unfiltered else node_names, - filtered_node_types=filtered_node_types, - ) - if read_unfiltered: - unfiltered_schemas[schema.lower()] = tuple( - (fingerprint_set.fingerprints_by_identity or {}).values() - ) - if node_names is not None: - fingerprint_set = select_latest_fingerprints( - fingerprint_set=fingerprint_set, - node_names=node_names, - filtered_node_types=filtered_node_types, - ) - node_name: str - fingerprint: Fingerprint - for node_name, fingerprint in fingerprint_set.fingerprints.items(): - if fingerprint.node_type == NODE_TYPE_MODEL: - model_fingerprints[node_name] = fingerprint - elif fingerprint.node_type in FUNCTION_NODE_TYPES: - function_fingerprints[node_name] = fingerprint - elif fingerprint.node_type == NODE_TYPE_SEED: - seed_fingerprints[node_name] = fingerprint - if fingerprint_set.fingerprints_by_identity is not None: - identity_key: tuple[str, str] - for identity_key, fingerprint in fingerprint_set.fingerprints_by_identity.items(): - if fingerprint.node_type not in { - NODE_TYPE_MODEL, - *FUNCTION_NODE_TYPES, - NODE_TYPE_SEED, - }: - python_fingerprints[identity_key] = fingerprint - return WarehouseFingerprints( - models=model_fingerprints, - functions=function_fingerprints, - seeds=seed_fingerprints, - python_nodes=python_fingerprints, - unfiltered_schemas=unfiltered_schemas, - unfiltered_database=database, - ) - - -def _selected_node_names( - selected_keys: frozenset[CompiledObjectKey] | None, -) -> tuple[str, ...] | None: - if selected_keys is None: - return None - return tuple(sorted({key.name for key in selected_keys})) - - -def _shares_unfiltered_fingerprints( - *, - project: CompiledProjectView, - selected_keys: frozenset[CompiledObjectKey] | None, -) -> bool: - """Return whether one unfiltered read can serve this selection and migration inspection.""" - - if selected_keys is None: - return True - selected: frozenset[str] = frozenset(key.name for key in selected_keys) - return all(model.name in selected for model in project.models) or any( - isinstance(model.config.values.get(MIGRATE_FROM_CONFIG_KEY), str) - for model in project.models - ) - - -def _gather_cursor_snapshots( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - execute: AdapterExecute[Any, Any], - existing_relations: dict[str, RelationInfo], - on_progress: Callable[[str], None] | None, - inputs: _CursorGatherInputs, -) -> dict[str, ModelCursorSnapshot]: - """Gather cursor MIN/MAX values for selected incremental models.""" - - model_map: dict[str, CompiledModelView] = {m.name: m for m in project.models} - source_map: dict[str, CompiledSourceView] = {s.name: s for s in project.sources} - - cursor_models: list[_CursorModelInfo] = _collect_cursor_models( - project=project, - adapter=adapter, - model_map=model_map, - source_map=source_map, - existing_relations=existing_relations, - inputs=inputs, - ) - if not cursor_models: - return {} - - queries: list[_PhysicalCursorQuery] = _build_cursor_queries(cursor_models) - cursor_start: float = time.monotonic() - concurrency: int = _inspection_concurrency(adapter=adapter, connection=connection) - query_results: _CursorQueryResults = _execute_cursor_queries( - queries=queries, - connection=connection, - execute=execute, - on_progress=on_progress, - concurrency=concurrency, - ) - results: dict[str, CursorScalar] = _gather_eligible_target_maxes( - cursor_models=cursor_models, - results=query_results.values, - adapter=adapter, - connection=connection, - execute=execute, - invocation_time=inputs.invocation_time, - ) - if on_progress is not None: - logical_total: int = sum(len(query.min_tags) + len(query.max_tags) for query in queries) - physical_total: int = len(queries) - on_progress( - f"Gathered cursor bounds ({len(results)}/{logical_total} logical values; " - f"{physical_total} physical relation reads). ({time.monotonic() - cursor_start:.2f}s)" - ) - - return _assemble_cursor_snapshots( - cursor_models=cursor_models, results=results, failed_tags=query_results.failed_tags - ) - - -def _collect_cursor_models( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - model_map: dict[str, CompiledModelView], - source_map: dict[str, CompiledSourceView], - existing_relations: dict[str, RelationInfo], - inputs: _CursorGatherInputs, -) -> list[_CursorModelInfo]: - """Identify selected incremental models and pre-resolve their cursor metadata.""" - - effective_runtime_producer_keys: frozenset[CompiledObjectKey] | None = ( - inputs.runtime_producer_keys - if inputs.runtime_producer_keys is not None - else inputs.selected_keys - ) - selected_names: frozenset[str] | None = ( - frozenset(k.name for k in effective_runtime_producer_keys) - if effective_runtime_producer_keys is not None - else None - ) - seed_map: dict[str, CompiledSeedView] = {seed.name: seed for seed in project.seeds} - function_map: dict[str, CompiledFunctionView] = { - function.name: function for function in project.functions - } - runtime_producer_names: frozenset[str] = _selected_runtime_producer_names( - project=project, - selected_keys=effective_runtime_producer_keys, - ) - cursor_models: list[_CursorModelInfo] = [] - model: CompiledModelView - for model in project.models: - if inputs.selected_keys is not None: - model_key: CompiledObjectKey = CompiledObjectKey( - resource_type=CompiledResourceType.MODEL, name=model.name - ) - if model_key not in inputs.selected_keys: - continue - - cursor_column: str | None = get_config_str(values=model.config.values, key="cursor") - materialized: str | None = get_config_str(values=model.config.values, key="materialized") - if materialized != MaterializationType.INCREMENTAL or cursor_column is None: - continue - - cursor_watermark_inputs: dict[str, str] = resolve_cursor_input_roles( - model=model - ).watermark_inputs - if model.name in inputs.full_refresh_model_names: - for input_name, upstream_cursor_col in cursor_watermark_inputs.items(): - ref: CompileSqlReference = resolve_lineage_reference( - model=model, - input_name=input_name, - models_by_name=model_map, - functions_by_name=function_map, - ) - _validate_watermark_contract_column( - model=model, - ref=ref, - cursor_column=upstream_cursor_col, - model_map=model_map, - source_map=source_map, - seed_map=seed_map, - ) - continue - - target_tag: str | None = None - target_relation: str | None = None - target_relation_info: RelationInfo | None = existing_relations.get(model.name) - redirected_target: str | None = (inputs.target_relation_overrides or {}).get(model.name) - if redirected_target is not None: - target_tag = f"{model.name}__target__max" - target_relation = redirected_target - elif ( - model.destination.qualified_name is not None - and target_relation_info is not None - and target_relation_info.name == model.name - ): - target_tag = ( - f"{model.name}__target__max" - if _target_has_cursor_column( - model_name=model.name, - cursor_column=cursor_column, - existing_columns=inputs.existing_columns, - ) - else None - ) - target_relation = model.destination.qualified_name - - cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") - cursor_grain: str | None = get_config_str(values=model.config.values, key="cursor_grain") - upstreams: list[_UpstreamCursorInfo] = [] - input_name: str - upstream_cursor_col: str - for input_name, upstream_cursor_col in cursor_watermark_inputs.items(): - ref: CompileSqlReference = resolve_lineage_reference( - model=model, - input_name=input_name, - models_by_name=model_map, - functions_by_name=function_map, - ) - _validate_watermark_contract_column( - model=model, - ref=ref, - cursor_column=upstream_cursor_col, - model_map=model_map, - source_map=source_map, - seed_map=seed_map, - ) - if ref.ref_name in runtime_producer_names: - continue - upstream_relation: str | None = _resolve_upstream_qualified_name( - ref=ref, - adapter=adapter, - model_map=model_map, - source_map=source_map, - seed_map=seed_map, - deferred_locations=inputs.deferred_locations, - selected_names=selected_names, - ) - if upstream_relation is None: - continue - upstream_exists: bool = ref.ref_name in existing_relations or ( - ref.ref_kind == SqlReferenceKind.SOURCE - ) - if not upstream_exists: - continue - upstreams.append( - _UpstreamCursorInfo( - tag_min=f"{model.name}__{ref.ref_name}__min", - tag_max=f"{model.name}__{ref.ref_name}__max", - relation=upstream_relation, - cursor_column=upstream_cursor_col, - input_name=ref.ref_name, - cursor_grain=( - CursorGrain(input_grain) - if (input_grain := _cursor_input_grain(ref=ref, model_map=model_map)) - is not None - else None - ), - terminal_cursor_start=( - _parse_optional_cursor( - value=get_config_cursor_bound( - values=model_map[ref.ref_name].config.values, key="cursor_start" - ), - cursor_type=cursor_type, - ) - if ref.ref_kind == SqlReferenceKind.REF and ref.ref_name in model_map - else None - ), - terminal_cursor_end=( - _parse_optional_cursor( - value=get_config_cursor_bound( - values=model_map[ref.ref_name].config.values, key="cursor_end" - ), - cursor_type=cursor_type, - ) - if ref.ref_kind == SqlReferenceKind.REF and ref.ref_name in model_map - else None - ), - ) - ) - - cursor_models.append( - _CursorModelInfo( - model_name=model.name, - target_tag=target_tag, - target_relation=target_relation, - origin_cursor_column=(inputs.origin_cursor_column_overrides or {}).get(model.name), - cursor_column=cursor_column, - upstreams=tuple(upstreams), - cursor_type=cursor_type, - cursor_grain=cursor_grain, - effective_cursor_grain=resolve_effective_timestamp_grain( - cursor_type=cursor_type, - downstream_grain=cursor_grain, - microbatch_strategy=get_config_str( - values=model.config.values, key="microbatch_strategy" - ), - cursor_input_grains=tuple(upstream.cursor_grain for upstream in upstreams), - ), - start_cursor_config=resolve_start_cursor_config( - model=model, project_config=inputs.start_cursor_config - ), - has_start_override=_has_start_override( - model=model, - cursor_overrides=inputs.cursor_overrides, - ), - cursor_watermark_mode=CursorWatermarkMode( - get_config_str(values=model.config.values, key="cursor_watermark_mode") or "all" - ), - microbatch_strategy=get_config_str( - values=model.config.values, key="microbatch_strategy" - ), - ) - ) - - return cursor_models - - -def _target_has_cursor_column( - *, - model_name: str, - cursor_column: str, - existing_columns: dict[str, tuple[ColumnInfo, ...]] | None, -) -> bool: - """Return whether the target may hold the cursor column, skipping reads known to fail.""" - - columns: tuple[ColumnInfo, ...] | None = (existing_columns or {}).get(model_name) - if not columns: - return True - wanted: str = cursor_column.strip('"`[]').lower() - if any(column.name.strip('"`[]').lower() == wanted for column in columns): - return True - log_debug_event( - logger=_DEBUG_LOGGER, - message="target relation lacks the cursor column; skipping its cursor bounds read", - sqlbuild_model=model_name, - sqlbuild_cursor_column=cursor_column, - ) - return False - - -def _build_cursor_queries(cursor_models: list[_CursorModelInfo]) -> list[_PhysicalCursorQuery]: - """Group logical cursor requests by physical relation and column.""" - - grouped_tags: dict[tuple[str, str, str], tuple[list[str], list[str]]] = {} - info: _CursorModelInfo - for info in cursor_models: - if info.target_tag is not None and info.target_relation is not None: - target_key: tuple[str, str, str] = ( - info.target_relation, - info.physical_cursor_column, - info.cursor_type or CursorType.TIMESTAMP, - ) - target_tags: tuple[list[str], list[str]] = grouped_tags.setdefault(target_key, ([], [])) - if info.target_tag not in target_tags[1]: - target_tags[1].append(info.target_tag) - upstream: _UpstreamCursorInfo - for upstream in info.upstreams: - upstream_key: tuple[str, str, str] = ( - upstream.relation, - upstream.cursor_column, - info.cursor_type or CursorType.TIMESTAMP, - ) - upstream_tags: tuple[list[str], list[str]] = grouped_tags.setdefault( - upstream_key, ([], []) - ) - if upstream.tag_min not in upstream_tags[0]: - upstream_tags[0].append(upstream.tag_min) - if upstream.tag_max not in upstream_tags[1]: - upstream_tags[1].append(upstream.tag_max) - - return [ - _PhysicalCursorQuery( - relation=relation, - cursor_column=cursor_column, - cursor_type=cursor_type, - min_tags=tuple(tags[0]), - max_tags=tuple(tags[1]), - ) - for (relation, cursor_column, cursor_type), tags in grouped_tags.items() - ] - - -def _cursor_input_grain( - *, ref: CompileSqlReference, model_map: dict[str, CompiledModelView] -) -> str | None: - if ref.ref_kind != SqlReferenceKind.REF: - return None - upstream_model: CompiledModelView | None = model_map.get(ref.ref_name) - return ( - get_config_str(values=upstream_model.config.values, key="cursor_grain") - if upstream_model is not None - else None - ) - - -def _parse_optional_cursor(*, value: str | None, cursor_type: str | None) -> CursorScalar | None: - if value is None: - return None - return parse(raw=value, cursor_type=cursor_type or CursorType.TIMESTAMP) - - -def _has_start_override( - *, model: CompiledModelView, cursor_overrides: CursorOverrides | None -) -> bool: - if cursor_overrides is None: - return False - cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") - if cursor_type == CursorType.TIMESTAMP: - return cursor_overrides.start_ts is not None - if cursor_type == CursorType.INTEGER: - return cursor_overrides.start_int is not None - return False - - -def _execute_cursor_queries( - *, - queries: list[_PhysicalCursorQuery], - connection: Any, - execute: AdapterExecute[Any, Any], - on_progress: Callable[[str], None] | None, - concurrency: int = 1, -) -> _CursorQueryResults: - """Run one unmerged MIN/MAX statement per relation, bounded-parallel, merged in query order.""" - - total: int = len(queries) - if concurrency <= 1 or total <= 1: - sequential_outcomes: list[_CursorQueryOutcome] = [] - query_index: int - query: _PhysicalCursorQuery - for query_index, query in enumerate(queries, start=1): - identity: str = _cursor_query_identity(query=query, index=query_index, total=total) - if on_progress is not None: - on_progress(f"Inspecting cursor bounds {identity}...") - outcome: _CursorQueryOutcome = _read_cursor_query( - query=query, connection=connection, execute=execute - ) - _report_cursor_query( - query=query, outcome=outcome, identity=identity, on_progress=on_progress - ) - sequential_outcomes.append(outcome) - return _merge_cursor_outcomes(queries=queries, outcomes=sequential_outcomes) - if on_progress is not None: - on_progress( - f"Inspecting cursor bounds for {total} relations " - f"({min(concurrency, total)} concurrent)..." - ) - progress: _CursorBoundsProgress = _CursorBoundsProgress( - queries=tuple(queries), on_progress=on_progress - ) - outcomes: list[_CursorQueryOutcome] = run_bounded_inspections( - tasks=tuple( - _bind_cursor_query(query=query, connection=connection, execute=execute) - for query in queries - ), - concurrency=concurrency, - on_complete=progress.report, - ) - return _merge_cursor_outcomes(queries=queries, outcomes=outcomes) - - -def _merge_cursor_outcomes( - *, queries: list[_PhysicalCursorQuery], outcomes: list[_CursorQueryOutcome] -) -> _CursorQueryResults: - merged: dict[str, CursorScalar] = {} - failed_tags: set[str] = set() - query: _PhysicalCursorQuery - outcome: _CursorQueryOutcome - for query, outcome in zip(queries, outcomes, strict=True): - if outcome.error is not None: - failed_tags.update((*query.min_tags, *query.max_tags)) - merged.update(_fan_out_cursor_row(query=query, row=outcome.row)) - return _CursorQueryResults(values=merged, failed_tags=frozenset(failed_tags)) - - -class _CursorBoundsProgress: - """Report concurrent cursor-bound completions in finish order from the planning thread.""" - - def __init__( - self, - *, - queries: tuple[_PhysicalCursorQuery, ...], - on_progress: Callable[[str], None] | None, - ) -> None: - self._queries: tuple[_PhysicalCursorQuery, ...] = queries - self._on_progress: Callable[[str], None] | None = on_progress - self._completed: int = 0 - - def report(self, *, index: int, result: _CursorQueryOutcome) -> None: - """Report one finished relation read.""" - - self._completed += 1 - _report_cursor_query( - query=self._queries[index], - outcome=result, - identity=_cursor_query_identity( - query=self._queries[index], index=self._completed, total=len(self._queries) - ), - on_progress=self._on_progress, - ) - - -def _inspection_concurrency(*, adapter: BaseAdapter, connection: Any) -> int: - catalog: InspectionCatalog | None = active_inspection_catalog( - adapter=adapter, connection=connection - ) - return 1 if catalog is None else catalog.concurrency - - -def _cursor_query_identity(*, query: _PhysicalCursorQuery, index: int, total: int) -> str: - return f"({index}/{total}): {query.relation}.{query.cursor_column} [{_cursor_bounds(query)}]" - - -def _cursor_bounds(query: _PhysicalCursorQuery) -> str: - bounds: str = ",".join(("min",) if query.min_tags else ()) - if query.max_tags: - bounds = f"{bounds},max" if bounds else "max" - return bounds - - -def _cursor_query_sql(query: _PhysicalCursorQuery) -> str: - select_parts: list[str] = [] - if query.min_tags: - select_parts.append(f"CAST(MIN({query.cursor_column}) AS VARCHAR) AS _min") - if query.max_tags: - select_parts.append(f"CAST(MAX({query.cursor_column}) AS VARCHAR) AS _max") - return f"SELECT {', '.join(select_parts)} FROM {query.relation}" - - -def _bind_cursor_query( - *, query: _PhysicalCursorQuery, connection: Any, execute: AdapterExecute[Any, Any] -) -> Callable[[], _CursorQueryOutcome]: - def read() -> _CursorQueryOutcome: - return _read_cursor_query(query=query, connection=connection, execute=execute) - - return read - - -def _read_cursor_query( - *, query: _PhysicalCursorQuery, connection: Any, execute: AdapterExecute[Any, Any] -) -> _CursorQueryOutcome: - """Run one physical cursor statement; a failure leaves only that relation unavailable.""" - - sql: str = _cursor_query_sql(query) - query_start: float = time.monotonic() - try: - result: Any = execute(connection=connection, sql=sql) - rows: list[Any] = result.fetchall() - except Exception as error: - elapsed: float = time.monotonic() - query_start - record_inspection_query( - record=InspectionQueryRecord( - sql=sql, elapsed_seconds=elapsed, row_count=None, error=str(error) - ) - ) - return _CursorQueryOutcome(row=None, elapsed_seconds=elapsed, error=error) - elapsed = time.monotonic() - query_start - record_inspection_query( - record=InspectionQueryRecord(sql=sql, elapsed_seconds=elapsed, row_count=len(rows)) - ) - return _CursorQueryOutcome(row=rows[0] if rows else None, elapsed_seconds=elapsed) - - -def _report_cursor_query( - *, - query: _PhysicalCursorQuery, - outcome: _CursorQueryOutcome, - identity: str, - on_progress: Callable[[str], None] | None, -) -> None: - if outcome.error is None: - if on_progress is not None: - on_progress(f"Inspected cursor bounds {identity} ({outcome.elapsed_seconds:.2f}s)") - return - if on_progress is not None: - on_progress( - f"Failed cursor bounds {identity} ({outcome.elapsed_seconds:.2f}s): {outcome.error}" - ) - log_debug_event( - logger=_DEBUG_LOGGER, - message="cursor bounds physical query failed; treating relation as unavailable", - sqlbuild_relation=query.relation, - sqlbuild_cursor_column=query.cursor_column, - sqlbuild_bounds=_cursor_bounds(query), - sqlbuild_elapsed_seconds=f"{outcome.elapsed_seconds:.2f}", - sqlbuild_error=str(outcome.error), - ) - - -def _fan_out_cursor_row(*, query: _PhysicalCursorQuery, row: Any) -> dict[str, CursorScalar]: - if row is None: - return {} - output: dict[str, CursorScalar] = {} - value_index: int = 0 - if query.min_tags: - min_value: Any = row[value_index] - if min_value is not None: - min_tag: str - for min_tag in query.min_tags: - output[min_tag] = _parse_warehouse_cursor(value=min_value) - value_index += 1 - if query.max_tags: - max_value: Any = row[value_index] - if max_value is not None: - max_tag: str - for max_tag in query.max_tags: - output[max_tag] = _parse_warehouse_cursor(value=max_value) - return output - - -def _parse_warehouse_cursor(*, value: object) -> CursorScalar: - """Parse a physical cursor value independently of a possibly incorrect declaration.""" - - try: - return parse(raw=value, cursor_type=CursorType.INTEGER) - except CursorAlgebraError: - return parse(raw=value, cursor_type=CursorType.TIMESTAMP) - - -def _gather_eligible_target_maxes( - *, - cursor_models: list[_CursorModelInfo], - results: dict[str, CursorScalar], - adapter: BaseAdapter, - connection: Any, - execute: AdapterExecute[Any, Any], - invocation_time: datetime | None, -) -> dict[str, CursorScalar]: - """Query highest eligible target cursors only when physical MAX is beyond policy.""" - - if invocation_time is None: - return results - eligible_results: dict[str, CursorScalar] = dict(results) - requests: list[tuple[_CursorModelInfo, str]] = [] - info: _CursorModelInfo - for info in cursor_models: - config: StartCursorsConfig | None = info.start_cursor_config - target_max: CursorScalar | None = results.get(info.target_tag) if info.target_tag else None - if ( - config is None - or config.max_ahead is None - or info.cursor_type != CursorType.TIMESTAMP - or target_max is None - or info.target_relation is None - or info.has_start_override - ): - continue - horizon: CursorScalar = parse( - raw=maximum_allowed_start( - discovered_value=render(value=target_max), - cursor_grain=info.effective_cursor_grain, - invocation_time=invocation_time, - max_ahead=config.max_ahead, - ), - cursor_type=CursorType.TIMESTAMP, - ) - if compare(left=target_max, right=horizon) <= 0: - continue - requests.append( - ( - info, - adapter.render_max_cursor_at_or_before( - relation=info.target_relation, - cursor_column=info.physical_cursor_column, - maximum_allowed=render(value=horizon), - cursor_type=info.cursor_type, - is_date=isinstance(horizon, DateValue), - ), - ) - ) - rows_by_request: list[list[Any]] = run_bounded_inspections( - tasks=tuple( - _bind_eligible_target_max( - info=request_info, sql=sql, connection=connection, execute=execute - ) - for request_info, sql in requests - ), - concurrency=_inspection_concurrency(adapter=adapter, connection=connection), - ) - rows: list[Any] - for (info, _sql), rows in zip(requests, rows_by_request, strict=True): - if rows and rows[0][0] is not None: - eligible_results[f"{info.model_name}__target__eligible_max"] = _normalize_cursor_grain( - value=parse(raw=rows[0][0], cursor_type=info.cursor_type or CursorType.TIMESTAMP), - cursor_type=info.cursor_type, - cursor_grain=info.effective_cursor_grain, - ) - return eligible_results - - -def _bind_eligible_target_max( - *, - info: _CursorModelInfo, - sql: str, - connection: Any, - execute: AdapterExecute[Any, Any], -) -> Callable[[], list[Any]]: - def read() -> list[Any]: - try: - return run_recorded_inspection_query( - sql=sql, - run=lambda: execute(connection=connection, sql=sql).fetchall(), - ) - except Exception as error: - raise PlannerInputError( - f"model '{info.model_name}': failed to query highest eligible target cursor " - f"for {info.target_relation}.{info.cursor_column}: {error}" - ) from error - - return read - - -def _normalize_cursor_grain( - *, value: CursorScalar, cursor_type: str | None, cursor_grain: str | None -) -> CursorScalar: - snapshot: ModelCursorSnapshot = normalize_cursor_snapshot_grain( - cursor_snapshot=ModelCursorSnapshot( - target_max=None, - upstream_mins=(), - upstream_maxes=(), - target_eligible_max=value, - ), - cursor_type=cursor_type, - effective_grain=cursor_grain, - ) - return snapshot.target_eligible_max or value - - -def _assemble_cursor_snapshots( - *, - cursor_models: list[_CursorModelInfo], - results: dict[str, CursorScalar], - failed_tags: frozenset[str] = frozenset(), -) -> dict[str, ModelCursorSnapshot]: - """Fan batch query results back into per-model cursor snapshots.""" - - snapshots: dict[str, ModelCursorSnapshot] = {} - info: _CursorModelInfo - for info in cursor_models: - target_max: CursorScalar | None = results.get(info.target_tag) if info.target_tag else None - - upstream_mins: list[CursorScalar] = [] - upstream_maxes: list[CursorScalar] = [] - input_evidence: list[CursorInputEvidence] = [] - unavailable_watermark_tags: list[str] = [] - terminal_starts: list[CursorScalar] = [] - terminal_ends: list[CursorScalar] = [] - end_inputs: list[tuple[CursorScalar | None, CursorScalar | None]] = [] - availability_ends: list[CursorScalar] = [] - empty_input_names: list[str] = [] - unreadable_input_names: list[str] = [] - upstream: _UpstreamCursorInfo - for upstream in info.upstreams: - min_val: CursorScalar | None = results.get(upstream.tag_min) - max_val: CursorScalar | None = results.get(upstream.tag_max) - if max_val is None and upstream.terminal_cursor_end is None: - input_label: str = ( - f"{upstream.input_name or upstream.relation}.{upstream.cursor_column}" - ) - if upstream.tag_max in failed_tags: - unreadable_input_names.append(input_label) - else: - empty_input_names.append(input_label) - effective_max: CursorScalar | None = ( - exclusive_to_inclusive( - value=upstream.terminal_cursor_end, - grain=( - None - if info.cursor_type == CursorType.INTEGER - else CursorGrain( - upstream.cursor_grain - or info.effective_cursor_grain - or CursorGrain.SECOND - ) - ), - ) - if upstream.terminal_cursor_end is not None - else max_val - ) - if min_val is not None: - upstream_mins.append(min_val) - elif upstream.terminal_cursor_start is not None: - upstream_mins.append(upstream.terminal_cursor_start) - elif target_max is None and info.cursor_watermark_mode == CursorWatermarkMode.ALL: - unavailable_watermark_tags.append(upstream.tag_min) - if effective_max is not None: - if max_val is not None: - upstream_maxes.append(max_val) - input_evidence.append( - CursorInputEvidence( - relation=upstream.relation, - cursor_column=upstream.cursor_column, - minimum=min_val or upstream.terminal_cursor_start, - maximum=effective_max, - ) - ) - elif ( - upstream.terminal_cursor_end is None - and info.cursor_watermark_mode == CursorWatermarkMode.ALL - ): - unavailable_watermark_tags.append(upstream.tag_max) - if upstream.terminal_cursor_start is not None: - terminal_starts.append(upstream.terminal_cursor_start) - if upstream.terminal_cursor_end is not None: - terminal_ends.append(upstream.terminal_cursor_end) - end_inputs.append((max_val, upstream.terminal_cursor_end)) - if info.microbatch_strategy == MicrobatchStrategy.WATERMARK: - availability_end: CursorScalar | None = upstream.terminal_cursor_end - if availability_end is None and max_val is not None: - availability_end = observed_partition( - value=max_val, - grain=( - None - if info.cursor_type == CursorType.INTEGER - else CursorGrain( - upstream.cursor_grain - or info.effective_cursor_grain - or CursorGrain.SECOND - ) - ), - ).end - if availability_end is not None: - availability_ends.append(availability_end) - - snapshots[info.model_name] = ModelCursorSnapshot( - target_max=target_max, - upstream_mins=tuple(upstream_mins), - upstream_maxes=tuple(upstream_maxes), - physical_target_max=target_max, - target_eligible_max=results.get(f"{info.model_name}__target__eligible_max"), - target_relation=info.target_relation, - destination_cursor_column=info.cursor_column, - input_evidence=tuple(input_evidence), - expected_watermark_count=len(info.upstreams), - unavailable_watermark_tags=tuple(unavailable_watermark_tags), - cursor_watermark_mode=info.cursor_watermark_mode, - upstream_terminal_starts=tuple(terminal_starts), - upstream_terminal_ends=tuple(terminal_ends), - upstream_end_inputs=tuple(end_inputs) if terminal_ends else (), - upstream_availability_ends=tuple(availability_ends), - empty_input_names=tuple(empty_input_names), - unreadable_input_names=tuple(unreadable_input_names), - ) - - return snapshots - - -def _resolve_upstream_qualified_name( - *, - ref: CompileSqlReference, - adapter: BaseAdapter, - model_map: dict[str, CompiledModelView], - source_map: dict[str, CompiledSourceView], - seed_map: dict[str, CompiledSeedView], - deferred_locations: dict[str, CompiledRelationLocation] | None = None, - selected_names: frozenset[str] | None = None, -) -> str | None: - """Resolve a reference to a qualified relation name for cursor reads.""" - - is_selected: bool = selected_names is not None and ref.ref_name in selected_names - if ( - ref.ref_kind in {SqlReferenceKind.REF, SqlReferenceKind.SEED} - and deferred_locations is not None - and ref.ref_name in deferred_locations - and not is_selected - ): - return deferred_locations[ref.ref_name].qualified_name - if ref.ref_kind == SqlReferenceKind.REF: - upstream_model: CompiledModelView | None = model_map.get(ref.ref_name) - if upstream_model is not None: - return upstream_model.destination.qualified_name - upstream_seed: CompiledSeedView | None = seed_map.get(ref.ref_name) - if upstream_seed is not None: - return upstream_seed.destination.qualified_name - elif ref.ref_kind == SqlReferenceKind.SEED: - seed: CompiledSeedView | None = seed_map.get(ref.ref_name) - if seed is not None: - return seed.destination.qualified_name - elif ref.ref_kind == SqlReferenceKind.SOURCE: - source: CompiledSourceView | None = source_map.get(ref.ref_name) - if source is not None: - entry: SourceEntry = source.source_entry - return render_source_relation(entry=entry, adapter=adapter) - return None - - -def _selected_runtime_producer_names( - *, project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] | None -) -> frozenset[str]: - """Return selected warehouse relations produced during this planner invocation.""" - - if selected_keys is None: - return frozenset() - source_load_names: frozenset[str] = frozenset( - source.name for source in project.sources if source.source_entry.loader is not None - ) - names: set[str] = set() - key: CompiledObjectKey - for key in selected_keys: - if key.resource_type in {CompiledResourceType.MODEL, CompiledResourceType.SEED}: - names.add(key.name) - elif key.resource_type == CompiledResourceType.SOURCE and key.name in source_load_names: - names.add(key.name) - return frozenset(names) - - -def _validate_watermark_contract_column( - *, - model: CompiledModelView, - ref: CompileSqlReference, - cursor_column: str, - model_map: dict[str, CompiledModelView], - source_map: dict[str, CompiledSourceView], - seed_map: dict[str, CompiledSeedView], -) -> None: - """Validate reliable watermark column contracts before warehouse inspection.""" - - declared_names: tuple[str, ...] = () - declared_types: dict[str, str] = {} - if ref.ref_kind == SqlReferenceKind.REF: - upstream_model: CompiledModelView | None = model_map.get(ref.ref_name) - if upstream_model is None: - return - upstream_cursor: str | None = get_config_str( - values=upstream_model.config.values, key="cursor" - ) - upstream_cursor_type: str | None = get_config_str( - values=upstream_model.config.values, key="cursor_type" - ) - consumer_cursor_type: str | None = get_config_str( - values=model.config.values, key="cursor_type" - ) - if ( - upstream_cursor is not None - and upstream_cursor.lower() == cursor_column.lower() - and upstream_cursor_type is not None - and consumer_cursor_type is not None - and upstream_cursor_type != consumer_cursor_type - ): - raise PlannerInputError( - f"model '{model.name}': cursor_inputs watermark '{ref.ref_name}." - f"{cursor_column}' has cursor_type={upstream_cursor_type}, incompatible with " - f"model cursor_type={consumer_cursor_type}", - code="S302", - ) - if upstream_model.config.values.get("contract") == ContractPolicy.ENFORCED: - if upstream_model.schema_entry is not None: - declared_names = tuple( - column.name for column in upstream_model.schema_entry.columns - ) - declared_types = { - column.name.lower(): column.type - for column in upstream_model.schema_entry.columns - if column.type is not None - } - else: - return - elif ref.ref_kind == SqlReferenceKind.SOURCE: - upstream_source: CompiledSourceView | None = source_map.get(ref.ref_name) - if ( - upstream_source is None - or upstream_source.source_entry.contract != ContractPolicy.ENFORCED - ): - return - declared_names = tuple(column.name for column in upstream_source.source_entry.columns) - declared_types = { - column.name.lower(): column.type - for column in upstream_source.source_entry.columns - if column.type is not None - } - elif ref.ref_kind == SqlReferenceKind.SEED: - upstream_seed: CompiledSeedView | None = seed_map.get(ref.ref_name) - if upstream_seed is None: - return - declared_names = tuple(column.name for column in upstream_seed.schema_entry.columns) - declared_types = { - column.name.lower(): column.type - for column in upstream_seed.schema_entry.columns - if column.type is not None - } - else: - return - if cursor_column.lower() in {name.lower() for name in declared_names}: - declared_type: str | None = declared_types.get(cursor_column.lower()) - cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") - if declared_type is not None and not _watermark_type_is_compatible( - declared_type=declared_type, cursor_type=cursor_type - ): - raise PlannerInputError( - f"model '{model.name}': cursor_inputs watermark '{ref.ref_name}." - f"{cursor_column}' type {declared_type} is incompatible with " - f"cursor_type={cursor_type}", - code="S302", - ) - return - declared_display: str = ", ".join(declared_names) or "none" - raise PlannerInputError( - f"model '{model.name}': cursor_inputs watermark references '{ref.ref_name}' column " - f"'{cursor_column}', but its enforced contract does not expose the column. " - f"Declared contract columns: {declared_display}", - code="S302", - ) - - -def _watermark_type_is_compatible(*, declared_type: str, cursor_type: str | None) -> bool: - try: - expected: CursorType = CursorType(cursor_type) - except (TypeError, ValueError): - return True - return classify_cursor_sql_type(declared_type) == expected diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_columns.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_columns.py deleted file mode 100644 index 99e1be5814..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_columns.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Warehouse source relation listing and column gathering for planning.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationLookup -from sqlbuild.adapter.relations.main.get_columns_for_inspection import get_columns_for_inspection -from sqlbuild.adapter.relations.main.list_relations_for_inspection import ( - list_relations_for_inspection, -) -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.constants import METADATA_NAME_FILTER_LIMIT -from sqlbuild.spec.contracts.models import SourceEntry - - -def gather_source_columns( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - source_entries: tuple[SourceEntry, ...] | None = None, -) -> dict[str, tuple[ColumnInfo, ...]]: - """Gather warehouse columns for all declared sources.""" - - return gather_source_inspection( - project=project, - adapter=adapter, - connection=connection, - source_entries=source_entries, - )[0] - - -def gather_source_inspection( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - source_entries: tuple[SourceEntry, ...] | None, -) -> tuple[dict[str, tuple[ColumnInfo, ...]], frozenset[str]]: - """Gather source columns and the names of sources the relation listing found.""" - - result: dict[str, tuple[ColumnInfo, ...]] = {} - listed_names: set[str] = set() - source_schemas: dict[str, set[str]] = {} - entries: tuple[SourceEntry, ...] = ( - source_entries - if source_entries is not None - else tuple(source.source_entry for source in project.sources) - ) - entry: SourceEntry - for entry in entries: - if entry.expression is not None: - if entry.type_enforcement: - column_names: tuple[str, ...] = adapter.query_column_names( - connection=connection, sql=entry.expression - ) - result[entry.name] = tuple(ColumnInfo(name=name, type="") for name in column_names) - continue - schema: str | None = entry.schema - if schema is None: - continue - db: str | None = entry.database - db_key: str = db or "" - source_schemas.setdefault(db_key, set()).add(schema) - - db_key_iter: str - schemas: set[str] - for db_key_iter, schemas in source_schemas.items(): - database: str | None = db_key_iter or None - scoped: dict[str, tuple[str | None, str | None, str]] = { - entry_iter.name: RelationLookup.key( - database=database, - schema=entry_iter.schema, - name=entry_iter.table if entry_iter.table is not None else entry_iter.name, - ) - for entry_iter in entries - if entry_iter.expression is None - and (entry_iter.database or None) == database - and entry_iter.schema in schemas - } - relations: tuple[RelationInfo, ...] = list_relations_for_inspection( - adapter=adapter, - connection=connection, - database=database, - schemas=tuple(sorted(schemas)), - names=_build_source_table_name_filter( - project=project, database=database, schemas=schemas, source_entries=entries - ), - ) - listed_identities: frozenset[tuple[str | None, str | None, str]] = frozenset( - relation.identity for relation in relations - ) - listed_names.update(name for name, key in scoped.items() if key in listed_identities) - wanted: frozenset[tuple[str | None, str | None, str]] = frozenset(scoped.values()) - all_columns: dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]] = ( - get_columns_for_inspection( - adapter=adapter, - connection=connection, - relations=tuple(relation for relation in relations if relation.identity in wanted), - ) - ) - result.update( - {name: all_columns[key] for name, key in scoped.items() if key in all_columns} - ) - - return result, frozenset(listed_names) - - -def _build_source_table_name_filter( - *, - project: CompiledProjectView, - database: str | None, - schemas: set[str], - source_entries: tuple[SourceEntry, ...] | None = None, -) -> tuple[str, ...] | None: - names: set[str] = set() - entries: tuple[SourceEntry, ...] = ( - source_entries - if source_entries is not None - else tuple(source.source_entry for source in project.sources) - ) - entry: SourceEntry - for entry in entries: - if entry.expression is not None or entry.schema not in schemas: - continue - if (entry.database or None) != database: - continue - names.add(entry.table if entry.table is not None else entry.name) - if not names or len(names) > METADATA_NAME_FILTER_LIMIT: - return None - return tuple(sorted(names)) diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py deleted file mode 100644 index 78a348b44a..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py +++ /dev/null @@ -1,261 +0,0 @@ -"""Source read deferral helpers for managed source loaders.""" - -from __future__ import annotations - -import re -from dataclasses import replace - -from sqlbuild._native import render_project_var_text, resolve_target_config -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, -) -from sqlbuild.compiler.compile.types import ( - CompiledAuditView, - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledSourceView, -) -from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.planner.constants import ( - MODEL_POST_HOOKS_CONFIG_KEY, - MODEL_PRE_HOOKS_CONFIG_KEY, - SOURCE_DEFERRAL_CONTEXT_FIELDS, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.references.constants import SOURCE_PATTERN -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceEntry, TargetConfig - -_CTX_PATTERN: re.Pattern[str] = re.compile(r"\$\{CTX:([^}]+)\}") -_VAR_PATTERN: re.Pattern[str] = re.compile(r"\$\{([^}:]+)\}") -_SOURCE_PATTERN: re.Pattern[str] = SOURCE_PATTERN - - -def build_source_read_map( - *, - project: CompiledProjectView, - source_map: dict[str, SourceEntry], - selected_keys: frozenset[CompiledObjectKey], - project_config: ProjectConfig | None, - local_config: LocalConfig | None, - defer_sources_to: str | None, - read_source_names: frozenset[str] | None = None, -) -> dict[str, SourceEntry]: - """Resolve source reads for selected SQL, or for ``read_source_names`` when given.""" - - managed_source_names: frozenset[str] = frozenset( - source.source_entry.name - for source in project.sources - if source.source_entry.loader is not None - ) - selected_managed_sources: tuple[str, ...] = ( - tuple(sorted(read_source_names & managed_source_names)) - if read_source_names is not None - else _selected_managed_source_refs( - project=project, - selected_keys=selected_keys, - managed_source_names=managed_source_names, - ) - ) - if not selected_managed_sources: - return source_map - source_target_name: str | None = _resolve_source_target_name( - project=project, - project_config=project_config, - local_config=local_config, - defer_sources_to=defer_sources_to, - ) - if source_target_name is None: - return source_map - if project_config is None or local_config is None: - return source_map - if ( - source_target_name not in project_config.targets - and source_target_name not in local_config.targets - ): - raise PlannerInputError(f"Unknown source deferral target '{source_target_name}'") - source_target: TargetConfig = resolve_target_config( - project_config=project_config, - local_config=local_config, - target_name=source_target_name, - ) - result: dict[str, SourceEntry] = dict(source_map) - source: CompiledSourceView - for source in project.sources: - if source.source_entry.name not in selected_managed_sources: - continue - raw_source_entry: SourceEntry = _raw_source_entry(source) - result[source.source_entry.name] = _source_entry_for_environment( - source_entry=raw_source_entry, - target_config=source_target, - effective_vars=project.effective_vars, - ) - return result - - -def with_declared_source_reads( - *, - project: CompiledProjectView, - source_map: dict[str, SourceEntry], - source_names: frozenset[str], -) -> dict[str, SourceEntry]: - """Add declared project sources that no selected SQL resource reads to a source map.""" - - missing: frozenset[str] = source_names - frozenset(source_map) - if not missing: - return source_map - result: dict[str, SourceEntry] = dict(source_map) - source: CompiledSourceView - for source in project.sources: - if source.source_entry.name in missing: - result[source.source_entry.name] = source.source_entry - return result - - -def _sql_hook_source_reads(model: CompiledModelView) -> tuple[str, ...]: - names: list[str] = [] - for hook_key in (MODEL_PRE_HOOKS_CONFIG_KEY, MODEL_POST_HOOKS_CONFIG_KEY): - value: object = model.config.values.get(hook_key) - entries: tuple[object, ...] = tuple(value) if isinstance(value, list | tuple) else (value,) - for entry in entries: - if isinstance(entry, SqlHookEntry): - names.extend( - ref.name for ref in entry.reads if ref.kind is SqlResourceRefKind.SOURCE - ) - return tuple(names) - - -def _selected_managed_source_refs( - *, - project: CompiledProjectView, - selected_keys: frozenset[CompiledObjectKey], - managed_source_names: frozenset[str], -) -> tuple[str, ...]: - names: set[str] = set() - model: CompiledModelView - for model in project.models: - if model.key not in selected_keys: - continue - names.update( - _direct_managed_source_refs( - sql=model.query_sql, managed_source_names=managed_source_names - ) - ) - names.update(name for name in _sql_hook_source_reads(model) if name in managed_source_names) - function: CompiledFunctionView - for function in project.functions: - if function.key not in selected_keys: - continue - names.update( - _direct_managed_source_refs( - sql=function.body_sql, managed_source_names=managed_source_names - ) - ) - audit: CompiledAuditView - for audit in project.audits: - if audit.scope_deps and not any(dep in selected_keys for dep in audit.scope_deps): - continue - for ref in audit.references: - if ref.ref_kind == SqlReferenceKind.SOURCE and ref.ref_name in managed_source_names: - names.add(ref.ref_name) - for match in _SOURCE_PATTERN.finditer(audit.sql_body): - source_name: str = match.group(1) - if source_name in managed_source_names: - names.add(source_name) - return tuple(sorted(names)) - - -def _direct_managed_source_refs( - *, sql: str, managed_source_names: frozenset[str] -) -> frozenset[str]: - names: set[str] = set() - match: re.Match[str] - for match in _SOURCE_PATTERN.finditer(sql): - source_name: str = match.group(1) - if source_name in managed_source_names: - names.add(source_name) - return frozenset(names) - - -def _resolve_source_target_name( - *, - project: CompiledProjectView, - project_config: ProjectConfig | None, - local_config: LocalConfig | None, - defer_sources_to: str | None, -) -> str | None: - if defer_sources_to is not None: - return defer_sources_to - if project.effective_target_name is None or project_config is None or local_config is None: - return None - active_target: TargetConfig = resolve_target_config( - project_config=project_config, - local_config=local_config, - target_name=project.effective_target_name, - ) - return active_target.defer_sources_to or project.effective_target_name - - -def _raw_source_entry(source: CompiledSourceView) -> SourceEntry: - source_entry: SourceEntry - for source_entry in source.source_file.source_entries: - if source_entry.name == source.source_entry.name: - return source_entry - return source.source_entry - - -def _source_entry_for_environment( - *, - source_entry: SourceEntry, - target_config: TargetConfig, - effective_vars: dict[str, object], -) -> SourceEntry: - if source_entry.expression is not None: - return source_entry - database: str | None = source_entry.database - if database is None: - database = _resolve_target_field( - target_value=target_config.database, - logical_value=source_entry.database, - effective_vars=effective_vars, - ) - schema: str | None = source_entry.schema - if schema is None: - schema = _resolve_target_field( - target_value=target_config.loader_schema or target_config.schema, - logical_value=source_entry.schema, - effective_vars=effective_vars, - ) - return replace( - source_entry, - database=database, - schema=schema, - ) - - -def _resolve_target_field( - *, target_value: str | None, logical_value: str | None, effective_vars: dict[str, object] -) -> str | None: - if target_value is None: - return logical_value - - def _replace_ctx(match: re.Match[str]) -> str: - ctx_key: str = match.group(1) - if ctx_key in SOURCE_DEFERRAL_CONTEXT_FIELDS: - return logical_value if logical_value is not None else "" - return match.group(0) - - result: str = _CTX_PATTERN.sub(_replace_ctx, target_value) - - def _replace_var(match: re.Match[str]) -> str: - var_name: str = match.group(1) - if var_name not in effective_vars: - return match.group(0) - return render_project_var_text( - value=effective_vars[var_name], - label=f"source deferral variable '${{{var_name}}}'", - ) - - return _VAR_PATTERN.sub(_replace_var, result) diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_freshness.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_freshness.py deleted file mode 100644 index 694a2469cc..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_freshness.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Direct planner source freshness orchestration helpers.""" - -from __future__ import annotations - -from dataclasses import replace -from datetime import UTC, datetime -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.planner.models import ( - PlannerRelationsContext, - PlannerScope, -) -from sqlbuild.compiler.source_freshness.main._planning import ( - build_direct_source_freshness_planning_result, -) -from sqlbuild.compiler.source_freshness.main._propagation import ( - build_direct_source_freshness_propagation_result, -) -from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult -from sqlbuild.runtime.observability.main.run_scope import run_scope - - -def build_planner_source_freshness_result( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - scope: PlannerScope, - relations: PlannerRelationsContext, - freshness_state_schemas: frozenset[str] = frozenset(), -) -> DirectSourceFreshnessPlanningResult: - """Build direct source freshness comparison data for planner output.""" - - state_schemas: tuple[str, ...] = _collect_state_schemas(project=project, scope=scope) - with run_scope(project.run_id): - source_freshness: DirectSourceFreshnessPlanningResult = ( - build_direct_source_freshness_planning_result( - adapter=adapter, - connection=connection, - sources=tuple(relations.source_read_map.values()), - state_database=_resolve_state_database(project), - state_schemas=state_schemas, - observed_at=datetime.now(UTC), - run_id="planning", - render_qualified_name=adapter.render_qualified_name, - state_table_exists_by_schema={ - state_schema: state_schema.lower() in freshness_state_schemas - for state_schema in state_schemas - }, - filter_previous_to_sources=True, - ) - ) - return replace( - source_freshness, - propagation=build_direct_source_freshness_propagation_result( - source_freshness=source_freshness, - scope=scope, - ), - ) - - -def _resolve_state_database(project: CompiledProjectView) -> str | None: - model_destination: CompiledRelationLocation - for model_destination in _iter_state_destinations(project=project): - if model_destination.database is not None: - return model_destination.database - return None - - -def _collect_state_schemas(*, project: CompiledProjectView, scope: PlannerScope) -> tuple[str, ...]: - selected_model_names: frozenset[str] = frozenset( - key.name for key in scope.selected_keys if key.resource_type == CompiledResourceType.MODEL - ) - schemas: set[str] = set() - model_destination: CompiledRelationLocation - for model_destination in _iter_state_destinations(project=project): - if model_destination.schema is not None: - schemas.add(model_destination.schema) - if not schemas: - model_name: str - for model_name in selected_model_names: - model: CompiledModelView | None = scope.models_by_name.get(model_name) - if model is not None and model.destination.schema is not None: - schemas.add(model.destination.schema) - return tuple(sorted(schemas)) - - -def _iter_state_destinations( - *, project: CompiledProjectView -) -> tuple[CompiledRelationLocation, ...]: - return ( - *(model.destination for model in project.models), - *(seed.destination for seed in project.seeds), - *(function.destination for function in project.functions), - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_freshness_warnings.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_freshness_warnings.py deleted file mode 100644 index 0d3636203d..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_freshness_warnings.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Plan warnings that explain why source freshness could not be observed.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.constants import SOURCE_FRESHNESS_UNKNOWN_WARNING_TITLE -from sqlbuild.compiler.planner.models import PlanWarning -from sqlbuild.compiler.planner.types import WarningSeverity -from sqlbuild.compiler.source_freshness.models import ( - DirectSourceFreshnessPlanningResult, - SourceFreshnessUnknown, -) -from sqlbuild.compiler.source_freshness.types import SourceFreshnessUnknownReason - -_NAME_LIMIT: int = 10 -_REASON_EXPLANATIONS: dict[SourceFreshnessUnknownReason, str] = { - SourceFreshnessUnknownReason.UNAVAILABLE: ( - "adapter freshness metadata is unavailable, for example for a view, an external table, " - "or a table without a last-modified time" - ), - SourceFreshnessUnknownReason.ERROR: "the freshness observation failed", -} - - -def build_source_freshness_unknown_warnings( - *, source_freshness: DirectSourceFreshnessPlanningResult -) -> tuple[PlanWarning, ...]: - """Return one warning per reason that left configured source freshness unknown.""" - - names_by_reason: dict[SourceFreshnessUnknownReason, list[str]] = {} - unknown: SourceFreshnessUnknown - for unknown in source_freshness.unknown_sources.values(): - if unknown.reason in _REASON_EXPLANATIONS: - names_by_reason.setdefault(unknown.reason, []).append(unknown.source_name) - return tuple( - PlanWarning( - model_name=None, - severity=WarningSeverity.WARNING, - message=( - f"{SOURCE_FRESHNESS_UNKNOWN_WARNING_TITLE} ({reason.value}) for " - f"{_format_names(names)}: {_REASON_EXPLANATIONS[reason]}; " - "run sqb freshness for each source's details" - ), - ) - for reason, names in names_by_reason.items() - ) - - -def _format_names(names: list[str]) -> str: - ordered: list[str] = sorted(names) - shown: str = ", ".join(ordered[:_NAME_LIMIT]) - if len(ordered) > _NAME_LIMIT: - return f"{shown} and {len(ordered) - _NAME_LIMIT} more" - return shown diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_tables.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_tables.py deleted file mode 100644 index 0ddfb7082e..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_tables.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Selected-scope validation that external source tables exist in the warehouse.""" - -from __future__ import annotations - -from dataclasses import dataclass -from functools import partial -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationReadProbe -from sqlbuild.adapter.contract.types import RelationReadStatus -from sqlbuild.adapter.relations.main.run_bounded_inspections import run_bounded_inspections -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import ( - CompiledProjectView, - CompiledResourceType, - CompiledSourceView, -) -from sqlbuild.compiler.discovery.main._yaml_entry_line import yaml_entry_line -from sqlbuild.compiler.planner._helpers.resolve.sources import render_source_read_relation -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope -from sqlbuild.spec.contracts.models import SourceEntry - -_MESSAGE_ITEM_LIMIT: int = 10 -_WILDCARD_TABLE_CHARACTER: str = "*" - - -@dataclass(frozen=True) -class _MissingSourceTable: - source: CompiledSourceView - entry: SourceEntry - read_by: tuple[str, ...] - probe: RelationReadProbe - - -def check_selected_source_tables_exist( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - scope: PlannerScope, - relations: PlannerRelationsContext, -) -> None: - """Raise when a selected resource reads an external source table that does not exist.""" - - readers_by_source: dict[str, list[str]] = _selected_source_readers(scope=scope) - candidates: list[tuple[CompiledSourceView, SourceEntry, tuple[str, ...]]] = [] - source: CompiledSourceView - for source in project.sources: - readers: list[str] | None = readers_by_source.get(source.name) - if not readers: - continue - entry: SourceEntry = relations.source_read_map.get(source.name, source.source_entry) - if not _is_external_table_source(source.source_entry) or not _is_external_table_source( - entry - ): - continue - candidates.append((source, entry, tuple(sorted(readers)))) - if not candidates: - return - unlisted: tuple[tuple[CompiledSourceView, SourceEntry, tuple[str, ...]], ...] = tuple( - candidate - for candidate in candidates - if candidate[0].name not in relations.listed_source_names - ) - probes: list[RelationReadProbe] = run_bounded_inspections( - tasks=tuple( - partial( - adapter.probe_relation_read, - connection=connection, - relation=render_source_read_relation(adapter=adapter, source_entry=entry), - ) - for _, entry, _ in unlisted - ), - concurrency=max(1, int(adapter.metadata_inspection_concurrency)), - ) - missing: tuple[_MissingSourceTable, ...] = tuple( - _MissingSourceTable(source=source, entry=entry, read_by=read_by, probe=probe) - for (source, entry, read_by), probe in zip(unlisted, probes, strict=True) - if probe.status != RelationReadStatus.READABLE - ) - if missing: - raise PlannerInputError( - _missing_source_tables_message(missing=missing, adapter=adapter), - code="S405", - help=_missing_source_tables_help(missing=missing), - ) - - -def _selected_source_readers(*, scope: PlannerScope) -> dict[str, list[str]]: - readers_by_source: dict[str, list[str]] = {} - selected_key: CompiledObjectKey - for selected_key in scope.selected_keys: - if selected_key.resource_type == CompiledResourceType.SOURCE: - continue - dep_key: CompiledObjectKey - for dep_key in scope.upstream_deps.get(selected_key, ()): - if dep_key.resource_type != CompiledResourceType.SOURCE: - continue - readers_by_source.setdefault(dep_key.name, []).append(selected_key.name) - return readers_by_source - - -def _is_external_table_source(entry: SourceEntry) -> bool: - return ( - not entry.managed - and entry.loader is None - and entry.integration_loader is None - and entry.expression is None - and entry.schema is not None - and _WILDCARD_TABLE_CHARACTER not in _table_name(entry) - ) - - -def _table_name(entry: SourceEntry) -> str: - return entry.table if entry.table is not None else entry.name - - -def _missing_source_tables_message( - *, missing: tuple[_MissingSourceTable, ...], adapter: BaseAdapter -) -> str: - noun: str = "source table" if len(missing) == 1 else "source tables" - verb: str = "does" if len(missing) == 1 else "do" - readability: str = ( - " or is not readable by the current role" - if len(missing) == 1 - else " or are not readable by the current role" - ) - lines: list[str] = [ - f"cannot build selected scope: {len(missing)} {noun} read by selected resources " - f"{verb} not exist in the warehouse" - f"{readability if _any_unreadable(missing) else ''}:" - ] - item: _MissingSourceTable - for item in missing[:_MESSAGE_ITEM_LIMIT]: - relation: str = render_source_read_relation(adapter=adapter, source_entry=item.entry) - readers: str = ", ".join(item.read_by) - lines.append( - f" - source '{item.source.name}' ({relation}), read by {readers}" - f"{_unreadable_suffix(item.probe)}" - ) - if len(missing) > _MESSAGE_ITEM_LIMIT: - lines.append(f" - ... and {len(missing) - _MESSAGE_ITEM_LIMIT} more") - return "\n".join(lines) - - -def _unreadable_suffix(probe: RelationReadProbe) -> str: - if probe.status != RelationReadStatus.MISSING_OR_UNREADABLE: - return "" - role: str = f"role {probe.role}" if probe.role is not None else "the current role" - return f"; the warehouse reports it does not exist or is not readable by {role}" - - -def _any_unreadable(missing: tuple[_MissingSourceTable, ...]) -> bool: - return any(item.probe.status == RelationReadStatus.MISSING_OR_UNREADABLE for item in missing) - - -def _missing_source_tables_help(*, missing: tuple[_MissingSourceTable, ...]) -> str: - locations: str = ", ".join( - f"'{item.source.name}' at {_declaration_location(item.source)}" - for item in missing[:_MESSAGE_ITEM_LIMIT] - ) - roles: tuple[str, ...] = tuple( - sorted({item.probe.role for item in missing if item.probe.role is not None}) - ) - role_text: str = f" ({', '.join(roles)})" if roles else "" - grant: str = ( - f"grant SELECT on it to the role in use{role_text}, " if _any_unreadable(missing) else "" - ) - return ( - f"create or load the table, {grant}correct its database, schema, or table in the " - f"source declaration, or stop reading it from the selected models; declared {locations}" - ) - - -def _declaration_location(source: CompiledSourceView) -> str: - path: str = source.source_file.relative_path.as_posix() - line: int | None = yaml_entry_line(contents=source.source_file.contents, name=source.name) - return path if line is None else f"{path}:{line}" diff --git a/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py b/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py deleted file mode 100644 index 57fd160d88..0000000000 --- a/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py +++ /dev/null @@ -1,234 +0,0 @@ -"""Migration fingerprints computed at most once per model definition and reused across runs.""" - -from __future__ import annotations - -import hashlib -import hmac -import json -import sqlite3 -from collections.abc import Mapping -from importlib.metadata import PackageNotFoundError, version -from pathlib import Path -from typing import Any, cast - -from sqlbuild._native import ( - build_migration_fingerprint, - migration_fingerprint_ref_names, - without_declared_columns_hash, -) -from sqlbuild.compiler.planner.constants import MIGRATION_FINGERPRINT_ALGORITHM - -type _FingerprintKey = tuple[str, str, str | None, tuple[tuple[str, str], ...]] - -_CACHE_VERSION: int = 1 -_CACHE_DIRECTORY_NAME: str = f"migration-fingerprints-v{_CACHE_VERSION}" -_CACHE_DATABASE_NAME: str = "migration-fingerprints.sqlite3" -_CACHE_ENTRY_SEPARATOR: str = "\n" -_MAX_CACHE_ENTRY_BYTES: int = 4_096 -_SQLITE_TIMEOUT_SECONDS: float = 0.1 -_CREATE_CACHE_TABLE_SQL: str = """ -CREATE TABLE IF NOT EXISTS migration_fingerprint ( - cache_key TEXT PRIMARY KEY, - payload TEXT NOT NULL -) -""" - - -class MigrationFingerprintCache: - """Reuse migration fingerprints across rename maps, planning phases, and invocations.""" - - def __init__(self, *, root: Path | None = None) -> None: - self._ref_names: dict[tuple[str, str], frozenset[str]] = {} - self._fingerprints: dict[_FingerprintKey, str | None] = {} - self._database_path: Path | None = ( - None if root is None else root / _CACHE_DIRECTORY_NAME / _CACHE_DATABASE_NAME - ) - self._connection: sqlite3.Connection | None = None - self._connection_opened: bool = False - self._pending: dict[str, str] = {} - self._shared_digest: str = _shared_digest() if root is not None else "" - - def fingerprint( - self, - *, - query_sql: str, - metadata_json: str, - ref_identities: Mapping[str, str], - dialect: str | None, - ) -> str | None: - """Return the migration fingerprint, keyed only by the renames this model can see.""" - - metadata_json = without_declared_columns_hash(metadata_json) - definition: tuple[str, str] = (query_sql, metadata_json) - names: frozenset[str] | None = self._ref_names.get(definition) - if names is None: - names = migration_fingerprint_ref_names( - query_sql=query_sql, metadata_json=metadata_json - ) - self._ref_names[definition] = names - visible: dict[str, str] = { - name: ref_identities[name] for name in sorted(names) if name in ref_identities - } - key: _FingerprintKey = (query_sql, metadata_json, dialect, tuple(visible.items())) - if key in self._fingerprints: - return self._fingerprints[key] - cache_key: str = self._cache_key(key) - found: bool - stored: str | None - found, stored = self._stored(cache_key) - if not found: - stored = build_migration_fingerprint( - query_sql=query_sql, - metadata_json=metadata_json, - ref_identities=visible, - dialect=dialect, - ) - self._remember(cache_key=cache_key, fingerprint=stored) - self._fingerprints[key] = stored - return stored - - def persist(self) -> None: - """Write fingerprints computed since the last persist; failures only skip reuse.""" - - pending: dict[str, str] = self._pending - self._pending = {} - connection: sqlite3.Connection | None = self._connection - self._connection = None - self._connection_opened = False - try: - if pending and self._database_path is not None: - if connection is None: - self._database_path.parent.mkdir(parents=True, exist_ok=True) - connection = sqlite3.connect( - self._database_path, timeout=_SQLITE_TIMEOUT_SECONDS - ) - _ = connection.execute(_CREATE_CACHE_TABLE_SQL) - _ = connection.executemany( - "INSERT OR REPLACE INTO migration_fingerprint (cache_key, payload) " - "VALUES (?, ?)", - pending.items(), - ) - connection.commit() - except (OSError, sqlite3.DatabaseError): - pass - finally: - _close(connection) - - def _cache_key(self, key: _FingerprintKey) -> str: - if self._database_path is None: - return "" - return hashlib.sha256( - json.dumps( - [self._shared_digest, *key[:3], [list(item) for item in key[3]]], - ensure_ascii=False, - separators=(",", ":"), - ).encode() - ).hexdigest() - - def _stored(self, cache_key: str) -> tuple[bool, str | None]: - connection: sqlite3.Connection | None = self._open() - if connection is None: - return False, None - try: - row: tuple[object] | None = connection.execute( - "SELECT payload FROM migration_fingerprint WHERE cache_key = ?", (cache_key,) - ).fetchone() - except sqlite3.DatabaseError: - self._connection = None - _close(connection) - return False, None - return ( - (False, None) - if row is None - else _fingerprint_from_contents(contents=row[0], expected_cache_key=cache_key) - ) - - def _open(self) -> sqlite3.Connection | None: - if self._connection_opened: - return self._connection - self._connection_opened = True - if self._database_path is None or not self._database_path.is_file(): - return None - try: - self._connection = sqlite3.connect(self._database_path, timeout=_SQLITE_TIMEOUT_SECONDS) - except (OSError, sqlite3.DatabaseError): - self._connection = None - return self._connection - - def _remember(self, *, cache_key: str, fingerprint: str | None) -> None: - if self._database_path is None: - return - serialized: str = json.dumps( - {"v": _CACHE_VERSION, "k": cache_key, "f": fingerprint}, - separators=(",", ":"), - sort_keys=True, - ) - self._pending[cache_key] = _CACHE_ENTRY_SEPARATOR.join( - (_entry_digest(cache_key=cache_key, serialized=serialized), serialized) - ) - - -def _fingerprint_from_contents( - *, contents: object, expected_cache_key: str -) -> tuple[bool, str | None]: - if not isinstance(contents, str) or len(contents) > _MAX_CACHE_ENTRY_BYTES: - return False, None - stored_digest, separator, serialized = contents.partition(_CACHE_ENTRY_SEPARATOR) - if ( - not separator - or not stored_digest.isascii() - or not hmac.compare_digest( - stored_digest, _entry_digest(cache_key=expected_cache_key, serialized=serialized) - ) - ): - return False, None - try: - payload: object = json.loads(serialized) - except (ValueError, RecursionError): - return False, None - values: dict[str, Any] = cast(dict[str, Any], payload) if isinstance(payload, dict) else {} - fingerprint: object = values.get("f") - if ( - values.get("v") != _CACHE_VERSION - or values.get("k") != expected_cache_key - or not (fingerprint is None or isinstance(fingerprint, str)) - ): - return False, None - return True, cast(str | None, fingerprint) - - -def _entry_digest(*, cache_key: str, serialized: str) -> str: - digest: Any = hashlib.sha256(cache_key.encode()) - digest.update(b"\0") - digest.update(serialized.encode()) - return digest.hexdigest() - - -def _shared_digest() -> str: - return hashlib.sha256( - json.dumps( - { - "algorithm": MIGRATION_FINGERPRINT_ALGORITHM, - "cache_version": _CACHE_VERSION, - "sqlbuild_version": _package_version("sqlbuild"), - "polyglot_version": _package_version("polyglot-sql-chio"), - }, - sort_keys=True, - ).encode() - ).hexdigest() - - -def _package_version(package: str) -> str: - try: - return version(package) - except PackageNotFoundError: - return "unknown" - - -def _close(connection: sqlite3.Connection | None) -> None: - if connection is None: - return - try: - connection.close() - except sqlite3.DatabaseError: - pass diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_queries.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_queries.py deleted file mode 100644 index cf37383578..0000000000 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_queries.py +++ /dev/null @@ -1,532 +0,0 @@ -from __future__ import annotations - -from typing import Any - -import pytest - -from sqlbuild._native import cursor_parse as parse -from sqlbuild._native import cursor_render as render -from sqlbuild.compiler.planner._helpers.resolve.cursor import cursor_inputs_without_rows -from sqlbuild.compiler.planner._helpers.warehouse import snapshot as snapshot_module -from sqlbuild.compiler.planner._helpers.warehouse.snapshot import ( - _assemble_cursor_snapshots, - _build_cursor_queries, - _CursorModelInfo, - _CursorQueryResults, - _execute_cursor_queries, - _PhysicalCursorQuery, - _UpstreamCursorInfo, -) -from sqlbuild.compiler.planner.models import EmptyCursorInputDecision, ModelCursorSnapshot -from sqlbuild.compiler.planner.types import CursorType -from sqlbuild.cursor_algebra.types import CursorScalar -from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( - CursorFetchFailureTestCase, - CursorQueryFailureTestCase, - CursorQueryGroupingTestCase, - CursorQueryShapeTestCase, - CursorSnapshotAvailabilityTestCase, - EmptyCursorInputsSnapshotTestCase, -) - - -class _RowsResult: - def __init__( - self, - *, - rows: list[tuple[str | None, ...]], - progress: list[str], - expected_progress_at_fetch: tuple[str, ...], - ) -> None: - self._rows: list[tuple[str | None, ...]] = rows - self._progress: list[str] = progress - self._expected_progress_at_fetch: tuple[str, ...] = expected_progress_at_fetch - - def fetchall(self) -> list[tuple[str | None, ...]]: - assert tuple(self._progress) == self._expected_progress_at_fetch - return self._rows - - -class _FailingRowsResult: - def __init__(self, *, progress: list[str], expected_progress_at_fetch: tuple[str, ...]) -> None: - self._progress: list[str] = progress - self._expected_progress_at_fetch: tuple[str, ...] = expected_progress_at_fetch - - def fetchall(self) -> list[tuple[str | None, ...]]: - assert tuple(self._progress) == self._expected_progress_at_fetch - raise RuntimeError("result transfer failed") - - -class _FailFirstExecute: - def __init__(self, *, progress: list[str]) -> None: - self.progress: list[str] = progress - self.sql: list[str] = [] - - def __call__(self, *, connection: Any, sql: str) -> _RowsResult: - self.sql.append(sql) - if "schema.bad" in sql: - assert self.progress[-1] == "Inspecting cursor bounds (1/2): schema.bad.ts [max]..." - raise RuntimeError("warehouse unavailable") - assert self.progress[-1] == "Inspecting cursor bounds (2/2): schema.good.ts [min,max]..." - return _RowsResult( - rows=[("2024-01-01", "2024-02-01")], - progress=self.progress, - expected_progress_at_fetch=tuple(self.progress), - ) - - -class _FailFirstFetch: - def __init__(self, *, progress: list[str]) -> None: - self.progress: list[str] = progress - self.sql: list[str] = [] - - def __call__(self, *, connection: Any, sql: str) -> _RowsResult | _FailingRowsResult: - self.sql.append(sql) - if "schema.bad" in sql: - return _FailingRowsResult( - progress=self.progress, - expected_progress_at_fetch=( - "Inspecting cursor bounds (1/2): schema.bad.ts [max]...", - ), - ) - return _RowsResult( - rows=[("2024-01-01", "2024-02-01")], - progress=self.progress, - expected_progress_at_fetch=( - "Inspecting cursor bounds (1/2): schema.bad.ts [max]...", - "Failed cursor bounds (1/2): schema.bad.ts [max] (0.25s): result transfer failed", - "Inspecting cursor bounds (2/2): schema.good.ts [min,max]...", - ), - ) - - -@pytest.mark.parametrize( - "test_case", - [ - CursorQueryGroupingTestCase( - description="deduplicates physical reads and fans out logical tags in stable order", - expected_physical_queries=( - ( - "analytics.first_target", - "event_time", - (), - ("first__target__max",), - ), - ( - "raw.events", - "event_time", - ("first__events__min", "second__events__min"), - ("first__events__max", "second__events__max"), - ), - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_duplicate_logical_requests_when_building_queries_then_groups_by_physical_relation( - test_case: CursorQueryGroupingTestCase, -) -> None: - shared_first: _UpstreamCursorInfo = _UpstreamCursorInfo( - tag_min="first__events__min", - tag_max="first__events__max", - relation="raw.events", - cursor_column="event_time", - ) - first: _CursorModelInfo = _CursorModelInfo( - model_name="first", - target_tag="first__target__max", - target_relation="analytics.first_target", - cursor_column="event_time", - upstreams=(shared_first, shared_first), - ) - second: _CursorModelInfo = _CursorModelInfo( - model_name="second", - target_tag=None, - target_relation=None, - cursor_column="event_time", - upstreams=( - _UpstreamCursorInfo( - tag_min="second__events__min", - tag_max="second__events__max", - relation="raw.events", - cursor_column="event_time", - ), - ), - ) - - queries: list[_PhysicalCursorQuery] = _build_cursor_queries([first, first, second]) - - actual: tuple[tuple[str, str, tuple[str, ...], tuple[str, ...]], ...] = tuple( - (query.relation, query.cursor_column, query.min_tags, query.max_tags) for query in queries - ) - assert actual == test_case.expected_physical_queries - - -@pytest.mark.parametrize( - "test_case", - [ - CursorQueryShapeTestCase( - description="min only", - min_tags=("model__source__min",), - max_tags=(), - row=("2024-01-01",), - expected_sql="SELECT CAST(MIN(event_time) AS VARCHAR) AS _min FROM raw.events", - expected_results={"model__source__min": "2024-01-01"}, - expected_bounds="min", - ), - CursorQueryShapeTestCase( - description="max only", - min_tags=(), - max_tags=("model__target__max",), - row=("2024-02-01",), - expected_sql="SELECT CAST(MAX(event_time) AS VARCHAR) AS _max FROM raw.events", - expected_results={"model__target__max": "2024-02-01"}, - expected_bounds="max", - ), - CursorQueryShapeTestCase( - description="min and max", - min_tags=("first__source__min", "second__source__min"), - max_tags=("first__source__max", "second__source__max"), - row=("2024-01-01", "2024-02-01"), - expected_sql=( - "SELECT CAST(MIN(event_time) AS VARCHAR) AS _min, " - "CAST(MAX(event_time) AS VARCHAR) AS _max FROM raw.events" - ), - expected_results={ - "first__source__min": "2024-01-01", - "second__source__min": "2024-01-01", - "first__source__max": "2024-02-01", - "second__source__max": "2024-02-01", - }, - expected_bounds="min,max", - ), - ], - ids=lambda case: case.description, -) -def test_given_requested_bounds_when_executing_then_uses_one_standalone_query_with_observability( - test_case: CursorQueryShapeTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - progress: list[str] = [] - statements: list[str] = [] - - def _execute(*, connection: Any, sql: str) -> _RowsResult: - assert progress == [ - f"Inspecting cursor bounds (1/1): raw.events.event_time [{test_case.expected_bounds}]..." - ] - statements.append(sql) - return _RowsResult( - rows=[test_case.row], - progress=progress, - expected_progress_at_fetch=( - f"Inspecting cursor bounds (1/1): raw.events.event_time " - f"[{test_case.expected_bounds}]...", - ), - ) - - monotonic_values: Any = iter((10.0, 10.42)) - monkeypatch.setattr(snapshot_module.time, "monotonic", monotonic_values.__next__) - query: _PhysicalCursorQuery = _PhysicalCursorQuery( - relation="raw.events", - cursor_column="event_time", - min_tags=test_case.min_tags, - max_tags=test_case.max_tags, - ) - - results: dict[str, CursorScalar] = _execute_cursor_queries( - queries=[query], - connection=object(), - execute=_execute, - on_progress=progress.append, - ).values - - assert { - key: render(value=value) for key, value in results.items() - } == test_case.expected_results - assert statements == [test_case.expected_sql] - assert "UNION ALL" not in statements[0] - assert progress[-1] == ( - f"Inspected cursor bounds (1/1): raw.events.event_time " - f"[{test_case.expected_bounds}] (0.42s)" - ) - - -@pytest.mark.parametrize( - "test_case", - [ - CursorQueryFailureTestCase( - description="failed relation remains unavailable and later relation succeeds", - expected_results={"good__min": "2024-01-01", "good__max": "2024-02-01"}, - expected_failure_progress=( - "Failed cursor bounds (1/2): schema.bad.ts [max] (0.25s): warehouse unavailable" - ), - expected_success_progress=( - "Inspected cursor bounds (2/2): schema.good.ts [min,max] (0.50s)" - ), - expected_failed_tags=frozenset({"bad__max"}), - ) - ], - ids=lambda case: case.description, -) -def test_given_one_failed_physical_read_when_executing_then_reports_failure_and_continues( - test_case: CursorQueryFailureTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - progress: list[str] = [] - execute: _FailFirstExecute = _FailFirstExecute(progress=progress) - monotonic_values: Any = iter((20.0, 20.25, 30.0, 30.50)) - monkeypatch.setattr(snapshot_module.time, "monotonic", monotonic_values.__next__) - queries: list[_PhysicalCursorQuery] = [ - _PhysicalCursorQuery( - relation="schema.bad", - cursor_column="ts", - min_tags=(), - max_tags=("bad__max",), - ), - _PhysicalCursorQuery( - relation="schema.good", - cursor_column="ts", - min_tags=("good__min",), - max_tags=("good__max",), - ), - ] - - query_results: _CursorQueryResults = _execute_cursor_queries( - queries=queries, - connection=object(), - execute=execute, - on_progress=progress.append, - ) - - assert { - key: render(value=value) for key, value in query_results.values.items() - } == test_case.expected_results - assert query_results.failed_tags == test_case.expected_failed_tags - assert len(execute.sql) == 2 - assert progress[1] == test_case.expected_failure_progress - assert progress[3] == test_case.expected_success_progress - - -@pytest.mark.parametrize( - "test_case", - [ - CursorFetchFailureTestCase( - description="fetch failure remains unavailable and later relation succeeds", - expected_results={"good__min": "2024-01-01", "good__max": "2024-02-01"}, - expected_progress=( - "Inspecting cursor bounds (1/2): schema.bad.ts [max]...", - "Failed cursor bounds (1/2): schema.bad.ts [max] (0.25s): result transfer failed", - "Inspecting cursor bounds (2/2): schema.good.ts [min,max]...", - "Inspected cursor bounds (2/2): schema.good.ts [min,max] (0.50s)", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_fetch_failure_when_executing_then_reports_failure_and_continues( - test_case: CursorFetchFailureTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - progress: list[str] = [] - execute: _FailFirstFetch = _FailFirstFetch(progress=progress) - monotonic_values: Any = iter((20.0, 20.25, 30.0, 30.50)) - monkeypatch.setattr(snapshot_module.time, "monotonic", monotonic_values.__next__) - queries: list[_PhysicalCursorQuery] = [ - _PhysicalCursorQuery( - relation="schema.bad", - cursor_column="ts", - min_tags=(), - max_tags=("bad__max",), - ), - _PhysicalCursorQuery( - relation="schema.good", - cursor_column="ts", - min_tags=("good__min",), - max_tags=("good__max",), - ), - ] - - results: dict[str, CursorScalar] = _execute_cursor_queries( - queries=queries, - connection=object(), - execute=execute, - on_progress=progress.append, - ).values - - assert { - key: render(value=value) for key, value in results.items() - } == test_case.expected_results - assert len(execute.sql) == 2 - assert tuple(progress) == test_case.expected_progress - - -@pytest.mark.parametrize( - "test_case", - [ - CursorSnapshotAvailabilityTestCase( - description="existing target still requires every watermark maximum", - target_tag="model__target__max", - target_relation="analytics.model", - results={ - "model__target__max": "2024-01-15", - "model__good__min": "2024-01-01", - "model__good__max": "2024-02-01", - "model__bad__min": "2024-01-01", - }, - expected_available=False, - expected_unavailable_tags=("model__bad__max",), - ), - CursorSnapshotAvailabilityTestCase( - description="first build requires every watermark minimum", - target_tag=None, - target_relation=None, - results={ - "model__good__min": "2024-01-01", - "model__good__max": "2024-02-01", - "model__bad__max": "2024-02-01", - }, - expected_available=False, - expected_unavailable_tags=("model__bad__min",), - ), - CursorSnapshotAvailabilityTestCase( - description="complete physical results resolve all declared watermarks", - target_tag=None, - target_relation=None, - results={ - "model__good__min": "2024-01-01", - "model__good__max": "2024-02-01", - "model__bad__min": "2024-01-02", - "model__bad__max": "2024-02-02", - }, - expected_available=True, - expected_unavailable_tags=(), - ), - ], - ids=lambda case: case.description, -) -def test_given_partial_watermark_results_when_assembling_then_model_snapshot_fails_closed( - test_case: CursorSnapshotAvailabilityTestCase, -) -> None: - info: _CursorModelInfo = _CursorModelInfo( - model_name="model", - target_tag=test_case.target_tag, - target_relation=test_case.target_relation, - cursor_column="event_time", - upstreams=( - _UpstreamCursorInfo( - tag_min="model__good__min", - tag_max="model__good__max", - relation="raw.good", - cursor_column="event_time", - ), - _UpstreamCursorInfo( - tag_min="model__bad__min", - tag_max="model__bad__max", - relation="raw.bad", - cursor_column="event_time", - ), - ), - ) - - snapshot: ModelCursorSnapshot = _assemble_cursor_snapshots( - cursor_models=[info], - results={ - key: parse(raw=value, cursor_type=CursorType.TIMESTAMP) - for key, value in test_case.results.items() - }, - )["model"] - - assert snapshot.watermarks_available is test_case.expected_available - assert snapshot.expected_watermark_count == 2 - assert snapshot.unavailable_watermark_tags == test_case.expected_unavailable_tags - - -@pytest.mark.parametrize( - "test_case", - [ - EmptyCursorInputsSnapshotTestCase( - description="one empty input leaves no cursor window", - results={"model__good__min": "2024-01-01", "model__good__max": "2024-02-01"}, - failed_tags=frozenset(), - expected_empty_input_names=("bad_events.event_time",), - expected_unreadable_input_names=(), - expected_decision=EmptyCursorInputDecision( - input_names=("bad_events.event_time",), waiting_on_empty_inputs=True - ), - ), - EmptyCursorInputsSnapshotTestCase( - description="every input empty leaves no cursor window", - results={}, - failed_tags=frozenset(), - expected_empty_input_names=("good_events.event_time", "bad_events.event_time"), - expected_unreadable_input_names=(), - expected_decision=EmptyCursorInputDecision( - input_names=("good_events.event_time", "bad_events.event_time") - ), - ), - EmptyCursorInputsSnapshotTestCase( - description="an unreadable input is a failure rather than an empty input", - results={}, - failed_tags=frozenset({"model__bad__min", "model__bad__max"}), - expected_empty_input_names=("good_events.event_time",), - expected_unreadable_input_names=("bad_events.event_time",), - expected_decision=EmptyCursorInputDecision(), - ), - EmptyCursorInputsSnapshotTestCase( - description="inputs with rows are neither empty nor unreadable", - results={ - "model__good__min": "2024-01-01", - "model__good__max": "2024-02-01", - "model__bad__min": "2024-01-02", - "model__bad__max": "2024-02-02", - }, - failed_tags=frozenset(), - expected_empty_input_names=(), - expected_unreadable_input_names=(), - expected_decision=EmptyCursorInputDecision(), - ), - ], - ids=lambda case: case.description, -) -def test_given_empty_or_failed_watermark_reads_when_assembling_then_names_the_inputs( - test_case: EmptyCursorInputsSnapshotTestCase, -) -> None: - info: _CursorModelInfo = _CursorModelInfo( - model_name="model", - target_tag=None, - target_relation=None, - cursor_column="event_time", - upstreams=( - _UpstreamCursorInfo( - tag_min="model__good__min", - tag_max="model__good__max", - relation="raw.good_events", - cursor_column="event_time", - input_name="good_events", - ), - _UpstreamCursorInfo( - tag_min="model__bad__min", - tag_max="model__bad__max", - relation="raw.bad_events", - cursor_column="event_time", - input_name="bad_events", - ), - ), - ) - - snapshot: ModelCursorSnapshot = _assemble_cursor_snapshots( - cursor_models=[info], - results={ - key: parse(raw=value, cursor_type=CursorType.TIMESTAMP) - for key, value in test_case.results.items() - }, - failed_tags=test_case.failed_tags, - )["model"] - - assert snapshot.empty_input_names == test_case.expected_empty_input_names - assert snapshot.unreadable_input_names == test_case.expected_unreadable_input_names - assert cursor_inputs_without_rows(cursor_snapshot=snapshot) == test_case.expected_decision - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_resolution.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_resolution.py deleted file mode 100644 index bd2850f379..0000000000 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_resolution.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Tests for cursor upstream qualified name resolution with deferred locations.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.compile.models import ( - CompiledRelationLocation, - CompileSqlReference, -) -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner._helpers.warehouse.snapshot import ( - _resolve_upstream_qualified_name, -) -from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( - CursorUpstreamResolutionTestCase, -) -from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( - PlannerTestAdapter, - build_cursor_deferred_locations, - build_cursor_model_map, - build_cursor_ref, -) - - -@pytest.mark.parametrize( - "test_case", - [ - CursorUpstreamResolutionTestCase( - description="non-deferred ref resolves to model target", - ref_name="orders", - model_qualified_name="staging.orders", - deferred_qualified_name=None, - selected_names=None, - expected_qualified_name="staging.orders", - ), - CursorUpstreamResolutionTestCase( - description="deferred non-selected ref resolves to deferred target", - ref_name="orders", - model_qualified_name="dev.orders", - deferred_qualified_name="prod.orders", - selected_names=frozenset({"fact_orders"}), - expected_qualified_name="prod.orders", - ), - CursorUpstreamResolutionTestCase( - description="deferred selected ref resolves to current target not deferred", - ref_name="orders", - model_qualified_name="dev.orders", - deferred_qualified_name="prod.orders", - selected_names=frozenset({"orders", "fact_orders"}), - expected_qualified_name="dev.orders", - ), - CursorUpstreamResolutionTestCase( - description="deferred with no selected_names uses deferred target", - ref_name="orders", - model_qualified_name="dev.orders", - deferred_qualified_name="prod.orders", - selected_names=None, - expected_qualified_name="prod.orders", - ), - ], - ids=lambda case: case.description, -) -def test_given_ref_and_deferred_locations_when_resolving_upstream_then_returns_expected( - test_case: CursorUpstreamResolutionTestCase, -) -> None: - ref: CompileSqlReference = build_cursor_ref(test_case.ref_name) - model_map: dict[str, CompiledModelView] = build_cursor_model_map( - test_case.ref_name, - test_case.model_qualified_name, - ) - deferred_locations: dict[str, CompiledRelationLocation] | None = ( - build_cursor_deferred_locations( - test_case.ref_name, - test_case.deferred_qualified_name, - ) - ) - - result: str | None = _resolve_upstream_qualified_name( - ref=ref, - adapter=PlannerTestAdapter(), - model_map=model_map, - source_map={}, - seed_map={}, - deferred_locations=deferred_locations, - selected_names=test_case.selected_names, - ) - - assert result == test_case.expected_qualified_name diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_warehouse_cursor_concurrency.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_warehouse_cursor_concurrency.py deleted file mode 100644 index f85047d1e8..0000000000 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_warehouse_cursor_concurrency.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Bounded-parallel cursor-bound reads must match sequential reads exactly.""" - -from __future__ import annotations - -from datetime import UTC, datetime -from typing import Any, cast - -import pytest - -from sqlbuild._native import cursor_parse as parse -from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog -from sqlbuild.compiler.planner._helpers.warehouse.snapshot import ( - _CursorModelInfo, - _execute_cursor_queries, - _gather_eligible_target_maxes, - _PhysicalCursorQuery, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.types import CursorType -from sqlbuild.cursor_algebra.types import CursorScalar -from tests.unit.src.sqlbuild.compiler.planner._helpers.warehouse._test_types import ( - ConcurrentCursorBoundsTestCase, - EligibleTargetMaxFailureTestCase, -) -from tests.unit.src.sqlbuild.compiler.planner._helpers.warehouse.helpers import ( - ConcurrencyTrackingCursorExecute, - EligibleMaxAdapter, - EligibleMaxExecute, - build_cursor_bound_queries, - build_eligible_max_cursor_models, -) - -_LATENCY_SECONDS: float = 0.01 - - -@pytest.mark.parametrize( - "test_case", - [ - ConcurrentCursorBoundsTestCase( - description="parallel reads stay within the bound", - relation_count=24, - concurrency=4, - failing_relations=frozenset(), - expected_failure_relations=(), - expected_start_progress="Inspecting cursor bounds for 24 relations (4 concurrent)...", - ), - ConcurrentCursorBoundsTestCase( - description="one failed relation stays unavailable and others succeed", - relation_count=12, - concurrency=8, - failing_relations=frozenset({"analytics.raw.orders_5"}), - expected_failure_relations=("analytics.raw.orders_5",), - expected_start_progress="Inspecting cursor bounds for 12 relations (8 concurrent)...", - ), - ], - ids=lambda case: case.description, -) -def test_given_concurrency_when_reading_cursor_bounds_then_matches_sequential_results( - test_case: ConcurrentCursorBoundsTestCase, -) -> None: - queries: list[_PhysicalCursorQuery] = build_cursor_bound_queries(test_case.relation_count) - sequential_execute: ConcurrencyTrackingCursorExecute = ConcurrencyTrackingCursorExecute( - failing_relations=test_case.failing_relations, latency_seconds=_LATENCY_SECONDS - ) - sequential: dict[str, CursorScalar] = _execute_cursor_queries( - queries=queries, connection=None, execute=sequential_execute, on_progress=None - ).values - parallel_execute: ConcurrencyTrackingCursorExecute = ConcurrencyTrackingCursorExecute( - failing_relations=test_case.failing_relations, latency_seconds=_LATENCY_SECONDS - ) - progress: list[str] = [] - - parallel: dict[str, CursorScalar] = _execute_cursor_queries( - queries=queries, - connection=None, - execute=parallel_execute, - on_progress=progress.append, - concurrency=test_case.concurrency, - ).values - - failures: tuple[str, ...] = tuple( - filter(lambda message: message.startswith("Failed"), progress) - ) - assert list(parallel.items()) == list(sequential.items()) - assert sequential_execute.max_active == 1 - assert 1 < parallel_execute.max_active <= test_case.concurrency - assert sorted(parallel_execute.sql) == sorted(sequential_execute.sql) - assert len(parallel_execute.sql) == test_case.relation_count - assert progress[0] == test_case.expected_start_progress - assert len(progress) == test_case.relation_count + 1 - assert ( - tuple(message.split(": ")[1].split(".ordered_at")[0] for message in failures) - == test_case.expected_failure_relations - ) - - -@pytest.mark.parametrize( - "test_case", - [ - EligibleTargetMaxFailureTestCase( - description="failure names the model and relation", - failing_relation="analytics.marts.orders_3", - expected_error_fragment=( - "model 'model_3': failed to query highest eligible target cursor for " - "analytics.marts.orders_3.ordered_at" - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_failing_eligible_max_read_when_reading_in_parallel_then_plan_fails_naming_relation( - test_case: EligibleTargetMaxFailureTestCase, -) -> None: - infos: list[_CursorModelInfo] = build_eligible_max_cursor_models(6) - results: dict[str, CursorScalar] = { - f"model_{index}__target__max": parse(raw="2099-01-01", cursor_type=CursorType.TIMESTAMP) - for index in range(6) - } - adapter: EligibleMaxAdapter = EligibleMaxAdapter() - connection: object = object() - - with ( - open_inspection_catalog(adapter=adapter, connection=connection), - pytest.raises(PlannerInputError, match=test_case.expected_error_fragment), - ): - _ = _gather_eligible_target_maxes( - cursor_models=infos, - results=results, - adapter=cast(Any, adapter), - connection=connection, - execute=EligibleMaxExecute( - failing_relation=test_case.failing_relation, latency_seconds=_LATENCY_SECONDS - ), - invocation_time=datetime(2026, 1, 20, tzinfo=UTC), - ) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) From 1681bbf231a5edb5c026c537e76801f9e832f190 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 03:34:20 +0100 Subject: [PATCH 02/11] refactor: host planner warehouse inspection and phases natively Move warehouse snapshot gathering, cursor bound reads, source columns, semantic source completion, source table checks, managed source deferral, planner source freshness, the execution planning phases, retention and table type planning, and the migration fingerprint cache into native code. Adapter, catalog and connection calls stay Python callbacks in the same order and count; the fingerprint cache keeps its SQLite file and row format. Delete the replaced Python and move tests of removed internals to integration tests over the native entry points. --- .../src/bindings/_helpers/mod.rs | 2 + .../_helpers/planner_migrations/constants.rs | 4 +- .../planner_migrations/planning_boundary.rs | 14 +- .../_helpers/planner_phases/buildability.rs | 73 + .../_helpers/planner_phases/constants.rs | 23 + .../planner_phases/execution_changes.rs | 226 ++++ .../_helpers/planner_phases/full_refresh.rs | 96 ++ .../_helpers/planner_phases/identities.rs | 179 +++ .../bindings/_helpers/planner_phases/mod.rs | 13 + .../_helpers/planner_phases/objects.rs | 178 +++ .../planner_phases/output_assembly.rs | 427 ++++++ .../_helpers/planner_phases/registration.rs | 88 ++ .../_helpers/planner_phases/retention.rs | 613 +++++++++ .../_helpers/planner_phases/scopes.rs | 101 ++ .../planner_phases/warehouse_state.rs | 194 +++ .../_helpers/planner_warehouse/constants.rs | 83 ++ .../_helpers/planner_warehouse/cursors.rs | 1194 +++++++++++++++++ .../planner_warehouse/fingerprint_cache.rs | 354 +++++ .../_helpers/planner_warehouse/mod.rs | 13 + .../planner_warehouse/registration.rs | 60 + .../planner_warehouse/semantic_sources.rs | 287 ++++ .../_helpers/planner_warehouse/snapshot.rs | 867 ++++++++++++ .../planner_warehouse/source_columns.rs | 220 +++ .../planner_warehouse/source_deferral.rs | 342 +++++ .../planner_warehouse/source_freshness.rs | 187 +++ .../planner_warehouse/source_tables.rs | 225 ++++ .../_helpers/planner_warehouse/watermarks.rs | 150 +++ .../src/bindings/main/register.rs | 2 + fensu.toml | 6 - python-boundary.toml | 6 +- src/sqlbuild/_native.pyi | 238 +++- .../_helpers/command_planning/static.py | 2 +- .../planner/_helpers/output/plan_entry.py | 20 +- .../compiler/planner/_helpers/reuse/clone.py | 2 +- .../planner/main/execution/display_plan.py | 2 +- .../planner/main/execution/execution.py | 41 +- .../main/selection/_resolve_planner_scopes.py | 4 +- src/sqlbuild/compiler/planner/models.py | 4 +- .../snowflake/test_table_type_conversion.py | 2 +- .../commands/main/model_migrations/helpers.py | 2 +- .../compiler/planner/_helpers/_test_types.py | 26 + .../compiler/planner/_helpers/helpers.py | 166 ++- .../_helpers/test_warehouse_cursor_reads.py | 208 +++ .../_helpers/test_warehouse_snapshot.py | 4 +- .../planner/main/test_build_execution_plan.py | 2 +- .../executor/build/test_table_type_routing.py | 2 +- .../_helpers/test_microbatch_redesign.py | 4 +- .../compiler/planner/_helpers/_test_types.py | 64 - .../_helpers/planning/test_full_refresh.py | 5 +- .../_helpers/planning/test_identities.py | 35 +- .../_helpers/planning/test_retention.py | 2 +- .../planner/_helpers/planning/test_scopes.py | 2 +- .../_helpers/test_selection_staleness.py | 2 +- .../planner/_helpers/test_source_columns.py | 2 +- .../planner/_helpers/test_source_freshness.py | 4 +- .../_helpers/test_source_load_nodes.py | 4 +- .../planner/_helpers/warehouse/_test_types.py | 17 - .../planner/_helpers/warehouse/helpers.py | 146 +- .../warehouse/test_semantic_sources.py | 13 +- .../compiler/planner/classes/helpers.py | 7 +- .../test_migration_fingerprint_cache.py | 9 +- 61 files changed, 6930 insertions(+), 338 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs create mode 100644 tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 2f8f0ebb07..ad1b9ea74f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -50,9 +50,11 @@ pub(crate) mod planner_identity; pub(crate) mod planner_migrations; pub(crate) mod planner_resolution; pub(crate) mod planner_scenario; +pub(crate) mod planner_phases; pub(crate) mod planner_scope; pub(crate) mod planner_selection; pub(crate) mod planner_strategy; +pub(crate) mod planner_warehouse; pub(crate) mod presentation_text; pub(crate) mod project_assembly; pub(crate) mod python_values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs index b11f9801d7..b686ee5250 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs @@ -8,10 +8,10 @@ pub(super) const NATIVE_MODULE: &str = "sqlbuild._native"; pub(super) const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; pub(super) const NODE_TYPE_MODEL: &str = "model"; pub(super) const LIST_RELATIONS_FOR_INSPECTION: (&str, &str) = ( - "sqlbuild.adapter.relations.main.list_relations_for_inspection", + "sqlbuild.adapter.relations.main._list_relations_for_inspection", "list_relations_for_inspection", ); pub(super) const GET_COLUMNS_FOR_INSPECTION: (&str, &str) = ( - "sqlbuild.adapter.relations.main.get_columns_for_inspection", + "sqlbuild.adapter.relations.main._get_columns_for_inspection", "get_columns_for_inspection", ); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs index b97b34d114..b3dc9a2486 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs @@ -14,11 +14,7 @@ fn migration_missing_origin_policy<'py>( runtime: &Bound<'py, PyAny>, ) -> PyResult> { call_with( - &imported( - runtime.py(), - "sqlbuild.compiler.planner._helpers.planning.retention", - "effective_target_config", - )?, + &imported(runtime.py(), "sqlbuild._native", "effective_target_config")?, &[("runtime", runtime.clone())], )? .getattr("missing_migration_origin") @@ -70,7 +66,7 @@ fn migration_redirected_cursor_snapshots<'py>( let full_refresh = call_with( &imported( py, - "sqlbuild.compiler.planner._helpers.planning.full_refresh", + "sqlbuild._native", "effectively_full_refreshed_model_names", )?, &[ @@ -79,11 +75,7 @@ fn migration_redirected_cursor_snapshots<'py>( ], )?; call_with( - &imported( - py, - "sqlbuild.compiler.planner._helpers.warehouse.snapshot", - "gather_redirected_cursor_snapshots", - )?, + &imported(py, "sqlbuild._native", "gather_redirected_cursor_snapshots")?, &[ ("project", project), ("adapter", runtime.getattr("adapter")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs new file mode 100644 index 0000000000..ae3394bacb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs @@ -0,0 +1,73 @@ +//! Selected-scope buildability validation before change planning. + +use crate::bindings::_helpers::planner_phases::constants::MISSING_UPSTREAM_CODE; +use crate::bindings::_helpers::planner_phases::objects::{frozenset, native, planner_error}; +use crate::bindings::_helpers::planner_warehouse::source_tables::check_selected_source_tables_exist; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; + +#[pyfunction] +#[pyo3(signature = (*, runtime, scopes, warehouse, deferral))] +pub(crate) fn check_selected_scope_buildability<'py>( + runtime: &Bound<'py, PyAny>, + scopes: &Bound<'py, PyAny>, + warehouse: &Bound<'py, PyAny>, + deferral: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py = runtime.py(); + let project = runtime.getattr("project")?; + let selected_scope = scopes.getattr("selected_scope")?; + let mut external_seeds: Vec<_> = Vec::new(); + for seed in project.getattr("seeds")?.try_iter()? { + let seed = seed?; + if seed.getattr("external")?.is_truthy()? { + external_seeds.push(seed.getattr("key")?); + } + } + let missing = native( + py, + "check_buildability", + &[ + ("selected_keys", selected_scope.getattr("selected_keys")?), + ("upstream_deps", selected_scope.getattr("upstream_deps")?), + ("snapshot", warehouse.getattr("snapshot")?), + ( + "deferred_relations", + deferral.getattr("deferred_relations")?, + ), + ("satisfied_keys", frozenset(py, &external_seeds)?), + ], + )?; + if missing.is_truthy()? { + let message: String = native( + py, + "missing_upstream_message", + &[ + ("missing", missing), + ( + "edge_origins", + native( + py, + "build_execution_edge_origins", + &[("project", project.clone())], + )?, + ), + ], + )? + .extract()?; + return Err(planner_error( + py, + &message, + (Some(MISSING_UPSTREAM_CODE), None), + )); + } + check_selected_source_tables_exist( + &project, + ( + &runtime.getattr("adapter")?, + &runtime.getattr("connection")?, + ), + &selected_scope, + &warehouse.getattr("inspection_relations")?, + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs new file mode 100644 index 0000000000..08dd8db18a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs @@ -0,0 +1,23 @@ +//! Python modules and names the planner phases exchange values with. + +pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; +pub(crate) const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +pub(crate) const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; +pub(crate) const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; +pub(crate) const ADAPTER_MODELS: &str = "sqlbuild.adapter.contract.models"; +pub(crate) const ADAPTER_TYPES: &str = "sqlbuild.adapter.contract.types"; +pub(crate) const SPEC_MODELS: &str = "sqlbuild.spec.contracts.models"; +pub(crate) const PLAN_ENTRY_MODULE: &str = "sqlbuild.compiler.planner._helpers.output.plan_entry"; +pub(crate) const PLAN_OUTPUT_MODULE: &str = "sqlbuild.compiler.planner._helpers.output.plan_output"; +pub(crate) const SNOWFLAKE_ADAPTER: &str = "snowflake"; +pub(crate) const BIGQUERY_ADAPTER: &str = "bigquery"; +pub(crate) const TRANSIENT_TABLE_TYPE: &str = "transient"; +pub(crate) const PERMANENT_TABLE_TYPE: &str = "permanent"; +pub(crate) const DECREASE_WARNING: &str = + "Retention decreases are irreversible once warehouse history expires."; +pub(crate) const DOWNGRADE_HISTORY_DAYS: i64 = 90; +pub(crate) const FULL_REFRESH_DISABLED_REBUILD_CODE: &str = "S203"; +pub(crate) const MISSING_UPSTREAM_CODE: &str = "S301"; +pub(crate) const RETENTION_DECREASE: &str = "decrease"; +pub(crate) const RETENTION_INCREASE: &str = "increase"; +pub(crate) const RETENTION_MATCH: &str = "match"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs new file mode 100644 index 0000000000..4106139b51 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs @@ -0,0 +1,226 @@ +//! Execution-scope pruning, write-hash reconciliation and plan entry construction. + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_phases::constants::PLAN_ENTRY_MODULE; +use crate::bindings::_helpers::planner_phases::objects::{native, planner_model}; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyDict as KeywordDict; +use pyo3::types::{PyFrozenSet, PyTuple}; + +#[pyfunction] +#[pyo3(signature = (*, scopes, resolved_actions))] +pub(crate) fn prune_planner_execution_scope<'py>( + scopes: &Bound<'py, PyAny>, + resolved_actions: &Bound<'py, PyAny>, +) -> PyResult> { + let py = scopes.py(); + let inspection = scopes.getattr("inspection_scope")?; + planner_model( + py, + "PlannerScopePruningResult", + &[ + ("inspection_scope", inspection.clone()), + ("execution_scope", inspection), + ("resolved_actions", resolved_actions.clone()), + ("pruned_direct_model_names", PyTuple::empty(py).into_any()), + ( + "direct_identity_stale_model_names", + PyFrozenSet::empty(py)?.into_any(), + ), + ], + ) +} + +#[pyfunction] +#[pyo3(signature = (*, warehouse, identities, pruning, changes))] +pub(crate) fn reconcile_execution_changes<'py>( + warehouse: &Bound<'py, PyAny>, + identities: &Bound<'py, PyAny>, + pruning: &Bound<'py, PyAny>, + changes: &Bound<'py, PyAny>, +) -> PyResult> { + let py = warehouse.py(); + let honest = native( + py, + "with_honest_model_write_hashes", + &[ + ("scope", pruning.getattr("execution_scope")?), + ("snapshot", warehouse.getattr("snapshot")?), + ("changes", changes.clone()), + ( + "version_identities", + identities.getattr("version_identities")?, + ), + ], + )?; + let resolved = native( + py, + "merge_recomputed_model_changes", + &[ + ("resolved_actions", pruning.getattr("resolved_actions")?), + ("changes", honest.clone()), + ], + )?; + planner_model( + py, + "PlannerChangeReconciliation", + &[("changes", honest), ("resolved_actions", resolved)], + ) +} + +fn config_or<'py>( + config: &Bound<'py, PyAny>, + path: &[&str], + fallback: Bound<'py, PyAny>, +) -> PyResult> { + if config.is_none() { + return Ok(fallback); + } + let mut value = config.clone(); + for name in path { + value = value.getattr(*name)?; + } + Ok(value) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_planner_entry_results<'py>( + py: pyo3::Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_planner_entry_results", + options, + ( + &[ + "runtime", + "warehouse", + "identities", + "overrides", + "policies", + "deferral", + "pruning", + "reconciliation", + "source_freshness", + ], + &[], + ), + )?; + let runtime = &args.value("runtime")?; + let warehouse = &args.value("warehouse")?; + let identities = &args.value("identities")?; + let overrides = &args.value("overrides")?; + let policies = &args.value("policies")?; + let deferral = &args.value("deferral")?; + let pruning = &args.value("pruning")?; + let reconciliation = &args.value("reconciliation")?; + let source_freshness = &args.value("source_freshness")?; + + let _ = (identities, policies); + let py = runtime.py(); + let project = runtime.getattr("project")?; + let config = runtime.getattr("project_config")?; + let execution_scope = pruning.getattr("execution_scope")?; + let relations = call_with( + &imported(py, PLAN_ENTRY_MODULE, "build_planner_relations_context")?, + &[ + ("project", project.clone()), + ("adapter", runtime.getattr("adapter")?), + ("connection", runtime.getattr("connection")?), + ("scope", execution_scope.clone()), + ("deferral", deferral.clone()), + ("project_config", config.clone()), + ("local_config", runtime.getattr("local_config")?), + ( + "known_source_columns", + warehouse + .getattr("inspection_relations")? + .getattr("source_warehouse_columns")?, + ), + ], + )?; + let propagation = source_freshness.getattr("propagation")?; + let blocked = if propagation.is_none() { + PyFrozenSet::empty(py)?.into_any() + } else { + propagation.getattr("blocked_model_names")? + }; + let none = || py.None().into_bound(py); + let max_override = overrides.getattr("max_microbatches")?; + let max_microbatches = if max_override.is_none() { + config_or(&config, &["microbatches", "limits", "max_batches"], none())? + } else { + max_override.clone() + }; + let limit_action = if config.is_none() { + planner_model(py, "PlanEntryBuildInputs", &[])?.getattr("microbatch_limit_action")? + } else { + config + .getattr("microbatches")? + .getattr("limits")? + .getattr("action")? + }; + let build_inputs = planner_model( + py, + "PlanEntryBuildInputs", + &[ + ("source_freshness_blocked_model_names", blocked), + ( + "external_blocked_model_names", + PyFrozenSet::new( + py, + overrides + .getattr("external_blocked_model_names")? + .try_iter()? + .collect::>>()?, + )? + .into_any(), + ), + ( + "future_cursor_config", + config_or(&config, &["cursors", "future"], none())?, + ), + ( + "start_cursor_config", + config_or(&config, &["cursors", "start"], none())?, + ), + ("invocation_time", runtime.getattr("invocation_time")?), + ("max_microbatches", max_microbatches), + ( + "max_microbatches_is_override", + (!max_override.is_none()) + .into_pyobject(py)? + .to_owned() + .into_any(), + ), + ("microbatch_limit_action", limit_action), + ], + )?; + let entries = call_with( + &imported(py, PLAN_ENTRY_MODULE, "build_plan_entries")?, + &[ + ("project", project), + ("adapter", runtime.getattr("adapter")?), + ("scope", execution_scope), + ("snapshot", warehouse.getattr("snapshot")?), + ("relations", relations), + ( + "resolved_actions", + reconciliation.getattr("resolved_actions")?, + ), + ("cursor_overrides", overrides.getattr("cursor_overrides")?), + ("full_refresh", overrides.getattr("full_refresh")?), + ("build_inputs", build_inputs), + ], + )?; + planner_model( + py, + "PlannerEntryResults", + &[("model_entry_results", entries)], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs new file mode 100644 index 0000000000..63b451f15c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs @@ -0,0 +1,96 @@ +//! Per-model full-refresh resolution and full-rebuild refusals. + +use crate::bindings::_helpers::planner_phases::constants::FULL_REFRESH_DISABLED_REBUILD_CODE; +use crate::bindings::_helpers::planner_phases::objects::{frozenset, planner_error, strings}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyBool; +use sqlbuild_analysis::planner_warehouse::main::full_rebuild_cause::full_rebuild_cause; +use sqlbuild_analysis::planner_warehouse::models::FullRebuildFacts; + +fn configured_full_refresh<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("full_refresh",)) +} + +/// dbt-compatible nullable model full-refresh semantics. +#[pyfunction] +#[pyo3(signature = (*, model, cli_full_refresh))] +pub(crate) fn resolve_model_full_refresh( + model: &Bound<'_, PyAny>, + cli_full_refresh: bool, +) -> PyResult { + let configured = configured_full_refresh(model)?; + if configured.is_instance_of::() { + return configured.extract(); + } + Ok(cli_full_refresh) +} + +/// Names of models effectively full refreshed for this invocation. +#[pyfunction] +#[pyo3(signature = (*, project, cli_full_refresh))] +pub(crate) fn effectively_full_refreshed_model_names<'py>( + project: &Bound<'py, PyAny>, + cli_full_refresh: bool, +) -> PyResult> { + let mut names: Vec<_> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + if resolve_model_full_refresh(&model, cli_full_refresh)? { + names.push(model.getattr("name")?); + } + } + frozenset(project.py(), &names) +} + +#[pyfunction] +#[pyo3(signature = (*, change_result, full_refresh))] +pub(crate) fn describe_full_rebuild_cause( + change_result: &Bound<'_, PyAny>, + full_refresh: bool, +) -> PyResult { + Ok(full_rebuild_cause(&FullRebuildFacts { + full_refresh, + change_kind: change_result + .getattr("change_kind")? + .str()? + .extract::()?, + query_changed: change_result.getattr("query_changed")?.is_truthy()?, + changed_functions: strings(&change_result.getattr("changed_functions")?)?, + has_schema_findings: change_result.getattr("schema_findings")?.is_truthy()?, + })) +} + +#[pyfunction] +#[pyo3(signature = (*, model, change_result, full_rebuild_cause))] +pub(crate) fn check_full_rebuild_allowed( + model: &Bound<'_, PyAny>, + change_result: &Bound<'_, PyAny>, + full_rebuild_cause: &str, +) -> PyResult<()> { + let py = model.py(); + if !configured_full_refresh(model)?.is(PyBool::new(py, false)) { + return Ok(()); + } + if change_result.getattr("change_kind")?.eq("first_run")? + || change_result.getattr("query_changed")?.is_truthy()? + { + return Ok(()); + } + let name: String = model.getattr("name")?.extract()?; + Err(planner_error( + py, + &format!( + "model '{name}' sets full_refresh false, but the plan would fully rebuild it ({full_rebuild_cause})" + ), + ( + Some(FULL_REFRESH_DISABLED_REBUILD_CODE), + Some( + "set replay_on_change to forward or bounded- on this model to keep its existing history, or remove full_refresh false to allow the rebuild", + ), + ), + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs new file mode 100644 index 0000000000..4df1960b1a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs @@ -0,0 +1,179 @@ +//! Expected version identities and stale-warning change detection. + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_phases::objects::{ + frozenset, native, planner_model, replace, +}; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyDict as KeywordDict; +use pyo3::types::{PyDict, PyDictMethods}; +use std::collections::HashSet; + +fn version_identities<'py>( + project: &Bound<'py, PyAny>, + scope: Bound<'py, PyAny>, +) -> PyResult> { + native( + project.py(), + "build_direct_model_version_identities", + &[ + ("functions", project.getattr("functions")?), + ("seeds", project.getattr("seeds")?), + ("scope", scope), + ("hook_functions", project.getattr("hook_functions")?), + ("dialect", project.getattr("sql_analysis_dialect")?), + ], + ) +} + +#[pyfunction] +#[pyo3(signature = (*, project, scopes, include_stale_warning_identities=true))] +pub(crate) fn build_planner_identity_context<'py>( + project: &Bound<'py, PyAny>, + scopes: &Bound<'py, PyAny>, + include_stale_warning_identities: bool, +) -> PyResult> { + let identities = version_identities(project, scopes.getattr("inspection_scope")?)?; + let stale = if include_stale_warning_identities { + version_identities(project, scopes.getattr("stale_warning_scope")?)? + } else { + identities.clone() + }; + planner_model( + project.py(), + "PlannerIdentityContext", + &[ + ("version_identities", identities), + ("stale_warning_identities", stale), + ], + ) +} + +fn reusable_model_names( + identities: &Bound<'_, PyAny>, + execution_changes: &Bound<'_, PyAny>, + query_change_tracking: bool, +) -> PyResult> { + let mut names: HashSet<_> = HashSet::new(); + if !query_change_tracking { + return Ok(names); + } + let execution = identities.getattr("version_identities")?; + let stale = identities.getattr("stale_warning_identities")?; + for name in execution_changes.getattr("models")?.try_iter()? { + let name = name?; + let same = |field: &str| -> PyResult { + execution + .getattr(field)? + .call_method1("get", (&name,))? + .eq(stale.getattr(field)?.call_method1("get", (&name,))?) + }; + if same("model_version_hashes")? && same("model_metadata_jsons")? { + names.insert(name.extract()?); + } + } + Ok(names) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn detect_stale_warning_changes<'py>( + py: pyo3::Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "detect_stale_warning_changes", + options, + ( + &[ + "project", + "scopes", + "snapshot", + "identities", + "execution_changes", + ], + &[], + ), + )?; + let project = &args.value("project")?; + let scopes = &args.value("scopes")?; + let snapshot = &args.value("snapshot")?; + let identities = &args.value("identities")?; + let execution_changes = &args.value("execution_changes")?; + + let py = project.py(); + let reusable = reusable_model_names( + identities, + execution_changes, + project + .getattr("settings")? + .getattr("query_change_tracking")? + .is_truthy()?, + )?; + let stale_scope = scopes.getattr("stale_warning_scope")?; + let mut remaining: Vec<_> = Vec::new(); + for key in stale_scope + .getattr("all_keys")? + .call_method0("values")? + .try_iter()? + { + let key = key?; + if key.getattr("resource_type")?.eq("model")? + && !reusable.contains(&key.getattr("name")?.extract::()?) + { + remaining.push(key); + } + } + let stale_identities = identities.getattr("stale_warning_identities")?; + let remaining_changes = native( + py, + "detect_model_changes_in_scope", + &[ + ("project", project.clone()), + ( + "scope", + replace( + &stale_scope, + &[("selected_keys", frozenset(py, &remaining)?)], + )?, + ), + ("snapshot", snapshot.clone()), + ( + "query_change_tracking", + true.into_pyobject(py)?.to_owned().into_any(), + ), + ( + "expected_version_hashes", + stale_identities.getattr("model_version_hashes")?, + ), + ( + "expected_metadata_jsons", + stale_identities.getattr("model_metadata_jsons")?, + ), + ], + )?; + let models = PyDict::new(py); + for (name, change) in execution_changes + .getattr("models")? + .downcast::()? + .iter() + { + if reusable.contains(&name.extract::()?) { + models.set_item(name, change)?; + } + } + for (name, change) in remaining_changes.downcast::()?.iter() { + models.set_item(name, change)?; + } + planner_model( + py, + "PlannerChangeResults", + &[ + ("models", models.into_any()), + ("functions", PyDict::new(py).into_any()), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs new file mode 100644 index 0000000000..32482259eb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs @@ -0,0 +1,13 @@ +//! Native execution-planning phases over Python planner models and adapter callbacks. + +pub(crate) mod buildability; +pub(crate) mod constants; +pub(crate) mod execution_changes; +pub(crate) mod full_refresh; +pub(crate) mod identities; +pub(crate) mod objects; +pub(crate) mod output_assembly; +pub(crate) mod registration; +pub(crate) mod retention; +pub(crate) mod scopes; +pub(crate) mod warehouse_state; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs new file mode 100644 index 0000000000..cc5e3514be --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs @@ -0,0 +1,178 @@ +//! Boundary helpers shared by the native planner phases. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::planner_phases::constants::{ + NATIVE_MODULE, PLANNER_EXCEPTIONS, PLANNER_MODELS, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyFrozenSet, PyString, PyTuple}; + +/// A Python `str`. +pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +/// `None`, or the text. +pub(crate) fn optional_text<'py>(py: Python<'py>, value: Option<&str>) -> Bound<'py, PyAny> { + value.map_or_else(|| py.None().into_bound(py), |value| text(py, value)) +} + +/// `module.name(**fields)`. +pub(crate) fn construct<'py>( + py: Python<'py>, + (module, name): (&str, &str), + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with(&imported(py, module, name)?, fields) +} + +/// A planner model dataclass. +pub(crate) fn planner_model<'py>( + py: Python<'py>, + name: &str, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + construct(py, (PLANNER_MODELS, name), fields) +} + +/// `sqlbuild._native.name(**fields)`. +pub(crate) fn native<'py>( + py: Python<'py>, + name: &str, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + construct(py, (NATIVE_MODULE, name), fields) +} + +/// `dataclasses.replace(value, **fields)`. +pub(crate) fn replace<'py>( + value: &Bound<'py, PyAny>, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + imported(value.py(), "dataclasses", "replace")? + .call((value,), Some(&keywords(value.py(), fields)?)) +} + +/// A Python enum member from its value. +pub(crate) fn member<'py>( + py: Python<'py>, + (module, name): (&str, &str), + value: &str, +) -> PyResult> { + imported(py, module, name)?.call1((value,)) +} + +/// `PlannerInputError(message, code=code, help=help)`. +pub(crate) fn planner_error( + py: Python<'_>, + message: &str, + (code, help): (Option<&str>, Option<&str>), +) -> PyErr { + let raised = imported(py, PLANNER_EXCEPTIONS, "PlannerInputError").and_then(|class| { + let fields = keywords( + py, + &[ + ("code", optional_text(py, code)), + ("help", optional_text(py, help)), + ], + )?; + class.call((message,), Some(&fields)) + }); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// A tuple of the items. +pub(crate) fn tuple<'py>( + py: Python<'py>, + items: Vec>, +) -> PyResult> { + Ok(PyTuple::new(py, items)?.into_any()) +} + +/// A frozenset of the items. +pub(crate) fn frozenset<'py>( + py: Python<'py>, + items: &[Bound<'py, PyAny>], +) -> PyResult> { + Ok(PyFrozenSet::new(py, items)?.into_any()) +} + +/// Names of model keys among `keys`. +pub(crate) fn model_key_names(keys: &Bound<'_, PyAny>) -> PyResult> { + let mut names: Vec<_> = Vec::new(); + for key in keys.try_iter()? { + let key = key?; + if key.getattr("resource_type")?.eq("model")? { + names.push(key.getattr("name")?.extract()?); + } + } + Ok(names) +} + +/// `sorted(items)` as a tuple of strings. +pub(crate) fn sorted_names<'py>( + py: Python<'py>, + mut names: Vec, +) -> PyResult> { + names.sort(); + tuple(py, names.iter().map(|name| text(py, name)).collect()) +} + +/// The strings of any Python iterable, in iteration order. +pub(crate) fn strings(values: &Bound<'_, PyAny>) -> PyResult> { + values + .try_iter()? + .map(|value| value?.extract::()) + .collect() +} + +/// The project nodes of `field` by name. +pub(crate) fn by_name<'py>( + project: &Bound<'py, PyAny>, + field: &str, +) -> PyResult> { + let nodes = pyo3::types::PyDict::new(project.py()); + for node in project.getattr(field)?.try_iter()? { + let node = node?; + pyo3::types::PyDictMethods::set_item(&nodes, node.getattr("name")?, node)?; + } + Ok(nodes) +} + +/// A model config value as text, `None` when absent or not text. +pub(crate) fn config_str(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { + native( + model.py(), + "get_config_str", + &[ + ("values", model.getattr("config")?.getattr("values")?), + ("key", text(model.py(), key)), + ], + )? + .extract() +} + +/// The compiled object key of the model `name`. +pub(crate) fn model_key<'py>( + py: Python<'py>, + name: &Bound<'py, PyAny>, +) -> PyResult> { + call_with( + &imported(py, "sqlbuild.compiler.compile.models", "CompiledObjectKey")?, + &[ + ( + "resource_type", + imported( + py, + "sqlbuild.compiler.compile.types", + "CompiledResourceType", + )? + .getattr("MODEL")?, + ), + ("name", name.clone()), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs new file mode 100644 index 0000000000..99fb843502 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -0,0 +1,427 @@ +//! Plan output assembly: storage policies, warnings and direct-mode metadata. + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_phases::constants::PLAN_OUTPUT_MODULE; +use crate::bindings::_helpers::planner_phases::objects::{ + model_key_names, native, planner_model, replace, sorted_names, strings, text, tuple, +}; +use crate::bindings::_helpers::planner_warehouse::source_freshness::source_freshness_unknown_warnings; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict as KeywordDict; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; +use std::collections::BTreeSet; + +fn concat<'py>(py: Python<'py>, parts: &[Bound<'py, PyAny>]) -> PyResult> { + let mut items: Vec<_> = Vec::new(); + for part in parts { + for item in part.try_iter()? { + items.push(item?); + } + } + tuple(py, items) +} + +fn with_metadata<'py>( + plan_output: &Bound<'py, PyAny>, + items: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let py = plan_output.py(); + let metadata = PyDict::new(py); + metadata.update( + plan_output + .getattr("metadata")? + .downcast::()?, + )?; + for (name, value) in items { + metadata.set_item(*name, value)?; + } + replace(plan_output, &[("metadata", metadata.into_any())]) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn assemble_base_plan_output<'py>( + py: pyo3::Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "assemble_base_plan_output", + options, + ( + &[ + "runtime", + "warehouse", + "identities", + "overrides", + "pruning", + "reconciliation", + "entries", + "source_freshness", + ], + &["planned_sql_tests"], + ), + )?; + let runtime = &args.value("runtime")?; + let warehouse = &args.value("warehouse")?; + let identities = &args.value("identities")?; + let overrides = &args.value("overrides")?; + let pruning = &args.value("pruning")?; + let reconciliation = &args.value("reconciliation")?; + let entries = &args.value("entries")?; + let source_freshness = &args.value("source_freshness")?; + let planned_sql_tests = &args.value("planned_sql_tests")?; + + let py = runtime.py(); + let version = identities.getattr("version_identities")?; + let extras = planner_model( + py, + "PlanOutputExtras", + &[ + ( + "seed_version_hashes", + version.getattr("seed_version_hashes")?, + ), + ( + "seed_metadata_jsons", + version.getattr("seed_metadata_jsons")?, + ), + ("planned_sql_tests", planned_sql_tests.clone()), + ], + )?; + let plan_output = call_with( + &imported(py, PLAN_OUTPUT_MODULE, "build_plan_output")?, + &[ + ("project", runtime.getattr("project")?), + ("adapter", runtime.getattr("adapter")?), + ("scope", pruning.getattr("execution_scope")?), + ("snapshot", warehouse.getattr("snapshot")?), + ("relations", warehouse.getattr("inspection_relations")?), + ("changes", reconciliation.getattr("changes")?), + ( + "model_entry_results", + entries.getattr("model_entry_results")?, + ), + ("reload_sources", overrides.getattr("reload_sources")?), + ("extras", extras), + ], + )?; + let plan_output = replace( + &plan_output, + &[ + ("source_freshness", source_freshness.clone()), + ("cursor_overrides", overrides.getattr("cursor_overrides")?), + ], + )?; + let pruned = pruning.getattr("pruned_direct_model_names")?; + if pruned.is_truthy()? { + return with_metadata(&plan_output, &[("direct_pruned_model_names", pruned)]); + } + Ok(plan_output) +} + +/// Attach near-match column rename hints to the model entries they describe. +#[pyfunction] +#[pyo3(signature = (*, entries, hints))] +pub(crate) fn with_column_rename_hints<'py>( + entries: &Bound<'py, PyAny>, + hints: &Bound<'py, PyAny>, +) -> PyResult> { + let py = entries.py(); + let by_model = PyDict::new(py); + for hint in hints.try_iter()? { + let hint = hint?; + let name = hint.getattr("model_name")?; + if let Some(existing) = by_model.get_item(&name)? { + existing.call_method1("append", (hint,))?; + } else { + by_model.set_item(name, pyo3::types::PyList::new(py, [hint])?)?; + } + } + let mut attached: Vec<_> = Vec::new(); + for entry in entries.try_iter()? { + let entry = entry?; + match by_model.get_item(entry.getattr("name")?)? { + Some(found) => attached.push(replace( + &entry, + &[( + "column_rename_hints", + PyTuple::new(py, found.try_iter()?.collect::>>()?)?.into_any(), + )], + )?), + None => attached.push(entry), + } + } + tuple(py, attached) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn with_storage_policies<'py>( + py: pyo3::Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "with_storage_policies", + options, + ( + &["plan_output", "runtime", "warehouse", "scopes", "policies"], + &[], + ), + )?; + let plan_output = &args.value("plan_output")?; + let runtime = &args.value("runtime")?; + let warehouse = &args.value("warehouse")?; + let scopes = &args.value("scopes")?; + let policies = &args.value("policies")?; + + let py = runtime.py(); + let mut entries = plan_output.getattr("model_entries")?; + if policies + .getattr("record_migration_fingerprints")? + .is_truthy()? + { + let models = PyDict::new(py); + for model in runtime.getattr("project")?.getattr("models")?.try_iter()? { + let model = model?; + models.set_item(model.getattr("name")?, model)?; + } + entries = native( + py, + "with_migration_fingerprints", + &[ + ("entries", entries), + ("models_by_name", models.into_any()), + ("fingerprints", warehouse.getattr("migration_fingerprints")?), + ( + "dialect", + runtime + .getattr("adapter")? + .call_method0("sql_analysis_dialect")?, + ), + ], + )?; + } + let entries = with_column_rename_hints(&entries, &warehouse.getattr("column_rename_hints")?)?; + let migration_entries = warehouse.getattr("migration_entries")?; + let column_entries = warehouse.getattr("column_migration_entries")?; + let old_name_entries = warehouse.getattr("old_name_view_entries")?; + let old_name_views = warehouse.getattr("old_name_views")?; + let warnings = concat( + py, + &[ + warehouse.getattr("migration_warnings")?, + plan_output.getattr("warnings")?, + ], + )?; + let selected = scopes.getattr("selected_scope")?; + let storage = [ + ("runtime", runtime.clone()), + ("warehouse", warehouse.clone()), + ("scope", selected), + ]; + let table_types = native(py, "plan_table_types", &storage)?; + let retention = native(py, "plan_retention", &storage)?; + replace( + plan_output, + &[ + ("model_entries", entries), + ("migration_entries", migration_entries), + ("column_migration_entries", column_entries), + ("old_name_view_entries", old_name_entries), + ("old_name_views", old_name_views), + ("warnings", warnings), + ("table_type_entries", table_types), + ("retention_entries", retention), + ], + ) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn with_plan_warnings<'py>( + py: pyo3::Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "with_plan_warnings", + options, + ( + &[ + "runtime", + "scopes", + "warehouse", + "identities", + "stale_warning_changes", + "pruning", + "source_freshness", + "plan_output", + "policies", + ], + &[], + ), + )?; + let runtime = &args.value("runtime")?; + let scopes = &args.value("scopes")?; + let warehouse = &args.value("warehouse")?; + let identities = &args.value("identities")?; + let stale_warning_changes = &args.value("stale_warning_changes")?; + let pruning = &args.value("pruning")?; + let source_freshness = &args.value("source_freshness")?; + let plan_output = &args.value("plan_output")?; + let policies = &args.value("policies")?; + + let py = runtime.py(); + let mut plan_output = plan_output.clone(); + let freshness = source_freshness_unknown_warnings(source_freshness)?; + if freshness.is_truthy()? { + let warnings = concat(py, &[plan_output.getattr("warnings")?, freshness])?; + plan_output = replace(&plan_output, &[("warnings", warnings)])?; + } + if !policies.getattr("selection_diagnostics")?.is_truthy()? { + return Ok(plan_output); + } + let stale = native( + py, + "build_stale_out_of_selection_warnings", + &[ + ("original_scope", scopes.getattr("stale_warning_scope")?), + ("execution_scope", pruning.getattr("execution_scope")?), + ("changes", stale_warning_changes.clone()), + ("snapshot", warehouse.getattr("snapshot")?), + ( + "version_identities", + identities.getattr("stale_warning_identities")?, + ), + ("source_freshness", source_freshness.clone()), + ( + "include_sources", + PyBool::new(py, false).to_owned().into_any(), + ), + ( + "model_changes_complete", + PyBool::new(py, true).to_owned().into_any(), + ), + ], + )?; + if stale.is_truthy()? { + let warnings = concat(py, &[plan_output.getattr("warnings")?, stale])?; + plan_output = replace(&plan_output, &[("warnings", warnings)])?; + } + Ok(plan_output) +} + +fn source_names<'py>(items: &Bound<'py, PyAny>) -> PyResult> { + let mut names: Vec<_> = Vec::new(); + for item in items.try_iter()? { + names.push(item?.getattr("source_name")?.extract::()?); + } + sorted_names(items.py(), names) +} + +fn age_names<'py>(statuses: &Bound<'py, PyAny>, wanted: &str) -> PyResult> { + let mut names: Vec<_> = Vec::new(); + for item in statuses.call_method0("items")?.try_iter()? { + let (identity, status): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + if status.eq(wanted)? { + names.push(identity.getattr("source_name")?.extract::()?); + } + } + sorted_names(statuses.py(), names) +} + +fn freshness_metadata<'py>(source_freshness: &Bound<'py, PyAny>) -> PyResult> { + let py = source_freshness.py(); + let propagation = source_freshness.getattr("propagation")?; + let propagated = |field: &str| -> PyResult> { + if propagation.is_none() { + return tuple(py, Vec::new()); + } + sorted_names(py, strings(&propagation.getattr(field)?)?) + }; + let mut details: Vec<_> = Vec::new(); + for unknown in source_freshness + .getattr("unknown_sources")? + .call_method0("values")? + .try_iter()? + { + let unknown = unknown?; + let detail = PyDict::new(py); + detail.set_item("source_name", unknown.getattr("source_name")?)?; + detail.set_item("reason", unknown.getattr("reason")?.getattr("value")?)?; + detail.set_item("message", unknown.getattr("message")?)?; + details.push(detail.into_any()); + } + let statuses = source_freshness.getattr("age_statuses")?; + let metadata = PyDict::new(py); + metadata.set_item( + "observed_source_names", + source_names(&source_freshness.getattr("observed_records")?)?, + )?; + metadata.set_item( + "changed_source_names", + source_names(&source_freshness.getattr("changed_identities")?)?, + )?; + metadata.set_item( + "unchanged_source_names", + source_names(&source_freshness.getattr("unchanged_identities")?)?, + )?; + metadata.set_item( + "unknown_source_names", + sorted_names( + py, + strings(&source_freshness.getattr("unknown_source_names")?)?, + )?, + )?; + metadata.set_item("unknown_source_details", tuple(py, details)?)?; + metadata.set_item("age_warning_source_names", age_names(&statuses, "warn")?)?; + metadata.set_item("age_error_source_names", age_names(&statuses, "error")?)?; + metadata.set_item("stale_model_names", propagated("stale_model_names")?)?; + metadata.set_item("blocked_model_names", propagated("blocked_model_names")?)?; + Ok(metadata.into_any()) +} + +#[pyfunction] +#[pyo3(signature = (*, plan_output, pruning, source_freshness, policies))] +pub(crate) fn with_plan_metadata<'py>( + plan_output: &Bound<'py, PyAny>, + pruning: &Bound<'py, PyAny>, + source_freshness: &Bound<'py, PyAny>, + policies: &Bound<'py, PyAny>, +) -> PyResult> { + let py = plan_output.py(); + let selected: BTreeSet = model_key_names( + &pruning + .getattr("inspection_scope")? + .getattr("selected_keys")?, + )? + .into_iter() + .collect(); + let remaining: Vec = strings(&pruning.getattr("direct_identity_stale_model_names")?)? + .into_iter() + .filter(|name| !selected.contains(name)) + .collect::>() + .into_iter() + .collect(); + let diagnostics = PyDict::new(py); + diagnostics.set_item("mode", text(py, "direct"))?; + diagnostics.set_item("enabled", policies.getattr("selection_diagnostics")?)?; + with_metadata( + plan_output, + &[ + ( + "direct_source_freshness", + freshness_metadata(source_freshness)?, + ), + ( + "direct_remaining_stale_model_names", + sorted_names(py, remaining)?, + ), + ("selection_diagnostics", diagnostics.into_any()), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs new file mode 100644 index 0000000000..fe50f5e71e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs @@ -0,0 +1,88 @@ +//! Register the native execution-planning phases. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + super::scopes::resolve_planner_scopes, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::identities::build_planner_identity_context, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::identities::detect_stale_warning_changes, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::buildability::check_selected_scope_buildability, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::execution_changes::prune_planner_execution_scope, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::execution_changes::reconcile_execution_changes, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::execution_changes::build_planner_entry_results, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::full_refresh::resolve_model_full_refresh, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::full_refresh::effectively_full_refreshed_model_names, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::full_refresh::describe_full_rebuild_cause, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::full_refresh::check_full_rebuild_allowed, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::retention::effective_target_config, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::retention::table_type_copy_name, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::retention::plan_table_types, + module + )?)?; + module.add_function(wrap_pyfunction!(super::retention::plan_retention, module)?)?; + module.add_function(wrap_pyfunction!( + super::output_assembly::assemble_base_plan_output, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::output_assembly::with_column_rename_hints, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::output_assembly::with_storage_policies, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::output_assembly::with_plan_warnings, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::output_assembly::with_plan_metadata, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::warehouse_state::gather_planner_warehouse_state, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs new file mode 100644 index 0000000000..c7ff944b97 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs @@ -0,0 +1,613 @@ +//! Warehouse-aware time-travel retention and Snowflake table-type planning. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_phases::constants::{ + ADAPTER_MODELS, ADAPTER_TYPES, BIGQUERY_ADAPTER, DECREASE_WARNING, DOWNGRADE_HISTORY_DAYS, + PERMANENT_TABLE_TYPE, PLANNER_TYPES, RETENTION_DECREASE, RETENTION_INCREASE, RETENTION_MATCH, + SNOWFLAKE_ADAPTER, SPEC_MODELS, TRANSIENT_TABLE_TYPE, +}; +use crate::bindings::_helpers::planner_phases::objects::{ + construct, member, model_key_names, native, optional_text, planner_error, planner_model, + replace, sorted_names, text, tuple, +}; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_analysis::planner_warehouse::main::retention_change::retention_change; +use sqlbuild_analysis::planner_warehouse::main::retention_direction::retention_direction; +use sqlbuild_analysis::planner_warehouse::main::table_type_copy_name::table_type_copy_name as copy_name; +use std::collections::{BTreeMap, BTreeSet}; + +/// The resolved config of the planned target, or defaults without one. +#[pyfunction] +#[pyo3(signature = (*, runtime))] +pub(crate) fn effective_target_config<'py>( + runtime: &Bound<'py, PyAny>, +) -> PyResult> { + let py = runtime.py(); + let project_config = runtime.getattr("project_config")?; + let local_config = runtime.getattr("local_config")?; + let target_name = runtime + .getattr("project")? + .getattr("effective_target_name")?; + if project_config.is_none() || local_config.is_none() || target_name.is_none() { + return construct(py, (SPEC_MODELS, "TargetConfig"), &[]); + } + native( + py, + "resolve_target_config", + &[ + ("project_config", project_config), + ("local_config", local_config), + ("target_name", target_name), + ], + ) +} + +/// A deterministic identifier-fitted conversion copy name. +#[pyfunction] +#[pyo3(signature = (*, target_name, identifier_limit))] +pub(crate) fn table_type_copy_name(target_name: &str, identifier_limit: i64) -> String { + copy_name(target_name, identifier_limit) +} + +fn selected_models<'py>( + runtime: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, +) -> PyResult>> { + let names: BTreeSet = model_key_names(&scope.getattr("selected_keys")?)? + .into_iter() + .collect(); + let mut models: Vec<_> = Vec::new(); + for model in runtime.getattr("project")?.getattr("models")?.try_iter()? { + let model = model?; + if names.contains(&model.getattr("name")?.extract::()?) { + models.push(model); + } + } + Ok(models) +} + +fn adapter_is(runtime: &Bound<'_, PyAny>, name: &str) -> PyResult { + runtime + .getattr("adapter")? + .getattr("adapter_name")? + .eq(name) +} + +/// Entries for selected Snowflake table-type drift. +#[pyfunction] +#[pyo3(signature = (*, runtime, warehouse, scope))] +pub(crate) fn plan_table_types<'py>( + runtime: &Bound<'py, PyAny>, + warehouse: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, +) -> PyResult> { + let py = runtime.py(); + let models = selected_models(runtime, scope)?; + let snowflake = adapter_is(runtime, SNOWFLAKE_ADAPTER)?; + let mut declared: Vec<_> = Vec::new(); + for model in &models { + if model + .getattr("config")? + .getattr("table_type")? + .getattr("declared")? + .is_truthy()? + { + declared.push(model.getattr("name")?.extract::()?); + } + } + if !declared.is_empty() && !snowflake { + return Err(planner_error( + py, + &format!( + "table_type is not supported on this adapter; declared by model(s): {}", + declared.join(", ") + ), + (None, None), + )); + } + if !snowflake { + return tuple(py, Vec::new()); + } + let target = effective_target_config(runtime)?; + let relations = warehouse + .getattr("snapshot")? + .getattr("existing_relations")?; + let adapter = runtime.getattr("adapter")?; + let materialization = imported(py, PLANNER_TYPES, "MaterializationType")?; + let mut entries: Vec<_> = Vec::new(); + for model in models { + let config = model.getattr("config")?; + let backed = call_with( + &materialization.getattr("is_table_backed")?, + &[( + "materialized", + config + .getattr("values")? + .call_method1("get", ("materialized",))?, + )], + )?; + if !backed.is_truthy()? { + continue; + } + let relation = relations.call_method1("get", (model.getattr("name")?,))?; + if relation.is_none() { + continue; + } + let table_type = config.getattr("table_type")?; + let desired_transient = table_type.getattr("value")?.eq(TRANSIENT_TABLE_TYPE)?; + let actual = relation.getattr("is_transient")?; + let actual: Option = if actual.is_none() { + None + } else { + Some(actual.is_truthy()?) + }; + if actual == Some(desired_transient) { + continue; + } + let downgrade = actual == Some(false) && desired_transient; + let destination = model.getattr("destination")?; + let limit: i64 = adapter + .call_method0("maximum_identifier_length")? + .extract()?; + let target_name: String = destination.getattr("name")?.extract()?; + entries.push(planner_model( + py, + "TableTypePlanEntry", + &[ + ("model_name", model.getattr("name")?), + ("destination", destination), + ("copy_name", text(py, ©_name(&target_name, limit))), + ("desired_type", table_type.getattr("value")?.getattr("value")?), + ( + "actual_type", + optional_text( + py, + actual.map(|transient| { + if transient { + TRANSIENT_TABLE_TYPE + } else { + PERMANENT_TABLE_TYPE + } + }), + ), + ), + ("source", table_type.getattr("source")?.getattr("value")?), + ("downgrade", downgrade.into_pyobject(py)?.to_owned().into_any()), + ( + "downgrade_policy", + target.getattr("table_type_downgrade")?.getattr("value")?, + ), + ( + "irreversible_warning", + optional_text( + py, + downgrade + .then(|| { + format!( + "Downgrading may discard up to {DOWNGRADE_HISTORY_DAYS} days of time-travel history." + ) + }) + .as_deref(), + ), + ), + ], + )?); + } + tuple(py, entries) +} + +struct RetentionEntry<'py> { + request: Bound<'py, PyAny>, + model_names: Bound<'py, PyAny>, + state: Option>, + source: Bound<'py, PyAny>, +} + +fn entry<'py>( + base: &RetentionEntry<'py>, + (direction, phase): (&str, &str), + statements: Bound<'py, PyAny>, +) -> PyResult> { + let py = base.request.py(); + let days = |name: &str| -> PyResult> { + base.state + .as_ref() + .map_or_else(|| Ok(py.None().into_bound(py)), |state| state.getattr(name)) + }; + planner_model( + py, + "RetentionPlanEntry", + &[ + ("request", base.request.clone()), + ("model_names", base.model_names.clone()), + ("actual_days", days("configured_days")?), + ("effective_days", days("effective_days")?), + ("source", base.source.clone()), + ( + "direction", + member(py, (PLANNER_TYPES, "RetentionDirection"), direction)?, + ), + ( + "phase", + member(py, (PLANNER_TYPES, "RetentionPlanPhase"), phase)?, + ), + ("statements", statements), + ( + "irreversible_warning", + optional_text( + py, + (direction == RETENTION_DECREASE).then_some(DECREASE_WARNING), + ), + ), + ], + ) +} + +fn flatten<'py>(changes: &Bound<'py, PyAny>) -> PyResult> { + let mut statements: Vec<_> = Vec::new(); + for change in changes.try_iter()? { + for statement in change?.getattr("statements")?.try_iter()? { + statements.push(statement?); + } + } + tuple(changes.py(), statements) +} + +fn render_changes<'py>( + runtime: &Bound<'py, PyAny>, + request: &Bound<'py, PyAny>, + state: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let mut fields = vec![("request", request.clone())]; + if let Some(state) = state { + fields.push(("state", state.clone())); + } + call_with( + &runtime + .getattr("adapter")? + .getattr("render_retention_changes")?, + &fields, + ) +} + +fn inspect<'py>( + runtime: &Bound<'py, PyAny>, + request: &Bound<'py, PyAny>, +) -> PyResult> { + call_with( + &runtime.getattr("adapter")?.getattr("inspect_retention")?, + &[ + ("connection", runtime.getattr("connection")?), + ("request", request.clone()), + ], + ) +} + +fn state_direction(state: &Bound<'_, PyAny>, desired_days: i64) -> PyResult<&'static str> { + let optional = |name: &str| -> PyResult> { state.getattr(name)?.extract() }; + Ok(retention_direction( + [ + optional("delta_log_retention_days")?, + optional("delta_deleted_file_retention_days")?, + ], + state.getattr("effective_days")?.extract()?, + desired_days, + )) +} + +fn transient_error(py: Python<'_>, name: &str, desired_days: i64, convert: &str) -> pyo3::PyErr { + planner_error( + py, + &format!( + "model '{name}': Snowflake transient tables cannot retain time travel for {desired_days} days; retention above one day requires a permanent table; {convert}table_type permanent on the model, its materialization default, or the target default_table_type" + ), + (None, None), + ) +} + +fn relation_retention<'py>( + runtime: &Bound<'py, PyAny>, + warehouse: &Bound<'py, PyAny>, + model: &Bound<'py, PyAny>, +) -> PyResult>> { + let py = runtime.py(); + let name: String = model.getattr("name")?.extract()?; + let config = model.getattr("config")?; + let retention = config.getattr("time_travel_retention")?; + if retention.getattr("desired_days")?.is_none() || retention.getattr("source")?.is_none() { + return Err(planner_error( + py, + &format!("model '{name}': managed retention is incomplete"), + (None, None), + )); + } + let desired_days: i64 = retention.getattr("desired_days")?.extract()?; + let destination = model.getattr("destination")?; + let schema = destination.getattr("schema")?; + if schema.is_none() { + return Err(planner_error( + py, + &format!("model '{name}': retention requires a destination schema"), + (None, None), + )); + } + let request = construct( + py, + (ADAPTER_MODELS, "RetentionRequest"), + &[ + ("request_id", model.getattr("name")?), + ( + "scope", + member(py, (ADAPTER_TYPES, "RetentionScope"), "relation")?, + ), + ("database", destination.getattr("database")?), + ("schema", schema.clone()), + ("name", destination.getattr("name")?), + ("desired_days", retention.getattr("desired_days")?), + ], + )?; + let mut base = RetentionEntry { + request: request.clone(), + model_names: tuple(py, vec![model.getattr("name")?])?, + state: None, + source: retention.getattr("source")?.getattr("value")?, + }; + let snowflake = adapter_is(runtime, SNOWFLAKE_ADAPTER)?; + let table_type = config.getattr("table_type")?.getattr("value")?; + let relations = warehouse + .getattr("snapshot")? + .getattr("existing_relations")?; + if !relations.contains(model.getattr("name")?)? { + if snowflake && desired_days > 1 && table_type.eq(TRANSIENT_TABLE_TYPE)? { + return Err(transient_error(py, &name, desired_days, "set ")); + } + let changes = render_changes(runtime, &request, None)?; + return Ok(vec![entry( + &base, + ("apply_after_create", "after_create"), + flatten(&changes)?, + )?]); + } + let relation = relations.get_item(model.getattr("name")?)?; + let relation_transient = relation.getattr("is_transient")?; + let lookup = call_with( + &imported(py, ADAPTER_MODELS, "RelationLookup")?.getattr("key")?, + &[ + ("database", destination.getattr("database")?), + ("schema", schema), + ("name", destination.getattr("name")?), + ], + )?; + let mut listed = py.None().into_bound(py); + if relation.getattr("identity")?.eq(&lookup)? { + listed = call_with( + &runtime + .getattr("adapter")? + .getattr("retention_state_from_relation")?, + &[("request", request.clone()), ("relation", relation.clone())], + )?; + } + let state = if listed.is_truthy()? { + listed + } else { + inspect(runtime, &request)? + }; + if state.getattr("is_transient")?.is_truthy()? + && desired_days > 1 + && table_type.eq(TRANSIENT_TABLE_TYPE)? + { + return Err(transient_error( + py, + &name, + desired_days, + "convert it by setting ", + )); + } + base.state = Some(state.clone()); + if snowflake + && desired_days > 1 + && relation_transient.is(pyo3::types::PyBool::new(py, true)) + && table_type.eq(PERMANENT_TABLE_TYPE)? + { + let changes = render_changes(runtime, &request, None)?; + return Ok(vec![entry( + &base, + ("apply_after_create", "after_create"), + flatten(&changes)?, + )?]); + } + let direction = state_direction(&state, desired_days)?; + if direction == RETENTION_MATCH { + return Ok(vec![entry( + &base, + (direction, "none"), + tuple(py, Vec::new())?, + )?]); + } + let changes = render_changes(runtime, &request, Some(&state))?; + let mut entries: Vec<_> = Vec::new(); + for change in changes.try_iter()? { + let change = change?; + let phase: String = change.getattr("phase")?.str()?.extract()?; + entries.push(entry( + &base, + retention_change(&phase, direction), + change.getattr("statements")?, + )?); + } + Ok(entries) +} + +fn bigquery_retention<'py>( + runtime: &Bound<'py, PyAny>, + models: &[Bound<'py, PyAny>], +) -> PyResult>> { + let py = runtime.py(); + let target = effective_target_config(runtime)?; + if !target + .getattr("owns_time_travel_retention_namespace")? + .is_truthy()? + { + return Err(planner_error( + py, + "BigQuery managed time_travel_retention requires target owns_time_travel_retention_namespace = true", + (None, None), + )); + } + type Scope = (Option, String); + let mut managed: BTreeMap>> = BTreeMap::new(); + for model in runtime.getattr("project")?.getattr("models")?.try_iter()? { + let model = model?; + let retention = model.getattr("config")?.getattr("time_travel_retention")?; + let destination = model.getattr("destination")?; + if retention.getattr("unmanaged")?.is_truthy()? + || retention.getattr("desired_days")?.is_none() + || destination.getattr("schema")?.is_none() + { + continue; + } + managed + .entry(( + destination.getattr("database")?.extract()?, + destination.getattr("schema")?.extract()?, + )) + .or_default() + .push(model); + } + let mut scopes: BTreeSet<(String, String, Option)> = BTreeSet::new(); + for model in models { + let destination = model.getattr("destination")?; + let database: Option = destination.getattr("database")?.extract()?; + let schema: String = destination.getattr("schema")?.str()?.extract()?; + scopes.insert((database.clone().unwrap_or_default(), schema, database)); + } + let mut entries: Vec<_> = Vec::new(); + for (_, schema, database) in scopes { + let scope_models = managed + .get(&(database.clone(), schema.clone())) + .cloned() + .unwrap_or_default(); + let mut desired_values: BTreeSet = BTreeSet::new(); + let mut details: Vec<_> = Vec::new(); + let mut sources: BTreeSet = BTreeSet::new(); + let mut names: Vec<_> = Vec::new(); + for item in &scope_models { + let retention = item.getattr("config")?.getattr("time_travel_retention")?; + let desired = retention.getattr("desired_days")?; + details.push(format!( + "{}={}d", + item.getattr("name")?.str()?, + desired.str()? + )); + if !desired.is_none() { + desired_values.insert(desired.extract()?); + } + let source = retention.getattr("source")?; + if !source.is_none() { + sources.insert(source.getattr("value")?.extract()?); + } + names.push(item.getattr("name")?.extract::()?); + } + let prefix = database + .as_ref() + .map_or_else(String::new, |database| format!("{database}.")); + if desired_values.len() != 1 { + return Err(planner_error( + py, + &format!( + "BigQuery dataset {prefix}{schema} has conflicting managed retention values: {}", + details.join(", ") + ), + (None, None), + )); + } + let desired_days = desired_values.into_iter().next().unwrap_or_default(); + let request = construct( + py, + (ADAPTER_MODELS, "RetentionRequest"), + &[ + ("request_id", text(py, &format!("{prefix}{schema}"))), + ( + "scope", + member(py, (ADAPTER_TYPES, "RetentionScope"), "namespace")?, + ), + ("database", optional_text(py, database.as_deref())), + ("schema", text(py, &schema)), + ("desired_days", desired_days.into_pyobject(py)?.into_any()), + ], + )?; + let state = inspect(runtime, &request)?; + let base = RetentionEntry { + request: request.clone(), + model_names: sorted_names(py, names)?, + state: Some(state.clone()), + source: text(py, &sources.into_iter().collect::>().join(",")), + }; + if !state.getattr("exists")?.is_truthy()? { + let changes = render_changes(runtime, &request, None)?; + entries.push(entry( + &base, + ("apply_after_create", "after_create"), + flatten(&changes)?, + )?); + continue; + } + let direction = state_direction(&state, desired_days)?; + let statements = if direction == RETENTION_MATCH { + tuple(py, Vec::new())? + } else { + flatten(&render_changes(runtime, &request, Some(&state))?)? + }; + let phase = if direction == RETENTION_MATCH { + "none" + } else if direction == RETENTION_INCREASE { + "pre" + } else { + "post" + }; + entries.push(entry(&base, (direction, phase), statements)?); + } + Ok(entries) +} + +/// Inspect managed selected retention scopes and produce independent plan entries. +#[pyfunction] +#[pyo3(signature = (*, runtime, warehouse, scope))] +pub(crate) fn plan_retention<'py>( + runtime: &Bound<'py, PyAny>, + warehouse: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, +) -> PyResult> { + let py = runtime.py(); + let mut models: Vec<_> = Vec::new(); + for model in selected_models(runtime, scope)? { + let retention = model.getattr("config")?.getattr("time_travel_retention")?; + if !retention.getattr("unmanaged")?.is_truthy()? + && !retention.getattr("desired_days")?.is_none() + { + models.push(model); + } + } + if models.is_empty() { + return tuple(py, Vec::new()); + } + let entries = if adapter_is(runtime, BIGQUERY_ADAPTER)? { + bigquery_retention(runtime, &models)? + } else { + let mut entries: Vec<_> = Vec::new(); + for model in &models { + entries.extend(relation_retention(runtime, warehouse, model)?); + } + entries + }; + let policy = effective_target_config(runtime)? + .getattr("time_travel_retention_decrease")? + .getattr("value")?; + let mut decided: Vec<_> = Vec::new(); + for entry in entries { + decided.push(replace(&entry, &[("decrease_policy", policy.clone())])?); + } + tuple(py, decided) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs new file mode 100644 index 0000000000..1d80a22359 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs @@ -0,0 +1,101 @@ +//! Resolve the selected, stale-warning and inspection scopes of one plan. + +use crate::bindings::_helpers::planner_phases::objects::{native, planner_model, replace, tuple}; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +#[pyfunction] +#[pyo3(signature = (*, project, selection, policies))] +pub(crate) fn resolve_planner_scopes<'py>( + project: &Bound<'py, PyAny>, + selection: &Bound<'py, PyAny>, + policies: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let auto_load = policies.getattr("auto_load_sources")?; + let selected = native( + py, + "build_planner_scope", + &[ + ("project", project.clone()), + ("select", selection.getattr("select")?), + ("exclude", selection.getattr("exclude")?), + ("auto_load_sources", auto_load.clone()), + ("selected_keys", selection.getattr("selected_keys")?), + ( + "python_read_source_names", + selection.getattr("python_read_source_names")?, + ), + ( + "sql_test_selection", + selection.getattr("sql_test_selection")?, + ), + ], + )?; + let empty = PyTuple::empty(py).into_any(); + let full = native( + py, + "build_planner_scope", + &[ + ("project", project.clone()), + ("select", empty.clone()), + ("exclude", empty), + ("auto_load_sources", auto_load), + ], + )?; + let stale_keys = native( + py, + "transitive_closure_many", + &[ + ("starts", selected.getattr("selected_keys")?), + ("edges", full.getattr("upstream_deps")?), + ( + "include_starts", + true.into_pyobject(py)?.to_owned().into_any(), + ), + ], + )?; + let all_keys = PyDict::new(py); + for (name, key) in full.getattr("all_keys")?.downcast::()?.iter() { + if stale_keys.contains(&key)? { + all_keys.set_item(name, key)?; + } + } + let models = PyDict::new(py); + for (name, model) in full.getattr("models_by_name")?.downcast::()?.iter() { + if stale_keys.contains(model.getattr("key")?)? { + models.set_item(name, model)?; + } + } + let mut order: Vec<_> = Vec::new(); + for key in full.getattr("execution_order")?.try_iter()? { + let key = key?; + if stale_keys.contains(&key)? { + order.push(key); + } + } + let stale_scope = replace( + &full, + &[ + ("all_keys", all_keys.into_any()), + ("models_by_name", models.into_any()), + ("selected_keys", selected.getattr("selected_keys")?), + ("execution_order", tuple(py, order)?), + ( + "user_selected_keys", + selected.getattr("user_selected_keys")?, + ), + ], + )?; + planner_model( + py, + "PlannerScopeResolution", + &[ + ("selected_scope", selected.clone()), + ("stale_warning_scope", stale_scope), + ("inspection_scope", selected), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs new file mode 100644 index 0000000000..d8cf02e822 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -0,0 +1,194 @@ +//! The warehouse inspection phase: snapshot, migrations and inspection relations in one pass. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_phases::constants::PLAN_ENTRY_MODULE; +use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; +use crate::bindings::_helpers::planner_phases::objects::{native, planner_model}; +use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; +use crate::bindings::_helpers::planner_warehouse::snapshot::{SnapshotRequest, warehouse_snapshot}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use std::time::Instant; + +fn progress(runtime: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { + let on_progress = runtime.getattr("on_progress")?; + if !on_progress.is_none() { + on_progress.call1((message,))?; + } + Ok(()) +} + +#[pyfunction] +#[pyo3(signature = (*, runtime, scopes, overrides, deferral))] +pub(crate) fn gather_planner_warehouse_state<'py>( + runtime: &Bound<'py, PyAny>, + scopes: &Bound<'py, PyAny>, + overrides: &Bound<'py, PyAny>, + deferral: &Bound<'py, PyAny>, +) -> PyResult> { + let py = runtime.py(); + let started = Instant::now(); + let project = runtime.getattr("project")?; + let fingerprints = Py::new( + py, + MigrationFingerprintCache::open(py, Some(project.getattr("compile_cache_dir")?))?, + )? + .into_bound(py) + .into_any(); + progress(runtime, "Inspecting warehouse state...")?; + let selected_scope = scopes.getattr("selected_scope")?; + let full_refresh: bool = overrides.getattr("full_refresh")?.extract()?; + let config = runtime.getattr("project_config")?; + let cursor_scope = planner_model( + py, + "CursorSnapshotScope", + &[ + ("model_keys", selected_scope.getattr("selected_keys")?), + ( + "runtime_producer_keys", + selected_scope.getattr("selected_keys")?, + ), + ("invocation_time", runtime.getattr("invocation_time")?), + ( + "start_cursor_config", + if config.is_none() { + py.None().into_bound(py) + } else { + config.getattr("cursors")?.getattr("start")? + }, + ), + ("cursor_overrides", overrides.getattr("cursor_overrides")?), + ], + )?; + let adapter = runtime.getattr("adapter")?; + let stale_keys = pyo3::types::PyFrozenSet::new( + py, + scopes + .getattr("stale_warning_scope")? + .getattr("all_keys")? + .call_method0("values")? + .try_iter()? + .collect::>>()?, + )? + .into_any(); + let snapshot = warehouse_snapshot(&SnapshotRequest { + project: project.clone(), + adapter: adapter.clone(), + connection: runtime.getattr("connection")?, + execute: adapter.getattr("execute")?, + selected_keys: stale_keys, + full_refresh: false, + full_refresh_model_names: effectively_full_refreshed_model_names(&project, full_refresh)?, + on_progress: runtime.getattr("on_progress")?, + deferred_locations: deferral.getattr("deferred_locations")?, + cursor_scope, + })?; + let migrations = native( + py, + "plan_model_migrations", + &[ + ("runtime", runtime.clone()), + ("scope", selected_scope.clone()), + ("snapshot", snapshot.clone()), + ("overrides", overrides.clone()), + ("deferral", deferral.clone()), + ("fingerprints", fingerprints.clone()), + ], + )?; + fingerprints.call_method0("persist")?; + let inspection_relations = call_with( + &imported(py, PLAN_ENTRY_MODULE, "build_planner_relations_context")?, + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("connection", runtime.getattr("connection")?), + ("scope", scopes.getattr("inspection_scope")?), + ("deferral", deferral.clone()), + ("project_config", config), + ("local_config", runtime.getattr("local_config")?), + ], + )?; + let physical = PyDict::new(py); + for entry in migrations.getattr("entries")?.try_iter()? { + let entry = entry?; + if entry + .getattr("decision")? + .getattr("moves_data")? + .is_truthy()? + { + let origin = entry.getattr("origin")?; + let qualified = origin.getattr("qualified_name")?; + physical.set_item( + entry.getattr("model_name")?, + if qualified.is_truthy()? { + qualified + } else { + origin.getattr("name")? + }, + )?; + } + } + let columns = native( + py, + "plan_column_migrations", + &[ + ("runtime", runtime.clone()), + ("scope", selected_scope.clone()), + ("snapshot", migrations.getattr("snapshot")?), + ( + "full_refresh_model_names", + effectively_full_refreshed_model_names(&project, full_refresh)?, + ), + ("physical_relations", physical.into_any()), + ("overrides", overrides.clone()), + ("deferral", deferral.clone()), + ( + "source_columns", + inspection_relations.getattr("source_warehouse_columns")?, + ), + ], + )?; + let old_names = native( + py, + "plan_old_name_views", + &[ + ("runtime", runtime.clone()), + ("scope", selected_scope), + ("snapshot", snapshot), + ("migration_entries", migrations.getattr("entries")?), + ("column_entries", columns.getattr("entries")?), + ], + )?; + if !runtime.getattr("on_progress")?.is_none() { + progress( + runtime, + &format!( + "Inspected warehouse state. ({:.2}s)", + started.elapsed().as_secs_f64() + ), + )?; + progress(runtime, "Generating plan...")?; + } + let mut warnings: Vec<_> = Vec::new(); + for source in [&migrations, &columns, &old_names] { + for warning in source.getattr("warnings")?.try_iter()? { + warnings.push(warning?); + } + } + planner_model( + py, + "PlannerWarehouseState", + &[ + ("snapshot", columns.getattr("snapshot")?), + ("inspection_relations", inspection_relations), + ("migration_entries", migrations.getattr("entries")?), + ("migration_warnings", PyTuple::new(py, warnings)?.into_any()), + ("column_migration_entries", columns.getattr("entries")?), + ("column_rename_hints", columns.getattr("hints")?), + ("old_name_view_entries", old_names.getattr("entries")?), + ("old_name_views", old_names.getattr("views")?), + ("migration_fingerprints", fingerprints), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs new file mode 100644 index 0000000000..21f6f9a2d8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs @@ -0,0 +1,83 @@ +//! Python modules and fixed names of warehouse inspection for planning. + +pub(crate) const RELATIONS_MAIN_LIST: (&str, &str) = ( + "sqlbuild.adapter.relations.main._list_relations_for_inspection", + "list_relations_for_inspection", +); +pub(crate) const RELATIONS_MAIN_COLUMNS: (&str, &str) = ( + "sqlbuild.adapter.relations.main._get_columns_for_inspection", + "get_columns_for_inspection", +); +pub(crate) const ACTIVE_CATALOG: (&str, &str) = ( + "sqlbuild.adapter.relations.main.active_inspection_catalog", + "active_inspection_catalog", +); +pub(crate) const RUN_BOUNDED: (&str, &str) = ( + "sqlbuild.adapter.relations.main.run_bounded_inspections", + "run_bounded_inspections", +); +pub(crate) const RECORD_QUERY: (&str, &str) = ( + "sqlbuild.adapter.relations.main.record_inspection_query", + "record_inspection_query", +); +pub(crate) const RUN_RECORDED_QUERY: (&str, &str) = ( + "sqlbuild.adapter.relations.main.run_recorded_inspection_query", + "run_recorded_inspection_query", +); +pub(crate) const QUERY_RECORD: (&str, &str) = + ("sqlbuild.adapter.relations.models", "InspectionQueryRecord"); +pub(crate) const COLUMN_INFO: (&str, &str) = ("sqlbuild.adapter.contract.models", "ColumnInfo"); +pub(crate) const RELATION_LOOKUP: (&str, &str) = + ("sqlbuild.adapter.contract.models", "RelationLookup"); +pub(crate) const LOG_DEBUG_EVENT: (&str, &str) = ( + "sqlbuild.diagnostics.main.log_debug_event", + "log_debug_event", +); +pub(crate) const CURSOR_RESOLVE_MODULE: &str = "sqlbuild.compiler.planner._helpers.resolve.cursor"; +pub(crate) const CURSOR_POLICIES: (&str, &str) = ( + "sqlbuild.compiler.planner._helpers.resolve.cursor_policies", + "resolve_start_cursor_config", +); +pub(crate) const MAXIMUM_START: (&str, &str) = ( + "sqlbuild.compiler.planner._helpers.resolve.maximum_start", + "maximum_allowed_start", +); +pub(crate) const RENDER_SOURCE_READ: (&str, &str) = ( + "sqlbuild.compiler.planner._helpers.resolve.sources", + "render_source_read_relation", +); +pub(crate) const YAML_ENTRY_LINE: (&str, &str) = ( + "sqlbuild.compiler.discovery.main._yaml_entry_line", + "yaml_entry_line", +); +pub(crate) const FRESHNESS_PLANNING: (&str, &str) = ( + "sqlbuild.compiler.source_freshness.main._planning", + "build_direct_source_freshness_planning_result", +); +pub(crate) const FRESHNESS_PROPAGATION: (&str, &str) = ( + "sqlbuild.compiler.source_freshness.main._propagation", + "build_direct_source_freshness_propagation_result", +); +pub(crate) const RUN_SCOPE: (&str, &str) = + ("sqlbuild.runtime.observability.main.run_scope", "run_scope"); +pub(crate) const SQL_HOOK_ENTRY: (&str, &str) = + ("sqlbuild.compiler.discovery.models", "SqlHookEntry"); +pub(crate) const CURSOR_ALGEBRA_ERROR: (&str, &str) = + ("sqlbuild.cursor_algebra.exceptions", "CursorAlgebraError"); +pub(crate) const DATE_VALUE: (&str, &str) = ("sqlbuild.cursor_algebra.models", "DateValue"); +pub(crate) const PLANNER_LOGGER: &str = "sqlbuild.planner"; +pub(crate) const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; +pub(crate) const SOURCE_FRESHNESS_TABLE_NAME: &str = "_sqlbuild_source_freshness"; +pub(crate) const OLD_NAME_VIEW_TABLE_NAME: &str = "_sqlbuild_old_name_views"; +pub(crate) const MIGRATION_TABLE_NAME: &str = "_sqlbuild_migrations"; +pub(crate) const METADATA_NAME_FILTER_LIMIT: usize = 250; +pub(crate) const NODE_TYPE_MODEL: &str = "model"; +pub(crate) const NODE_TYPE_SEED: &str = "seed"; +pub(crate) const FUNCTION_NODE_TYPES: [&str; 2] = ["udf", "table_fn"]; +pub(crate) const WILDCARD_TABLE_CHARACTER: &str = "*"; +pub(crate) const MIGRATE_FROM_CONFIG_KEY: &str = "migrate_from"; +pub(crate) const MODEL_HOOK_CONFIG_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; +pub(crate) const CURSOR_TYPE_TIMESTAMP: &str = "timestamp"; +pub(crate) const CURSOR_TYPE_INTEGER: &str = "integer"; +pub(crate) const MISSING_SOURCE_TABLE_CODE: &str = "S405"; +pub(crate) const WATERMARK_CONTRACT_CODE: &str = "S302"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs new file mode 100644 index 0000000000..35a18d4420 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -0,0 +1,1194 @@ +//! Cursor MIN/MAX inspection of selected incremental models and their watermark inputs. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::planner_phases::constants::PLANNER_TYPES; +use crate::bindings::_helpers::planner_phases::objects::{ + by_name, config_str, member, model_key, native, optional_text, planner_error, planner_model, + text, tuple, +}; +use crate::bindings::_helpers::planner_warehouse::constants::{ + CURSOR_ALGEBRA_ERROR, CURSOR_POLICIES, CURSOR_RESOLVE_MODULE, CURSOR_TYPE_INTEGER, + CURSOR_TYPE_TIMESTAMP, DATE_VALUE, LOG_DEBUG_EVENT, MAXIMUM_START, PLANNER_LOGGER, + QUERY_RECORD, RECORD_QUERY, RUN_BOUNDED, RUN_RECORDED_QUERY, +}; +use crate::bindings::_helpers::planner_warehouse::snapshot::{ + SnapshotRequest, inspection_concurrency, +}; +use crate::bindings::_helpers::planner_warehouse::watermarks::validate_watermark_column; +use pyo3::IntoPyObject; +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyFloat, PyTuple}; +use sqlbuild_analysis::planner_warehouse::main::cursor_bounds::cursor_bounds; +use sqlbuild_analysis::planner_warehouse::main::cursor_query_groups::cursor_query_groups; +use sqlbuild_analysis::planner_warehouse::main::cursor_query_identity::cursor_query_identity; +use sqlbuild_analysis::planner_warehouse::main::cursor_query_sql::cursor_query_sql; +use sqlbuild_analysis::planner_warehouse::models::{CursorTagRequest, PhysicalCursorQuery}; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Instant; + +/// Selection and policy inputs for cursor warehouse inspection. +pub(crate) struct CursorInputs<'py> { + pub(crate) selected_keys: Bound<'py, PyAny>, + pub(crate) full_refresh_model_names: Bound<'py, PyAny>, + pub(crate) deferred_locations: Bound<'py, PyAny>, + pub(crate) runtime_producer_keys: Bound<'py, PyAny>, + pub(crate) invocation_time: Bound<'py, PyAny>, + pub(crate) start_cursor_config: Bound<'py, PyAny>, + pub(crate) cursor_overrides: Bound<'py, PyAny>, + pub(crate) target_relation_overrides: Option>, + pub(crate) origin_cursor_column_overrides: Option>, + pub(crate) existing_columns: Option>, +} + +struct UpstreamCursor<'py> { + tag_min: String, + tag_max: String, + relation: String, + cursor_column: String, + input_name: String, + cursor_grain: Option, + terminal_start: Option>, + terminal_end: Option>, +} + +struct CursorModel<'py> { + model_name: String, + target_tag: Option, + target_relation: Option, + cursor_column: String, + origin_cursor_column: Option, + upstreams: Vec>, + cursor_type: Option, + effective_grain: Bound<'py, PyAny>, + start_cursor_config: Bound<'py, PyAny>, + has_start_override: bool, + watermark_mode: Bound<'py, PyAny>, + watermark_all: bool, + watermark_strategy: bool, +} + +impl CursorModel<'_> { + fn physical_cursor_column(&self) -> &str { + self.origin_cursor_column + .as_deref() + .filter(|column| !column.is_empty()) + .unwrap_or(&self.cursor_column) + } +} + +type Values<'py> = HashMap>; + +struct ModelMaps<'py> { + models: Bound<'py, PyDict>, + sources: Bound<'py, PyDict>, + seeds: Bound<'py, PyDict>, + functions: Bound<'py, PyDict>, +} + +fn parse<'py>( + py: Python<'py>, + raw: &Bound<'py, PyAny>, + cursor_type: &str, +) -> PyResult> { + native( + py, + "cursor_parse", + &[("raw", raw.clone()), ("cursor_type", text(py, cursor_type))], + ) +} + +fn render<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + native(value.py(), "cursor_render", &[("value", value.clone())]) +} + +fn grain<'py>(py: Python<'py>, value: &str) -> PyResult> { + member(py, (PLANNER_TYPES, "CursorGrain"), value) +} + +fn parse_warehouse_cursor<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py = value.py(); + match parse(py, value, CURSOR_TYPE_INTEGER) { + Ok(parsed) => Ok(parsed), + Err(error) + if error.is_instance( + py, + &imported(py, CURSOR_ALGEBRA_ERROR.0, CURSOR_ALGEBRA_ERROR.1)?, + ) => + { + parse(py, value, CURSOR_TYPE_TIMESTAMP) + } + Err(error) => Err(error), + } +} + +fn log_debug<'py>( + py: Python<'py>, + message: &str, + context: &[(&str, Bound<'py, PyAny>)], +) -> PyResult<()> { + let logger = imported(py, "logging", "getLogger")?.call1((PLANNER_LOGGER,))?; + let mut fields = vec![("logger", logger), ("message", text(py, message))]; + fields.extend(context.iter().cloned()); + call_with( + &imported(py, LOG_DEBUG_EVENT.0, LOG_DEBUG_EVENT.1)?, + &fields, + )?; + Ok(()) +} + +fn target_has_cursor_column( + py: Python<'_>, + (model_name, cursor_column): (&str, &str), + existing_columns: Option<&Bound<'_, PyDict>>, +) -> PyResult { + let columns = match existing_columns { + Some(columns) => columns.get_item(model_name)?, + None => None, + }; + let Some(columns) = columns.filter(|columns| columns.is_truthy().unwrap_or(false)) else { + return Ok(true); + }; + let strip = |value: &Bound<'_, PyAny>| -> PyResult { + value + .call_method1("strip", ("\"`[]",))? + .call_method0("lower")? + .extract() + }; + let wanted = strip(&text(py, cursor_column))?; + for column in columns.try_iter()? { + if strip(&column?.getattr("name")?)? == wanted { + return Ok(true); + } + } + log_debug( + py, + "target relation lacks the cursor column; skipping its cursor bounds read", + &[ + ("sqlbuild_model", text(py, model_name)), + ("sqlbuild_cursor_column", text(py, cursor_column)), + ], + )?; + Ok(false) +} + +fn runtime_producer_names( + project: &Bound<'_, PyAny>, + selected: &Bound<'_, PyAny>, +) -> PyResult> { + let mut names: HashSet<_> = HashSet::new(); + if selected.is_none() { + return Ok(names); + } + let mut loaded: HashSet = HashSet::new(); + for source in project.getattr("sources")?.try_iter()? { + let source = source?; + if !source.getattr("source_entry")?.getattr("loader")?.is_none() { + loaded.insert(source.getattr("name")?.extract()?); + } + } + for key in selected.try_iter()? { + let key = key?; + let kind = key.getattr("resource_type")?; + let name: String = key.getattr("name")?.extract()?; + if kind.eq("model")? || kind.eq("seed")? || (kind.eq("source")? && loaded.contains(&name)) { + names.insert(name); + } + } + Ok(names) +} + +fn upstream_relation<'py>( + reference: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + maps: &ModelMaps<'py>, + (deferred, selected): (&Bound<'py, PyAny>, Option<&HashSet>), +) -> PyResult> { + let py = reference.py(); + let kind = reference.getattr("ref_kind")?; + let name = reference.getattr("ref_name")?; + let name_text: String = name.extract()?; + let is_selected = selected.is_some_and(|names| names.contains(&name_text)); + let qualified = + |node: Bound<'py, PyAny>| node.getattr("destination")?.getattr("qualified_name"); + if (kind.eq("ref")? || kind.eq("seed")?) + && !deferred.is_none() + && deferred.contains(&name)? + && !is_selected + { + return deferred.get_item(&name)?.getattr("qualified_name"); + } + if kind.eq("ref")? { + if let Some(model) = maps.models.get_item(&name)? { + return qualified(model); + } + if let Some(seed) = maps.seeds.get_item(&name)? { + return qualified(seed); + } + } else if kind.eq("seed")? { + if let Some(seed) = maps.seeds.get_item(&name)? { + return qualified(seed); + } + } else if kind.eq("source")? + && let Some(source) = maps.sources.get_item(&name)? + { + return native( + py, + "render_source_relation", + &[ + ("entry", source.getattr("source_entry")?), + ("adapter", adapter.clone()), + ], + ); + } + Ok(py.None().into_bound(py)) +} + +fn lineage_reference<'py>( + model: &Bound<'py, PyAny>, + input_name: &Bound<'py, PyAny>, + maps: &ModelMaps<'py>, +) -> PyResult> { + native( + model.py(), + "resolve_lineage_reference", + &[ + ("model", model.clone()), + ("input_name", input_name.clone()), + ("models_by_name", maps.models.clone().into_any()), + ("functions_by_name", maps.functions.clone().into_any()), + ], + ) +} + +fn terminal_bound<'py>( + reference: &Bound<'py, PyAny>, + maps: &ModelMaps<'py>, + (key, cursor_type): (&str, Option<&str>), +) -> PyResult>> { + let py = reference.py(); + if !reference.getattr("ref_kind")?.eq("ref")? { + return Ok(None); + } + let Some(upstream) = maps.models.get_item(reference.getattr("ref_name")?)? else { + return Ok(None); + }; + let value = native( + py, + "get_config_cursor_bound", + &[ + ("values", upstream.getattr("config")?.getattr("values")?), + ("key", text(py, key)), + ], + )?; + if value.is_none() { + return Ok(None); + } + Ok(Some(parse( + py, + &value, + cursor_type + .filter(|value| !value.is_empty()) + .unwrap_or(CURSOR_TYPE_TIMESTAMP), + )?)) +} + +fn input_grain(reference: &Bound<'_, PyAny>, maps: &ModelMaps<'_>) -> PyResult> { + if !reference.getattr("ref_kind")?.eq("ref")? { + return Ok(None); + } + match maps.models.get_item(reference.getattr("ref_name")?)? { + Some(upstream) => config_str(&upstream, "cursor_grain"), + None => Ok(None), + } +} + +fn start_override_set(model: &Bound<'_, PyAny>, overrides: &Bound<'_, PyAny>) -> PyResult { + if overrides.is_none() { + return Ok(false); + } + match config_str(model, "cursor_type")?.as_deref() { + Some(CURSOR_TYPE_TIMESTAMP) => Ok(!overrides.getattr("start_ts")?.is_none()), + Some(CURSOR_TYPE_INTEGER) => Ok(!overrides.getattr("start_int")?.is_none()), + _ => Ok(false), + } +} + +fn collect_models<'py>( + request: &SnapshotRequest<'py>, + existing_relations: &Bound<'py, PyDict>, + inputs: &CursorInputs<'py>, + maps: &ModelMaps<'py>, +) -> PyResult>> { + let py = request.project.py(); + let project = &request.project; + let producers = if inputs.runtime_producer_keys.is_none() { + inputs.selected_keys.clone() + } else { + inputs.runtime_producer_keys.clone() + }; + let selected_names: Option> = if producers.is_none() { + None + } else { + Some( + producers + .try_iter()? + .map(|key| key?.getattr("name")?.extract::()) + .collect::>()?, + ) + }; + let runtime_names = runtime_producer_names(project, &producers)?; + let mut models: Vec<_> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + let name_value = model.getattr("name")?; + let model_name: String = name_value.extract()?; + if !inputs.selected_keys.is_none() + && !inputs.selected_keys.contains(model_key(py, &name_value)?)? + { + continue; + } + let cursor_column = config_str(&model, "cursor")?; + let materialized = config_str(&model, "materialized")?; + let Some(cursor_column) = + cursor_column.filter(|_| materialized.as_deref() == Some("incremental")) + else { + continue; + }; + let watermark_inputs = native( + py, + "resolve_cursor_input_roles", + &[("model", model.clone())], + )? + .getattr("watermark_inputs")?; + let watermark_inputs: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = watermark_inputs + .call_method0("items")? + .try_iter()? + .map(|item| item?.extract()) + .collect::>()?; + if inputs.full_refresh_model_names.contains(&name_value)? { + for (input_name, upstream_column) in &watermark_inputs { + let reference = lineage_reference(&model, input_name, maps)?; + validate_watermark_column( + &model, + &reference, + &upstream_column.extract::()?, + maps_tuple(maps), + )?; + } + continue; + } + let mut target_tag = None; + let mut target_relation = None; + let redirected = match &inputs.target_relation_overrides { + Some(overrides) => overrides.get_item(&name_value)?, + None => None, + }; + let target_info = existing_relations.get_item(&name_value)?; + let qualified = model.getattr("destination")?.getattr("qualified_name")?; + if let Some(redirected) = redirected.filter(|value| !value.is_none()) { + target_tag = Some(format!("{model_name}__target__max")); + target_relation = Some(redirected.extract::()?); + } else if !qualified.is_none() + && let Some(info) = target_info + && info.getattr("name")?.eq(&name_value)? + { + if target_has_cursor_column( + py, + (&model_name, &cursor_column), + inputs.existing_columns.as_ref(), + )? { + target_tag = Some(format!("{model_name}__target__max")); + } + target_relation = Some(qualified.extract::()?); + } + let cursor_type = config_str(&model, "cursor_type")?; + let cursor_grain = config_str(&model, "cursor_grain")?; + let mut upstreams: Vec<_> = Vec::new(); + for (input_name, upstream_column) in &watermark_inputs { + let upstream_column: String = upstream_column.extract()?; + let reference = lineage_reference(&model, input_name, maps)?; + validate_watermark_column(&model, &reference, &upstream_column, maps_tuple(maps))?; + let ref_name: String = reference.getattr("ref_name")?.extract()?; + if runtime_names.contains(&ref_name) { + continue; + } + let relation = upstream_relation( + &reference, + &request.adapter, + maps, + (&inputs.deferred_locations, selected_names.as_ref()), + )?; + if relation.is_none() { + continue; + } + if !existing_relations.contains(&ref_name)? + && !reference.getattr("ref_kind")?.eq("source")? + { + continue; + } + upstreams.push(UpstreamCursor { + tag_min: format!("{model_name}__{ref_name}__min"), + tag_max: format!("{model_name}__{ref_name}__max"), + relation: relation.extract()?, + cursor_column: upstream_column, + input_name: ref_name, + cursor_grain: input_grain(&reference, maps)?, + terminal_start: terminal_bound( + &reference, + maps, + ("cursor_start", cursor_type.as_deref()), + )?, + terminal_end: terminal_bound( + &reference, + maps, + ("cursor_end", cursor_type.as_deref()), + )?, + }); + } + let strategy = config_str(&model, "microbatch_strategy")?; + let mut input_grains: Vec<_> = Vec::new(); + for upstream in &upstreams { + input_grains.push(match &upstream.cursor_grain { + Some(value) => grain(py, value)?, + None => py.None().into_bound(py), + }); + } + let effective_grain = call_with( + &imported( + py, + CURSOR_RESOLVE_MODULE, + "resolve_effective_timestamp_grain", + )?, + &[ + ("cursor_type", optional_text(py, cursor_type.as_deref())), + ( + "downstream_grain", + optional_text(py, cursor_grain.as_deref()), + ), + ( + "microbatch_strategy", + optional_text(py, strategy.as_deref()), + ), + ("cursor_input_grains", tuple(py, input_grains)?), + ], + )?; + let mode = config_str(&model, "cursor_watermark_mode")? + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| "all".to_owned()); + let watermark_mode = member(py, (PLANNER_TYPES, "CursorWatermarkMode"), &mode)?; + models.push(CursorModel { + target_tag, + target_relation, + origin_cursor_column: match &inputs.origin_cursor_column_overrides { + Some(overrides) => overrides + .get_item(&name_value)? + .map(|value| value.extract()) + .transpose()?, + None => None, + }, + cursor_column, + upstreams, + effective_grain, + start_cursor_config: call_with( + &imported(py, CURSOR_POLICIES.0, CURSOR_POLICIES.1)?, + &[ + ("model", model.clone()), + ("project_config", inputs.start_cursor_config.clone()), + ], + )?, + has_start_override: start_override_set(&model, &inputs.cursor_overrides)?, + watermark_all: watermark_mode.eq("all")?, + watermark_mode, + watermark_strategy: strategy.as_deref() == Some("watermark"), + cursor_type, + model_name, + }); + } + Ok(models) +} + +fn maps_tuple<'a, 'py>( + maps: &'a ModelMaps<'py>, +) -> ( + &'a Bound<'py, PyDict>, + &'a Bound<'py, PyDict>, + &'a Bound<'py, PyDict>, +) { + (&maps.models, &maps.sources, &maps.seeds) +} + +fn physical_queries(models: &[CursorModel<'_>]) -> Vec { + let mut requests: Vec<_> = Vec::new(); + for info in models { + if let (Some(tag), Some(relation)) = (&info.target_tag, &info.target_relation) { + requests.push(CursorTagRequest { + relation: relation.clone(), + cursor_column: info.physical_cursor_column().to_owned(), + cursor_type: info.cursor_type.clone(), + min_tag: None, + max_tag: tag.clone(), + }); + } + for upstream in &info.upstreams { + requests.push(CursorTagRequest { + relation: upstream.relation.clone(), + cursor_column: upstream.cursor_column.clone(), + cursor_type: info.cursor_type.clone(), + min_tag: Some(upstream.tag_min.clone()), + max_tag: upstream.tag_max.clone(), + }); + } + } + cursor_query_groups(&requests) +} + +/// Run one physical cursor statement; a failure leaves only that relation unavailable. +fn read_cursor_query<'py>( + py: Python<'py>, + sql: &str, + (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let started = Instant::now(); + let outcome = call_with( + execute, + &[("connection", connection.clone()), ("sql", text(py, sql))], + ) + .and_then(|result| result.call_method0("fetchall")); + let elapsed = started.elapsed().as_secs_f64(); + let (row, error, count) = match outcome { + Ok(rows) => { + let count = rows.len()?; + let row = if count > 0 { + rows.get_item(0)? + } else { + py.None().into_bound(py) + }; + ( + row, + py.None().into_bound(py), + count.into_pyobject(py)?.into_any(), + ) + } + Err(error) if error.is_instance_of::(py) => { + let value = error.value(py).clone().into_any(); + (py.None().into_bound(py), value, py.None().into_bound(py)) + } + Err(error) => return Err(error), + }; + let error_text = if error.is_none() { + py.None().into_bound(py) + } else { + error.str()?.into_any() + }; + let record = call_with( + &imported(py, QUERY_RECORD.0, QUERY_RECORD.1)?, + &[ + ("sql", text(py, sql)), + ("elapsed_seconds", PyFloat::new(py, elapsed).into_any()), + ("row_count", count), + ("error", error_text), + ], + )?; + call_with( + &imported(py, RECORD_QUERY.0, RECORD_QUERY.1)?, + &[("record", record)], + )?; + Ok(PyTuple::new(py, [row, PyFloat::new(py, elapsed).into_any(), error])?.into_any()) +} + +fn report_cursor_query( + py: Python<'_>, + query: &PhysicalCursorQuery, + (outcome, identity): (&Bound<'_, PyAny>, &str), + on_progress: &Bound<'_, PyAny>, +) -> PyResult<()> { + let elapsed: f64 = outcome.get_item(1)?.extract()?; + let error = outcome.get_item(2)?; + if error.is_none() { + if !on_progress.is_none() { + on_progress.call1((format!( + "Inspected cursor bounds {identity} ({elapsed:.2}s)" + ),))?; + } + return Ok(()); + } + let error_text: String = error.str()?.extract()?; + if !on_progress.is_none() { + on_progress.call1((format!( + "Failed cursor bounds {identity} ({elapsed:.2}s): {error_text}" + ),))?; + } + log_debug( + py, + "cursor bounds physical query failed; treating relation as unavailable", + &[ + ("sqlbuild_relation", text(py, &query.relation)), + ("sqlbuild_cursor_column", text(py, &query.cursor_column)), + ("sqlbuild_bounds", text(py, &cursor_bounds(query))), + ( + "sqlbuild_elapsed_seconds", + text(py, &format!("{elapsed:.2}")), + ), + ("sqlbuild_error", text(py, &error_text)), + ], + ) +} + +fn bound_task<'py>( + py: Python<'py>, + sql: String, + (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let connection: Py = connection.clone().unbind(); + let execute: Py = execute.clone().unbind(); + Ok(PyCFunction::new_closure( + py, + None, + None, + move |args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>| + -> PyResult> { + let py = args.py(); + Ok(read_cursor_query(py, &sql, (connection.bind(py), execute.bind(py)))?.unbind()) + }, + )? + .into_any()) +} + +fn completion_reporter<'py>( + py: Python<'py>, + queries: Vec, + on_progress: &Bound<'py, PyAny>, +) -> PyResult> { + let completed = Arc::new(AtomicUsize::new(0)); + let on_progress: Py = on_progress.clone().unbind(); + Ok(PyCFunction::new_closure( + py, + None, + None, + move |args: &Bound<'_, PyTuple>, kwargs: Option<&Bound<'_, PyDict>>| -> PyResult<()> { + let py = args.py(); + let kwargs = kwargs.cloned().unwrap_or_else(|| PyDict::new(py)); + let index: usize = kwargs + .get_item("index")? + .map_or(Ok(0), |value| value.extract())?; + let result = kwargs + .get_item("result")? + .unwrap_or_else(|| py.None().into_bound(py)); + let done = completed.fetch_add(1, Ordering::SeqCst) + 1; + let query = &queries[index]; + let identity = cursor_query_identity(query, (done, queries.len())); + report_cursor_query(py, query, (&result, &identity), on_progress.bind(py)) + }, + )? + .into_any()) +} + +fn fan_out<'py>(query: &PhysicalCursorQuery, row: &Bound<'py, PyAny>) -> PyResult> { + let mut values = Values::new(); + if row.is_none() { + return Ok(values); + } + let mut index = 0; + if !query.min_tags.is_empty() { + let value = row.get_item(index)?; + if !value.is_none() { + for tag in &query.min_tags { + values.insert(tag.clone(), parse_warehouse_cursor(&value)?); + } + } + index += 1; + } + if !query.max_tags.is_empty() { + let value = row.get_item(index)?; + if !value.is_none() { + for tag in &query.max_tags { + values.insert(tag.clone(), parse_warehouse_cursor(&value)?); + } + } + } + Ok(values) +} + +fn execute_queries<'py>( + request: &SnapshotRequest<'py>, + queries: &[PhysicalCursorQuery], + concurrency: i64, +) -> PyResult<(Values<'py>, HashSet)> { + let py = request.project.py(); + let total = queries.len(); + let on_progress = &request.on_progress; + let handles = (&request.connection, &request.execute); + let outcomes: Vec> = if concurrency <= 1 || total <= 1 { + let mut outcomes: Vec<_> = Vec::new(); + for (position, query) in queries.iter().enumerate() { + let identity = cursor_query_identity(query, (position + 1, total)); + if !on_progress.is_none() { + on_progress.call1((format!("Inspecting cursor bounds {identity}..."),))?; + } + let outcome = read_cursor_query(py, &cursor_query_sql(query), handles)?; + report_cursor_query(py, query, (&outcome, &identity), on_progress)?; + outcomes.push(outcome); + } + outcomes + } else { + if !on_progress.is_none() { + on_progress.call1((format!( + "Inspecting cursor bounds for {total} relations ({} concurrent)...", + (concurrency as usize).min(total) + ),))?; + } + let mut tasks: Vec<_> = Vec::new(); + for query in queries { + tasks.push(bound_task(py, cursor_query_sql(query), handles)?); + } + call_with( + &imported(py, RUN_BOUNDED.0, RUN_BOUNDED.1)?, + &[ + ("tasks", tuple(py, tasks)?), + ("concurrency", concurrency.into_pyobject(py)?.into_any()), + ( + "on_complete", + completion_reporter(py, queries.to_vec(), on_progress)?, + ), + ], + )? + .try_iter()? + .collect::>()? + }; + let mut values = Values::new(); + let mut failed: HashSet<_> = HashSet::new(); + for (query, outcome) in queries.iter().zip(outcomes) { + if !outcome.get_item(2)?.is_none() { + failed.extend(query.min_tags.iter().cloned()); + failed.extend(query.max_tags.iter().cloned()); + } + values.extend(fan_out(query, &outcome.get_item(0)?)?); + } + Ok((values, failed)) +} + +fn eligible_task<'py>( + py: Python<'py>, + (model_name, target, cursor_column): (String, String, String), + sql: String, + (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let connection: Py = connection.clone().unbind(); + let execute: Py = execute.clone().unbind(); + Ok(PyCFunction::new_closure( + py, + None, + None, + move |args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>| -> PyResult> { + let py = args.py(); + let run_connection = connection.clone_ref(py); + let run_execute = execute.clone_ref(py); + let run_sql = sql.clone(); + let run = PyCFunction::new_closure( + py, + None, + None, + move |args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>| -> PyResult> { + let py = args.py(); + Ok(run_execute + .bind(py) + .call((), Some(&keywords(py, &[ + ("connection", run_connection.bind(py).clone()), + ("sql", text(py, &run_sql)), + ])?))? + .call_method0("fetchall")? + .unbind()) + }, + )?; + let rows = call_with( + &imported(py, RUN_RECORDED_QUERY.0, RUN_RECORDED_QUERY.1)?, + &[("sql", text(py, &sql)), ("run", run.into_any())], + ); + match rows { + Ok(rows) => Ok(rows.unbind()), + Err(error) if error.is_instance_of::(py) => { + let wrapped: PyErr = planner_error( + py, + &format!( + "model '{model_name}': failed to query highest eligible target cursor for {target}.{cursor_column}: {}", + error.value(py).str()? + ), + (None, None), + ); + wrapped.set_cause(py, Some(error)); + Err(wrapped) + } + Err(error) => Err(error), + } + }, + )? + .into_any()) +} + +fn eligible_target_maxes<'py>( + request: &SnapshotRequest<'py>, + models: &[CursorModel<'py>], + mut values: Values<'py>, + invocation_time: &Bound<'py, PyAny>, +) -> PyResult> { + let py = request.project.py(); + if invocation_time.is_none() { + return Ok(values); + } + let mut requests: Vec<(&CursorModel<'py>, Bound<'py, PyAny>)> = Vec::new(); + for info in models { + let config = &info.start_cursor_config; + let target_max = info.target_tag.as_ref().and_then(|tag| values.get(tag)); + let Some(target_max) = target_max else { + continue; + }; + if config.is_none() + || config.getattr("max_ahead")?.is_none() + || info.cursor_type.as_deref() != Some(CURSOR_TYPE_TIMESTAMP) + || info.target_relation.is_none() + || info.has_start_override + { + continue; + } + let allowed = call_with( + &imported(py, MAXIMUM_START.0, MAXIMUM_START.1)?, + &[ + ("discovered_value", render(target_max)?), + ("cursor_grain", info.effective_grain.clone()), + ("invocation_time", invocation_time.clone()), + ("max_ahead", config.getattr("max_ahead")?), + ], + )?; + let horizon = parse(py, &allowed, CURSOR_TYPE_TIMESTAMP)?; + let order: i64 = native( + py, + "cursor_compare", + &[("left", target_max.clone()), ("right", horizon.clone())], + )? + .extract()?; + if order <= 0 { + continue; + } + let sql = call_with( + &request.adapter.getattr("render_max_cursor_at_or_before")?, + &[ + ( + "relation", + optional_text(py, info.target_relation.as_deref()), + ), + ("cursor_column", text(py, info.physical_cursor_column())), + ("maximum_allowed", render(&horizon)?), + ( + "cursor_type", + optional_text(py, info.cursor_type.as_deref()), + ), + ( + "is_date", + horizon + .is_instance(&imported(py, DATE_VALUE.0, DATE_VALUE.1)?)? + .into_pyobject(py)? + .to_owned() + .into_any(), + ), + ], + )?; + requests.push((info, sql)); + } + let mut tasks: Vec<_> = Vec::new(); + for (info, sql) in &requests { + tasks.push(eligible_task( + py, + ( + info.model_name.clone(), + info.target_relation.clone().unwrap_or_default(), + info.cursor_column.clone(), + ), + sql.extract()?, + (&request.connection, &request.execute), + )?); + } + let rows_by_request = call_with( + &imported(py, RUN_BOUNDED.0, RUN_BOUNDED.1)?, + &[ + ("tasks", tuple(py, tasks)?), + ( + "concurrency", + inspection_concurrency(&request.adapter, &request.connection)? + .into_pyobject(py)? + .into_any(), + ), + ], + )?; + for ((info, _), rows) in requests.iter().zip(rows_by_request.try_iter()?) { + let rows = rows?; + if !rows.is_truthy()? { + continue; + } + let first = rows.get_item(0)?.get_item(0)?; + if first.is_none() { + continue; + } + let cursor_type = info + .cursor_type + .as_deref() + .filter(|value| !value.is_empty()) + .unwrap_or(CURSOR_TYPE_TIMESTAMP); + let value = parse(py, &first, cursor_type)?; + let normalized = call_with( + &imported(py, CURSOR_RESOLVE_MODULE, "normalize_cursor_snapshot_grain")?, + &[ + ( + "cursor_snapshot", + planner_model( + py, + "ModelCursorSnapshot", + &[ + ("target_max", py.None().into_bound(py)), + ("upstream_mins", PyTuple::empty(py).into_any()), + ("upstream_maxes", PyTuple::empty(py).into_any()), + ("target_eligible_max", value.clone()), + ], + )?, + ), + ( + "cursor_type", + optional_text(py, info.cursor_type.as_deref()), + ), + ("effective_grain", info.effective_grain.clone()), + ], + )? + .getattr("target_eligible_max")?; + values.insert( + format!("{}__target__eligible_max", info.model_name), + if normalized.is_truthy()? { + normalized + } else { + value + }, + ); + } + Ok(values) +} + +fn cursor_grain<'py>( + py: Python<'py>, + info: &CursorModel<'py>, + upstream: &UpstreamCursor<'py>, +) -> PyResult> { + if info.cursor_type.as_deref() == Some(CURSOR_TYPE_INTEGER) { + return Ok(py.None().into_bound(py)); + } + let value = match upstream + .cursor_grain + .as_deref() + .filter(|value| !value.is_empty()) + { + Some(value) => value.to_owned(), + None if info.effective_grain.is_truthy()? => info.effective_grain.str()?.extract()?, + None => "second".to_owned(), + }; + grain(py, &value) +} + +fn optional<'py>(py: Python<'py>, value: Option<&Bound<'py, PyAny>>) -> Bound<'py, PyAny> { + value.cloned().unwrap_or_else(|| py.None().into_bound(py)) +} + +fn assemble<'py>( + py: Python<'py>, + models: &[CursorModel<'py>], + (values, failed): (&Values<'py>, &HashSet), +) -> PyResult> { + let snapshots = PyDict::new(py); + for info in models { + let target_max = info.target_tag.as_ref().and_then(|tag| values.get(tag)); + let mut mins: Vec<_> = Vec::new(); + let mut maxes: Vec<_> = Vec::new(); + let mut evidence: Vec<_> = Vec::new(); + let mut unavailable: Vec<_> = Vec::new(); + let mut starts: Vec<_> = Vec::new(); + let mut ends: Vec<_> = Vec::new(); + let mut end_inputs: Vec<_> = Vec::new(); + let mut availability: Vec<_> = Vec::new(); + let mut empty: Vec<_> = Vec::new(); + let mut unreadable: Vec<_> = Vec::new(); + for upstream in &info.upstreams { + let min_value = values.get(&upstream.tag_min); + let max_value = values.get(&upstream.tag_max); + if max_value.is_none() && upstream.terminal_end.is_none() { + let input = if upstream.input_name.is_empty() { + &upstream.relation + } else { + &upstream.input_name + }; + let label = text(py, &format!("{input}.{}", upstream.cursor_column)); + if failed.contains(&upstream.tag_max) { + unreadable.push(label); + } else { + empty.push(label); + } + } + let effective_max = match &upstream.terminal_end { + Some(end) => Some(native( + py, + "cursor_exclusive_to_inclusive", + &[ + ("value", end.clone()), + ("grain", cursor_grain(py, info, upstream)?), + ], + )?), + None => max_value.cloned(), + }; + if let Some(min_value) = min_value { + mins.push(min_value.clone()); + } else if let Some(start) = &upstream.terminal_start { + mins.push(start.clone()); + } else if target_max.is_none() && info.watermark_all { + unavailable.push(text(py, &upstream.tag_min)); + } + if let Some(effective_max) = effective_max { + if let Some(max_value) = max_value { + maxes.push(max_value.clone()); + } + let minimum = match min_value { + Some(min_value) if min_value.is_truthy()? => min_value.clone(), + _ => optional(py, upstream.terminal_start.as_ref()), + }; + evidence.push(planner_model( + py, + "CursorInputEvidence", + &[ + ("relation", text(py, &upstream.relation)), + ("cursor_column", text(py, &upstream.cursor_column)), + ("minimum", minimum), + ("maximum", effective_max), + ], + )?); + } else if upstream.terminal_end.is_none() && info.watermark_all { + unavailable.push(text(py, &upstream.tag_max)); + } + if let Some(start) = &upstream.terminal_start { + starts.push(start.clone()); + } + if let Some(end) = &upstream.terminal_end { + ends.push(end.clone()); + } + end_inputs.push( + PyTuple::new( + py, + [ + optional(py, max_value), + optional(py, upstream.terminal_end.as_ref()), + ], + )? + .into_any(), + ); + if info.watermark_strategy { + let mut available = upstream.terminal_end.clone(); + if available.is_none() + && let Some(max_value) = max_value + { + available = Some( + native( + py, + "cursor_observed_partition", + &[ + ("value", max_value.clone()), + ("grain", cursor_grain(py, info, upstream)?), + ], + )? + .getattr("end")?, + ); + } + if let Some(available) = available.filter(|value| !value.is_none()) { + availability.push(available); + } + } + } + let end_inputs = if ends.is_empty() { + Vec::new() + } else { + end_inputs + }; + snapshots.set_item( + &info.model_name, + planner_model( + py, + "ModelCursorSnapshot", + &[ + ("target_max", optional(py, target_max)), + ("upstream_mins", tuple(py, mins)?), + ("upstream_maxes", tuple(py, maxes)?), + ("physical_target_max", optional(py, target_max)), + ( + "target_eligible_max", + optional( + py, + values.get(&format!("{}__target__eligible_max", info.model_name)), + ), + ), + ( + "target_relation", + optional_text(py, info.target_relation.as_deref()), + ), + ("destination_cursor_column", text(py, &info.cursor_column)), + ("input_evidence", tuple(py, evidence)?), + ( + "expected_watermark_count", + info.upstreams.len().into_pyobject(py)?.into_any(), + ), + ("unavailable_watermark_tags", tuple(py, unavailable)?), + ("cursor_watermark_mode", info.watermark_mode.clone()), + ("upstream_terminal_starts", tuple(py, starts)?), + ("upstream_terminal_ends", tuple(py, ends)?), + ("upstream_end_inputs", tuple(py, end_inputs)?), + ("upstream_availability_ends", tuple(py, availability)?), + ("empty_input_names", tuple(py, empty)?), + ("unreadable_input_names", tuple(py, unreadable)?), + ], + )?, + )?; + } + Ok(snapshots) +} + +/// Cursor MIN/MAX values of the selected incremental models, by model name. +pub(crate) fn cursor_snapshots<'py>( + request: &SnapshotRequest<'py>, + existing_relations: &Bound<'py, PyDict>, + inputs: &CursorInputs<'py>, +) -> PyResult> { + let py = request.project.py(); + let maps = ModelMaps { + models: by_name(&request.project, "models")?, + sources: by_name(&request.project, "sources")?, + seeds: by_name(&request.project, "seeds")?, + functions: by_name(&request.project, "functions")?, + }; + let models = collect_models(request, existing_relations, inputs, &maps)?; + if models.is_empty() { + return Ok(PyDict::new(py)); + } + let queries = physical_queries(&models); + let started = Instant::now(); + let concurrency = inspection_concurrency(&request.adapter, &request.connection)?; + let (values, failed) = execute_queries(request, &queries, concurrency)?; + let values = eligible_target_maxes(request, &models, values, &inputs.invocation_time)?; + if !request.on_progress.is_none() { + let logical: usize = queries + .iter() + .map(|query| query.min_tags.len() + query.max_tags.len()) + .sum(); + request.on_progress.call1((format!( + "Gathered cursor bounds ({}/{logical} logical values; {} physical relation reads). ({:.2}s)", + values.len(), + queries.len(), + started.elapsed().as_secs_f64() + ),))?; + } + assemble(py, &models, (&values, &failed)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs new file mode 100644 index 0000000000..7f970753fc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs @@ -0,0 +1,354 @@ +//! Migration fingerprints computed at most once per model definition and reused across runs. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_phases::objects::{native, optional_text, text}; +use pyo3::exceptions::{PyOSError, PyValueError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyFloat, PyList, PyListMethods, PyTuple}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_entry::fingerprint_cache_entry; +use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_key::fingerprint_cache_key; +use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_lookup::fingerprint_cache_lookup; +use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_shared_digest::fingerprint_cache_shared_digest; +use sqlbuild_analysis::planner_warehouse::models::FingerprintCacheKey; +use std::collections::HashMap; + +const CACHE_DIRECTORY_NAME: &str = "migration-fingerprints-v1"; +const CACHE_DATABASE_NAME: &str = "migration-fingerprints.sqlite3"; +const SQLITE_TIMEOUT_SECONDS: f64 = 0.1; +const CREATE_CACHE_TABLE_SQL: &str = "\nCREATE TABLE IF NOT EXISTS migration_fingerprint (\n cache_key TEXT PRIMARY KEY,\n payload TEXT NOT NULL\n)\n"; +const INSERT_SQL: &str = + "INSERT OR REPLACE INTO migration_fingerprint (cache_key, payload) VALUES (?, ?)"; +const SELECT_SQL: &str = "SELECT payload FROM migration_fingerprint WHERE cache_key = ?"; + +type MemoKey = (String, String, Option, Vec<(String, String)>); + +/// Reuse migration fingerprints across rename maps, planning phases and invocations. +#[pyclass(module = "sqlbuild._native")] +pub(crate) struct MigrationFingerprintCache { + ref_names: HashMap<(String, String), Vec>, + fingerprints: HashMap>, + database_path: Option>, + connection: Option>, + connection_opened: bool, + pending: Vec<(String, String)>, + shared_digest: String, +} + +fn sqlite_error(py: Python<'_>, error: &PyErr) -> PyResult { + Ok(error.is_instance(py, &imported(py, "sqlite3", "DatabaseError")?)) +} + +fn close(py: Python<'_>, connection: Option>) -> PyResult<()> { + let Some(connection) = connection else { + return Ok(()); + }; + match connection.bind(py).call_method0("close") { + Ok(_) => Ok(()), + Err(error) if sqlite_error(py, &error)? => Ok(()), + Err(error) => Err(error), + } +} + +fn connect<'py>(path: &Bound<'py, PyAny>) -> PyResult> { + let py = path.py(); + call_with( + &imported(py, "sqlite3", "connect")?, + &[ + ("database", path.clone()), + ( + "timeout", + PyFloat::new(py, SQLITE_TIMEOUT_SECONDS).into_any(), + ), + ], + ) +} + +fn package_version(py: Python<'_>, package: &str) -> PyResult { + match imported(py, "importlib.metadata", "version")?.call1((package,)) { + Ok(version) => version.extract(), + Err(error) + if error.is_instance( + py, + &imported(py, "importlib.metadata", "PackageNotFoundError")?, + ) => + { + Ok("unknown".to_owned()) + } + Err(error) => Err(error), + } +} + +impl MigrationFingerprintCache { + /// A cache persisting under `root`, or an in-memory cache without one. + pub(crate) fn open(py: Python<'_>, root: Option>) -> PyResult { + let root = root.filter(|root| !root.is_none()); + let database_path = match &root { + Some(root) => Some( + root.call_method1("__truediv__", (CACHE_DIRECTORY_NAME,))? + .call_method1("__truediv__", (CACHE_DATABASE_NAME,))? + .unbind(), + ), + None => None, + }; + let shared_digest = if root.is_some() { + fingerprint_cache_shared_digest( + &package_version(py, "sqlbuild")?, + &package_version(py, "polyglot-sql-chio")?, + ) + .map_err(|error| PyValueError::new_err(format!("{error:?}")))? + } else { + String::new() + }; + Ok(Self { + ref_names: HashMap::new(), + fingerprints: HashMap::new(), + database_path, + connection: None, + connection_opened: false, + pending: Vec::new(), + shared_digest, + }) + } + + fn cache_key(&self, key: &MemoKey) -> PyResult { + if self.database_path.is_none() { + return Ok(String::new()); + } + fingerprint_cache_key(&FingerprintCacheKey { + shared_digest: self.shared_digest.clone(), + query_sql: key.0.clone(), + metadata_json: key.1.clone(), + dialect: key.2.clone(), + visible_renames: key.3.clone(), + }) + .map_err(|error| PyValueError::new_err(format!("{error:?}"))) + } + + fn open_connection(&mut self, py: Python<'_>) -> PyResult>> { + if self.connection_opened { + return Ok(self + .connection + .as_ref() + .map(|connection| connection.clone_ref(py))); + } + self.connection_opened = true; + let Some(path) = &self.database_path else { + return Ok(None); + }; + let path = path.bind(py); + if !path.call_method0("is_file")?.is_truthy()? { + return Ok(None); + } + self.connection = match connect(path) { + Ok(connection) => Some(connection.unbind()), + Err(error) if error.is_instance_of::(py) || sqlite_error(py, &error)? => { + None + } + Err(error) => return Err(error), + }; + Ok(self + .connection + .as_ref() + .map(|connection| connection.clone_ref(py))) + } + + fn stored(&mut self, py: Python<'_>, cache_key: &str) -> PyResult>> { + let Some(connection) = self.open_connection(py)? else { + return Ok(None); + }; + let row = connection + .bind(py) + .call_method1("execute", (SELECT_SQL, PyTuple::new(py, [cache_key])?)) + .and_then(|cursor| cursor.call_method0("fetchone")); + let row = match row { + Ok(row) => row, + Err(error) if sqlite_error(py, &error)? => { + self.connection = None; + close(py, Some(connection))?; + return Ok(None); + } + Err(error) => return Err(error), + }; + if row.is_none() { + return Ok(None); + } + let contents = row.get_item(0)?; + let Ok(contents) = contents.extract::() else { + return Ok(None); + }; + Ok(fingerprint_cache_lookup(&contents, cache_key)) + } + + fn remember(&mut self, cache_key: &str, fingerprint: Option<&str>) -> PyResult<()> { + if self.database_path.is_none() { + return Ok(()); + } + let entry = fingerprint_cache_entry(cache_key, fingerprint) + .map_err(|error| PyValueError::new_err(format!("{error:?}")))?; + match self.pending.iter_mut().find(|(key, _)| key == cache_key) { + Some(existing) => existing.1 = entry, + None => self.pending.push((cache_key.to_owned(), entry)), + } + Ok(()) + } + + fn write_pending( + &self, + py: Python<'_>, + pending: &[(String, String)], + connection: Option>, + ) -> (Option>, PyResult<()>) { + let Some(path) = &self.database_path else { + return (connection, Ok(())); + }; + if pending.is_empty() { + return (connection, Ok(())); + } + let path = path.bind(py); + let connection = match connection { + Some(connection) => connection, + None => { + let created = call_with( + &match path + .getattr("parent") + .and_then(|parent| parent.getattr("mkdir")) + { + Ok(mkdir) => mkdir, + Err(error) => return (None, Err(error)), + }, + &[ + ("parents", PyBool::new(py, true).to_owned().into_any()), + ("exist_ok", PyBool::new(py, true).to_owned().into_any()), + ], + ) + .and_then(|_| connect(path)); + match created { + Ok(created) => created.unbind(), + Err(error) => return (None, Err(error)), + } + } + }; + let written = (|| -> PyResult<()> { + let open = connection.bind(py); + open.call_method1("execute", (CREATE_CACHE_TABLE_SQL,))?; + let rows = PyList::empty(py); + for (key, entry) in pending { + rows.append(PyTuple::new(py, [key.as_str(), entry.as_str()])?)?; + } + open.call_method1("executemany", (INSERT_SQL, rows))?; + open.call_method0("commit")?; + Ok(()) + })(); + (Some(connection), written) + } +} + +#[pymethods] +impl MigrationFingerprintCache { + #[new] + #[pyo3(signature = (*, root=None))] + fn new(py: Python<'_>, root: Option>) -> PyResult { + Self::open(py, root) + } + + /// The migration fingerprint, keyed only by the renames this model can see. + #[pyo3(signature = (**options))] + fn fingerprint( + &mut self, + py: Python<'_>, + options: Option>, + ) -> PyResult> { + let args = KeywordArguments::read( + py, + "fingerprint", + options, + ( + &["query_sql", "metadata_json", "ref_identities", "dialect"], + &[], + ), + )?; + let query_sql: String = args.value("query_sql")?.extract()?; + let metadata_json: String = args.value("metadata_json")?.extract()?; + let ref_identities = &args.value("ref_identities")?; + let dialect: Option = args.value("dialect")?.extract()?; + let metadata_json: String = + imported(py, "sqlbuild._native", "without_declared_columns_hash")? + .call1((metadata_json,))? + .extract()?; + let definition = (query_sql.clone(), metadata_json.clone()); + let names = match self.ref_names.get(&definition) { + Some(names) => names.clone(), + None => { + let found = native( + py, + "migration_fingerprint_ref_names", + &[ + ("query_sql", text(py, &query_sql)), + ("metadata_json", text(py, &metadata_json)), + ], + )?; + let mut names: Vec = found + .try_iter()? + .map(|name| name?.extract::()) + .collect::>()?; + names.sort(); + self.ref_names.insert(definition, names.clone()); + names + } + }; + let mut visible: Vec<_> = Vec::new(); + for name in names { + if ref_identities.contains(&name)? { + let identity: String = ref_identities.get_item(&name)?.extract()?; + visible.push((name, identity)); + } + } + let key: MemoKey = (query_sql, metadata_json, dialect, visible); + if let Some(found) = self.fingerprints.get(&key) { + return Ok(found.clone()); + } + let cache_key = self.cache_key(&key)?; + let stored = match self.stored(py, &cache_key)? { + Some(stored) => stored, + None => { + let identities = PyDict::new(py); + for (name, identity) in &key.3 { + pyo3::types::PyDictMethods::set_item(&identities, name, identity)?; + } + let computed: Option = native( + py, + "build_migration_fingerprint", + &[ + ("query_sql", text(py, &key.0)), + ("metadata_json", text(py, &key.1)), + ("ref_identities", identities.into_any()), + ("dialect", optional_text(py, key.2.as_deref())), + ], + )? + .extract()?; + self.remember(&cache_key, computed.as_deref())?; + computed + } + }; + self.fingerprints.insert(key, stored.clone()); + Ok(stored) + } + + /// Write fingerprints computed since the last persist; failures only skip reuse. + fn persist(&mut self, py: Python<'_>) -> PyResult<()> { + let pending = std::mem::take(&mut self.pending); + let connection = self.connection.take(); + self.connection_opened = false; + let (connection, written) = self.write_pending(py, &pending, connection); + let closed = close(py, connection); + match written { + Err(error) if error.is_instance_of::(py) || sqlite_error(py, &error)? => {} + Err(error) => return Err(error), + Ok(()) => {} + } + closed + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs new file mode 100644 index 0000000000..b48621b451 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs @@ -0,0 +1,13 @@ +//! Native warehouse inspection for planning over Python adapter callbacks. + +pub(crate) mod constants; +pub(crate) mod cursors; +pub(crate) mod fingerprint_cache; +pub(crate) mod registration; +pub(crate) mod semantic_sources; +pub(crate) mod snapshot; +pub(crate) mod source_columns; +pub(crate) mod source_deferral; +pub(crate) mod source_freshness; +pub(crate) mod source_tables; +pub(crate) mod watermarks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs new file mode 100644 index 0000000000..cbe7d5efed --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs @@ -0,0 +1,60 @@ +//! Register native warehouse inspection for planning. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!( + super::snapshot::gather_warehouse_snapshot, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::snapshot::gather_redirected_cursor_snapshots, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::watermarks::watermark_type_is_compatible, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::source_columns::gather_source_columns, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::snapshot::planner_metadata_name_filter, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::semantic_sources::list_source_candidates, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::source_columns::gather_source_inspection, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::semantic_sources::semantic_source_columns, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::source_tables::check_selected_source_tables_exist_binding, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::source_deferral::build_source_read_map, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::source_deferral::with_declared_source_reads, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::source_freshness::source_freshness_unknown_warnings, + module + )?)?; + module.add_function(wrap_pyfunction!( + super::source_freshness::build_planner_source_freshness_result, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs new file mode 100644 index 0000000000..747681446e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs @@ -0,0 +1,287 @@ +//! Complete missing source metadata in bounded, database-grouped batches. + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_phases::objects::{text, tuple}; +use crate::bindings::_helpers::planner_warehouse::constants::ACTIVE_CATALOG; +use crate::bindings::_helpers::planner_warehouse::snapshot::{columns_for, list_relations}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict as KeywordDict; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use std::collections::BTreeSet; + +fn casefold(value: &Bound<'_, PyAny>) -> PyResult { + value.call_method0("casefold")?.extract() +} + +fn wanted_name<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { + let table = entry.getattr("table")?; + if table.is_truthy()? { + Ok(table) + } else { + entry.getattr("name") + } +} + +fn matches(entry: &Bound<'_, PyAny>, relation: &Bound<'_, PyAny>) -> PyResult { + if casefold(&relation.getattr("name")?)? != casefold(&wanted_name(entry)?)? { + return Ok(false); + } + let schema = entry.getattr("schema")?; + if schema.is_none() { + return Ok(true); + } + let relation_schema = relation.getattr("schema")?; + let relation_schema = if relation_schema.is_truthy()? { + relation_schema + } else { + text(relation.py(), "") + }; + Ok(casefold(&relation_schema)? == casefold(&schema)?) +} + +fn sorted_texts<'py>(py: Python<'py>, values: BTreeSet) -> PyResult> { + tuple(py, values.iter().map(|value| text(py, value)).collect()) +} + +fn source_candidates<'py>( + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + database: &Bound<'py, PyAny>, + entries: &[Bound<'py, PyAny>], +) -> PyResult>> { + let py = adapter.py(); + let mut pending: Vec> = entries.to_vec(); + let mut relations: Vec> = Vec::new(); + let catalog = call_with( + &imported(py, ACTIVE_CATALOG.0, ACTIVE_CATALOG.1)?, + &[ + ("adapter", adapter.clone()), + ("connection", connection.clone()), + ], + )?; + let mut scoped: Vec<_> = Vec::new(); + for entry in entries { + if !entry.getattr("schema")?.is_none() { + scoped.push(entry.clone()); + } + } + if !catalog.is_none() && catalog.getattr("schema_scoped")?.is_truthy()? && !scoped.is_empty() { + let mut schemas: BTreeSet<_> = BTreeSet::new(); + let mut names: BTreeSet<_> = BTreeSet::new(); + for entry in &scoped { + schemas.insert( + entry + .getattr("schema")? + .extract::>()? + .unwrap_or_default(), + ); + names.insert(wanted_name(entry)?.extract::()?); + } + let listed = list_relations( + (adapter, connection), + database, + sorted_texts(py, schemas)?, + sorted_texts(py, names)?, + )?; + for relation in listed.try_iter()? { + relations.push(relation?); + } + let mut remaining: Vec<_> = Vec::new(); + for entry in entries { + let mut found = false; + for relation in &relations { + if matches(entry, relation)? { + found = true; + break; + } + } + if !found { + remaining.push(entry.clone()); + } + } + pending = remaining; + } + if !pending.is_empty() { + let mut names: BTreeSet<_> = BTreeSet::new(); + for entry in &pending { + names.insert(wanted_name(entry)?.extract::()?); + } + let listed = list_relations( + (adapter, connection), + database, + py.None().into_bound(py), + sorted_texts(py, names)?, + )?; + for relation in listed.try_iter()? { + relations.push(relation?); + } + } + let unique = PyDict::new(py); + for relation in relations { + let identity = relation.getattr("identity")?; + if !unique.contains(&identity)? { + unique.set_item(identity, relation)?; + } + } + unique.values().try_iter()?.collect() +} + +/// Source relation candidates by name; schema listings first answer schema-scoped catalogs. +#[pyfunction] +#[pyo3(signature = (*, adapter, connection, database, entries))] +pub(crate) fn list_source_candidates<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + database: &Bound<'py, PyAny>, + entries: &Bound<'py, PyAny>, +) -> PyResult> { + let entries: Vec> = entries.try_iter()?.collect::>()?; + tuple( + adapter.py(), + source_candidates((adapter, connection), database, &entries)?, + ) +} + +/// Reuse collected columns and inspect only selected, unresolved table sources. +#[pyfunction] +#[pyo3(name = "get_semantic_source_columns")] +#[pyo3(signature = (**options))] +pub(crate) fn semantic_source_columns<'py>( + py: pyo3::Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "get_semantic_source_columns", + options, + ( + &[ + "project", + "adapter", + "connection", + "source_read_map", + "columns", + "selected_keys", + ], + &[], + ), + )?; + let project = &args.value("project")?; + let adapter = &args.value("adapter")?; + let connection = &args.value("connection")?; + let source_read_map = &args.value("source_read_map")?; + let columns = args.value("columns")?.downcast_into::()?; + let columns = &columns; + let selected_keys = &args.value("selected_keys")?; + + let py = project.py(); + if !project + .getattr("settings")? + .getattr("sql_analysis")? + .is_truthy()? + || connection.is_none() + { + return Ok(columns.clone()); + } + let mut required: BTreeSet = BTreeSet::new(); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + let analysis = model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("sql_analysis",))?; + if !selected_keys.contains(model.getattr("key")?)? + || analysis.is(pyo3::types::PyBool::new(py, false)) + { + continue; + } + for reference in model.getattr("references")?.try_iter()? { + let reference = reference?; + if reference.getattr("ref_kind")?.eq("source")? { + required.insert(reference.getattr("ref_name")?.extract()?); + } + } + } + let mut groups: Vec<(Bound<'py, PyAny>, Vec>)> = Vec::new(); + for name in required { + if columns.contains(&name)? { + continue; + } + let entry = source_read_map.call_method1("get", (&name,))?; + if entry.is_none() + || !entry.getattr("expression")?.is_none() + || entry.getattr("contract")?.eq("enforced")? + { + continue; + } + let database = entry.getattr("database")?; + let mut placed = false; + for (group, entries) in groups.iter_mut() { + if group.eq(&database)? { + entries.push(entry.clone()); + placed = true; + break; + } + } + if !placed { + groups.push((database, vec![entry])); + } + } + let result = columns.copy()?; + for (database, entries) in groups { + let relations = source_candidates((adapter, connection), &database, &entries)?; + let mut chosen: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); + for entry in &entries { + let mut candidates: Vec<_> = Vec::new(); + for relation in &relations { + if matches(entry, relation)? { + candidates.push(relation.clone()); + } + } + if candidates.len() == 1 { + let name = entry.getattr("name")?; + match chosen + .iter_mut() + .find(|(chosen_name, _)| chosen_name.eq(&name).unwrap_or(false)) + { + Some(existing) => existing.1 = candidates.remove(0), + None => chosen.push((name, candidates.remove(0))), + } + } + } + let inspected = if chosen.is_empty() { + PyDict::new(py).into_any() + } else { + let unique = PyDict::new(py); + for (_, relation) in &chosen { + unique.set_item(relation, py.None())?; + } + columns_for( + (adapter, connection), + PyTuple::new(py, unique.keys())?.into_any(), + )? + }; + for (name, relation) in chosen { + let lower = |value: Bound<'py, PyAny>| -> PyResult> { + if value.is_truthy()? { + value.call_method0("lower") + } else { + Ok(py.None().into_bound(py)) + } + }; + let identity = PyTuple::new( + py, + [ + lower(relation.getattr("database")?)?, + lower(relation.getattr("schema")?)?, + relation.getattr("name")?.call_method0("lower")?, + ], + )?; + if inspected.contains(&identity)? { + result.set_item(name, inspected.get_item(identity)?)?; + } + } + } + Ok(result) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs new file mode 100644 index 0000000000..fbfeebf1e9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs @@ -0,0 +1,867 @@ +//! Gather a frozen point-in-time warehouse snapshot for planning. + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; +use crate::bindings::_helpers::planner_phases::objects::{ + by_name, frozenset, model_key, native, planner_model, text, tuple, +}; +use crate::bindings::_helpers::planner_warehouse::constants::{ + ACTIVE_CATALOG, FINGERPRINT_TABLE_NAME, FUNCTION_NODE_TYPES, METADATA_NAME_FILTER_LIMIT, + MIGRATE_FROM_CONFIG_KEY, MIGRATION_TABLE_NAME, NODE_TYPE_MODEL, NODE_TYPE_SEED, + OLD_NAME_VIEW_TABLE_NAME, RELATIONS_MAIN_COLUMNS, RELATIONS_MAIN_LIST, + SOURCE_FRESHNESS_TABLE_NAME, +}; +use crate::bindings::_helpers::planner_warehouse::cursors::{CursorInputs, cursor_snapshots}; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict as KeywordDict; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use std::collections::{BTreeSet, HashSet}; + +/// Everything one snapshot gather needs. +pub(crate) struct SnapshotRequest<'py> { + pub(crate) project: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) connection: Bound<'py, PyAny>, + pub(crate) execute: Bound<'py, PyAny>, + pub(crate) selected_keys: Bound<'py, PyAny>, + pub(crate) full_refresh: bool, + pub(crate) full_refresh_model_names: Bound<'py, PyAny>, + pub(crate) on_progress: Bound<'py, PyAny>, + pub(crate) deferred_locations: Bound<'py, PyAny>, + pub(crate) cursor_scope: Bound<'py, PyAny>, +} + +#[derive(Default)] +struct StateSchemas { + fingerprints: BTreeSet, + freshness: BTreeSet, + old_names: BTreeSet, + migrations: BTreeSet, +} + +pub(crate) fn lower(value: &Bound<'_, PyAny>) -> PyResult { + value.call_method0("lower")?.extract() +} + +fn names_set<'py>(py: Python<'py>, names: &BTreeSet) -> PyResult> { + let items: Vec> = names.iter().map(|name| text(py, name)).collect(); + frozenset(py, &items) +} + +/// Nodes of the project whose destinations hold state: models, seeds and functions. +pub(crate) fn destination_nodes<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult>> { + let mut nodes: Vec<_> = Vec::new(); + for field in ["models", "seeds", "functions"] { + for node in project.getattr(field)?.try_iter()? { + nodes.push(node?); + } + } + Ok(nodes) +} + +fn selected_node(selected: &Bound<'_, PyAny>, node: &Bound<'_, PyAny>) -> PyResult { + Ok(selected.is_none() || selected.contains(node.getattr("key")?)?) +} + +fn relevant_state_keys<'py>( + project: &Bound<'py, PyAny>, + selected: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + if selected.is_none() { + return Ok(selected.clone()); + } + let deps = + imported(py, "sqlbuild._native", "build_execution_upstream_deps")?.call1((project,))?; + native( + py, + "transitive_closure_many", + &[ + ("starts", selected.clone()), + ("edges", deps), + ( + "include_starts", + true.into_pyobject(py)?.to_owned().into_any(), + ), + ], + ) +} + +fn resolve_database<'py>( + project: &Bound<'py, PyAny>, + selected: &Bound<'py, PyAny>, +) -> PyResult> { + for node in destination_nodes(project)? { + if !selected_node(selected, &node)? { + continue; + } + let database = node.getattr("destination")?.getattr("database")?; + if !database.is_none() { + return Ok(database); + } + } + project.getattr("effective_target_database") +} + +fn target_schemas( + project: &Bound<'_, PyAny>, + selected: &Bound<'_, PyAny>, +) -> PyResult> { + let mut schemas: BTreeSet<_> = BTreeSet::new(); + for node in destination_nodes(project)? { + if !selected_node(selected, &node)? { + continue; + } + let schema = node.getattr("destination")?.getattr("schema")?; + if !schema.is_none() { + schemas.insert(schema.extract()?); + } + } + let fallback = project.getattr("effective_target_schema")?; + if schemas.is_empty() && !fallback.is_none() { + schemas.insert(fallback.extract()?); + } + Ok(schemas) +} + +fn source_table_name(entry: &Bound<'_, PyAny>) -> PyResult { + let table = entry.getattr("table")?; + if table.is_truthy()? { + table.extract() + } else { + entry.getattr("name")?.extract() + } +} + +fn model_upstream_names( + model: &Bound<'_, PyAny>, + (models, seeds): (&Bound<'_, PyDict>, &Bound<'_, PyDict>), + selected_names: &HashSet, +) -> PyResult> { + let mut names: BTreeSet = BTreeSet::new(); + for reference in model.getattr("references")?.try_iter()? { + let reference = reference?; + let kind = reference.getattr("ref_kind")?; + let name: String = reference.getattr("ref_name")?.extract()?; + let is_ref = kind.eq("ref")?; + if !(is_ref || kind.eq("seed")?) || selected_names.contains(&name) { + continue; + } + let lookup = if is_ref { models } else { seeds }; + if let Some(node) = lookup.get_item(&name)? { + names.insert(node.getattr("destination")?.getattr("name")?.extract()?); + } + } + Ok(names) +} + +fn metadata_name_filter( + project: &Bound<'_, PyAny>, + selected: &Bound<'_, PyAny>, +) -> PyResult>> { + let py = project.py(); + let mut names: BTreeSet = BTreeSet::new(); + if selected.is_none() { + for field in ["models", "seeds"] { + for node in project.getattr(field)?.try_iter()? { + names.insert(node?.getattr("destination")?.getattr("name")?.extract()?); + } + } + } else { + let selected_names: HashSet = selected + .try_iter()? + .map(|key| key?.getattr("name")?.extract::()) + .collect::>()?; + let models = by_name(project, "models")?; + let seeds = by_name(project, "seeds")?; + let sources = PyDict::new(py); + for source in project.getattr("sources")?.try_iter()? { + let entry = source?.getattr("source_entry")?; + sources.set_item(entry.getattr("name")?, entry)?; + } + for key in selected.try_iter()? { + let name = key?.getattr("name")?; + if let Some(model) = models.get_item(&name)? { + names.insert(model.getattr("destination")?.getattr("name")?.extract()?); + names.extend(model_upstream_names( + &model, + (&models, &seeds), + &selected_names, + )?); + } else if let Some(seed) = seeds.get_item(&name)? { + names.insert(seed.getattr("destination")?.getattr("name")?.extract()?); + } else if let Some(entry) = sources.get_item(&name)? { + names.insert(source_table_name(&entry)?); + } + } + let intermediate = native( + py, + "build_upstream_intermediate_source_map", + &[ + ("project", project.clone()), + ("selected_keys", selected.clone()), + ], + )?; + for entry in intermediate.call_method0("values")?.try_iter()? { + names.insert(source_table_name(&entry?)?); + } + } + if names.is_empty() || names.len() > METADATA_NAME_FILTER_LIMIT { + return Ok(None); + } + for state in [ + FINGERPRINT_TABLE_NAME, + SOURCE_FRESHNESS_TABLE_NAME, + OLD_NAME_VIEW_TABLE_NAME, + MIGRATION_TABLE_NAME, + ] { + names.insert(state.to_owned()); + } + Ok(Some(names)) +} + +fn active_catalog<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, +) -> PyResult> { + call_with( + &imported(adapter.py(), ACTIVE_CATALOG.0, ACTIVE_CATALOG.1)?, + &[ + ("adapter", adapter.clone()), + ("connection", connection.clone()), + ], + ) +} + +/// The concurrency of the active inspection catalog, one without a catalog. +pub(crate) fn inspection_concurrency( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, +) -> PyResult { + let catalog = active_catalog(adapter, connection)?; + if catalog.is_none() { + return Ok(1); + } + catalog.getattr("concurrency")?.extract() +} + +fn prefetch_schemas<'py>( + request: &SnapshotRequest<'py>, + database: &Bound<'py, PyAny>, + schemas: &BTreeSet, + relevant: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py = request.project.py(); + let catalog = active_catalog(&request.adapter, &request.connection)?; + if catalog.is_none() { + return Ok(()); + } + let mut scopes: Vec> = Vec::new(); + for schema in schemas { + scopes.push(PyTuple::new(py, [database.clone(), text(py, schema)])?.into_any()); + } + for source in request.project.getattr("sources")?.try_iter()? { + let source = source?; + let entry = source.getattr("source_entry")?; + if (!relevant.is_none() && !relevant.contains(source.getattr("key")?)?) + || !entry.getattr("expression")?.is_none() + || entry.getattr("schema")?.is_none() + { + continue; + } + scopes.push( + PyTuple::new(py, [entry.getattr("database")?, entry.getattr("schema")?])?.into_any(), + ); + } + call_with( + &catalog.getattr("prefetch_schema_listings")?, + &[ + ("scopes", tuple(py, scopes)?), + ("best_effort", true.into_pyobject(py)?.to_owned().into_any()), + ], + )?; + Ok(()) +} + +/// List relations through the inspection catalog wrappers. +pub(crate) fn list_relations<'py>( + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + database: &Bound<'py, PyAny>, + schemas: Bound<'py, PyAny>, + names: Bound<'py, PyAny>, +) -> PyResult> { + call_with( + &imported(adapter.py(), RELATIONS_MAIN_LIST.0, RELATIONS_MAIN_LIST.1)?, + &[ + ("adapter", adapter.clone()), + ("connection", connection.clone()), + ("database", database.clone()), + ("schemas", schemas), + ("names", names), + ], + ) +} + +/// Columns of the relations through the inspection catalog wrappers. +pub(crate) fn columns_for<'py>( + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + relations: Bound<'py, PyAny>, +) -> PyResult> { + call_with( + &imported( + adapter.py(), + RELATIONS_MAIN_COLUMNS.0, + RELATIONS_MAIN_COLUMNS.1, + )?, + &[ + ("adapter", adapter.clone()), + ("connection", connection.clone()), + ("relations", relations), + ], + ) +} + +fn optional_names<'py>( + py: Python<'py>, + names: Option<&BTreeSet>, +) -> PyResult> { + match names { + None => Ok(py.None().into_bound(py)), + Some(names) => tuple(py, names.iter().map(|name| text(py, name)).collect()), + } +} + +fn location_identity<'py>(location: &Bound<'py, PyAny>) -> PyResult> { + let py = location.py(); + let part = |name: &str| -> PyResult> { + let value = location.getattr(name)?; + if value.is_none() { + Ok(value) + } else { + value.call_method0("lower") + } + }; + Ok(PyTuple::new(py, [part("database")?, part("schema")?, part("name")?])?.into_any()) +} + +fn gather_relations<'py>( + request: &SnapshotRequest<'py>, + database: &Bound<'py, PyAny>, + schemas: Bound<'py, PyAny>, + names: Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyDict>, StateSchemas)> { + let py = request.project.py(); + let relations = list_relations( + (&request.adapter, &request.connection), + database, + schemas, + names, + )?; + let logical = PyDict::new(py); + for node in destination_nodes(&request.project)? { + logical.set_item( + location_identity(&node.getattr("destination")?)?, + node.getattr("name")?, + )?; + } + let result = PyDict::new(py); + let mut states = StateSchemas::default(); + for relation in relations.try_iter()? { + let relation = relation?; + let name: String = relation.getattr("name")?.extract()?; + let schema = relation.getattr("schema")?; + let schema = if schema.is_none() { + None + } else { + Some(lower(&schema)?) + }; + if name == MIGRATION_TABLE_NAME + && let Some(schema) = &schema + { + states.migrations.insert(schema.clone()); + } + let state = if name == OLD_NAME_VIEW_TABLE_NAME { + Some(&mut states.old_names) + } else if name == FINGERPRINT_TABLE_NAME { + Some(&mut states.fingerprints) + } else if name == SOURCE_FRESHNESS_TABLE_NAME { + Some(&mut states.freshness) + } else { + None + }; + if let Some(state) = state { + if let Some(schema) = schema { + state.insert(schema); + } + continue; + } + let identity = relation.getattr("identity")?; + let key = logical + .get_item(&identity)? + .unwrap_or_else(|| PyString::new(py, &name).into_any()); + result.set_item(key, relation)?; + } + Ok((result, states)) +} + +fn gather_columns<'py>( + request: &SnapshotRequest<'py>, + relations: &Bound<'py, PyDict>, +) -> PyResult> { + let py = request.project.py(); + let mut node_names: HashSet = HashSet::new(); + for node in destination_nodes(&request.project)? { + node_names.insert(node.getattr("name")?.extract()?); + } + let mut needed: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); + for (logical, relation) in relations.iter() { + if node_names.contains(&logical.extract::()?) { + needed.push((logical, relation)); + } + } + let result = PyDict::new(py); + if needed.is_empty() { + return Ok(result); + } + let all = columns_for( + (&request.adapter, &request.connection), + tuple( + py, + needed + .iter() + .map(|(_, relation)| relation.clone()) + .collect(), + )?, + )?; + for (logical, relation) in needed { + let identity = relation.getattr("identity")?; + if all.contains(&identity)? { + result.set_item(logical, all.get_item(identity)?)?; + } + } + Ok(result) +} + +fn shares_unfiltered(project: &Bound<'_, PyAny>, selected: &Bound<'_, PyAny>) -> PyResult { + if selected.is_none() { + return Ok(true); + } + let names: HashSet = selected + .try_iter()? + .map(|key| key?.getattr("name")?.extract::()) + .collect::>()?; + let mut all_selected = true; + let mut any_migration = false; + for model in project.getattr("models")?.try_iter()? { + let model = model?; + if !names.contains(&model.getattr("name")?.extract::()?) { + all_selected = false; + } + if model + .getattr("config")? + .getattr("values")? + .call_method1("get", (MIGRATE_FROM_CONFIG_KEY,))? + .is_instance_of::() + { + any_migration = true; + } + } + Ok(all_selected || any_migration) +} + +fn gather_fingerprints<'py>( + request: &SnapshotRequest<'py>, + database: &Bound<'py, PyAny>, + schemas: Option<&BTreeSet>, + (state_schemas, relevant): (&BTreeSet, &Bound<'py, PyAny>), +) -> PyResult> { + let py = request.project.py(); + let Some(schemas) = schemas else { + return planner_model(py, "WarehouseFingerprints", &[]); + }; + let node_names: Option> = if relevant.is_none() { + None + } else { + Some( + relevant + .try_iter()? + .map(|key| key?.getattr("name")?.extract::()) + .collect::>()?, + ) + }; + let node_names_value = optional_names(py, node_names.as_ref())?; + let read_unfiltered = shares_unfiltered(&request.project, relevant)?; + let models = PyDict::new(py); + let functions = PyDict::new(py); + let seeds = PyDict::new(py); + let python_nodes = PyDict::new(py); + let unfiltered = PyDict::new(py); + let filtered_types = tuple( + py, + [ + NODE_TYPE_MODEL, + FUNCTION_NODE_TYPES[0], + FUNCTION_NODE_TYPES[1], + NODE_TYPE_SEED, + ] + .iter() + .map(|name| text(py, name)) + .collect(), + )?; + for schema in schemas { + let schema_lower = lower(&text(py, schema))?; + let mut set = native( + py, + "read_latest_fingerprints", + &[ + ("connection", request.connection.clone()), + ("execute", request.execute.clone()), + ( + "table_exists", + state_schemas + .contains(&schema_lower) + .into_pyobject(py)? + .to_owned() + .into_any(), + ), + ("database", database.clone()), + ("schema", text(py, schema)), + ( + "render_qualified_name", + request.adapter.getattr("render_qualified_name")?, + ), + ( + "render_read_latest_sql", + request + .adapter + .getattr("render_read_latest_fingerprints_sql")?, + ), + ( + "node_names", + if read_unfiltered { + py.None().into_bound(py) + } else { + node_names_value.clone() + }, + ), + ("filtered_node_types", filtered_types.clone()), + ], + )?; + if read_unfiltered { + let by_identity = set.getattr("fingerprints_by_identity")?; + let values: Vec> = if by_identity.is_none() { + Vec::new() + } else { + by_identity + .call_method0("values")? + .try_iter()? + .collect::>()? + }; + unfiltered.set_item(&schema_lower, tuple(py, values)?)?; + if node_names.is_some() { + set = native( + py, + "select_latest_fingerprints", + &[ + ("fingerprint_set", set), + ("node_names", node_names_value.clone()), + ("filtered_node_types", filtered_types.clone()), + ], + )?; + } + } + for (name, fingerprint) in set.getattr("fingerprints")?.downcast::()?.iter() { + let node_type: String = fingerprint.getattr("node_type")?.extract()?; + if node_type == NODE_TYPE_MODEL { + models.set_item(name, fingerprint)?; + } else if FUNCTION_NODE_TYPES.contains(&node_type.as_str()) { + functions.set_item(name, fingerprint)?; + } else if node_type == NODE_TYPE_SEED { + seeds.set_item(name, fingerprint)?; + } + } + let by_identity = set.getattr("fingerprints_by_identity")?; + if !by_identity.is_none() { + for item in by_identity.call_method0("items")?.try_iter()? { + let (key, fingerprint): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let node_type: String = fingerprint.getattr("node_type")?.extract()?; + if node_type != NODE_TYPE_MODEL + && node_type != NODE_TYPE_SEED + && !FUNCTION_NODE_TYPES.contains(&node_type.as_str()) + { + python_nodes.set_item(key, fingerprint)?; + } + } + } + } + planner_model( + py, + "WarehouseFingerprints", + &[ + ("models", models.into_any()), + ("functions", functions.into_any()), + ("seeds", seeds.into_any()), + ("python_nodes", python_nodes.into_any()), + ("unfiltered_schemas", unfiltered.into_any()), + ("unfiltered_database", database.clone()), + ], + ) +} + +fn scope_field<'py>(scope: &Bound<'py, PyAny>, name: &str) -> PyResult> { + if scope.is_none() { + Ok(scope.clone()) + } else { + scope.getattr(name) + } +} + +/// Gather relations, columns, fingerprints and cursor bounds of the planned target schemas. +pub(crate) fn warehouse_snapshot<'py>( + request: &SnapshotRequest<'py>, +) -> PyResult> { + let py = request.project.py(); + let project = &request.project; + let relevant = relevant_state_keys(project, &request.selected_keys)?; + let database = resolve_database(project, &relevant)?; + let schemas = target_schemas(project, &relevant)?; + let metadata_names = metadata_name_filter(project, &relevant)?; + if schemas.is_empty() && metadata_names.is_none() { + return planner_model(py, "WarehouseSnapshot", &[]); + } + let query_schemas = (!schemas.is_empty()).then_some(&schemas); + prefetch_schemas(request, &database, &schemas, &relevant)?; + let (relations, states) = gather_relations( + request, + &database, + optional_names(py, query_schemas)?, + optional_names(py, metadata_names.as_ref())?, + )?; + let columns = gather_columns(request, &relations)?; + let fingerprints = gather_fingerprints( + request, + &database, + query_schemas, + (&states.fingerprints, &relevant), + )?; + let full_refresh_names = if request.full_refresh_model_names.is_none() { + effectively_full_refreshed_model_names(project, request.full_refresh)? + } else { + request.full_refresh_model_names.clone() + }; + let scope = &request.cursor_scope; + let cursor_selected = if scope.is_none() { + request.selected_keys.clone() + } else { + scope.getattr("model_keys")? + }; + let cursors = cursor_snapshots( + request, + &relations, + &CursorInputs { + selected_keys: cursor_selected, + full_refresh_model_names: full_refresh_names, + deferred_locations: request.deferred_locations.clone(), + runtime_producer_keys: scope_field(scope, "runtime_producer_keys")?, + invocation_time: scope_field(scope, "invocation_time")?, + start_cursor_config: scope_field(scope, "start_cursor_config")?, + cursor_overrides: scope_field(scope, "cursor_overrides")?, + target_relation_overrides: None, + origin_cursor_column_overrides: None, + existing_columns: Some(columns.clone()), + }, + )?; + let listed = match query_schemas { + None => py.None().into_bound(py), + Some(schemas) => { + let mut lowered: BTreeSet<_> = BTreeSet::new(); + for schema in schemas { + lowered.insert(lower(&text(py, schema))?); + } + names_set(py, &lowered)? + } + }; + planner_model( + py, + "WarehouseSnapshot", + &[ + ("existing_relations", relations.into_any()), + ("existing_columns", columns.into_any()), + ("fingerprints", fingerprints), + ("cursor_snapshots", cursors.into_any()), + ( + "source_freshness_state_schemas", + names_set(py, &states.freshness)?, + ), + ( + "column_dialect", + request.adapter.call_method0("sql_analysis_dialect")?, + ), + ( + "old_name_view_state_schemas", + names_set(py, &states.old_names)?, + ), + ( + "migration_state_schemas", + names_set(py, &states.migrations)?, + ), + ("listed_state_schemas", listed), + ], + ) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn gather_warehouse_snapshot<'py>( + py: pyo3::Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "gather_warehouse_snapshot", + options, + ( + &["project", "adapter", "connection", "execute"], + &[ + "selected_keys", + "full_refresh", + "full_refresh_model_names", + "on_progress", + "deferred_locations", + "cursor_scope", + ], + ), + )?; + let project = &args.value("project")?; + let adapter = &args.value("adapter")?; + let connection = &args.value("connection")?; + let execute = &args.value("execute")?; + let selected_keys = &args.value("selected_keys")?; + let full_refresh: bool = args.flag("full_refresh")?; + let full_refresh_model_names = &args.value("full_refresh_model_names")?; + let on_progress = &args.value("on_progress")?; + let deferred_locations = &args.value("deferred_locations")?; + let cursor_scope = &args.value("cursor_scope")?; + + warehouse_snapshot(&SnapshotRequest { + project: project.clone(), + adapter: adapter.clone(), + connection: connection.clone(), + execute: execute.clone(), + selected_keys: selected_keys.clone(), + full_refresh, + full_refresh_model_names: full_refresh_model_names.clone(), + on_progress: on_progress.clone(), + deferred_locations: deferred_locations.clone(), + cursor_scope: cursor_scope.clone(), + }) +} + +/// Cursor snapshots of models whose history lives in another relation or column. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn gather_redirected_cursor_snapshots<'py>( + py: pyo3::Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "gather_redirected_cursor_snapshots", + options, + ( + &[ + "project", + "adapter", + "connection", + "existing_relations", + "target_relations", + "cursor_scope", + "full_refresh_model_names", + ], + &["deferred_locations", "on_progress", "origin_cursor_columns"], + ), + )?; + let project = &args.value("project")?; + let adapter = &args.value("adapter")?; + let connection = &args.value("connection")?; + let existing_relations = args.value("existing_relations")?; + let existing_relations = existing_relations.downcast::()?; + let target_relations = args.value("target_relations")?.downcast_into::()?; + let cursor_scope = &args.value("cursor_scope")?; + let full_refresh_model_names = &args.value("full_refresh_model_names")?; + let deferred_locations = &args.value("deferred_locations")?; + let origin_cursor_columns = args.value("origin_cursor_columns")?; + let result = PyDict::new(py); + if target_relations.is_empty() { + return Ok(result.into_any()); + } + let mut keys: Vec> = Vec::new(); + for name in target_relations.keys() { + keys.push(model_key(py, &name)?); + } + let request = SnapshotRequest { + project: project.clone(), + adapter: adapter.clone(), + connection: connection.clone(), + execute: adapter.getattr("execute")?, + selected_keys: frozenset(py, &keys)?, + full_refresh: false, + full_refresh_model_names: full_refresh_model_names.clone(), + on_progress: args.value("on_progress")?, + deferred_locations: deferred_locations.clone(), + cursor_scope: cursor_scope.clone(), + }; + let snapshots = cursor_snapshots( + &request, + existing_relations, + &CursorInputs { + selected_keys: frozenset(py, &keys)?, + full_refresh_model_names: full_refresh_model_names.clone(), + deferred_locations: deferred_locations.clone(), + runtime_producer_keys: cursor_scope.getattr("runtime_producer_keys")?, + invocation_time: cursor_scope.getattr("invocation_time")?, + start_cursor_config: cursor_scope.getattr("start_cursor_config")?, + cursor_overrides: cursor_scope.getattr("cursor_overrides")?, + target_relation_overrides: Some(target_relations), + origin_cursor_column_overrides: if origin_cursor_columns.is_none() { + None + } else { + Some(origin_cursor_columns.downcast_into::()?) + }, + existing_columns: None, + }, + )?; + let models = by_name(project, "models")?; + for (name, snapshot) in snapshots.iter() { + if let Some(model) = models.get_item(&name)? { + result.set_item( + name, + crate::bindings::_helpers::planner_phases::objects::replace( + &snapshot, + &[( + "target_relation", + model.getattr("destination")?.getattr("qualified_name")?, + )], + )?, + )?; + } + } + Ok(result.into_any()) +} + +/// The relation names one metadata listing is filtered to, or `None` for an unfiltered listing. +#[pyfunction] +#[pyo3(signature = (*, project, selected_keys=None))] +pub(crate) fn planner_metadata_name_filter<'py>( + py: Python<'py>, + project: &Bound<'py, PyAny>, + selected_keys: Option>, +) -> PyResult> { + let selected = selected_keys.unwrap_or_else(|| py.None().into_bound(py)); + optional_names(py, metadata_name_filter(project, &selected)?.as_ref()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs new file mode 100644 index 0000000000..36f4cdcce9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs @@ -0,0 +1,220 @@ +//! Warehouse source relation listing and column gathering for planning. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_phases::objects::{frozenset, text, tuple}; +use crate::bindings::_helpers::planner_warehouse::constants::{ + COLUMN_INFO, METADATA_NAME_FILTER_LIMIT, RELATION_LOOKUP, +}; +use crate::bindings::_helpers::planner_warehouse::snapshot::{columns_for, list_relations}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use std::collections::{BTreeSet, HashSet}; + +fn entries_of<'py>( + project: &Bound<'py, PyAny>, + source_entries: Option>, +) -> PyResult>> { + match source_entries.filter(|entries| !entries.is_none()) { + Some(entries) => entries.try_iter()?.collect(), + None => project + .getattr("sources")? + .try_iter()? + .map(|source| source?.getattr("source_entry")) + .collect(), + } +} + +fn table_name(entry: &Bound<'_, PyAny>) -> PyResult { + let table = entry.getattr("table")?; + if table.is_none() { + entry.getattr("name")?.extract() + } else { + table.extract() + } +} + +fn database_of(entry: &Bound<'_, PyAny>) -> PyResult> { + let database = entry.getattr("database")?; + if database.is_truthy()? { + Ok(Some(database.extract()?)) + } else { + Ok(None) + } +} + +fn name_filter( + entries: &[Bound<'_, PyAny>], + database: Option<&str>, + schemas: &BTreeSet, +) -> PyResult>> { + let mut names: BTreeSet<_> = BTreeSet::new(); + for entry in entries { + if !entry.getattr("expression")?.is_none() { + continue; + } + let schema: Option = entry.getattr("schema")?.extract()?; + if !schema.is_some_and(|schema| schemas.contains(&schema)) { + continue; + } + if database_of(entry)?.as_deref() != database { + continue; + } + names.insert(table_name(entry)?); + } + if names.is_empty() || names.len() > METADATA_NAME_FILTER_LIMIT { + return Ok(None); + } + Ok(Some(names)) +} + +fn names_tuple<'py>(py: Python<'py>, names: &BTreeSet) -> PyResult> { + tuple(py, names.iter().map(|name| text(py, name)).collect()) +} + +/// Source columns and the names of sources the relation listing found. +pub(crate) fn source_inspection<'py>( + project: &Bound<'py, PyAny>, + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + source_entries: Option>, +) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyAny>)> { + let py = project.py(); + let entries = entries_of(project, source_entries)?; + let result = PyDict::new(py); + let mut listed: Vec> = Vec::new(); + let mut listed_names: HashSet = HashSet::new(); + let mut groups: Vec<(String, BTreeSet)> = Vec::new(); + let column_info = imported(py, COLUMN_INFO.0, COLUMN_INFO.1)?; + for entry in &entries { + let expression = entry.getattr("expression")?; + if !expression.is_none() { + if entry.getattr("type_enforcement")?.is_truthy()? { + let names = call_with( + &adapter.getattr("query_column_names")?, + &[("connection", connection.clone()), ("sql", expression)], + )?; + let mut columns: Vec<_> = Vec::new(); + for name in names.try_iter()? { + columns.push(call_with( + &column_info, + &[("name", name?), ("type", text(py, ""))], + )?); + } + result.set_item(entry.getattr("name")?, tuple(py, columns)?)?; + } + continue; + } + let schema = entry.getattr("schema")?; + if schema.is_none() { + continue; + } + let key: String = entry + .getattr("database")? + .extract::>()? + .unwrap_or_default(); + let schema: String = schema.extract()?; + match groups.iter_mut().find(|(group, _)| *group == key) { + Some((_, schemas)) => { + schemas.insert(schema); + } + None => groups.push((key, BTreeSet::from([schema]))), + } + } + let lookup = imported(py, RELATION_LOOKUP.0, RELATION_LOOKUP.1)?.getattr("key")?; + for (key, schemas) in groups { + let database = (!key.is_empty()).then_some(key.as_str()); + let database_value = match database { + Some(database) => text(py, database), + None => py.None().into_bound(py), + }; + let mut scoped: Vec<(String, Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); + for entry in &entries { + if !entry.getattr("expression")?.is_none() || database_of(entry)?.as_deref() != database + { + continue; + } + let schema: Option = entry.getattr("schema")?.extract()?; + if !schema.is_some_and(|schema| schemas.contains(&schema)) { + continue; + } + let identity = call_with( + &lookup, + &[ + ("database", database_value.clone()), + ("schema", entry.getattr("schema")?), + ("name", text(py, &table_name(entry)?)), + ], + )?; + let entry_name: String = entry.getattr("name")?.extract()?; + match scoped.iter_mut().find(|(name, _, _)| *name == entry_name) { + Some(existing) => existing.2 = identity, + None => scoped.push((entry_name, entry.getattr("name")?, identity)), + } + } + let relations = list_relations( + (adapter, connection), + &database_value, + names_tuple(py, &schemas)?, + match name_filter(&entries, database, &schemas)? { + Some(names) => names_tuple(py, &names)?, + None => py.None().into_bound(py), + }, + )?; + let relations: Vec> = relations.try_iter()?.collect::>()?; + let identities = frozenset( + py, + &relations + .iter() + .map(|relation| relation.getattr("identity")) + .collect::>>()?, + )?; + for (text_name, name, identity) in &scoped { + if identities.contains(identity)? && listed_names.insert(text_name.clone()) { + listed.push(name.clone()); + } + } + let wanted = frozenset( + py, + &scoped + .iter() + .map(|(_, _, identity)| identity.clone()) + .collect::>(), + )?; + let mut chosen: Vec<_> = Vec::new(); + for relation in &relations { + if wanted.contains(relation.getattr("identity")?)? { + chosen.push(relation.clone()); + } + } + let all = columns_for((adapter, connection), tuple(py, chosen)?)?; + for (_, name, identity) in &scoped { + if all.contains(identity)? { + result.set_item(name, all.get_item(identity)?)?; + } + } + } + Ok((result, frozenset(py, &listed)?)) +} + +#[pyfunction] +#[pyo3(signature = (*, project, adapter, connection, source_entries))] +pub(crate) fn gather_source_inspection<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + source_entries: Option>, +) -> PyResult> { + let (columns, listed) = source_inspection(project, (adapter, connection), source_entries)?; + Ok(PyTuple::new(project.py(), [columns.into_any(), listed])?.into_any()) +} + +#[pyfunction] +#[pyo3(signature = (*, project, adapter, connection, source_entries=None))] +pub(crate) fn gather_source_columns<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + source_entries: Option>, +) -> PyResult> { + Ok(source_inspection(project, (adapter, connection), source_entries)?.0) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs new file mode 100644 index 0000000000..be0af28693 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs @@ -0,0 +1,342 @@ +//! Source read deferral for managed source loaders. + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::planner_phases::objects::{ + native, optional_text, planner_error, replace, text, +}; +use crate::bindings::_helpers::planner_warehouse::constants::{ + MODEL_HOOK_CONFIG_KEYS, SQL_HOOK_ENTRY, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyDict as KeywordDict; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use sqlbuild_analysis::planner_warehouse::main::source_call_names::source_call_names; +use sqlbuild_analysis::planner_warehouse::main::source_deferral_field::source_deferral_field; +use std::collections::BTreeSet; + +fn managed_names(project: &Bound<'_, PyAny>) -> PyResult> { + let mut names: BTreeSet<_> = BTreeSet::new(); + for source in project.getattr("sources")?.try_iter()? { + let entry = source?.getattr("source_entry")?; + if !entry.getattr("loader")?.is_none() { + names.insert(entry.getattr("name")?.extract()?); + } + } + Ok(names) +} + +fn sql_source_reads(sql: &Bound<'_, PyAny>, managed: &BTreeSet) -> PyResult> { + let mut names: Vec = Vec::new(); + for name in source_call_names(&sql.extract::()?) { + if managed.contains(&name) { + names.push(name); + } + } + Ok(names) +} + +fn hook_source_reads( + model: &Bound<'_, PyAny>, + managed: &BTreeSet, +) -> PyResult> { + let mut names: Vec = Vec::new(); + let py = model.py(); + let hook_entry = imported(py, SQL_HOOK_ENTRY.0, SQL_HOOK_ENTRY.1)?; + let source_kind = + imported(py, "sqlbuild.python_nodes.types", "SqlResourceRefKind")?.getattr("SOURCE")?; + let values = model.getattr("config")?.getattr("values")?; + for key in MODEL_HOOK_CONFIG_KEYS { + let value = values.call_method1("get", (key,))?; + let entries: Vec> = + if value.is_instance_of::() || value.is_instance_of::() { + value.try_iter()?.collect::>()? + } else { + vec![value] + }; + for entry in entries { + if !entry.is_instance(&hook_entry)? { + continue; + } + for reference in entry.getattr("reads")?.try_iter()? { + let reference = reference?; + if reference.getattr("kind")?.is(&source_kind) { + let name: String = reference.getattr("name")?.extract()?; + if managed.contains(&name) { + names.push(name); + } + } + } + } + } + Ok(names) +} + +fn selected_managed_refs( + project: &Bound<'_, PyAny>, + selected: &Bound<'_, PyAny>, + managed: &BTreeSet, +) -> PyResult> { + let mut names: BTreeSet<_> = BTreeSet::new(); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + if !selected.contains(model.getattr("key")?)? { + continue; + } + names.extend(sql_source_reads(&model.getattr("query_sql")?, managed)?); + names.extend(hook_source_reads(&model, managed)?); + } + for function in project.getattr("functions")?.try_iter()? { + let function = function?; + if selected.contains(function.getattr("key")?)? { + names.extend(sql_source_reads(&function.getattr("body_sql")?, managed)?); + } + } + for audit in project.getattr("audits")?.try_iter()? { + let audit = audit?; + let deps = audit.getattr("scope_deps")?; + if deps.is_truthy()? { + let mut any = false; + for dep in deps.try_iter()? { + if selected.contains(dep?)? { + any = true; + break; + } + } + if !any { + continue; + } + } + for reference in audit.getattr("references")?.try_iter()? { + let reference = reference?; + let name: String = reference.getattr("ref_name")?.extract()?; + if reference.getattr("ref_kind")?.eq("source")? && managed.contains(&name) { + names.insert(name); + } + } + names.extend(sql_source_reads(&audit.getattr("sql_body")?, managed)?); + } + Ok(names) +} + +fn resolve_target<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, + target_name: Bound<'py, PyAny>, +) -> PyResult> { + native( + project_config.py(), + "resolve_target_config", + &[ + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ("target_name", target_name), + ], + ) +} + +fn source_target_name<'py>( + project: &Bound<'py, PyAny>, + (project_config, local_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + defer_sources_to: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + if !defer_sources_to.is_none() { + return Ok(defer_sources_to.clone()); + } + let active = project.getattr("effective_target_name")?; + if active.is_none() || project_config.is_none() || local_config.is_none() { + return Ok(py.None().into_bound(py)); + } + let deferred = resolve_target(project_config, local_config, active.clone())? + .getattr("defer_sources_to")?; + Ok(if deferred.is_truthy()? { + deferred + } else { + active + }) +} + +fn raw_source_entry<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + let declared = source.getattr("source_entry")?; + let name = declared.getattr("name")?; + for entry in source + .getattr("source_file")? + .getattr("source_entries")? + .try_iter()? + { + let entry = entry?; + if entry.getattr("name")?.eq(&name)? { + return Ok(entry); + } + } + Ok(declared) +} + +fn target_field<'py>( + target_value: &Bound<'py, PyAny>, + logical_value: &Bound<'py, PyAny>, + effective_vars: &Bound<'py, PyAny>, +) -> PyResult> { + let py = target_value.py(); + let target: Option = target_value.extract()?; + let logical: Option = logical_value.extract()?; + let resolved = source_deferral_field( + (target.as_deref(), logical.as_deref()), + |name: &str| -> PyResult> { + if !effective_vars.contains(name)? { + return Ok(None); + } + Ok(Some( + native( + py, + "render_project_var_text", + &[ + ("value", effective_vars.get_item(name)?), + ( + "label", + text(py, &format!("source deferral variable '${{{name}}}'")), + ), + ], + )? + .extract()?, + )) + }, + )?; + Ok(optional_text(py, resolved.as_deref())) +} + +fn entry_for_environment<'py>( + entry: &Bound<'py, PyAny>, + target: &Bound<'py, PyAny>, + effective_vars: &Bound<'py, PyAny>, +) -> PyResult> { + if !entry.getattr("expression")?.is_none() { + return Ok(entry.clone()); + } + let mut database = entry.getattr("database")?; + if database.is_none() { + database = target_field(&target.getattr("database")?, &database, effective_vars)?; + } + let mut schema = entry.getattr("schema")?; + if schema.is_none() { + let loader_schema = target.getattr("loader_schema")?; + let target_schema = if loader_schema.is_truthy()? { + loader_schema + } else { + target.getattr("schema")? + }; + schema = target_field(&target_schema, &schema, effective_vars)?; + } + replace(entry, &[("database", database), ("schema", schema)]) +} + +/// Resolve source reads for selected SQL, or for `read_source_names` when given. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_source_read_map<'py>( + py: pyo3::Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_source_read_map", + options, + ( + &[ + "project", + "source_map", + "selected_keys", + "project_config", + "local_config", + "defer_sources_to", + ], + &["read_source_names"], + ), + )?; + let project = &args.value("project")?; + let source_map = args.value("source_map")?.downcast_into::()?; + let source_map = &source_map; + let selected_keys = &args.value("selected_keys")?; + let project_config = &args.value("project_config")?; + let local_config = &args.value("local_config")?; + let defer_sources_to = &args.value("defer_sources_to")?; + let read_source_names = &args.value("read_source_names")?; + + let py = project.py(); + let managed = managed_names(project)?; + let selected: BTreeSet = match Some(read_source_names).filter(|names| !names.is_none()) + { + Some(names) => names + .try_iter()? + .map(|name| name?.extract::()) + .collect::>>()? + .intersection(&managed) + .cloned() + .collect(), + None => selected_managed_refs(project, selected_keys, &managed)?, + }; + if selected.is_empty() { + return Ok(source_map.clone()); + } + let target_name = + source_target_name(project, (project_config, local_config), defer_sources_to)?; + if target_name.is_none() || project_config.is_none() || local_config.is_none() { + return Ok(source_map.clone()); + } + if !project_config.getattr("targets")?.contains(&target_name)? + && !local_config.getattr("targets")?.contains(&target_name)? + { + return Err(planner_error( + py, + &format!("Unknown source deferral target '{}'", target_name.str()?), + (None, None), + )); + } + let target = resolve_target(project_config, local_config, target_name)?; + let effective_vars = project.getattr("effective_vars")?; + let result = source_map.copy()?; + for source in project.getattr("sources")?.try_iter()? { + let source = source?; + let name: String = source.getattr("source_entry")?.getattr("name")?.extract()?; + if !selected.contains(&name) { + continue; + } + let entry = raw_source_entry(&source)?; + result.set_item( + name, + entry_for_environment(&entry, &target, &effective_vars)?, + )?; + } + Ok(result) +} + +/// Add declared project sources that no selected SQL resource reads to a source map. +#[pyfunction] +#[pyo3(signature = (*, project, source_map, source_names))] +pub(crate) fn with_declared_source_reads<'py>( + project: &Bound<'py, PyAny>, + source_map: &Bound<'py, PyDict>, + source_names: &Bound<'py, PyAny>, +) -> PyResult> { + let mut missing: BTreeSet<_> = BTreeSet::new(); + for name in source_names.try_iter()? { + let name: String = name?.extract()?; + if !source_map.contains(&name)? { + missing.insert(name); + } + } + if missing.is_empty() { + return Ok(source_map.clone()); + } + let result = source_map.copy()?; + for source in project.getattr("sources")?.try_iter()? { + let entry = source?.getattr("source_entry")?; + let name: String = entry.getattr("name")?.extract()?; + if missing.contains(&name) { + result.set_item(name, entry)?; + } + } + Ok(result) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs new file mode 100644 index 0000000000..0a688192c6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs @@ -0,0 +1,187 @@ +//! Direct planner source freshness observation and the warnings for unknown freshness. + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::planner_phases::objects::{ + model_key_names, planner_model, replace, text, tuple, +}; +use crate::bindings::_helpers::planner_warehouse::constants::{ + FRESHNESS_PLANNING, FRESHNESS_PROPAGATION, RUN_SCOPE, +}; +use crate::bindings::_helpers::planner_warehouse::snapshot::{destination_nodes, lower}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyDict as KeywordDict; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use sqlbuild_analysis::planner_warehouse::main::freshness_unknown_message::freshness_unknown_message; +use std::collections::BTreeSet; + +/// One warning per reason that left configured source freshness unknown. +#[pyfunction] +#[pyo3(name = "build_source_freshness_unknown_warnings")] +#[pyo3(signature = (*, source_freshness))] +pub(crate) fn source_freshness_unknown_warnings<'py>( + source_freshness: &Bound<'py, PyAny>, +) -> PyResult> { + let py = source_freshness.py(); + let mut reasons: Vec<(String, Vec)> = Vec::new(); + for unknown in source_freshness + .getattr("unknown_sources")? + .call_method0("values")? + .try_iter()? + { + let unknown = unknown?; + let reason: String = unknown.getattr("reason")?.str()?.extract()?; + if freshness_unknown_message(&reason, &[]).is_none() { + continue; + } + let name: String = unknown.getattr("source_name")?.extract()?; + match reasons.iter_mut().find(|(existing, _)| *existing == reason) { + Some((_, names)) => names.push(name), + None => reasons.push((reason, vec![name])), + } + } + let mut warnings: Vec<_> = Vec::new(); + for (reason, names) in reasons { + if let Some(message) = freshness_unknown_message(&reason, &names) { + warnings.push(planner_model( + py, + "PlanWarning", + &[ + ("model_name", py.None().into_bound(py)), + ( + "severity", + imported(py, "sqlbuild.compiler.planner.types", "WarningSeverity")? + .getattr("WARNING")?, + ), + ("message", text(py, &message)), + ], + )?); + } + } + tuple(py, warnings) +} + +fn state_database<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + for node in destination_nodes(project)? { + let database = node.getattr("destination")?.getattr("database")?; + if !database.is_none() { + return Ok(database); + } + } + Ok(project.py().None().into_bound(project.py())) +} + +fn state_schemas( + project: &Bound<'_, PyAny>, + scope: &Bound<'_, PyAny>, +) -> PyResult> { + let mut schemas: BTreeSet<_> = BTreeSet::new(); + for node in destination_nodes(project)? { + let schema = node.getattr("destination")?.getattr("schema")?; + if !schema.is_none() { + schemas.insert(schema.extract()?); + } + } + if schemas.is_empty() { + let models = scope.getattr("models_by_name")?; + for name in model_key_names(&scope.getattr("selected_keys")?)? { + let model = models.call_method1("get", (name,))?; + if model.is_none() { + continue; + } + let schema = model.getattr("destination")?.getattr("schema")?; + if !schema.is_none() { + schemas.insert(schema.extract()?); + } + } + } + Ok(schemas) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_planner_source_freshness_result<'py>( + py: pyo3::Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_planner_source_freshness_result", + options, + ( + &["project", "adapter", "connection", "scope", "relations"], + &["freshness_state_schemas"], + ), + )?; + let project = &args.value("project")?; + let adapter = &args.value("adapter")?; + let connection = &args.value("connection")?; + let scope = &args.value("scope")?; + let relations = &args.value("relations")?; + let freshness_state_schemas = &args.value("freshness_state_schemas")?; + + let py = project.py(); + let schemas = state_schemas(project, scope)?; + let mut lowered_state: BTreeSet = BTreeSet::new(); + if !freshness_state_schemas.is_none() { + let existing = freshness_state_schemas; + for schema in existing.try_iter()? { + lowered_state.insert(schema?.extract()?); + } + } + let exists = PyDict::new(py); + for schema in &schemas { + exists.set_item( + schema, + PyBool::new(py, lowered_state.contains(&lower(&text(py, schema))?)), + )?; + } + let sources = tuple( + py, + relations + .getattr("source_read_map")? + .call_method0("values")? + .try_iter()? + .collect::>()?, + )?; + let observed_at = imported(py, "datetime", "datetime")? + .call_method1("now", (imported(py, "datetime", "UTC")?,))?; + let result = within( + &imported(py, RUN_SCOPE.0, RUN_SCOPE.1)?.call1((project.getattr("run_id")?,))?, + |_| { + call_with( + &imported(py, FRESHNESS_PLANNING.0, FRESHNESS_PLANNING.1)?, + &[ + ("adapter", adapter.clone()), + ("connection", connection.clone()), + ("sources", sources), + ("state_database", state_database(project)?), + ( + "state_schemas", + tuple(py, schemas.iter().map(|schema| text(py, schema)).collect())?, + ), + ("observed_at", observed_at), + ("run_id", text(py, "planning")), + ( + "render_qualified_name", + adapter.getattr("render_qualified_name")?, + ), + ("state_table_exists_by_schema", exists.into_any()), + ( + "filter_previous_to_sources", + PyBool::new(py, true).to_owned().into_any(), + ), + ], + ) + }, + )?; + let propagation = call_with( + &imported(py, FRESHNESS_PROPAGATION.0, FRESHNESS_PROPAGATION.1)?, + &[ + ("source_freshness", result.clone()), + ("scope", scope.clone()), + ], + )?; + replace(&result, &[("propagation", propagation)]) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs new file mode 100644 index 0000000000..60dcbb90af --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs @@ -0,0 +1,225 @@ +//! Selected-scope validation that external source tables exist in the warehouse. + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::planner_phases::objects::{planner_error, tuple}; +use crate::bindings::_helpers::planner_warehouse::constants::{ + MISSING_SOURCE_TABLE_CODE, RENDER_SOURCE_READ, RUN_BOUNDED, WILDCARD_TABLE_CHARACTER, + YAML_ENTRY_LINE, +}; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyDict as KeywordDict; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_analysis::planner_warehouse::main::missing_source_tables_help::missing_source_tables_help; +use sqlbuild_analysis::planner_warehouse::main::missing_source_tables_message::missing_source_tables_message; +use sqlbuild_analysis::planner_warehouse::models::MissingSourceTable; +use std::collections::HashMap; + +fn selected_source_readers(scope: &Bound<'_, PyAny>) -> PyResult>> { + let mut readers: HashMap> = HashMap::new(); + let deps = scope.getattr("upstream_deps")?; + for key in scope.getattr("selected_keys")?.try_iter()? { + let key = key?; + if key.getattr("resource_type")?.eq("source")? { + continue; + } + let reader: String = key.getattr("name")?.extract()?; + for dep in deps + .call_method1("get", (&key, PyTuple::empty(key.py())))? + .try_iter()? + { + let dep = dep?; + if dep.getattr("resource_type")?.eq("source")? { + readers + .entry(dep.getattr("name")?.extract()?) + .or_default() + .push(reader.clone()); + } + } + } + Ok(readers) +} + +fn table_name(entry: &Bound<'_, PyAny>) -> PyResult { + let table = entry.getattr("table")?; + if table.is_none() { + entry.getattr("name")?.extract() + } else { + table.extract() + } +} + +fn external_table(entry: &Bound<'_, PyAny>) -> PyResult { + Ok(!entry.getattr("managed")?.is_truthy()? + && entry.getattr("loader")?.is_none() + && entry.getattr("integration_loader")?.is_none() + && entry.getattr("expression")?.is_none() + && !entry.getattr("schema")?.is_none() + && !table_name(entry)?.contains(WILDCARD_TABLE_CHARACTER)) +} + +fn read_relation<'py>( + adapter: &Bound<'py, PyAny>, + entry: &Bound<'py, PyAny>, +) -> PyResult> { + call_with( + &imported(adapter.py(), RENDER_SOURCE_READ.0, RENDER_SOURCE_READ.1)?, + &[ + ("adapter", adapter.clone()), + ("source_entry", entry.clone()), + ], + ) +} + +fn declaration_location(source: &Bound<'_, PyAny>) -> PyResult { + let file = source.getattr("source_file")?; + let path: String = file + .getattr("relative_path")? + .call_method0("as_posix")? + .extract()?; + let line = call_with( + &imported(source.py(), YAML_ENTRY_LINE.0, YAML_ENTRY_LINE.1)?, + &[ + ("contents", file.getattr("contents")?), + ("name", source.getattr("name")?), + ], + )?; + if line.is_none() { + Ok(path) + } else { + Ok(format!("{path}:{}", line.str()?)) + } +} + +/// Raise S405 when a selected resource reads an external source table that does not exist. +pub(crate) fn check_selected_source_tables_exist<'py>( + project: &Bound<'py, PyAny>, + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + scope: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py = project.py(); + let readers = selected_source_readers(scope)?; + let read_map = relations.getattr("source_read_map")?; + let listed = relations.getattr("listed_source_names")?; + let mut unlisted: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>, Vec)> = Vec::new(); + for source in project.getattr("sources")?.try_iter()? { + let source = source?; + let name: String = source.getattr("name")?.extract()?; + let Some(source_readers) = readers.get(&name).filter(|readers| !readers.is_empty()) else { + continue; + }; + let declared = source.getattr("source_entry")?; + let entry = read_map.call_method1("get", (&name, &declared))?; + if !external_table(&declared)? || !external_table(&entry)? { + continue; + } + if listed.contains(&name)? { + continue; + } + let mut sorted = source_readers.clone(); + sorted.sort(); + unlisted.push((source, entry, sorted)); + } + if unlisted.is_empty() { + return Ok(()); + } + let probe = adapter.getattr("probe_relation_read")?; + let mut tasks: Vec<_> = Vec::new(); + for (_, entry, _) in &unlisted { + let fields = keywords( + py, + &[ + ("connection", connection.clone()), + ("relation", read_relation(adapter, entry)?), + ], + )?; + tasks.push(imported(py, "functools", "partial")?.call((&probe,), Some(&fields))?); + } + let concurrency: i64 = imported(py, "builtins", "int")? + .call1((adapter.getattr("metadata_inspection_concurrency")?,))? + .extract()?; + let probes = call_with( + &imported(py, RUN_BOUNDED.0, RUN_BOUNDED.1)?, + &[ + ("tasks", tuple(py, tasks)?), + ( + "concurrency", + concurrency.max(1).into_pyobject(py)?.into_any(), + ), + ], + )?; + let mut missing: Vec<_> = Vec::new(); + let mut entries: Vec<_> = Vec::new(); + for ((source, entry, read_by), probe) in unlisted.into_iter().zip(probes.try_iter()?) { + let probe = probe?; + let status = probe.getattr("status")?; + if status.eq("readable")? { + continue; + } + let role = probe.getattr("role")?; + missing.push(MissingSourceTable { + source_name: source.getattr("name")?.extract()?, + relation: String::new(), + read_by, + unreadable: status.eq("missing_or_unreadable")?, + role: if role.is_none() { + None + } else { + Some(role.str()?.extract()?) + }, + location: String::new(), + }); + entries.push(entry); + } + if missing.is_empty() { + return Ok(()); + } + for (item, entry) in missing.iter_mut().zip(&entries).take(10) { + item.relation = read_relation(adapter, entry)?.str()?.extract()?; + } + let message = missing_source_tables_message(&missing); + let sources = PyDict::new(py); + for source in project.getattr("sources")?.try_iter()? { + let source = source?; + sources.set_item(source.getattr("name")?, source)?; + } + for item in missing.iter_mut().take(10) { + if let Some(source) = sources.get_item(&item.source_name)? { + item.location = declaration_location(&source)?; + } + } + let help = missing_source_tables_help(&missing); + Err(planner_error( + py, + &message, + (Some(MISSING_SOURCE_TABLE_CODE), Some(&help)), + )) +} + +#[pyfunction] +#[pyo3(name = "check_selected_source_tables_exist")] +#[pyo3(signature = (**options))] +pub(crate) fn check_selected_source_tables_exist_binding<'py>( + py: pyo3::Python<'py>, + options: Option>, +) -> PyResult<()> { + let args = KeywordArguments::read( + py, + "check_selected_source_tables_exist", + options, + ( + &["project", "adapter", "connection", "scope", "relations"], + &[], + ), + )?; + let project = &args.value("project")?; + let adapter = &args.value("adapter")?; + let connection = &args.value("connection")?; + let scope = &args.value("scope")?; + let relations = &args.value("relations")?; + + check_selected_source_tables_exist(project, (adapter, connection), scope, relations) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs new file mode 100644 index 0000000000..42ab6ce201 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs @@ -0,0 +1,150 @@ +//! Validate reliable watermark column contracts before warehouse inspection. + +use crate::bindings::_helpers::planner_phases::objects::{config_str, planner_error, text}; +use crate::bindings::_helpers::planner_warehouse::constants::{ + CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, WATERMARK_CONTRACT_CODE, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; +use std::collections::HashMap; + +/// Whether a declared column type can hold the model's cursor type. +#[pyfunction] +#[pyo3(signature = (*, declared_type, cursor_type))] +pub(crate) fn watermark_type_is_compatible( + declared_type: &Bound<'_, PyAny>, + cursor_type: Option<&str>, +) -> PyResult { + let Some(expected) = cursor_type + .filter(|value| *value == CURSOR_TYPE_TIMESTAMP || *value == CURSOR_TYPE_INTEGER) + else { + return Ok(true); + }; + let classified = crate::bindings::_helpers::compile_driver::python_calls::imported( + declared_type.py(), + "sqlbuild._native", + "classify_cursor_sql_type", + )? + .call1((declared_type,))?; + classified.eq(expected) +} + +type DeclaredColumns<'py> = (Vec, HashMap>); + +fn declared_columns<'py>(columns: &Bound<'py, PyAny>) -> PyResult> { + let mut names: Vec<_> = Vec::new(); + let mut types: HashMap<_, _> = HashMap::new(); + for column in columns.try_iter()? { + let column = column?; + let name: String = column.getattr("name")?.extract()?; + let column_type = column.getattr("type")?; + if !column_type.is_none() { + types.insert( + column.getattr("name")?.call_method0("lower")?.extract()?, + column_type, + ); + } + names.push(name); + } + Ok((names, types)) +} + +/// Raise S302 when a reliable upstream contract lacks or mistypes a watermark column. +pub(crate) fn validate_watermark_column<'py>( + model: &Bound<'py, PyAny>, + reference: &Bound<'py, PyAny>, + cursor_column: &str, + (models, sources, seeds): ( + &Bound<'py, PyDict>, + &Bound<'py, PyDict>, + &Bound<'py, PyDict>, + ), +) -> Result<(), PyErr> { + let py = model.py(); + let kind = reference.getattr("ref_kind")?; + let ref_name: String = reference.getattr("ref_name")?.extract()?; + let model_name: String = model.getattr("name")?.extract()?; + let lowered = + |value: &str| -> PyResult { text(py, value).call_method0("lower")?.extract() }; + let (names, types) = if kind.eq("ref")? { + let Some(upstream) = models.get_item(&ref_name)? else { + return Ok(()); + }; + let upstream_cursor = config_str(&upstream, "cursor")?; + let upstream_type = config_str(&upstream, "cursor_type")?; + let consumer_type = config_str(model, "cursor_type")?; + if let (Some(upstream_cursor), Some(upstream_type), Some(consumer_type)) = + (&upstream_cursor, &upstream_type, &consumer_type) + && lowered(upstream_cursor)? == lowered(cursor_column)? + && upstream_type != consumer_type + { + return Err(planner_error( + py, + &format!( + "model '{model_name}': cursor_inputs watermark '{ref_name}.{cursor_column}' has cursor_type={upstream_type}, incompatible with model cursor_type={consumer_type}" + ), + (Some(WATERMARK_CONTRACT_CODE), None), + )); + } + let values = upstream.getattr("config")?.getattr("values")?; + if !values.call_method1("get", ("contract",))?.eq("enforced")? { + return Ok(()); + } + let schema = upstream.getattr("schema_entry")?; + if schema.is_none() { + (Vec::new(), HashMap::new()) + } else { + declared_columns(&schema.getattr("columns")?)? + } + } else if kind.eq("source")? { + let Some(source) = sources.get_item(&ref_name)? else { + return Ok(()); + }; + let entry = source.getattr("source_entry")?; + if !entry.getattr("contract")?.eq("enforced")? { + return Ok(()); + } + declared_columns(&entry.getattr("columns")?)? + } else if kind.eq("seed")? { + let Some(seed) = seeds.get_item(&ref_name)? else { + return Ok(()); + }; + declared_columns(&seed.getattr("schema_entry")?.getattr("columns")?)? + } else { + return Ok(()); + }; + let wanted = lowered(cursor_column)?; + let mut lowered_names: Vec<_> = Vec::new(); + for name in &names { + lowered_names.push(lowered(name)?); + } + if lowered_names.contains(&wanted) { + if let Some(declared_type) = types.get(&wanted) { + let cursor_type = config_str(model, "cursor_type")?; + if !watermark_type_is_compatible(declared_type, cursor_type.as_deref())? { + return Err(planner_error( + py, + &format!( + "model '{model_name}': cursor_inputs watermark '{ref_name}.{cursor_column}' type {} is incompatible with cursor_type={}", + declared_type.str()?, + cursor_type.as_deref().unwrap_or("None") + ), + (Some(WATERMARK_CONTRACT_CODE), None), + )); + } + } + return Ok(()); + } + let mut declared = names.join(", "); + if declared.is_empty() { + declared = "none".to_owned(); + } + Err(planner_error( + py, + &format!( + "model '{model_name}': cursor_inputs watermark references '{ref_name}' column '{cursor_column}', but its enforced contract does not expose the column. Declared contract columns: {declared}" + ), + (Some(WATERMARK_CONTRACT_CODE), None), + )) +} diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index d141f8a041..c06c2bb17a 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -48,6 +48,8 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { plan_output::registration::register(module)?; planner_changes::registration::register(module)?; crate::bindings::_helpers::planner_migrations::registration::register(module)?; + crate::bindings::_helpers::planner_phases::registration::register(module)?; + crate::bindings::_helpers::planner_warehouse::registration::register(module)?; planner_identity::registration::register(module)?; planner_selection::registration::register(module)?; planner_scope::registration::register(module)?; diff --git a/fensu.toml b/fensu.toml index 1e559ae0a3..5c8cc82a29 100644 --- a/fensu.toml +++ b/fensu.toml @@ -188,12 +188,6 @@ rule = "FFR601" path = "src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py" reason = "The SQL Server adapter intentionally keeps overrideable backend behavior on one class." -[[targets.python.rule_exceptions]] -rule = "XSB051" -path = "src/sqlbuild/compiler/planner/_helpers/warehouse/source_columns.py" -symbols = ["gather_source_inspection"] -reason = "Each type-enforced expression source is its own query whose output columns can only be read by running it; table sources are listed in one batch per database." - [[targets.python.rule_exceptions]] rule = "XSB051" path = "src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py" diff --git a/python-boundary.toml b/python-boundary.toml index 3449eeb06b..ba895ed983 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 27784 +remaining_lines = 23650 [[stays]] path = "executor" @@ -613,10 +613,6 @@ note = "Public facade." location = "compiler/planner/models.py::CursorBounds._parse_inferred" note = "A bound that is not integer text is parsed as a timestamp instead; CursorAlgebraError is an input error, and native compiler failures are not caught." -[[handled_native_errors]] -location = "compiler/planner/_helpers/warehouse/snapshot.py::_parse_warehouse_cursor" -note = "A warehouse cursor value that is not an integer is parsed as a timestamp instead; CursorAlgebraError is an input error, and native compiler failures are not caught." - [[handled_native_errors]] location = "cli/commands/_helpers/refactor/compile_facts.py::compile_for_refactor" note = "Reports the Python stage errors the native compile driver propagates as refactor diagnostics; a native compiler failure is a RuntimeError and is not caught." diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 20bb2d0946..58f3eaa344 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -206,7 +206,6 @@ from sqlbuild.compiler.pipeline.models import ( PythonRunPlanOutputs, StaticCommandContext, ) -from sqlbuild.compiler.planner.classes.migration_fingerprint_cache import MigrationFingerprintCache from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( AuditPlanEntry, @@ -220,6 +219,7 @@ from sqlbuild.compiler.planner.models import ( CursorInputRelation, CursorOverridePair, CursorOverrides, + CursorSnapshotScope, DeferralInputs, DirectModelVersionIdentities, EmptyCursorInputDecision, @@ -246,12 +246,21 @@ from sqlbuild.compiler.planner.models import ( ParsedSelector, PathSelector, PlanEntryBuildInputs, + PlannedSqlTests, + PlannerChangeReconciliation, PlannerChangeResults, + PlannerEntryResults, + PlannerIdentityContext, PlannerOverrides, + PlannerPolicies, PlannerRelationsContext, PlannerResolvedActions, PlannerRuntime, PlannerScope, + PlannerScopePruningResult, + PlannerScopeResolution, + PlannerSelection, + PlannerWarehouseState, PlanOutput, PlanWarning, QueryShape, @@ -259,6 +268,7 @@ from sqlbuild.compiler.planner.models import ( RelationFixtureCompletion, RelationFixtureDiagnostic, RelationFixturePlanningContext, + RetentionPlanEntry, ScenarioArtifactIdentity, ScenarioExecutionPlan, ScenarioFixturePlan, @@ -275,6 +285,7 @@ from sqlbuild.compiler.planner.models import ( SqlTestPlanEntry, SqlTestPlanResult, SqlTestSelection, + TableTypePlanEntry, UnitTestSelectorSplit, WarehouseFingerprints, WarehouseSnapshot, @@ -5325,3 +5336,228 @@ def format_model_migrations( *, lines: list[str], plan: PlanOutput, display_options: DisplayOptions ) -> list[str]: ... def old_name_rows(*, entry: OldNameViewPlanEntry, style: CliStyle) -> list[str]: ... + +class MigrationFingerprintCache: + def __init__(self, *, root: Path | None = None) -> None: ... + def fingerprint( + self, + *, + query_sql: str, + metadata_json: str, + ref_identities: Mapping[str, str], + dialect: str | None, + ) -> str | None: ... + def persist(self) -> None: ... + +def gather_warehouse_snapshot( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + connection: Any, + execute: AdapterExecute[Any, Any], + selected_keys: frozenset[CompiledObjectKey] | None = None, + full_refresh: bool = False, + full_refresh_model_names: frozenset[str] | None = None, + on_progress: Callable[[str], None] | None = None, + deferred_locations: dict[str, CompiledRelationLocation] | None = None, + cursor_scope: CursorSnapshotScope | None = None, +) -> WarehouseSnapshot: ... +def gather_redirected_cursor_snapshots( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + connection: Any, + existing_relations: dict[str, RelationInfo], + target_relations: dict[str, str], + cursor_scope: CursorSnapshotScope, + full_refresh_model_names: frozenset[str], + deferred_locations: dict[str, CompiledRelationLocation] | None = None, + on_progress: Callable[[str], None] | None = None, + origin_cursor_columns: dict[str, str] | None = None, +) -> dict[str, ModelCursorSnapshot]: ... +def watermark_type_is_compatible(*, declared_type: str, cursor_type: str | None) -> bool: ... +def gather_source_columns( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + connection: Any, + source_entries: tuple[SourceEntry, ...] | None = None, +) -> dict[str, tuple[ColumnInfo, ...]]: ... +def gather_source_inspection( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + connection: Any, + source_entries: tuple[SourceEntry, ...] | None, +) -> tuple[dict[str, tuple[ColumnInfo, ...]], frozenset[str]]: ... +def get_semantic_source_columns( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + connection: Any, + source_read_map: dict[str, SourceEntry], + columns: dict[str, tuple[ColumnInfo, ...]], + selected_keys: frozenset[CompiledObjectKey], +) -> dict[str, tuple[ColumnInfo, ...]]: ... +def check_selected_source_tables_exist( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + connection: Any, + scope: PlannerScope, + relations: PlannerRelationsContext, +) -> None: ... +def build_source_read_map( + *, + project: CompiledProjectView, + source_map: dict[str, SourceEntry], + selected_keys: frozenset[CompiledObjectKey], + project_config: ProjectConfig | None, + local_config: LocalConfig | None, + defer_sources_to: str | None, + read_source_names: frozenset[str] | None = None, +) -> dict[str, SourceEntry]: ... +def with_declared_source_reads( + *, + project: CompiledProjectView, + source_map: dict[str, SourceEntry], + source_names: frozenset[str], +) -> dict[str, SourceEntry]: ... +def build_source_freshness_unknown_warnings( + *, source_freshness: DirectSourceFreshnessPlanningResult +) -> tuple[PlanWarning, ...]: ... +def build_planner_source_freshness_result( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + connection: Any, + scope: PlannerScope, + relations: PlannerRelationsContext, + freshness_state_schemas: frozenset[str] | None = None, +) -> DirectSourceFreshnessPlanningResult: ... +def resolve_planner_scopes( + *, + project: CompiledProjectView, + selection: PlannerSelection, + policies: PlannerPolicies, +) -> PlannerScopeResolution: ... +def build_planner_identity_context( + *, + project: CompiledProjectView, + scopes: PlannerScopeResolution, + include_stale_warning_identities: bool = True, +) -> PlannerIdentityContext: ... +def detect_stale_warning_changes( + *, + project: CompiledProjectView, + scopes: PlannerScopeResolution, + snapshot: WarehouseSnapshot, + identities: PlannerIdentityContext, + execution_changes: PlannerChangeResults, +) -> PlannerChangeResults: ... +def check_selected_scope_buildability( + *, + runtime: PlannerRuntime, + scopes: PlannerScopeResolution, + warehouse: PlannerWarehouseState, + deferral: DeferralInputs, +) -> None: ... +def prune_planner_execution_scope( + *, scopes: PlannerScopeResolution, resolved_actions: PlannerResolvedActions +) -> PlannerScopePruningResult: ... +def reconcile_execution_changes( + *, + warehouse: PlannerWarehouseState, + identities: PlannerIdentityContext, + pruning: PlannerScopePruningResult, + changes: PlannerChangeResults, +) -> PlannerChangeReconciliation: ... +def build_planner_entry_results( + *, + runtime: PlannerRuntime, + warehouse: PlannerWarehouseState, + identities: PlannerIdentityContext, + overrides: PlannerOverrides, + policies: PlannerPolicies, + deferral: DeferralInputs, + pruning: PlannerScopePruningResult, + reconciliation: PlannerChangeReconciliation, + source_freshness: DirectSourceFreshnessPlanningResult, +) -> PlannerEntryResults: ... +def resolve_model_full_refresh(*, model: CompiledModelView, cli_full_refresh: bool) -> bool: ... +def effectively_full_refreshed_model_names( + *, project: CompiledProjectView, cli_full_refresh: bool +) -> frozenset[str]: ... +def describe_full_rebuild_cause( + *, change_result: ChangeDetectionResult, full_refresh: bool +) -> str: ... +def check_full_rebuild_allowed( + *, model: CompiledModelView, change_result: ChangeDetectionResult, full_rebuild_cause: str +) -> None: ... +def effective_target_config(*, runtime: PlannerRuntime) -> TargetConfig: ... +def table_type_copy_name(*, target_name: str, identifier_limit: int) -> str: ... +def plan_table_types( + *, runtime: PlannerRuntime, warehouse: PlannerWarehouseState, scope: PlannerScope +) -> tuple[TableTypePlanEntry, ...]: ... +def plan_retention( + *, runtime: PlannerRuntime, warehouse: PlannerWarehouseState, scope: PlannerScope +) -> tuple[RetentionPlanEntry, ...]: ... +def assemble_base_plan_output( + *, + runtime: PlannerRuntime, + warehouse: PlannerWarehouseState, + identities: PlannerIdentityContext, + overrides: PlannerOverrides, + pruning: PlannerScopePruningResult, + reconciliation: PlannerChangeReconciliation, + entries: PlannerEntryResults, + source_freshness: DirectSourceFreshnessPlanningResult, + planned_sql_tests: PlannedSqlTests | None = None, +) -> PlanOutput: ... +def with_column_rename_hints( + *, entries: tuple[ModelPlanEntry, ...], hints: tuple[ColumnRenameHint, ...] +) -> tuple[ModelPlanEntry, ...]: ... +def with_storage_policies( + *, + plan_output: PlanOutput, + runtime: PlannerRuntime, + warehouse: PlannerWarehouseState, + scopes: PlannerScopeResolution, + policies: PlannerPolicies, +) -> PlanOutput: ... +def with_plan_warnings( + *, + runtime: PlannerRuntime, + scopes: PlannerScopeResolution, + warehouse: PlannerWarehouseState, + identities: PlannerIdentityContext, + stale_warning_changes: PlannerChangeResults, + pruning: PlannerScopePruningResult, + source_freshness: DirectSourceFreshnessPlanningResult, + plan_output: PlanOutput, + policies: PlannerPolicies, +) -> PlanOutput: ... +def with_plan_metadata( + *, + plan_output: PlanOutput, + pruning: PlannerScopePruningResult, + source_freshness: DirectSourceFreshnessPlanningResult, + policies: PlannerPolicies, +) -> PlanOutput: ... +def gather_planner_warehouse_state( + *, + runtime: PlannerRuntime, + scopes: PlannerScopeResolution, + overrides: PlannerOverrides, + deferral: DeferralInputs, +) -> PlannerWarehouseState: ... +def planner_metadata_name_filter( + *, project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] | None = None +) -> tuple[str, ...] | None: ... +def list_source_candidates( + *, + adapter: BaseAdapter, + connection: Any, + database: str | None, + entries: tuple[SourceEntry, ...], +) -> tuple[RelationInfo, ...]: ... diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py index 3b0738dc89..06b82aa8e6 100644 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py +++ b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py @@ -8,6 +8,7 @@ build_selected_audit_entries, extract_seed_columns, get_materialization_type, + resolve_planner_scopes, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey @@ -19,7 +20,6 @@ plan_function, sql_test_is_selected, ) -from sqlbuild.compiler.planner._helpers.planning.scopes import resolve_planner_scopes from sqlbuild.compiler.planner.models import ( AuditPlanEntry, DeferralInputs, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index c398c92029..6dac3632fa 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -13,19 +13,26 @@ audit_is_selected, build_cursor_input_relations, build_source_load_map, + build_source_read_map, + check_full_rebuild_allowed, + describe_full_rebuild_cause, detect_model_changes, + gather_source_inspection, get_config_cursor_bound, get_config_str, get_materialization_type, + get_semantic_source_columns, microbatch_guidance, replaces_incremental_relation, resolve_bounded_cursor_override, resolve_microbatch_lookback, + resolve_model_full_refresh, resolve_model_plan_action, runtime_cursor_producer_names, source_binding_diagnostics, validate_capped_watermark_inputs, validate_source_cursor_input_columns, + with_declared_source_reads, ) from sqlbuild._native import cursor_render as render from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token @@ -43,11 +50,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( - check_full_rebuild_allowed, - describe_full_rebuild_cause, - resolve_model_full_refresh, -) from sqlbuild.compiler.planner._helpers.resolve.cursor import ( compute_cursor_bounds, cursor_inputs_without_rows, @@ -69,14 +71,6 @@ resolve_model_hook_entries, resolve_model_sql, ) -from sqlbuild.compiler.planner._helpers.warehouse.semantic_sources import ( - get_semantic_source_columns, -) -from sqlbuild.compiler.planner._helpers.warehouse.source_columns import gather_source_inspection -from sqlbuild.compiler.planner._helpers.warehouse.source_deferral import ( - build_source_read_map, - with_declared_source_reads, -) from sqlbuild.compiler.planner.constants import MICROBATCH_CONCURRENCY_MESSAGE from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety diff --git a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py index b735c1ee9d..18cbcfb135 100644 --- a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py +++ b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py @@ -8,6 +8,7 @@ build_compiled_function_fingerprint_sql, build_downstream_deps, build_execution_upstream_deps, + gather_source_columns, get_materialization_type, resolve_graph_selection, topologically_order_keys, @@ -32,7 +33,6 @@ resolve_function_sql, resolve_model_sql, ) -from sqlbuild.compiler.planner._helpers.warehouse.source_columns import gather_source_columns from sqlbuild.compiler.planner.models import ( BackfillResult, CloneSourcePlanEntry, diff --git a/src/sqlbuild/compiler/planner/main/execution/display_plan.py b/src/sqlbuild/compiler/planner/main/execution/display_plan.py index 224227c80b..8dab9c5583 100644 --- a/src/sqlbuild/compiler/planner/main/execution/display_plan.py +++ b/src/sqlbuild/compiler/planner/main/execution/display_plan.py @@ -7,12 +7,12 @@ build_execution_upstream_deps, get_config_str, get_materialization_type, + resolve_model_full_refresh, ) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.planning.full_refresh import resolve_model_full_refresh from sqlbuild.compiler.planner.models import BackfillResult, ModelPlanEntry, PlanOutput from sqlbuild.compiler.planner.types import ( BackfillAction, diff --git a/src/sqlbuild/compiler/planner/main/execution/execution.py b/src/sqlbuild/compiler/planner/main/execution/execution.py index d2977a7cb6..637548b5ff 100644 --- a/src/sqlbuild/compiler/planner/main/execution/execution.py +++ b/src/sqlbuild/compiler/planner/main/execution/execution.py @@ -7,40 +7,33 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import detect_changes, resolve_model_actions -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.planner._helpers.planning.buildability import ( - check_selected_scope_buildability, -) -from sqlbuild.compiler.planner._helpers.planning.entries import build_planner_entry_results -from sqlbuild.compiler.planner._helpers.planning.identities import ( +from sqlbuild._native import ( + assemble_base_plan_output, + build_planner_entry_results, build_planner_identity_context, + build_planner_source_freshness_result, + check_selected_scope_buildability, + detect_changes, detect_stale_warning_changes, -) -from sqlbuild.compiler.planner._helpers.planning.output_assembly import ( - assemble_base_plan_output, + gather_planner_warehouse_state, + prune_planner_execution_scope, + reconcile_execution_changes, + resolve_model_actions, + resolve_planner_scopes, with_plan_metadata, with_plan_warnings, with_storage_policies, ) -from sqlbuild.compiler.planner._helpers.planning.reconciliation import reconcile_execution_changes -from sqlbuild.compiler.planner._helpers.planning.scope_pruning import prune_planner_execution_scope -from sqlbuild.compiler.planner._helpers.planning.scopes import resolve_planner_scopes -from sqlbuild.compiler.planner._helpers.planning.warehouse_state import ( - gather_planner_warehouse_state, +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, ) from sqlbuild.compiler.planner._helpers.resolve.cursor import ( type_cursor_overrides, ) -from sqlbuild.compiler.planner._helpers.warehouse.source_freshness import ( - build_planner_source_freshness_result, -) from sqlbuild.compiler.planner.classes.background_sql_test_planning import ( BackgroundSqlTestPlanning, ) diff --git a/src/sqlbuild/compiler/planner/main/selection/_resolve_planner_scopes.py b/src/sqlbuild/compiler/planner/main/selection/_resolve_planner_scopes.py index bc7586b9df..c1f60c120f 100644 --- a/src/sqlbuild/compiler/planner/main/selection/_resolve_planner_scopes.py +++ b/src/sqlbuild/compiler/planner/main/selection/_resolve_planner_scopes.py @@ -2,10 +2,8 @@ from __future__ import annotations +from sqlbuild._native import resolve_planner_scopes as _resolve_planner_scopes from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.planning.scopes import ( - resolve_planner_scopes as _resolve_planner_scopes, -) from sqlbuild.compiler.planner.models import ( PlannerPolicies, PlannerScopeResolution, diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 135520c10a..6a8ba4b1f8 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Any +from sqlbuild._native import MigrationFingerprintCache from sqlbuild._native import cursor_parse as parse from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RetentionRequest @@ -44,9 +45,6 @@ OldNameViewAction, ) from sqlbuild.compiler.planner.classes.fixture_column_inferences import FixtureColumnInferences -from sqlbuild.compiler.planner.classes.migration_fingerprint_cache import ( - MigrationFingerprintCache, -) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.types import ( BackfillAction, diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py b/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py index 9cb255c9cc..467aaba1a1 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py @@ -6,9 +6,9 @@ import pytest +from sqlbuild._native import table_type_copy_name from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner._helpers.planning.retention import table_type_copy_name from sqlbuild.compiler.planner.models import TableTypePlanEntry from sqlbuild.executor.build._helpers.retention import apply_table_type_conversion from sqlbuild.executor.clone.main._clone_relation_operation import clone_relation_by_names diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 2286abfaee..3146c1624c 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -13,7 +13,7 @@ import pytest from _pytest.capture import CaptureResult -import sqlbuild.compiler.planner.classes.migration_fingerprint_cache as fingerprint_cache_module +import sqlbuild._native as fingerprint_cache_module from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.models import MigrationStagePlan from sqlbuild.adapter.contract.types import MigrationTransfer diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 5a0cdf10b5..8173500d6b 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -130,3 +130,29 @@ class FixtureSchemaIntegrationTestCase: contract: str expected_authoritative: bool expected_diagnostics: tuple[str, ...] + + +@dataclass(frozen=True) +class ConcurrentCursorReadsTestCase: + description: str + relation_count: int + concurrency: int + failing_relations: frozenset[str] + expected_start_progress: str + expected_unreadable: dict[str, tuple[str, ...]] + + +@dataclass(frozen=True) +class FailedCursorReadTestCase: + description: str + fail_on_fetch: bool + expected_unreadable_input_names: tuple[str, ...] + expected_empty_input_names: tuple[str, ...] + expected_unavailable_watermark_tags: tuple[str, ...] + + +@dataclass(frozen=True) +class EligibleMaxFailureTestCase: + description: str + model_count: int + expected_error_fragment: str diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py index f48e3a7d86..775ecf7c84 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -2,15 +2,20 @@ from __future__ import annotations +import threading +import time +from collections.abc import Callable from dataclasses import dataclass, field, replace from pathlib import Path -from typing import Any +from typing import Any, ClassVar from sqlbuild._native import ( compiled_model_record, + gather_warehouse_snapshot, project_with, ) from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -34,6 +39,7 @@ DiscoveredSeedFile, DiscoveredSourceFile, ) +from sqlbuild.compiler.planner.models import WarehouseSnapshot from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import ( SchemaColumn, @@ -258,3 +264,161 @@ def build_deferred_locations_from_map( ) for name, qualified in targets.items() } + + +class ConcurrentDuckDbAdapter(DuckDbAdapter): + """DuckDB adapter allowing four concurrent inspection reads.""" + + metadata_inspection_concurrency: ClassVar[int] = 4 + + +class _FailingFetch: + def fetchall(self) -> list[Any]: + raise RuntimeError("fetch failed") + + +class _Rows: + def __init__(self, rows: list[Any]) -> None: + self._rows: list[Any] = rows + + def fetchall(self) -> list[Any]: + return self._rows + + +def _read_rows(connection: Any, sql: str, lock: threading.Lock) -> Any: + with lock: + return _Rows(connection.execute(sql).fetchall()) + + +def _raise_unreadable(connection: Any, sql: str, lock: threading.Lock) -> Any: + del connection, lock + raise RuntimeError(f"cannot read {sql.split(' FROM ', 1)[1]}") + + +def _failing_fetch(connection: Any, sql: str, lock: threading.Lock) -> Any: + del connection, sql, lock + return _FailingFetch() + + +_CURSOR_ANSWERS: dict[tuple[bool, bool], Callable[[Any, str, threading.Lock], Any]] = { + (False, False): _read_rows, + (False, True): _read_rows, + (True, False): _raise_unreadable, + (True, True): _failing_fetch, +} + + +class TrackingCursorExecute: + """Execute double that tracks overlapping cursor reads and fails chosen relations.""" + + def __init__( + self, + *, + failing_relations: frozenset[str] = frozenset(), + failing_eligible_relations: frozenset[str] = frozenset(), + fail_on_fetch: bool = False, + ) -> None: + self._failing: tuple[frozenset[str], frozenset[str]] = ( + failing_relations, + failing_eligible_relations, + ) + self._fail_on_fetch: bool = fail_on_fetch + self._lock: threading.Lock = threading.Lock() + self._active: int = 0 + self.max_active: int = 0 + self.cursor_sql: list[str] = [] + + def __call__(self, *, connection: Any, sql: str) -> Any: + eligible: bool = "<=" in sql + tracked: bool = eligible or "AS _max" in sql + return (self._untracked, self._tracked)[tracked](connection, sql, eligible) + + def _untracked(self, connection: Any, sql: str, eligible: bool) -> Any: + del eligible + return _read_rows(connection, sql, self._lock) + + def _tracked(self, connection: Any, sql: str, eligible: bool) -> Any: + with self._lock: + self._active += 1 + self.max_active = max(self.max_active, self._active) + self.cursor_sql.append(sql) + try: + time.sleep(0.01) + relation: str = sql.split(" FROM ", 1)[1].split(" ", 1)[0] + failing: bool = relation in self._failing[eligible] + return _CURSOR_ANSWERS[(failing, self._fail_on_fetch)](connection, sql, self._lock) + finally: + with self._lock: + self._active -= 1 + + +def create_events(connection: Any, count: int) -> None: + """Create one small events table per fanned model.""" + + index: int + for index in range(count): + connection.execute(f"CREATE TABLE staging.events_{index} (event_time TIMESTAMP)") + connection.execute( + f"INSERT INTO staging.events_{index} VALUES ('2024-01-01'), " + f"('2024-03-0{index % 9 + 1}')" + ) + + +def gather_with_catalog( + *, + project: CompiledProjectView, + adapter: DuckDbAdapter, + connection: Any, + execute: TrackingCursorExecute, + progress: list[str], +) -> WarehouseSnapshot: + """Gather a snapshot inside an inspection catalog of the adapter.""" + + with open_inspection_catalog(adapter=adapter, connection=connection): + return gather_warehouse_snapshot( + project=project, + adapter=adapter, + connection=connection, + execute=execute, + on_progress=progress.append, + ) + + +def unreadable_inputs(snapshot: WarehouseSnapshot) -> dict[str, tuple[str, ...]]: + """Return the unreadable watermark inputs of every model that has some.""" + + return dict( + filter( + lambda item: bool(item[1]), + ( + (name, cursor.unreadable_input_names) + for name, cursor in snapshot.cursor_snapshots.items() + ), + ) + ) + + +def cursor_progress(progress: list[str]) -> list[str]: + """Return the cursor-bound progress messages.""" + + return list(filter(lambda message: "cursor bounds" in message, progress)) + + +def build_fanned_cursor_project( + count: int, *, extra_config: dict[str, object] | None = None +) -> CompiledProjectView: + """Build one incremental model per upstream events table.""" + + return build_project_with_targets( + model_locations={f"events_{index}": "staging" for index in range(count)}, + incremental_models=tuple( + _IncrementalModelSpec( + name=f"daily_{index}", + schema="marts", + cursor="event_time", + ref_names=(f"events_{index}",), + extra_config=extra_config or {}, + ) + for index in range(count) + ), + ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py new file mode 100644 index 0000000000..27bf62fe45 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +import pytest + +from sqlbuild._native import gather_warehouse_snapshot +from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.planner.exceptions import PlannerInputError +from sqlbuild.compiler.planner.models import ( + CursorSnapshotScope, + ModelCursorSnapshot, + WarehouseSnapshot, +) +from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( + ConcurrentCursorReadsTestCase, + EligibleMaxFailureTestCase, + FailedCursorReadTestCase, +) +from tests.integration.src.sqlbuild.compiler.planner._helpers.helpers import ( + ConcurrentDuckDbAdapter, + TrackingCursorExecute, + build_fanned_cursor_project, + create_events, + cursor_progress, + gather_with_catalog, + unreadable_inputs, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ConcurrentCursorReadsTestCase( + description="parallel reads stay within the bound", + relation_count=12, + concurrency=4, + failing_relations=frozenset(), + expected_start_progress="Inspecting cursor bounds for 12 relations (4 concurrent)...", + expected_unreadable={}, + ), + ConcurrentCursorReadsTestCase( + description="one failed relation stays unavailable and others succeed", + relation_count=8, + concurrency=4, + failing_relations=frozenset({"staging.events_5"}), + expected_start_progress="Inspecting cursor bounds for 8 relations (4 concurrent)...", + expected_unreadable={"daily_5": ("events_5.event_time",)}, + ), + ], + ids=lambda case: case.description, +) +def test_given_concurrent_catalog_when_reading_cursor_bounds_then_matches_sequential_reads( + test_case: ConcurrentCursorReadsTestCase, + adapter: DuckDbAdapter, + connection: Any, +) -> None: + create_events(connection, test_case.relation_count) + project: CompiledProjectView = build_fanned_cursor_project(test_case.relation_count) + sequential_execute: TrackingCursorExecute = TrackingCursorExecute( + failing_relations=test_case.failing_relations + ) + parallel_execute: TrackingCursorExecute = TrackingCursorExecute( + failing_relations=test_case.failing_relations + ) + sequential_progress: list[str] = [] + parallel_progress: list[str] = [] + + sequential: WarehouseSnapshot = gather_with_catalog( + project=project, + adapter=adapter, + connection=connection, + execute=sequential_execute, + progress=sequential_progress, + ) + parallel: WarehouseSnapshot = gather_with_catalog( + project=project, + adapter=ConcurrentDuckDbAdapter(), + connection=connection, + execute=parallel_execute, + progress=parallel_progress, + ) + + unreadable: dict[str, tuple[str, ...]] = unreadable_inputs(parallel) + reads: list[str] = cursor_progress(parallel_progress) + assert parallel.cursor_snapshots == sequential.cursor_snapshots + assert sequential_execute.max_active == 1 + assert 1 < parallel_execute.max_active <= test_case.concurrency + assert sorted(parallel_execute.cursor_sql) == sorted(sequential_execute.cursor_sql) + assert reads[0] == test_case.expected_start_progress + assert len(reads) == test_case.relation_count + 2 + assert unreadable == test_case.expected_unreadable + + +@pytest.mark.parametrize( + "test_case", + [ + FailedCursorReadTestCase( + description="failed statement names the unreadable input", + fail_on_fetch=False, + expected_unreadable_input_names=("events_0.event_time",), + expected_empty_input_names=(), + expected_unavailable_watermark_tags=( + "daily_0__events_0__min", + "daily_0__events_0__max", + ), + ), + FailedCursorReadTestCase( + description="failed fetch names the unreadable input", + fail_on_fetch=True, + expected_unreadable_input_names=("events_0.event_time",), + expected_empty_input_names=(), + expected_unavailable_watermark_tags=( + "daily_0__events_0__min", + "daily_0__events_0__max", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_failed_cursor_read_when_gathering_then_reports_failure_and_continues( + test_case: FailedCursorReadTestCase, + adapter: DuckDbAdapter, + connection: Any, +) -> None: + create_events(connection, 2) + progress: list[str] = [] + + snapshot: WarehouseSnapshot = gather_with_catalog( + project=build_fanned_cursor_project(2), + adapter=adapter, + connection=connection, + execute=TrackingCursorExecute( + failing_relations=frozenset({"staging.events_0"}), + fail_on_fetch=test_case.fail_on_fetch, + ), + progress=progress, + ) + + failed: ModelCursorSnapshot = snapshot.cursor_snapshots["daily_0"] + assert failed.unreadable_input_names == test_case.expected_unreadable_input_names + assert failed.empty_input_names == test_case.expected_empty_input_names + assert failed.unavailable_watermark_tags == test_case.expected_unavailable_watermark_tags + assert snapshot.cursor_snapshots["daily_1"].upstream_maxes + assert any(message.startswith("Failed cursor bounds (1/2)") for message in progress) + + +@pytest.mark.parametrize( + "test_case", + [ + EligibleMaxFailureTestCase( + description="failure names the model and relation", + model_count=4, + expected_error_fragment=( + "model 'daily_2': failed to query highest eligible target cursor for " + "marts.daily_2.event_time" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_failing_eligible_max_read_when_reading_in_parallel_then_plan_fails_naming_relation( + test_case: EligibleMaxFailureTestCase, + connection: Any, +) -> None: + create_events(connection, test_case.model_count) + index: int + for index in range(test_case.model_count): + connection.execute(f"CREATE TABLE marts.daily_{index} (event_time TIMESTAMP)") + connection.execute(f"INSERT INTO marts.daily_{index} VALUES ('2099-01-01')") + keys: frozenset[CompiledObjectKey] = frozenset( + CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=f"daily_{index}") + for index in range(test_case.model_count) + ) + adapter: ConcurrentDuckDbAdapter = ConcurrentDuckDbAdapter() + + with ( + open_inspection_catalog(adapter=adapter, connection=connection), + pytest.raises(PlannerInputError, match=test_case.expected_error_fragment) as raised, + ): + _ = gather_warehouse_snapshot( + project=build_fanned_cursor_project( + test_case.model_count, + extra_config={ + "cursor_type": "timestamp", + "cursor_grain": "day", + "cursor_start_max_ahead": "1d", + }, + ), + adapter=adapter, + connection=connection, + execute=TrackingCursorExecute(failing_eligible_relations=frozenset({"marts.daily_2"})), + cursor_scope=CursorSnapshotScope( + model_keys=keys, + runtime_producer_keys=keys, + invocation_time=datetime(2026, 1, 20, tzinfo=UTC), + ), + ) + + assert isinstance(raised.value.__cause__, RuntimeError) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py index cdaf0e433c..3e2f071bd2 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py @@ -8,6 +8,8 @@ from sqlbuild._native import ( compiled_model_record, + gather_source_columns, + gather_warehouse_snapshot, project_with, write_fingerprint, ) @@ -23,8 +25,6 @@ CompiledResourceType, ) from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.warehouse.snapshot import gather_warehouse_snapshot -from sqlbuild.compiler.planner._helpers.warehouse.source_columns import gather_source_columns from sqlbuild.compiler.planner.constants import METADATA_NAME_FILTER_LIMIT from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py index 8212321405..ce7415f003 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py @@ -7,6 +7,7 @@ import pytest +import sqlbuild._native as identity_planning from sqlbuild._native import ( compiled_model_record, format_plan, @@ -15,7 +16,6 @@ from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.planning import identities as identity_planning from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( CursorBounds, diff --git a/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py b/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py index b35c539ac7..08999a5dab 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py +++ b/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py @@ -8,7 +8,7 @@ import duckdb import pytest -import sqlbuild.compiler.planner._helpers.planning.output_assembly as planner_assembly_module +import sqlbuild._native as planner_assembly_module import sqlbuild.executor.build.classes.build_scheduler as build_scheduler_module from tests.integration.src.sqlbuild.executor.build._test_types import TableTypeRoutingTestCase from tests.integration.src.sqlbuild.executor.build.helpers import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py index bf6eac5e51..0ad9a64bbe 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py @@ -4,13 +4,13 @@ import pytest +from sqlbuild._native import watermark_type_is_compatible from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileModelConfig from sqlbuild.compiler.planner._helpers.resolve.cursor import ( compute_cursor_bounds, resolve_effective_timestamp_grain, ) -from sqlbuild.compiler.planner._helpers.warehouse.snapshot import _watermark_type_is_compatible from sqlbuild.compiler.planner.models import CursorBounds, CursorInputRelation, ModelCursorSnapshot from sqlbuild.cursor_algebra.models import DateValue, TimestampValue from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( @@ -576,7 +576,7 @@ def test_given_known_watermark_contract_type_when_validating_then_cursor_domain_ test_case: MicrobatchCursorTypeTestCase, ) -> None: assert ( - _watermark_type_is_compatible( + watermark_type_is_compatible( declared_type=test_case.declared_type, cursor_type=test_case.cursor_type ) is test_case.expected_compatible diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 345612fbe9..7144fe00cb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -15,7 +15,6 @@ from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import ( CursorBounds, - EmptyCursorInputDecision, MissingUpstream, ParsedSelector, PathSelector, @@ -121,32 +120,6 @@ class MetadataNameFilterTestCase: expected_physical_names: frozenset[str] -@dataclass(frozen=True) -class CursorQueryShapeTestCase: - description: str - min_tags: tuple[str, ...] - max_tags: tuple[str, ...] - row: tuple[str | None, ...] - expected_sql: str - expected_results: dict[str, str] - expected_bounds: str - - -@dataclass(frozen=True) -class CursorQueryGroupingTestCase: - description: str - expected_physical_queries: tuple[tuple[str, str, tuple[str, ...], tuple[str, ...]], ...] - - -@dataclass(frozen=True) -class CursorQueryFailureTestCase: - description: str - expected_results: dict[str, str] - expected_failure_progress: str - expected_success_progress: str - expected_failed_tags: frozenset[str] = frozenset() - - @dataclass(frozen=True) class MicrobatchCursorEndPlanTestCase: description: str @@ -161,13 +134,6 @@ class MicrobatchCursorEndPlanTestCase: expected_microbatch_range: CursorBounds -@dataclass(frozen=True) -class CursorFetchFailureTestCase: - description: str - expected_results: dict[str, str] - expected_progress: tuple[str, ...] - - @dataclass(frozen=True) class SeedIdentityTestCase: description: str @@ -272,16 +238,6 @@ class WatermarkLineageTestCase: expected_error_fragment: str | None = None -@dataclass(frozen=True) -class CursorSnapshotAvailabilityTestCase: - description: str - target_tag: str | None - target_relation: str | None - results: dict[str, str] - expected_available: bool - expected_unavailable_tags: tuple[str, ...] - - @dataclass(frozen=True) class RuntimeCursorProducerNamesTestCase: description: str @@ -652,16 +608,6 @@ class ApplyDeferredTargetsTestCase: expected_seed_qualified_names: dict[str, str | None] -@dataclass(frozen=True) -class CursorUpstreamResolutionTestCase: - description: str - ref_name: str - model_qualified_name: str | None - deferred_qualified_name: str | None - selected_names: frozenset[str] | None - expected_qualified_name: str | None - - @dataclass(frozen=True) class CursorOverrideResolutionTestCase: description: str @@ -864,16 +810,6 @@ class MissingUpstreamMessageTestCase: expected_message: str -@dataclass(frozen=True) -class EmptyCursorInputsSnapshotTestCase: - description: str - results: dict[str, str] - failed_tags: frozenset[str] - expected_empty_input_names: tuple[str, ...] - expected_unreadable_input_names: tuple[str, ...] - expected_decision: EmptyCursorInputDecision - - @dataclass(frozen=True) class LoaderDependencyNamesTestCase: """Expected first-seen loader names after resolving Python function dependencies.""" diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py index 5d599a3301..672a3357a0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py @@ -4,10 +4,7 @@ import pytest -from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( - check_full_rebuild_allowed, - describe_full_rebuild_cause, -) +from sqlbuild._native import check_full_rebuild_allowed, describe_full_rebuild_cause from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import BackfillResult, ChangeDetectionResult from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py index b4d4755c1f..62e07557bb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py @@ -1,15 +1,19 @@ from __future__ import annotations +from types import SimpleNamespace + import pytest -from sqlbuild.compiler.planner._helpers.planning.identities import ( - _reusable_execution_model_names, -) +import sqlbuild._native as native_module +from sqlbuild._native import detect_stale_warning_changes from sqlbuild.compiler.planner.models import ( ChangeDetectionResult, DirectModelVersionIdentities, PlannerChangeResults, PlannerIdentityContext, + PlannerScope, + PlannerScopeResolution, + WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import ChangeKind from tests.unit.src.sqlbuild.compiler.planner._helpers.planning._test_types import ( @@ -46,6 +50,7 @@ ) def test_given_execution_change_when_checking_reuse_then_requires_equivalent_semantics( test_case: ReusableExecutionModelNamesTestCase, + monkeypatch: pytest.MonkeyPatch, ) -> None: execution_identities: DirectModelVersionIdentities = DirectModelVersionIdentities( function_local_hashes={}, @@ -64,7 +69,26 @@ def test_given_execution_change_when_checking_reuse_then_requires_equivalent_sem model_version_hashes={"orders": test_case.stale_warning_hash}, ) - reusable_names: frozenset[str] = _reusable_execution_model_names( + monkeypatch.setattr(native_module, "detect_model_changes_in_scope", lambda **_: {}) + stale_scope: PlannerScope = PlannerScope( + upstream_deps={}, + downstream_deps={}, + all_keys={}, + models_by_name={}, + selected_keys=frozenset(), + execution_order=(), + ) + + reused: PlannerChangeResults = detect_stale_warning_changes( + project=SimpleNamespace( + settings=SimpleNamespace(query_change_tracking=test_case.query_change_tracking) + ), + scopes=PlannerScopeResolution( + selected_scope=stale_scope, + stale_warning_scope=stale_scope, + inspection_scope=stale_scope, + ), + snapshot=WarehouseSnapshot(), identities=PlannerIdentityContext( version_identities=execution_identities, stale_warning_identities=stale_identities, @@ -78,10 +102,9 @@ def test_given_execution_change_when_checking_reuse_then_requires_equivalent_sem }, functions={}, ), - query_change_tracking=test_case.query_change_tracking, ) - assert reusable_names == test_case.expected_names + assert frozenset(reused.models) == test_case.expected_names if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py index 0e73a4f65a..f6efbc2626 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py @@ -6,6 +6,7 @@ import pytest +from sqlbuild._native import plan_retention, plan_table_types from sqlbuild.adapter.contract.models import ( RelationInfo, RenderedRetentionChange, @@ -18,7 +19,6 @@ ) from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.planning.retention import plan_retention, plan_table_types from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( PlannerRuntime, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py index d5460e3284..f92acf0a35 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py @@ -2,8 +2,8 @@ import pytest +from sqlbuild._native import resolve_planner_scopes from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.planning.scopes import resolve_planner_scopes from sqlbuild.compiler.planner.models import ( PlannerPolicies, PlannerScopeResolution, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index b2b494bb98..689b7fb73f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( build_stale_out_of_selection_warnings, compiled_model_record, + resolve_planner_scopes, ) from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject from sqlbuild.compiler.compile.types import ( @@ -17,7 +18,6 @@ ) from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_SEED from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.planning.scopes import resolve_planner_scopes from sqlbuild.compiler.planner.models import ( ChangeDetectionResult, DirectModelVersionIdentities, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py index f89f17f262..ca6736d1da 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( build_cursor_input_relations, + gather_source_columns, runtime_cursor_producer_names, validate_source_cursor_input_columns, ) @@ -25,7 +26,6 @@ CompiledResourceType, ) from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context -from sqlbuild.compiler.planner._helpers.warehouse.source_columns import gather_source_columns from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( CursorInputRelation, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py index 91898b9c3a..e186a8891f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py @@ -4,11 +4,9 @@ import pytest +from sqlbuild._native import build_planner_source_freshness_result from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.planner._helpers.warehouse.source_freshness import ( - build_planner_source_freshness_result, -) from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py index 06ad20a51d..7a23ab1127 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py @@ -12,6 +12,7 @@ build_source_load_entries, build_source_load_map, expand_selected_loader_dependencies, + planner_metadata_name_filter, project_with, upstream_loader_dependency_names, ) @@ -28,7 +29,6 @@ CompiledResourceType, ) from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.planner._helpers.warehouse.snapshot import _build_metadata_name_filter from sqlbuild.compiler.planner.models import SourceLoadPlanEntry from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.spec.contracts.models import SourceEntry @@ -213,7 +213,7 @@ def test_given_unselected_upstream_in_closure_when_filtering_metadata_then_uses_ models=(selected_model, upstream_model), ) - names: tuple[str, ...] | None = _build_metadata_name_filter( + names: tuple[str, ...] | None = planner_metadata_name_filter( project=project, selected_keys=frozenset({selected_key, upstream_key}), ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/_test_types.py index 93a280e580..a9a2f1a7f9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/_test_types.py @@ -12,23 +12,6 @@ class SemanticSourceBatchCase: expected_sources: frozenset[str] -@dataclass(frozen=True) -class ConcurrentCursorBoundsTestCase: - description: str - relation_count: int - concurrency: int - failing_relations: frozenset[str] - expected_failure_relations: tuple[str, ...] - expected_start_progress: str - - -@dataclass(frozen=True) -class EligibleTargetMaxFailureTestCase: - description: str - failing_relation: str - expected_error_fragment: str - - @dataclass(frozen=True) class SemanticSourceSchemaCaseCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py index 140cd1ade2..701313e83b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py @@ -2,158 +2,18 @@ from __future__ import annotations -import threading -import time -from collections.abc import Callable from pathlib import Path from types import SimpleNamespace -from typing import Any, cast +from typing import cast +from sqlbuild._native import check_selected_source_tables_exist from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompiledSource from sqlbuild.compiler.compile.types import CompiledResourceType, CompiledSourceView from sqlbuild.compiler.discovery.models import DiscoveredSourceFile -from sqlbuild.compiler.planner._helpers.warehouse.snapshot import ( - _CursorModelInfo, - _PhysicalCursorQuery, -) -from sqlbuild.compiler.planner._helpers.warehouse.source_tables import ( - check_selected_source_tables_exist, -) from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope -from sqlbuild.compiler.planner.types import CursorType -from sqlbuild.spec.contracts.models import SourceEntry, StartCursorsConfig - - -class CursorBoundRows: - """Fetchable result of one cursor-bound statement double.""" - - def __init__(self, rows: list[tuple[str | None, ...]]) -> None: - self._rows: list[tuple[str | None, ...]] = rows - - def fetchall(self) -> list[tuple[str | None, ...]]: - return self._rows - - -def _answer_cursor_relation(relation: str) -> CursorBoundRows: - day: int = int(relation.rsplit("_", 1)[1]) % 28 + 1 - return CursorBoundRows([(f"2026-01-{day:02d}", f"2026-02-{day:02d}")]) - - -def _reject_cursor_relation(relation: str) -> CursorBoundRows: - raise RuntimeError(f"Object '{relation}' does not exist or not authorized.") - - -_CURSOR_RELATION_ANSWERS: dict[bool, Callable[[str], CursorBoundRows]] = { - False: _answer_cursor_relation, - True: _reject_cursor_relation, -} - - -class ConcurrencyTrackingCursorExecute: - """Execute double answering one MIN/MAX statement per relation while counting overlap.""" - - def __init__(self, *, failing_relations: frozenset[str], latency_seconds: float) -> None: - self._failing_relations: frozenset[str] = failing_relations - self._latency_seconds: float = latency_seconds - self._lock: threading.Lock = threading.Lock() - self._active: int = 0 - self.max_active: int = 0 - self.sql: list[str] = [] - - def __call__(self, *, connection: Any, sql: str) -> CursorBoundRows: - del connection - with self._lock: - self._active += 1 - self.max_active = max(self.max_active, self._active) - self.sql.append(sql) - try: - time.sleep(self._latency_seconds) - relation: str = sql.rsplit(" FROM ", 1)[1] - return _CURSOR_RELATION_ANSWERS[relation in self._failing_relations](relation) - finally: - with self._lock: - self._active -= 1 - - -def build_cursor_bound_queries(count: int) -> list[_PhysicalCursorQuery]: - """Return one MIN/MAX physical cursor query per synthetic orders relation.""" - - return [ - _PhysicalCursorQuery( - relation=f"analytics.raw.orders_{index}", - cursor_column="ordered_at", - min_tags=(f"model_{index}__orders__min",), - max_tags=(f"model_{index}__orders__max",), - ) - for index in range(count) - ] - - -class EligibleMaxAdapter: - """Adapter double rendering bounded MAX statements and allowing parallel reads.""" - - metadata_inspection_concurrency: int = 4 - - def render_max_cursor_at_or_before( - self, - *, - relation: str, - cursor_column: str, - maximum_allowed: str, - cursor_type: str, - is_date: bool, - ) -> str: - del cursor_type, is_date - return ( - f"SELECT CAST(MAX({cursor_column}) AS VARCHAR) FROM {relation} " - f"WHERE {cursor_column} <= '{maximum_allowed}'" - ) - - -def _answer_eligible_max(relation: str) -> CursorBoundRows: - del relation - return CursorBoundRows([("2026-01-15",)]) - - -_ELIGIBLE_MAX_ANSWERS: dict[bool, Callable[[str], CursorBoundRows]] = { - False: _answer_eligible_max, - True: _reject_cursor_relation, -} - - -class EligibleMaxExecute: - """Execute double failing the bounded MAX read of one relation.""" - - def __init__(self, *, failing_relation: str, latency_seconds: float) -> None: - self._failing_relation: str = failing_relation - self._latency_seconds: float = latency_seconds - - def __call__(self, *, connection: Any, sql: str) -> CursorBoundRows: - del connection - time.sleep(self._latency_seconds) - relation: str = sql.split(" FROM ", 1)[1].split(" ", 1)[0] - return _ELIGIBLE_MAX_ANSWERS[relation == self._failing_relation](relation) - - -def build_eligible_max_cursor_models(count: int) -> list[_CursorModelInfo]: - """Return timestamp cursor models whose target MAX is beyond a one-day horizon.""" - - return [ - _CursorModelInfo( - model_name=f"model_{index}", - target_tag=f"model_{index}__target__max", - target_relation=f"analytics.marts.orders_{index}", - cursor_column="ordered_at", - upstreams=(), - cursor_type=CursorType.TIMESTAMP, - cursor_grain="day", - effective_cursor_grain="day", - start_cursor_config=StartCursorsConfig(max_ahead="1d"), - ) - for index in range(count) - ] +from sqlbuild.spec.contracts.models import SourceEntry def column_names(columns: tuple[ColumnInfo, ...]) -> tuple[str, ...]: diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py index 5ccf38ab2f..973e792d11 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py @@ -10,15 +10,16 @@ import duckdb import pytest -from sqlbuild._native import build_compiled_project, discover_project_inputs +from sqlbuild._native import ( + build_compiled_project, + discover_project_inputs, + get_semantic_source_columns, + list_source_candidates, +) from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.warehouse.semantic_sources import ( - _list_source_candidates, - get_semantic_source_columns, -) from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import ( FakeRelation, @@ -185,7 +186,7 @@ def test_given_snowflake_source_schema_when_listing_candidates_then_finds_relati ) with open_inspection_catalog(adapter=adapter, connection=connection): - candidates: tuple[RelationInfo, ...] = _list_source_candidates( + candidates: tuple[RelationInfo, ...] = list_source_candidates( adapter=adapter, connection=connection, database="analytics", diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py index 71a660d9a8..12b3a8f69f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import importlib.metadata import json import sqlite3 from collections.abc import Mapping @@ -9,8 +10,8 @@ import pytest +import sqlbuild._native as native_module import sqlbuild.compiler.planner.classes.fixture_column_inferences as fixture_inferences -import sqlbuild.compiler.planner.classes.migration_fingerprint_cache as fingerprint_cache from sqlbuild._native import build_migration_fingerprint, infer_fixture_column_facts from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.models import FixtureColumnInference @@ -48,7 +49,7 @@ def counting( dialect=dialect, ) - monkeypatch.setattr(fingerprint_cache, "build_migration_fingerprint", counting) + monkeypatch.setattr(native_module, "build_migration_fingerprint", counting) return computations @@ -98,7 +99,7 @@ def upgrade_sqlbuild(root: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Report a different installed sqlbuild version for the next run.""" del root - monkeypatch.setattr(fingerprint_cache, "_package_version", lambda package: f"{package}-next") + monkeypatch.setattr(importlib.metadata, "version", lambda package: f"{package}-next") FIXTURE_PROFILES: dict[str, ExpressionInferenceProfile] = { diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_migration_fingerprint_cache.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_migration_fingerprint_cache.py index 0c6851289e..021e1b6932 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_migration_fingerprint_cache.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_migration_fingerprint_cache.py @@ -8,11 +8,8 @@ import pytest -import sqlbuild.compiler.planner.classes.migration_fingerprint_cache as fingerprint_cache -from sqlbuild._native import build_migration_fingerprint -from sqlbuild.compiler.planner.classes.migration_fingerprint_cache import ( - MigrationFingerprintCache, -) +import sqlbuild._native as native_module +from sqlbuild._native import MigrationFingerprintCache, build_migration_fingerprint from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import ( MigrationFingerprintCacheTestCase, PersistedMigrationFingerprintTestCase, @@ -106,7 +103,7 @@ def counting( dialect=dialect, ) - monkeypatch.setattr(fingerprint_cache, "build_migration_fingerprint", counting) + monkeypatch.setattr(native_module, "build_migration_fingerprint", counting) cache: MigrationFingerprintCache = MigrationFingerprintCache() results: list[str | None] = [ From 6e289561d6a8176b71d579df9e487e679d819289 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 03:52:56 +0100 Subject: [PATCH 03/11] style: order integrated planner binding modules --- crates/sqlbuild-python/src/bindings/_helpers/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index ad1b9ea74f..18d12c951f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -48,9 +48,9 @@ pub(crate) mod plan_output; pub(crate) mod planner_changes; pub(crate) mod planner_identity; pub(crate) mod planner_migrations; +pub(crate) mod planner_phases; pub(crate) mod planner_resolution; pub(crate) mod planner_scenario; -pub(crate) mod planner_phases; pub(crate) mod planner_scope; pub(crate) mod planner_selection; pub(crate) mod planner_strategy; From 70e978dcd731a32cc2018ebda35e28a88b155c39 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 04:06:03 +0100 Subject: [PATCH 04/11] refactor: group planner binding registration --- .../src/bindings/_helpers/mod.rs | 1 + .../_helpers/planner_registration/mod.rs | 1 + .../planner_registration/registration.rs | 22 +++++++++++++++++++ .../src/bindings/main/register.rs | 18 ++++----------- 4 files changed, 28 insertions(+), 14 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_registration/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 18d12c951f..e7a77b676b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -49,6 +49,7 @@ pub(crate) mod planner_changes; pub(crate) mod planner_identity; pub(crate) mod planner_migrations; pub(crate) mod planner_phases; +pub(crate) mod planner_registration; pub(crate) mod planner_resolution; pub(crate) mod planner_scenario; pub(crate) mod planner_scope; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/mod.rs new file mode 100644 index 0000000000..7444bd0374 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/mod.rs @@ -0,0 +1 @@ +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs new file mode 100644 index 0000000000..cfffe8f312 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs @@ -0,0 +1,22 @@ +//! Registers every planner binding group. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::{ + planner_changes, planner_identity, planner_migrations, planner_phases, planner_resolution, + planner_scenario, planner_scope, planner_selection, planner_strategy, planner_warehouse, +}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + planner_changes::registration::register(module)?; + planner_migrations::registration::register(module)?; + planner_phases::registration::register(module)?; + planner_warehouse::registration::register(module)?; + planner_identity::registration::register(module)?; + planner_selection::registration::register(module)?; + planner_scope::registration::register(module)?; + planner_scenario::registration::register(module)?; + planner_strategy::registration::register(module)?; + planner_resolution::registration::register(module)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index c06c2bb17a..46f3b76be5 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -7,10 +7,9 @@ use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, contracts, cursor, dag_artifact, diagnostics, discovery, fingerprints, frontier, graph, - lineage, macro_bridge, manifest, model_config, plan_output, planner_changes, planner_identity, - planner_scope, planner_selection, planner_strategy, project_assembly, refactor, references, - render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, - type_system, + lineage, macro_bridge, manifest, model_config, plan_output, planner_registration, + project_assembly, refactor, references, render, rules, scopes, semantic_checks, + semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, }; use crate::bindings::models; @@ -46,16 +45,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { attachments::registration::register(module)?; refactor::registration::register(module)?; plan_output::registration::register(module)?; - planner_changes::registration::register(module)?; - crate::bindings::_helpers::planner_migrations::registration::register(module)?; - crate::bindings::_helpers::planner_phases::registration::register(module)?; - crate::bindings::_helpers::planner_warehouse::registration::register(module)?; - planner_identity::registration::register(module)?; - planner_selection::registration::register(module)?; - planner_scope::registration::register(module)?; - crate::bindings::_helpers::planner_scenario::registration::register(module)?; - planner_strategy::registration::register(module)?; - crate::bindings::_helpers::planner_resolution::registration::register(module)?; + planner_registration::registration::register(module)?; references::registration::register(module)?; authored_values::registration::register(module)?; auditing::registration::register(module)?; From 4e1ea99f2ed240704d09aa0924a61db8419bfedf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 03:02:22 +0100 Subject: [PATCH 05/11] refactor: host dbt interop argument, config, profile and reference glue natively Parse sqb dbt execution flags with the native argparse engine over an exported parser table, resolve dbt config overrides, vars, plan options, manifest path, interop adapter, dbt-profile connections and the compile-time dbt reference resolver in Rust, and render dbt init progress with a native reporter. The replaced Python modules are deleted. --- .../src/argv/_helpers/dbt_execution.rs | 77 ++++ crates/sqlbuild-cli/src/argv/_helpers/mod.rs | 1 + .../sqlbuild-cli/src/argv/_helpers/parsing.rs | 29 +- .../sqlbuild-cli/src/argv/_helpers/table.rs | 20 +- crates/sqlbuild-cli/src/argv/constants.rs | 6 + .../src/argv/dbt_execution_table.json | 230 +++++++++++ crates/sqlbuild-cli/src/argv/main/mod.rs | 1 + .../main/parse_dbt_execution_arguments.rs | 48 +++ crates/sqlbuild-cli/src/argv/models.rs | 20 + crates/sqlbuild-cli/src/argv/tests.rs | 2 + crates/sqlbuild-cli/src/argv/tests/helpers.rs | 5 + .../test_parse_dbt_execution_arguments.rs | 78 ++++ .../sqlbuild-cli/src/argv/tests/test_types.rs | 11 +- .../src/dbt_cli/main/dbt_flag_value.rs | 8 + crates/sqlbuild-core/src/dbt_cli/main/mod.rs | 1 + crates/sqlbuild-core/src/dbt_cli/tests.rs | 2 + .../src/dbt_cli/tests/test_dbt_flag_value.rs | 36 ++ .../src/dbt_cli/tests/test_types.rs | 7 + .../_helpers/audit_command/command.rs | 2 +- .../_helpers/build_command/planning.rs | 2 +- .../bindings/_helpers/check/check_command.rs | 2 +- .../_helpers/cli/progress_reporters.rs | 56 +++ .../bindings/_helpers/clone/clone_command.rs | 2 +- .../_helpers/command_runtime/connection.rs | 6 +- .../_helpers/dbt_command/auto_init.rs | 6 +- .../bindings/_helpers/dbt_command/command.rs | 7 +- .../src/bindings/_helpers/dbt_command/init.rs | 12 +- .../src/bindings/_helpers/dbt_interop/cli.rs | 2 +- .../src/bindings/_helpers/dbt_interop/mod.rs | 3 + .../bindings/_helpers/dbt_interop/pipeline.rs | 13 +- .../_helpers/dbt_interop/pipeline_prologue.rs | 99 ++--- .../_helpers/dbt_interop/plan_output.rs | 30 +- .../dbt_interop/profile_connection.rs | 206 ++++++++++ .../dbt_interop/reference_resolver.rs | 376 ++++++++++++++++++ .../_helpers/dbt_interop/registration.rs | 3 + .../_helpers/dbt_interop/runtime_config.rs | 367 +++++++++++++++++ .../bindings/_helpers/diff_report/command.rs | 2 +- .../bindings/_helpers/plan_command/plan.rs | 2 +- .../bindings/_helpers/query_command/dag.rs | 2 +- .../_helpers/scenario_command/setup.rs | 2 +- .../bindings/_helpers/seed_command/command.rs | 2 +- .../_helpers/sql_test_command/test_command.rs | 2 +- src/sqlbuild/_native.pyi | 44 ++ ...classes.py => _builtin_adapter_classes.py} | 0 .../adapter/discovery/main/builtins.py | 2 +- .../_helpers/build_planning/defer_clone.py | 4 +- .../_helpers/planning/external_refs.py | 46 --- .../classes/dbt_init_progress_reporter.py | 35 -- .../dbt/_helpers/cli/arg_parser.py | 124 ------ .../dbt/_helpers/planning/runtime.py | 162 -------- .../dbt/_helpers/profile/connection.py | 154 ------- .../dbt/_helpers/runtime/progress.py | 12 - .../classes/dbt_compile_reference_resolver.py | 123 ------ .../integrations/dbt/main/cli/__init__.py | 1 - .../dbt/main/cli/_parse_execution_args.py | 15 - .../dbt/main/cli/_validate_execution_args.py | 23 -- .../integrations/dbt/main/config/__init__.py | 1 - .../main/config/_parse_config_overrides.py | 10 - .../dbt/main/config/_resolve_plan_options.py | 15 - .../dbt/main/config/_resolve_vars_mapping.py | 12 - .../dbt/main/manifest/__init__.py | 1 - .../main/manifest/_resolve_manifest_path.py | 14 - .../build_compile_reference_resolver.py | 21 - .../integrations/dbt/main/profile/__init__.py | 1 - .../profile/_resolve_connection_config.py | 25 -- .../dbt/main/profile/profile_connection.py | 28 -- .../dbt/main/profile/profile_init.py | 21 - .../dbt/main/runtime/_report_progress.py | 11 - .../main/runtime/_resolve_interop_adapter.py | 16 - .../src/sqlbuild/integrations/dbt/helpers.py | 4 +- .../_helpers/runtime/test_connection.py | 6 +- .../cli/commands/main/dbt/test_dbt.py | 4 +- .../cli/commands/main/dbt/test_init.py | 22 +- .../cli/commands/main/execution/helpers.py | 5 +- .../compiler/compile/_test_helpers.py | 4 +- .../planner/_helpers/resolve/test_refs.py | 5 +- .../integrations/dbt/profile/_test_types.py | 16 + .../integrations/dbt/profile/helpers.py | 30 ++ .../dbt/profile/test_connection.py | 84 ++++ .../sqlbuild/integrations/dbt/test_args.py | 3 +- .../sqlbuild/integrations/dbt/test_config.py | 9 +- 81 files changed, 1857 insertions(+), 1044 deletions(-) create mode 100644 crates/sqlbuild-cli/src/argv/_helpers/dbt_execution.rs create mode 100644 crates/sqlbuild-cli/src/argv/dbt_execution_table.json create mode 100644 crates/sqlbuild-cli/src/argv/main/parse_dbt_execution_arguments.rs create mode 100644 crates/sqlbuild-cli/src/argv/tests/test_parse_dbt_execution_arguments.rs create mode 100644 crates/sqlbuild-core/src/dbt_cli/main/dbt_flag_value.rs create mode 100644 crates/sqlbuild-core/src/dbt_cli/tests/test_dbt_flag_value.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_connection.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs rename src/sqlbuild/adapter/discovery/main/{builtin_adapter_classes.py => _builtin_adapter_classes.py} (100%) delete mode 100644 src/sqlbuild/cli/commands/_helpers/planning/external_refs.py delete mode 100644 src/sqlbuild/cli/commands/classes/dbt_init_progress_reporter.py delete mode 100644 src/sqlbuild/integrations/dbt/_helpers/cli/arg_parser.py delete mode 100644 src/sqlbuild/integrations/dbt/_helpers/planning/runtime.py delete mode 100644 src/sqlbuild/integrations/dbt/_helpers/profile/connection.py delete mode 100644 src/sqlbuild/integrations/dbt/_helpers/runtime/progress.py delete mode 100644 src/sqlbuild/integrations/dbt/classes/dbt_compile_reference_resolver.py delete mode 100644 src/sqlbuild/integrations/dbt/main/cli/__init__.py delete mode 100644 src/sqlbuild/integrations/dbt/main/cli/_parse_execution_args.py delete mode 100644 src/sqlbuild/integrations/dbt/main/cli/_validate_execution_args.py delete mode 100644 src/sqlbuild/integrations/dbt/main/config/__init__.py delete mode 100644 src/sqlbuild/integrations/dbt/main/config/_parse_config_overrides.py delete mode 100644 src/sqlbuild/integrations/dbt/main/config/_resolve_plan_options.py delete mode 100644 src/sqlbuild/integrations/dbt/main/config/_resolve_vars_mapping.py delete mode 100644 src/sqlbuild/integrations/dbt/main/manifest/__init__.py delete mode 100644 src/sqlbuild/integrations/dbt/main/manifest/_resolve_manifest_path.py delete mode 100644 src/sqlbuild/integrations/dbt/main/manifest/build_compile_reference_resolver.py delete mode 100644 src/sqlbuild/integrations/dbt/main/profile/__init__.py delete mode 100644 src/sqlbuild/integrations/dbt/main/profile/_resolve_connection_config.py delete mode 100644 src/sqlbuild/integrations/dbt/main/profile/profile_connection.py delete mode 100644 src/sqlbuild/integrations/dbt/main/profile/profile_init.py delete mode 100644 src/sqlbuild/integrations/dbt/main/runtime/_report_progress.py delete mode 100644 src/sqlbuild/integrations/dbt/main/runtime/_resolve_interop_adapter.py create mode 100644 tests/unit/src/sqlbuild/integrations/dbt/profile/helpers.py create mode 100644 tests/unit/src/sqlbuild/integrations/dbt/profile/test_connection.py diff --git a/crates/sqlbuild-cli/src/argv/_helpers/dbt_execution.rs b/crates/sqlbuild-cli/src/argv/_helpers/dbt_execution.rs new file mode 100644 index 0000000000..9e6c50435b --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/_helpers/dbt_execution.rs @@ -0,0 +1,77 @@ +//! The plain host and namespace reading of the `sqb dbt` execution parser. + +use sqlbuild_core::dbt_cli::models::DbtParsedFlags; +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +use crate::argv::models::{Namespace, TypeFailure, Value}; +use crate::argv::types::ArgvHost; + +/// The execution parser declares no converters and no version. +pub(crate) struct PlainHost; + +impl ArgvHost for PlainHost { + fn convert(&mut self, _converter: &str, text: &str) -> Result { + Ok(Value::Str(text.to_owned())) + } + + fn repr(&self, text: &str) -> String { + python_str_repr(text) + } + + fn program_version(&mut self) -> String { + String::new() + } +} + +fn texts(namespace: &Namespace, name: &str) -> Vec { + match namespace.value(name) { + Value::List(items) => items.into_iter().filter_map(string).collect(), + _ => Vec::new(), + } +} + +fn text(namespace: &Namespace, name: &str) -> Option { + string(namespace.value(name)) +} + +fn string(value: Value) -> Option { + match value { + Value::Str(text) => Some(text), + _ => None, + } +} + +fn flag(namespace: &Namespace, name: &str) -> bool { + namespace.value(name) == Value::Bool(true) +} + +/// The parsed flags, the strings after `--` appended to the passthrough positional. +pub(crate) fn parsed_flags(namespace: &Namespace, tail: Vec) -> DbtParsedFlags { + let mut dbt_passthrough: Vec = texts(namespace, "dbt_passthrough"); + dbt_passthrough.extend(tail); + DbtParsedFlags { + select: texts(namespace, "select"), + exclude: texts(namespace, "exclude"), + vars: text(namespace, "vars"), + threads: text(namespace, "threads"), + full_refresh: flag(namespace, "full_refresh"), + event_time_start: text(namespace, "event_time_start"), + event_time_end: text(namespace, "event_time_end"), + project_dir: text(namespace, "project_dir"), + profiles_dir: text(namespace, "profiles_dir"), + profile: text(namespace, "profile"), + target: text(namespace, "target"), + target_path: text(namespace, "target_path"), + state: text(namespace, "state"), + indirect_selection: text(namespace, "indirect_selection"), + defer: flag(namespace, "defer"), + start_cursor_ts: text(namespace, "start_cursor_ts"), + end_cursor_ts: text(namespace, "end_cursor_ts"), + start_cursor_int: text(namespace, "start_cursor_int"), + end_cursor_int: text(namespace, "end_cursor_int"), + defer_to: text(namespace, "defer_to"), + fail_fast: flag(namespace, "fail_fast"), + hard_copy: flag(namespace, "hard_copy"), + dbt_passthrough, + } +} diff --git a/crates/sqlbuild-cli/src/argv/_helpers/mod.rs b/crates/sqlbuild-cli/src/argv/_helpers/mod.rs index 4575f034fb..22763ff51e 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/mod.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod dbt_execution; pub(crate) mod formatting; pub(crate) mod namespace; pub(crate) mod normalization; diff --git a/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs b/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs index 7f7476b267..4ff0d45fc2 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs @@ -81,8 +81,22 @@ impl Run<'_> { &mut self, parser: &ParserSpec, args: Vec, - mut namespace: Namespace, + namespace: Namespace, ) -> Result { + match self.known_args(parser, args, namespace) { + Ok(known) => Ok(known), + Err(Failure::Argument(message)) => Err(self.error(parser, &message)), + Err(Failure::Exit(exit)) => Err(exit), + } + } + + /// `parse_known_args` for a parser whose `error` raises: the `ArgumentError` text, not an exit. + pub(crate) fn known_args( + &mut self, + parser: &ParserSpec, + args: Vec, + mut namespace: Namespace, + ) -> Result { for action in &parser.actions { let (Some(dest), Default::Value(default)) = (&action.dest, &action.default) else { continue; @@ -92,16 +106,11 @@ impl Run<'_> { } } let level: Level<'_, '_, '_> = Level::new(self, parser, args, namespace); - match level.parse() { - Ok((mut namespace, mut extras)) => { - if let Some(unrecognized) = namespace.unrecognized.take() { - extras.extend(unrecognized); - } - Ok((namespace, extras)) - } - Err(Failure::Argument(message)) => Err(self.error(parser, &message)), - Err(Failure::Exit(exit)) => Err(exit), + let (mut namespace, mut extras): KnownArgs = level.parse()?; + if let Some(unrecognized) = namespace.unrecognized.take() { + extras.extend(unrecognized); } + Ok((namespace, extras)) } } diff --git a/crates/sqlbuild-cli/src/argv/_helpers/table.rs b/crates/sqlbuild-cli/src/argv/_helpers/table.rs index 0783155dbe..94b61666df 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/table.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/table.rs @@ -5,12 +5,15 @@ use std::sync::OnceLock; use serde_json::Value as Json; -use crate::argv::constants::{COMMAND_TABLE, SUPPRESS_DEST}; +use crate::argv::constants::{ + COMMAND_TABLE, DBT_COMMAND_PLACEHOLDER, DBT_EXECUTION_TABLE, SUPPRESS_DEST, +}; use crate::argv::models::{ ActionKind, ActionSpec, ChoiceHelp, Default, GroupSpec, HelpText, Nargs, ParserSpec, Value, }; static ROOT: OnceLock = OnceLock::new(); +static DBT_EXECUTION: OnceLock = OnceLock::new(); /// The root `sqb` parser. pub(crate) fn root_parser() -> &'static ParserSpec { @@ -20,6 +23,21 @@ pub(crate) fn root_parser() -> &'static ParserSpec { }) } +/// The `sqb dbt ` execution parser with the command named in its texts. +pub(crate) fn dbt_execution_parser(command: &str) -> ParserSpec { + let mut parser: ParserSpec = DBT_EXECUTION + .get_or_init(|| { + let table: Json = serde_json::from_str(DBT_EXECUTION_TABLE).unwrap_or(Json::Null); + TableReader { next_id: 0 }.parser(&table) + }) + .clone(); + parser.prog = parser.prog.replace(DBT_COMMAND_PLACEHOLDER, command); + parser.description = parser + .description + .map(|description| description.replace(DBT_COMMAND_PLACEHOLDER, command)); + parser +} + /// Reads parser specs, numbering every action across the whole tree. struct TableReader { next_id: usize, diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 1094ff3510..4ddca6dcee 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -2,6 +2,12 @@ /// The exported `sqb` parser tree: every parser, group and action in `add_argument` order. pub(crate) const COMMAND_TABLE: &str = include_str!("command_table.json"); +/// The exported `sqb dbt plan|run|build` execution parser, its command name a placeholder. +pub(crate) const DBT_EXECUTION_TABLE: &str = include_str!("dbt_execution_table.json"); +/// The placeholder of the dbt command name in the execution parser's texts. +pub(crate) const DBT_COMMAND_PLACEHOLDER: &str = "{command}"; +/// `sqb dbt` execution arguments after this separator go to dbt verbatim. +pub(crate) const DBT_EXECUTION_SEPARATOR: &str = "--"; /// `argparse.SUPPRESS` as a destination. pub(crate) const SUPPRESS_DEST: &str = "==SUPPRESS=="; /// The placeholder of the installed version in the `--version` text. diff --git a/crates/sqlbuild-cli/src/argv/dbt_execution_table.json b/crates/sqlbuild-cli/src/argv/dbt_execution_table.json new file mode 100644 index 0000000000..5b8395e8e8 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/dbt_execution_table.json @@ -0,0 +1,230 @@ +{ + "prog": "sqb dbt {command}", + "description": "Run `dbt {command}` with SQLBuild interop.", + "epilog": "Pass raw dbt operational flags after `--`, e.g. `-- --log-level debug`.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "extend", + "options": [ + "--select", + "-s" + ], + "dest": "select", + "nargs": "+", + "default": [], + "group": 1 + }, + { + "kind": "extend", + "options": [ + "--exclude" + ], + "dest": "exclude", + "nargs": "+", + "default": [], + "group": 1 + }, + { + "kind": "store", + "options": [ + "--vars" + ], + "dest": "vars", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--threads" + ], + "dest": "threads", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--full-refresh" + ], + "dest": "full_refresh", + "nargs": 0, + "const": true, + "default": false, + "group": 1 + }, + { + "kind": "store", + "options": [ + "--event-time-start" + ], + "dest": "event_time_start", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--event-time-end" + ], + "dest": "event_time_end", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--project-dir" + ], + "dest": "project_dir", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--profiles-dir" + ], + "dest": "profiles_dir", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--profile" + ], + "dest": "profile", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--target" + ], + "dest": "target", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--target-path" + ], + "dest": "target_path", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--state" + ], + "dest": "state", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--indirect-selection" + ], + "dest": "indirect_selection", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--defer" + ], + "dest": "defer", + "nargs": 0, + "const": true, + "default": false, + "group": 1 + }, + { + "kind": "store", + "options": [ + "--defer-to" + ], + "dest": "defer_to", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--start-cursor-ts" + ], + "dest": "start_cursor_ts", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--end-cursor-ts" + ], + "dest": "end_cursor_ts", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--start-cursor-int" + ], + "dest": "start_cursor_int", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--end-cursor-int" + ], + "dest": "end_cursor_int", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--fail-fast" + ], + "dest": "fail_fast", + "nargs": 0, + "const": true, + "default": false, + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--hard-copy" + ], + "dest": "hard_copy", + "nargs": 0, + "const": true, + "default": false, + "group": 1 + }, + { + "kind": "store", + "dest": "dbt_passthrough", + "nargs": "*", + "help": "raw dbt flags after `--`, forwarded verbatim", + "metavar": "-- DBT_ARGS", + "group": 0 + } + ] +} diff --git a/crates/sqlbuild-cli/src/argv/main/mod.rs b/crates/sqlbuild-cli/src/argv/main/mod.rs index cb9812b068..42e5fe2b00 100644 --- a/crates/sqlbuild-cli/src/argv/main/mod.rs +++ b/crates/sqlbuild-cli/src/argv/main/mod.rs @@ -1 +1,2 @@ pub mod parse_cli_arguments; +pub mod parse_dbt_execution_arguments; diff --git a/crates/sqlbuild-cli/src/argv/main/parse_dbt_execution_arguments.rs b/crates/sqlbuild-cli/src/argv/main/parse_dbt_execution_arguments.rs new file mode 100644 index 0000000000..203bba8896 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/main/parse_dbt_execution_arguments.rs @@ -0,0 +1,48 @@ +//! Parse `sqb dbt plan|run|build` arguments as their `argparse` parser did. + +use sqlbuild_core::dbt_cli::models::DbtParsedFlags; + +use crate::argv::_helpers::dbt_execution::{PlainHost, parsed_flags}; +use crate::argv::_helpers::parsing::{Failure, KnownArgs, Run}; +use crate::argv::_helpers::table::dbt_execution_parser; +use crate::argv::constants::DBT_EXECUTION_SEPARATOR; +use crate::argv::models::{ + ArgvSettings, DbtExecutionFailure, DbtExecutionOutcome, Namespace, ParserSpec, +}; + +/// Parse the declared flags before the first `--`; everything after it is passed to dbt. +pub fn parse_dbt_execution_arguments( + command: &str, + args: &[String], + settings: ArgvSettings, +) -> DbtExecutionOutcome { + let (head, tail): (&[String], &[String]) = + match args.iter().position(|arg| arg == DBT_EXECUTION_SEPARATOR) { + Some(separator) => (&args[..separator], &args[separator + 1..]), + None => (args, &[]), + }; + let parser: ParserSpec = dbt_execution_parser(command); + let mut host: PlainHost = PlainHost; + let mut run: Run<'_> = Run { + settings: &settings, + host: &mut host, + writes: Vec::new(), + }; + let known: Result = + run.known_args(&parser, head.to_vec(), Namespace::default()); + let parsed: Result = match known { + Ok((namespace, unknown)) if unknown.is_empty() => { + Ok(parsed_flags(&namespace, tail.to_vec())) + } + Ok((_, unknown)) => Err(DbtExecutionFailure::Unrecognized(unknown)), + Err(Failure::Argument(message)) => Err(DbtExecutionFailure::Argument(format!( + "{}: {message}", + parser.prog + ))), + Err(Failure::Exit(exit)) => Err(DbtExecutionFailure::Exit(exit.0)), + }; + DbtExecutionOutcome { + writes: run.writes, + parsed, + } +} diff --git a/crates/sqlbuild-cli/src/argv/models.rs b/crates/sqlbuild-cli/src/argv/models.rs index 70042fdbea..f7da7be220 100644 --- a/crates/sqlbuild-cli/src/argv/models.rs +++ b/crates/sqlbuild-cli/src/argv/models.rs @@ -2,6 +2,8 @@ use std::collections::HashMap; +use sqlbuild_core::dbt_cli::models::DbtParsedFlags; + /// How many command-line strings an action consumes. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Nargs { @@ -160,3 +162,21 @@ pub struct ArgvOutcome { pub writes: Vec<(Stream, String)>, pub parsed: Result, } + +/// Why a `sqb dbt` execution parse stopped before producing flags. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DbtExecutionFailure { + /// `-h` printed the help: `SystemExit` with this status. + Exit(i64), + /// `ArgumentParser.error`: the parser's `prog`, then the `ArgumentError` text. + Argument(String), + /// Strings before `--` that no declared flag consumed. + Unrecognized(Vec), +} + +/// The writes of a `sqb dbt` execution parse, then its flags or why it stopped. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DbtExecutionOutcome { + pub writes: Vec<(Stream, String)>, + pub parsed: Result, +} diff --git a/crates/sqlbuild-cli/src/argv/tests.rs b/crates/sqlbuild-cli/src/argv/tests.rs index 2499094a55..0289a8fd3d 100644 --- a/crates/sqlbuild-cli/src/argv/tests.rs +++ b/crates/sqlbuild-cli/src/argv/tests.rs @@ -2,6 +2,8 @@ mod helpers; #[path = "tests/test_parse_cli_arguments.rs"] mod parse_cli_arguments; +#[path = "tests/test_parse_dbt_execution_arguments.rs"] +mod parse_dbt_execution_arguments; #[path = "tests/test_types.rs"] mod test_types; #[path = "tests/test_text_wrap.rs"] diff --git a/crates/sqlbuild-cli/src/argv/tests/helpers.rs b/crates/sqlbuild-cli/src/argv/tests/helpers.rs index ee18b96a67..b735659aeb 100644 --- a/crates/sqlbuild-cli/src/argv/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/argv/tests/helpers.rs @@ -21,3 +21,8 @@ pub(super) fn strings(items: &[&str]) -> Value { .collect(), ) } + +/// Owned copies of command-line strings. +pub(super) fn owned(items: &[&str]) -> Vec { + items.iter().map(|item| (*item).to_owned()).collect() +} diff --git a/crates/sqlbuild-cli/src/argv/tests/test_parse_dbt_execution_arguments.rs b/crates/sqlbuild-cli/src/argv/tests/test_parse_dbt_execution_arguments.rs new file mode 100644 index 0000000000..16ae463877 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/tests/test_parse_dbt_execution_arguments.rs @@ -0,0 +1,78 @@ +use sqlbuild_core::dbt_cli::models::DbtParsedFlags; + +use crate::argv::main::parse_dbt_execution_arguments::parse_dbt_execution_arguments; +use crate::argv::models::{ArgvSettings, DbtExecutionFailure}; +use crate::argv::tests::helpers::owned; +use crate::argv::tests::test_types::DbtExecutionTestCase; + +#[test] +fn given_dbt_execution_arguments_when_parsing_then_flags_or_failures_match_argparse() { + let test_cases = [ + DbtExecutionTestCase { + description: "declared flags, stray positionals and the -- tail", + args: &[ + "-s", + "a", + "b", + "--full-refresh", + "extra", + "--", + "--log-level", + "debug", + ], + expected_parsed: Ok(DbtParsedFlags { + select: owned(&["a", "b"]), + full_refresh: true, + dbt_passthrough: owned(&["extra", "--log-level", "debug"]), + ..DbtParsedFlags::default() + }), + }, + DbtExecutionTestCase { + description: "an ambiguous abbreviation lists its matches", + args: &["--targ=prod"], + expected_parsed: Err(DbtExecutionFailure::Argument( + "sqb dbt run: ambiguous option: --targ=prod could match --target, --target-path" + .to_owned(), + )), + }, + DbtExecutionTestCase { + description: "unique abbreviations extend the selection", + args: &["x", "--sel", "a", "--se", "y"], + expected_parsed: Ok(DbtParsedFlags { + select: owned(&["a", "y"]), + dbt_passthrough: owned(&["x"]), + ..DbtParsedFlags::default() + }), + }, + DbtExecutionTestCase { + description: "an unknown option is unrecognized", + args: &["--bogus", "--", "--bogus-too"], + expected_parsed: Err(DbtExecutionFailure::Unrecognized(owned(&["--bogus"]))), + }, + DbtExecutionTestCase { + description: "a missing value is an argument error under the command's prog", + args: &["--select"], + expected_parsed: Err(DbtExecutionFailure::Argument( + "sqb dbt run: argument --select/-s: expected at least one argument".to_owned(), + )), + }, + DbtExecutionTestCase { + description: "help exits successfully", + args: &["-h"], + expected_parsed: Err(DbtExecutionFailure::Exit(0)), + }, + ]; + + for test_case in test_cases { + let settings: ArgvSettings = ArgvSettings { + columns: 100, + use_color: false, + }; + let outcome = parse_dbt_execution_arguments("run", &owned(test_case.args), settings); + assert_eq!( + outcome.parsed, test_case.expected_parsed, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/argv/tests/test_types.rs b/crates/sqlbuild-cli/src/argv/tests/test_types.rs index d500108f92..1be086a89b 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_types.rs @@ -1,6 +1,8 @@ use sqlbuild_core::text::main::python_str_repr::python_str_repr; -use crate::argv::models::{TypeFailure, Value}; +use sqlbuild_core::dbt_cli::models::DbtParsedFlags; + +use crate::argv::models::{DbtExecutionFailure, TypeFailure, Value}; use crate::argv::types::ArgvHost; /// Integer conversion and a fixed version, standing in for the interpreter. @@ -46,3 +48,10 @@ pub(super) struct WrapTestCase { pub(super) indent: &'static str, pub(super) expected_lines: &'static [&'static str], } + +/// `sqb dbt` execution arguments and the parse result they produce. +pub(super) struct DbtExecutionTestCase { + pub(super) description: &'static str, + pub(super) args: &'static [&'static str], + pub(super) expected_parsed: Result, +} diff --git a/crates/sqlbuild-core/src/dbt_cli/main/dbt_flag_value.rs b/crates/sqlbuild-core/src/dbt_cli/main/dbt_flag_value.rs new file mode 100644 index 0000000000..269517d216 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/main/dbt_flag_value.rs @@ -0,0 +1,8 @@ +//! The value after a dbt flag in routed dbt arguments. + +/// The argument after the first exact `flag`, when there is one. +#[must_use] +pub fn dbt_flag_value(args: &[String], flag: &str) -> Option { + let index: usize = args.iter().position(|arg| arg == flag)?; + args.get(index + 1).cloned() +} diff --git a/crates/sqlbuild-core/src/dbt_cli/main/mod.rs b/crates/sqlbuild-core/src/dbt_cli/main/mod.rs index c7e471d790..c9c9248b42 100644 --- a/crates/sqlbuild-core/src/dbt_cli/main/mod.rs +++ b/crates/sqlbuild-core/src/dbt_cli/main/mod.rs @@ -1,5 +1,6 @@ pub mod build_dbt_argv; pub mod choose_dbt_config; pub mod dbt_debug_args; +pub mod dbt_flag_value; pub mod resolve_dbt_executable; pub mod route_dbt_interop_args; diff --git a/crates/sqlbuild-core/src/dbt_cli/tests.rs b/crates/sqlbuild-core/src/dbt_cli/tests.rs index 0a5b3d14a0..fa2de38fca 100644 --- a/crates/sqlbuild-core/src/dbt_cli/tests.rs +++ b/crates/sqlbuild-core/src/dbt_cli/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/test_dbt_cli.rs"] mod dbt_cli; +#[path = "tests/test_dbt_flag_value.rs"] +mod dbt_flag_value; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-core/src/dbt_cli/tests/test_dbt_flag_value.rs b/crates/sqlbuild-core/src/dbt_cli/tests/test_dbt_flag_value.rs new file mode 100644 index 0000000000..c26a3aac40 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/tests/test_dbt_flag_value.rs @@ -0,0 +1,36 @@ +use crate::dbt_cli::main::dbt_flag_value::dbt_flag_value; +use crate::dbt_cli::tests::test_types::FlagValueTestCase; + +#[test] +fn given_routed_dbt_args_when_reading_a_flag_then_returns_the_first_value() { + let test_cases = [ + FlagValueTestCase { + description: "the value after the first occurrence", + args: &["--target", "dev", "--target", "prod"], + flag: "--target", + expected_value: Some("dev"), + }, + FlagValueTestCase { + description: "a trailing flag has no value", + args: &["--select", "a", "--target"], + flag: "--target", + expected_value: None, + }, + FlagValueTestCase { + description: "only exact flags match", + args: &["--target=dev"], + flag: "--target", + expected_value: None, + }, + ]; + + for test_case in test_cases { + let args: Vec = test_case.args.iter().map(|arg| (*arg).to_owned()).collect(); + assert_eq!( + dbt_flag_value(&args, test_case.flag).as_deref(), + test_case.expected_value, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/dbt_cli/tests/test_types.rs b/crates/sqlbuild-core/src/dbt_cli/tests/test_types.rs index 037be62406..c8a161337f 100644 --- a/crates/sqlbuild-core/src/dbt_cli/tests/test_types.rs +++ b/crates/sqlbuild-core/src/dbt_cli/tests/test_types.rs @@ -20,3 +20,10 @@ pub(super) struct ConfigTestCase { pub(super) choice: Result, pub(super) expected_choice: Result, } + +pub(super) struct FlagValueTestCase { + pub(super) description: &'static str, + pub(super) args: &'static [&'static str], + pub(super) flag: &'static str, + pub(super) expected_value: Option<&'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index 67cdf32b3f..c6c24751b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const AUDIT_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._audit"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index e486e528b1..da72502d25 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const CONSTANTS_MODULE: &str = "sqlbuild.cli.commands.constants"; const DEFER_CLONE_MODULE: &str = "sqlbuild.cli.commands._helpers.build_planning.defer_clone"; -const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 09b1c82ae6..6638a667b3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -24,7 +24,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const CHECK_CORE_MODULE: &str = "sqlbuild.cli.commands._helpers.check.core"; -const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs index f73973bdc3..3e519738d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs @@ -1,6 +1,8 @@ //! Warehouse connection and planning phase progress reporters. use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Mutex, PoisonError}; +use std::time::Instant; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; @@ -199,7 +201,61 @@ impl PlanningProgressReporter { } } +/// Render timed `sqb dbt init` phase progress. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct DbtInitProgressReporter { + stream: Py, + status: Py, + started: AtomicBool, + phase_start: Mutex>, +} + +#[pymethods] +impl DbtInitProgressReporter { + #[new] + #[pyo3(signature = (*, stream, use_color))] + fn new(stream: &Bound<'_, PyAny>, use_color: bool) -> PyResult { + Ok(Self { + stream: stream.clone().unbind(), + status: status_reporter(stream, use_color)?.unbind(), + started: AtomicBool::new(false), + phase_start: Mutex::new(None), + }) + } + + fn start(&self, py: Python<'_>, message: &str) -> PyResult<()> { + if !self.started.load(Ordering::SeqCst) { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + stream.call_method1("write", ("\n",))?; + stream.call_method0("flush")?; + self.started.store(true, Ordering::SeqCst); + } + *self + .phase_start + .lock() + .unwrap_or_else(PoisonError::into_inner) = Some(Instant::now()); + self.status.bind(py).call_method1("start", (message,))?; + Ok(()) + } + + fn complete(&self, py: Python<'_>, message: &str) -> PyResult<()> { + let phase_start: Option = self + .phase_start + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + let elapsed_seconds: f64 = phase_start.map_or(0.0, |start| start.elapsed().as_secs_f64()); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("message", format!("{message} ({elapsed_seconds:.2}s)"))?; + self.status + .bind(py) + .call_method("complete", (), Some(&keywords))?; + Ok(()) + } +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; + module.add_class::()?; module.add_class::() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index 0a41321cf0..b7d33d8a59 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -33,7 +33,7 @@ const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const NO_DESTINATION: (&str, &str) = ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 0ab1a951c6..094d94104d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -18,7 +18,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -const DBT_PROFILE_MODULE: &str = "sqlbuild.integrations.dbt.main.profile.profile_connection"; +const DBT_PROFILE_MODULE: &str = "sqlbuild._native"; const DBT_ERRORS_MODULE: &str = "sqlbuild.integrations.dbt.exceptions"; const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; @@ -60,7 +60,7 @@ fn dbt_profile_error(py: Python<'_>) -> PyErr { } /// The config with a referenced dbt profile resolved and user overrides kept on top. -fn with_dbt_profile<'py>( +pub(crate) fn with_dbt_profile<'py>( raw_config: &Bound<'py, PyDict>, config: Bound<'py, PyDict>, (project_dir, adapter_name): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -100,7 +100,7 @@ fn with_dbt_profile<'py>( } /// Make a relative DuckDB database path relative to the project directory. -fn anchor_duckdb_database( +pub(crate) fn anchor_duckdb_database( config: &Bound<'_, PyDict>, (project_dir, adapter_name): (&Bound<'_, PyAny>, &str), ) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs index 7f00d3c1dd..7d505418ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs @@ -15,8 +15,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::dbt_command::support::{color_enabled, flag, path, stream}; const DISCOVERY_CONSTANTS: &str = "sqlbuild.compiler.discovery.constants"; -const PROGRESS_REPORTER_MODULE: &str = "sqlbuild.cli.commands.classes.dbt_init_progress_reporter"; -const PROFILE_INIT_MODULE: &str = "sqlbuild.integrations.dbt.main.profile.profile_init"; +const PROGRESS_REPORTER_MODULE: &str = "sqlbuild._native"; +const PROFILE_INIT_MODULE: &str = "sqlbuild.integrations.dbt._helpers.profile.init"; const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; fn project_config_found(directory: &Bound<'_, PyAny>) -> PyResult { @@ -122,7 +122,7 @@ pub(crate) fn ensure_twin_project<'py>( ], )?; let result: Bound<'py, PyAny> = call_with( - &imported(py, PROFILE_INIT_MODULE, "run_dbt_profile_init")?, + &imported(py, PROFILE_INIT_MODULE, "build_dbt_init_project")?, &[("request", request)], )?; let project_file: String = result.getattr("project_file")?.str()?.to_string(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs index c32e8340ba..5c41b0c1cb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs @@ -12,9 +12,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::dbt_command::auto_init::{args_tuple, ensure_twin_project}; use crate::bindings::_helpers::dbt_command::debug::run_dbt_debug; use crate::bindings::_helpers::dbt_command::support::{color_enabled, effective_dir, flag, stream}; +use crate::bindings::_helpers::dbt_interop::runtime_config::validate_dbt_execution_args; const NATIVE_MODULE: &str = "sqlbuild._native"; -const VALIDATE_ARGS_MODULE: &str = "sqlbuild.integrations.dbt.main.cli._validate_execution_args"; const PLAN_MODULE: &str = "sqlbuild._native"; const EXECUTE_MODULE: &str = "sqlbuild._native"; const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; @@ -148,10 +148,7 @@ fn run_dbt_command(py: Python<'_>, options: Option>) -> PyResu let command: Bound<'_, PyAny> = arguments.value("command")?; let args_value: Bound<'_, PyAny> = arguments.value("args")?; let no_color: bool = arguments.flag("no_color")?; - call_with( - &imported(py, VALIDATE_ARGS_MODULE, "validate_dbt_execution_args")?, - &[("command", command.clone()), ("args", args_value.clone())], - )?; + validate_dbt_execution_args(&command, &args_value)?; let args: Vec = args_value .try_iter()? .map(|arg| arg?.extract()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs index 32b16cc2e4..17cf8c3bcc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs @@ -15,8 +15,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::dbt_command::support::{color_enabled, flag, path, stream}; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const PROGRESS_REPORTER_MODULE: &str = "sqlbuild.cli.commands.classes.dbt_init_progress_reporter"; -const PROFILE_INIT_MODULE: &str = "sqlbuild.integrations.dbt.main.profile.profile_init"; +const PROGRESS_REPORTER_MODULE: &str = "sqlbuild._native"; +const PROFILE_INIT_MODULE: &str = "sqlbuild.integrations.dbt._helpers.profile.init"; const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; fn missing_project(py: Python<'_>) -> PyErr { @@ -118,15 +118,11 @@ fn run_dbt_init_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult< ], )?; call_with( - &imported( - py, - PROFILE_INIT_MODULE, - "_validate_dbt_profile_init_request", - )?, + &imported(py, PROFILE_INIT_MODULE, "validate_dbt_init_project")?, &[("request", init_request.clone())], )?; let result: Bound<'_, PyAny> = call_with( - &imported(py, PROFILE_INIT_MODULE, "run_dbt_profile_init")?, + &imported(py, PROFILE_INIT_MODULE, "build_dbt_init_project")?, &[("request", init_request)], )?; let style: CliStyle = CliStyle { use_color }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs index 534a637084..eabf25abdb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs @@ -312,7 +312,7 @@ fn route_dbt_interop_args<'py>( } /// `Path(raw).expanduser()` resolved, relative paths against the resolved project root. -fn resolved_path<'py>( +pub(crate) fn resolved_path<'py>( project_root: &Bound<'py, PyAny>, raw_value: Option, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs index c512d186d8..f60b21c5ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs @@ -10,7 +10,10 @@ pub(crate) mod pipeline_prologue; pub(crate) mod plan; pub(crate) mod plan_output; pub(crate) mod profile; +pub(crate) mod profile_connection; +pub(crate) mod reference_resolver; pub(crate) mod registration; +pub(crate) mod runtime_config; pub(crate) mod runtime_execution; pub(crate) mod runtime_results; pub(crate) mod sqlbuild_work; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs index 4212c4663c..04fbb52289 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs @@ -13,6 +13,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::dbt_interop::pipeline_prologue::progress; +use crate::bindings::_helpers::dbt_interop::runtime_config::config_overrides; const MODELS: &str = "sqlbuild.integrations.dbt.models"; const PLAN_OPTIONS: (&[&str], &[&str]) = ( @@ -323,12 +324,12 @@ fn debug_dbt_from_project(py: Python<'_>, options: Option>) -> "discover_project_inputs", &[("project_dir", args.value("project_dir")?)], )?; - let overrides = imported( - py, - "sqlbuild.integrations.dbt.main.config._parse_config_overrides", - "parse_dbt_config_overrides", - )? - .call1((args.value("args")?,))?; + let dbt_args: Vec = args + .value("args")? + .try_iter()? + .map(|arg| arg?.extract()) + .collect::>()?; + let overrides = config_overrides(py, &dbt_args)?; let resolved = native( py, "resolve_dbt_config", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs index f1c0637a8d..1c7f11b482 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs @@ -7,6 +7,11 @@ use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::PyDict; + +use crate::bindings::_helpers::dbt_interop::reference_resolver::DbtCompileReferenceResolver; +use crate::bindings::_helpers::dbt_interop::runtime_config::{ + interop_adapter, manifest_path, parse_dbt_execution_args, plan_options, vars_mapping, +}; use pyo3::{pyfunction, wrap_pyfunction}; use std::time::Instant; @@ -66,55 +71,29 @@ pub(crate) fn resolve_dbt_execution_invocation<'py>( if !stream.is_truthy()? { stream = py.import("sys")?.getattr("stdout")?; } - let parsed = call_with( - &imported( - py, - "sqlbuild.integrations.dbt.main.cli._parse_execution_args", - "parse_dbt_execution_args", - )?, - &[ - ("command", request.getattr("command")?), - ("args", request.getattr("args")?), - ], - )?; + let command = request.getattr("command")?; + let parsed = parse_dbt_execution_args(&command, &request.getattr("args")?)?; let routed = call_with( &imported(py, "sqlbuild._native", "route_dbt_interop_args")?, - &[("command", request.getattr("command")?), ("parsed", parsed)], + &[("command", command), ("parsed", parsed)], )?; + let project_dir = request.getattr("project_dir")?; let discovered = call_with( &imported(py, "sqlbuild._native", "discover_project_inputs")?, - &[("project_dir", request.getattr("project_dir")?)], - )?; - let options = call_with( - &imported( - py, - "sqlbuild.integrations.dbt.main.config._resolve_plan_options", - "resolve_dbt_plan_options", - )?, - &[ - ("project_dir", request.getattr("project_dir")?), - ("discovered_inputs", discovered.clone()), - ("dbt_args", routed.getattr("dbt_args")?), - ], - )?; - let vars = call_with( - &imported( - py, - "sqlbuild.integrations.dbt.main.config._resolve_vars_mapping", - "resolve_dbt_vars_mapping", - )?, - &[ - ( - "project_config", - discovered.getattr("project_config")?.getattr("dbt")?, - ), - ( - "local_config", - discovered.getattr("local_config")?.getattr("dbt")?, - ), - ("dbt_args", routed.getattr("dbt_args")?), - ], + &[("project_dir", project_dir.clone())], )?; + let dbt_args: Vec = routed + .getattr("dbt_args")? + .try_iter()? + .map(|arg| arg?.extract()) + .collect::>()?; + let options = plan_options(&project_dir, &discovered, &dbt_args)?; + let vars = vars_mapping( + &discovered.getattr("project_config")?.getattr("dbt")?, + &discovered.getattr("local_config")?.getattr("dbt")?, + &dbt_args, + )? + .into_any(); let mut runner = request.getattr("dbt_runner")?; if !runner.is_truthy()? { runner = call_with( @@ -189,14 +168,7 @@ fn load_compiled_dbt_manifest<'py>( completed(&callback, "Compiled dbt project.", start)?; let start = Instant::now(); progress(&callback, "Loading dbt manifest...")?; - let path = call_with( - &imported( - py, - "sqlbuild.integrations.dbt.main.manifest._resolve_manifest_path", - "resolve_dbt_manifest_path", - )?, - &[("options", args.value("dbt_options")?)], - )?; + let path = manifest_path(&args.value("dbt_options")?)?; let manifest = call_with( &imported(py, "sqlbuild._native", "load_dbt_manifest_index")?, &[("manifest_path", path)], @@ -224,25 +196,12 @@ fn compile_dbt_interop_project<'py>( ("local_config", discovered.getattr("local_config")?), ], )?; - let adapter = call_with( - &imported( - py, - "sqlbuild.integrations.dbt.main.runtime._resolve_interop_adapter", - "resolve_dbt_interop_adapter", - )?, - &[ - ("adapter_name", name.clone()), - ("project_dir", args.value("project_dir")?), - ], - )?; - let resolver = call_with( - &imported( - py, - "sqlbuild.integrations.dbt.classes.dbt_compile_reference_resolver", - "DbtCompileReferenceResolver", - )?, - &[("dbt_manifest", args.value("manifest")?)], - )?; + let adapter = interop_adapter(&name, &args.value("project_dir")?)?; + let resolver = Bound::new( + py, + DbtCompileReferenceResolver::new(Some(args.value("manifest")?.unbind())), + )? + .into_any(); let project = call_with( &imported(py, "sqlbuild._native", "build_compiled_project")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index 897a2de7c8..7e9b95a669 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -1,10 +1,13 @@ //! Plan selected SQLBuild work against the live adapter and attach it to the interop plan. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::PyDict; +use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::build_command::support::{boolean, keyword_relay, none}; +use crate::bindings::_helpers::command_runtime::connection::{ + anchor_duckdb_database, with_dbt_profile, +}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; @@ -35,19 +38,20 @@ pub(crate) fn connection_config<'py>( &imported(py, "sqlbuild._native", "build_effective_connection_config")?, &[("discovered_inputs", discovered.clone())], )?; - let config = call_with( - &imported( - py, - "sqlbuild.integrations.dbt.main.profile._resolve_connection_config", - "resolve_connection_config", - )?, - &[ - ("raw_config", raw), - ("project_dir", environment.getattr("project_dir")?), - ("adapter_name", environment.getattr("adapter_name")?), - ("discovered_inputs", discovered.clone()), - ], + let raw: Bound<'_, PyDict> = raw.cast_into()?; + let project_dir = environment.getattr("project_dir")?; + let adapter_name = environment.getattr("adapter_name")?; + let config: Bound<'_, PyDict> = with_dbt_profile( + &raw, + raw.copy()?, + (&project_dir, &adapter_name), + (&discovered, &py.None().into_bound(py)), + )?; + anchor_duckdb_database( + &config, + (&project_dir, &adapter_name.str()?.extract::()?), )?; + let config = config.into_any(); call_with( &imported(py, "sqlbuild._native", "apply_command_warehouse")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_connection.rs new file mode 100644 index 0000000000..4e451be153 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_connection.rs @@ -0,0 +1,206 @@ +//! Resolve a `source = "dbt_profile"` connection through the user's dbt profile. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::dbt_interop::cli::resolved_path; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const PROFILE_LOAD_MODULE: &str = "sqlbuild.integrations.dbt._helpers.profile.load"; +const PROFILE_RENDER_MODULE: &str = "sqlbuild.integrations.dbt._helpers.profile.render"; +const DBT_ERRORS_MODULE: &str = "sqlbuild.integrations.dbt.exceptions"; +const DBT_PROFILE_SOURCE: &str = "dbt_profile"; +const PROFILE_CONNECTION_OPTIONS: (&[&str], &[&str]) = ( + &[ + "raw_config", + "project_dir", + "adapter_name", + "project_config", + ], + &["cli_vars"], +); +const NON_EMPTY_MESSAGE: &str = "dbt_profile connection values must be non-empty strings"; + +fn profile_error(py: Python<'_>, message: String) -> PyErr { + match imported(py, DBT_ERRORS_MODULE, "DbtProfileError") + .and_then(|class| class.call1((message,))) + { + Ok(instance) => PyErr::from_value(instance), + Err(error) => error, + } +} + +/// A raw connection value as a stripped non-empty string, or `None` when absent. +fn optional_str(config: &Bound<'_, PyDict>, key: &str) -> PyResult> { + let py: Python<'_> = config.py(); + let Some(value) = config.get_item(key)? else { + return Ok(None); + }; + if value.is_none() { + return Ok(None); + } + let Ok(value) = value.cast_into::() else { + return Err(profile_error(py, NON_EMPTY_MESSAGE.to_owned())); + }; + let text: String = value.extract()?; + let stripped: &str = python_strip(&text); + if stripped.is_empty() { + return Err(profile_error(py, NON_EMPTY_MESSAGE.to_owned())); + } + Ok(Some(stripped.to_owned())) +} + +fn or_config(value: Option, fallback: &Bound<'_, PyAny>) -> PyResult> { + if value.is_some() || fallback.is_none() { + return Ok(value); + } + Ok(Some(fallback.extract()?)) +} + +fn load<'py>(py: Python<'py>, name: &str) -> PyResult> { + imported(py, PROFILE_LOAD_MODULE, name) +} + +/// The dbt profile reference rendered and normalized into SQLBuild adapter config. +fn profile_connection<'py>( + sqlbuild_project_dir: &Bound<'py, PyAny>, + (dbt_project_dir, profiles_dir): (Bound<'py, PyAny>, Bound<'py, PyAny>), + (profile_name, target_name): (Option, Option), + cli_vars: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = sqlbuild_project_dir.py(); + let dbt_project_dir: Bound<'py, PyAny> = if dbt_project_dir.is_none() { + sqlbuild_project_dir.clone() + } else { + dbt_project_dir + }; + let profiles_dir: Bound<'py, PyAny> = if profiles_dir.is_truthy()? { + profiles_dir + } else { + load(py, "default_profiles_dir")?.call0()? + }; + let profile_name: Bound<'py, PyAny> = match profile_name { + Some(name) => PyString::new(py, &name).into_any(), + None => call_with( + &load(py, "load_dbt_project_metadata")?, + &[("project_dir", dbt_project_dir.clone())], + )? + .getattr("profile_name")?, + }; + let profile: Bound<'py, PyAny> = call_with( + &load(py, "load_raw_dbt_profile")?, + &[ + ("profiles_dir", profiles_dir.clone()), + ("profile_name", profile_name), + ], + )?; + let target_name: Bound<'py, PyAny> = target_name.map_or_else( + || py.None().into_bound(py), + |name| PyString::new(py, &name).into_any(), + ); + let selected: Bound<'py, PyAny> = call_with( + &load(py, "select_dbt_profile_output")?, + &[("profile", profile), ("target_name", target_name)], + )?; + let resolved: Bound<'py, PyAny> = call_with( + &imported( + py, + PROFILE_RENDER_MODULE, + "render_selected_dbt_profile_output", + )?, + &[ + ("selected", selected), + ("project_dir", dbt_project_dir), + ("profiles_dir", profiles_dir), + ("cli_vars", cli_vars), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "normalize_dbt_profile_connection")?, + &[("resolved", resolved)], + ) +} + +/// Resolve a raw connection config if it references a dbt profile. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_raw_dbt_profile_connection<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_raw_dbt_profile_connection", + options, + PROFILE_CONNECTION_OPTIONS, + )?; + let raw_config: Bound<'py, PyDict> = arguments.value("raw_config")?.cast_into()?; + let project_dir: Bound<'py, PyAny> = arguments.value("project_dir")?; + let adapter_name: Bound<'py, PyAny> = arguments.value("adapter_name")?; + let project_config: Bound<'py, PyAny> = arguments.value("project_config")?; + let cli_vars: Bound<'py, PyAny> = arguments.value("cli_vars")?; + let is_profile: bool = match raw_config.get_item("source")? { + Some(source) => source.eq(DBT_PROFILE_SOURCE)?, + None => false, + }; + if !is_profile { + return Ok(py.None().into_bound(py)); + } + let dbt_config: Bound<'py, PyAny> = project_config.getattr("dbt")?; + let dbt_project_dir: Bound<'py, PyAny> = resolved_path( + &project_dir, + or_config( + optional_str(&raw_config, "project_dir")?, + &dbt_config.getattr("project_dir")?, + )?, + )?; + let profiles_dir: Bound<'py, PyAny> = resolved_path( + &project_dir, + or_config( + optional_str(&raw_config, "profiles_dir")?, + &dbt_config.getattr("profiles_dir")?, + )?, + )?; + let profile_name: Option = optional_str(&raw_config, "profile")?; + let target_name: Option = or_config( + optional_str(&raw_config, "target")?, + &dbt_config.getattr("target")?, + )?; + let cli_vars: Bound<'py, PyAny> = if cli_vars.is_none() { + PyDict::new(py).into_any() + } else { + cli_vars + }; + let resolved: Bound<'py, PyAny> = profile_connection( + &project_dir, + (dbt_project_dir, profiles_dir), + (profile_name, target_name), + cli_vars, + )?; + let resolved_adapter: Bound<'py, PyAny> = resolved.getattr("adapter")?; + if !resolved_adapter.eq(&adapter_name)? { + return Err(profile_error( + py, + format!( + "dbt profile adapter type does not match SQLBuild project adapter: dbt={}, sqlbuild={}", + resolved_adapter.str()?, + adapter_name.str()? + ), + )); + } + Ok(resolved) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + resolve_raw_dbt_profile_connection, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs new file mode 100644 index 0000000000..1eb5d3c1a8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs @@ -0,0 +1,376 @@ +//! The compile-time external reference resolver backed by a dbt manifest index. + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PySet, PySetMethods, PyString, PyTuple, PyTupleMethods}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use std::collections::HashSet; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, within, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; +const VALIDATE_OPTIONS: (&[&str], &[&str]) = ( + &[ + "ref_kind", + "ref_name", + "ref_package", + "owner_relative_sql_path", + ], + &[], +); +const MANIFEST_FILENAME: &str = "manifest.json"; +const MODEL_CONFLICT_CODE: &str = "C218"; +const SOURCE_CONFLICT_CODE: &str = "C216"; +const SEED_CONFLICT_CODE: &str = "C217"; + +fn compile_input_error(py: Python<'_>, message: String, code: &str) -> PyErr { + let created = || -> PyResult { + let error: Bound<'_, PyAny> = imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")? + .call( + (message,), + Some(&keywords( + py, + &[("code", PyString::new(py, code).into_any())], + )?), + )?; + Ok(PyErr::from_value(error)) + }; + created().unwrap_or_else(|failure| failure) +} + +fn text(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + value.getattr(name)?.extract() +} + +/// External reference resolver backed by a dbt manifest index. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct DbtCompileReferenceResolver { + dbt_manifest: Option>, +} + +impl DbtCompileReferenceResolver { + pub(crate) fn new(dbt_manifest: Option>) -> Self { + Self { dbt_manifest } + } + + fn manifest<'py>(&self, py: Python<'py>) -> Option> { + self.dbt_manifest + .as_ref() + .map(|manifest| manifest.bind(py).clone()) + .filter(|manifest| !manifest.is_none()) + } + + fn manifest_or_none<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { + self.manifest(py) + .unwrap_or_else(|| py.None().into_bound(py)) + } +} + +/// Bare `(source_name, name)` or seed names that occur more than once, in a first pass. +fn duplicates(keys: Vec) -> HashSet { + let mut seen: HashSet = HashSet::new(); + let mut duplicate: HashSet = HashSet::new(); + for key in keys { + if !seen.insert(key.clone()) { + duplicate.insert(key); + } + } + duplicate +} + +#[pymethods] +impl DbtCompileReferenceResolver { + #[new] + #[pyo3(signature = (*, dbt_manifest))] + fn py_new(dbt_manifest: Option>) -> Self { + Self::new(dbt_manifest) + } + + #[pyo3(signature = (*, known_model_names))] + fn validate_model_names( + &self, + py: Python<'_>, + known_model_names: Bound<'_, PyAny>, + ) -> Result<(), PyErr> { + call_with( + &imported(py, NATIVE_MODULE, "validate_compile_dbt_model_names")?, + &[ + ("known_model_names", known_model_names), + ("dbt_manifest", self.manifest_or_none(py)), + ], + )?; + Ok(()) + } + + #[pyo3(signature = (*, known_model_names))] + fn extend_sql_test_model_names<'py>( + &self, + py: Python<'py>, + known_model_names: Bound<'py, PyAny>, + ) -> PyResult> { + let names: Bound<'py, PySet> = PySet::empty(py)?; + let Some(manifest) = self.manifest(py) else { + return Ok(names); + }; + for item in manifest + .getattr("models_by_name")? + .call_method0("items")? + .try_iter()? + { + let (model_name, matches): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + if matches.len()? != 1 { + continue; + } + if known_model_names.contains(&model_name)? { + return Err(compile_input_error( + py, + format!( + "dbt model fixture '__ref__{}' conflicts with a SQLBuild model", + model_name.str()? + ), + MODEL_CONFLICT_CODE, + )); + } + names.add(model_name)?; + } + for key in manifest.getattr("models_by_package_and_name")?.try_iter()? { + let key: Bound<'py, PyTuple> = key?.cast_into()?; + let qualified: String = + format!("{}__{}", key.get_item(0)?.str()?, key.get_item(1)?.str()?); + if known_model_names.contains(&qualified)? { + return Err(compile_input_error( + py, + format!( + "dbt model fixture '__ref__{qualified}' conflicts with a SQLBuild model" + ), + MODEL_CONFLICT_CODE, + )); + } + names.add(qualified)?; + } + Ok(names) + } + + #[pyo3(signature = (*, known_source_names))] + fn extend_sql_test_source_names<'py>( + &self, + py: Python<'py>, + known_source_names: Bound<'py, PyAny>, + ) -> PyResult> { + let names: Bound<'py, PySet> = PySet::empty(py)?; + let Some(manifest) = self.manifest(py) else { + return Ok(names); + }; + let sources: Vec> = manifest + .getattr("sources_by_unique_id")? + .call_method0("values")? + .try_iter()? + .collect::>()?; + let mut keys: Vec = Vec::with_capacity(sources.len()); + for source in &sources { + keys.push(format!( + "{}\0{}", + text(source, "source_name")?, + text(source, "name")? + )); + } + let duplicate: HashSet = duplicates(keys.clone()); + for (source, key) in sources.iter().zip(keys) { + let source_name: String = text(source, "source_name")?; + let name: String = text(source, "name")?; + let bare: String = format!("{source_name}__{name}"); + if known_source_names.contains(&bare)? { + return Err(compile_input_error( + py, + format!( + "dbt source fixture '__source__{bare}' conflicts with a SQLBuild source" + ), + SOURCE_CONFLICT_CODE, + )); + } + if !duplicate.contains(&key) { + names.add(bare)?; + } + names.add(format!( + "{}__{source_name}__{name}", + text(source, "package_name")? + ))?; + } + Ok(names) + } + + #[pyo3(signature = (*, known_seed_names))] + fn extend_sql_test_seed_names<'py>( + &self, + py: Python<'py>, + known_seed_names: Bound<'py, PyAny>, + ) -> PyResult> { + let names: Bound<'py, PySet> = PySet::empty(py)?; + let Some(manifest) = self.manifest(py) else { + return Ok(names); + }; + let seeds: Vec> = manifest + .getattr("seeds_by_unique_id")? + .call_method0("values")? + .try_iter()? + .collect::>()?; + let mut seed_names: Vec = Vec::with_capacity(seeds.len()); + for seed in &seeds { + seed_names.push(text(seed, "name")?); + } + let duplicate: HashSet = duplicates(seed_names.clone()); + for (seed, name) in seeds.iter().zip(seed_names) { + if known_seed_names.contains(&name)? { + return Err(compile_input_error( + py, + format!("dbt seed fixture '__seed__{name}' conflicts with a SQLBuild seed"), + SEED_CONFLICT_CODE, + )); + } + if !duplicate.contains(&name) { + names.add(&name)?; + } + names.add(format!("{}__{name}", text(seed, "package_name")?))?; + } + Ok(names) + } + + #[pyo3(signature = (**options))] + fn validate_reference( + &self, + py: Python<'_>, + options: Option>, + ) -> Result<(), PyErr> { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "validate_reference", options, VALIDATE_OPTIONS)?; + let reference: Bound<'_, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "CompileSqlReference")?, + &[ + ("ref_kind", arguments.value("ref_kind")?), + ("ref_name", arguments.value("ref_name")?), + ("ref_package", arguments.value("ref_package")?), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "validate_compile_dbt_model_reference")?, + &[ + ("reference", reference), + ( + "model_relative_path", + arguments.value("owner_relative_sql_path")?, + ), + ("dbt_manifest", self.manifest_or_none(py)), + ], + )?; + Ok(()) + } + + #[pyo3(signature = (*, ref_kind, ref_name, ref_package))] + fn resolve_reference<'py>( + &self, + py: Python<'py>, + ref_kind: Bound<'py, PyAny>, + ref_name: Bound<'py, PyAny>, + ref_package: Bound<'py, PyAny>, + ) -> PyResult> { + call_with( + &imported(py, NATIVE_MODULE, "resolve_dbt_reference_relation")?, + &[ + ("manifest", self.manifest_or_none(py)), + ("ref_kind", ref_kind), + ("ref_name", ref_name), + ("ref_package", ref_package), + ], + ) + } +} + +/// A dbt-backed external reference resolver from manifest contents, or `None` without them. +#[pyfunction] +#[pyo3(signature = (*, manifest_contents))] +pub(crate) fn build_compile_reference_resolver<'py>( + py: Python<'py>, + manifest_contents: Bound<'py, PyAny>, +) -> PyResult> { + if manifest_contents.is_none() { + return Ok(py.None().into_bound(py)); + } + let manifest: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "build_compile_dbt_manifest_index")?, + &[("manifest_contents", manifest_contents)], + )?; + Ok(Bound::new( + py, + DbtCompileReferenceResolver::new(Some(manifest.unbind())), + )? + .into_any()) +} + +/// `/manifest.json` under the project, when that file exists. +fn manifest_contents<'py>( + project_dir: &Bound<'py, PyAny>, + target_path: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let raw: Bound<'py, PyAny> = imported(py, "pathlib", "Path")?.call1((target_path,))?; + let resolved: Bound<'py, PyAny> = if raw.call_method0("is_absolute")?.is_truthy()? { + raw + } else { + project_dir.div(raw)? + }; + let manifest: Bound<'py, PyAny> = resolved.div(MANIFEST_FILENAME)?; + if !manifest.call_method0("is_file")?.is_truthy()? { + return Ok(py.None().into_bound(py)); + } + call_with( + &manifest.getattr("read_text")?, + &[("encoding", PyString::new(py, "utf-8").into_any())], + ) +} + +/// Build the external reference resolver configured for a discovered project. +#[pyfunction] +#[pyo3(signature = (*, project_dir, discovered_inputs))] +pub(crate) fn resolve_external_sql_reference_resolver<'py>( + project_dir: Bound<'py, PyAny>, + discovered_inputs: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let target_path: Bound<'py, PyAny> = discovered_inputs + .getattr("project_config")? + .getattr("dbt")? + .getattr("target_path")?; + let contents: Bound<'py, PyAny> = if target_path.is_none() { + py.None().into_bound(py) + } else { + let lifecycle: Bound<'py, PyAny> = call_with( + &imported(py, LIFECYCLE_MODULE, "OperationLifecycle")?, + &[ + ("operation_kind", PyString::new(py, "project").into_any()), + ( + "operation_name", + PyString::new(py, "external_manifest_discovery").into_any(), + ), + ], + )?; + within(&lifecycle, |_| { + manifest_contents(&project_dir, &target_path) + })? + }; + build_compile_reference_resolver(py, contents) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(build_compile_reference_resolver, module)?)?; + module.add_function(wrap_pyfunction!( + resolve_external_sql_reference_resolver, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs index fc5cb1f0ef..0b9b4076f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs @@ -14,6 +14,9 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { orchestration::register(module)?; plan::register(module)?; profile::register(module)?; + super::profile_connection::register(module)?; + super::reference_resolver::register(module)?; + super::runtime_config::register(module)?; super::runtime_execution::register(module)?; super::pipeline_prologue::register(module)?; super::plan_output::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs new file mode 100644 index 0000000000..574b733d81 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs @@ -0,0 +1,367 @@ +//! `sqb dbt` execution flags, routed dbt config overrides, vars, plan options and the adapter. + +use pyo3::exceptions::PySystemExit; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_cli::argv::main::parse_dbt_execution_arguments::parse_dbt_execution_arguments; +use sqlbuild_cli::argv::models::{ArgvSettings, DbtExecutionFailure, DbtExecutionOutcome, Stream}; +use sqlbuild_core::dbt_cli::errors::DbtCliError; +use sqlbuild_core::dbt_cli::main::dbt_flag_value::dbt_flag_value; +use sqlbuild_core::dbt_cli::models::DbtParsedFlags; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::dbt_interop::cli::{raised, resolved_path}; + +const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; +const NATIVE_MODULE: &str = "sqlbuild._native"; +const BUILTIN_ADAPTERS_MODULE: &str = "sqlbuild.adapter.discovery.main.builtins"; +const PROJECT_ADAPTERS_MODULE: &str = "sqlbuild.adapter.discovery.main.project_adapters"; +const ARGUMENT_ERROR: &str = "DbtInteropArgumentError"; +const CONFIG_ERROR: &str = "DbtInteropConfigError"; +const ARGUMENT_CODE: &str = "C238"; +const UNRECOGNIZED_HELP: &str = "SQLBuild flags go before `--`; pass raw dbt flags after `--`."; +const PARSER_HELP: &str = "Run `-h` to list supported flags; pass raw dbt flags after `--`."; +const EXECUTION_COMMANDS: [&str; 3] = ["plan", "run", "build"]; +const DISPLAY_FLAGS: [&str; 3] = ["--json", "--verbose", "-v"]; +const DEFER_FLAG: &str = "--defer"; +const VARS_FLAG: &str = "--vars"; +const STATE_FLAG: &str = "--state"; +const MANIFEST_FILENAME: &str = "manifest.json"; +const DEFAULT_TARGET_DIR: &str = "target"; + +fn none(py: Python<'_>) -> Bound<'_, PyAny> { + py.None().into_bound(py) +} + +fn optional_text<'py>(py: Python<'py>, value: Option) -> Bound<'py, PyAny> { + value.map_or_else(|| none(py), |text| PyString::new(py, &text).into_any()) +} + +fn flag(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { + PyBool::new(py, value).to_owned().into_any() +} + +fn argument_error(py: Python<'_>, message: String, help: &'static str) -> PyErr { + raised( + py, + ARGUMENT_ERROR, + DbtCliError { + message, + code: Some(ARGUMENT_CODE), + help: Some(help), + }, + ) +} + +fn write_outputs(py: Python<'_>, outcome: &DbtExecutionOutcome) -> PyResult<()> { + let sys: Bound<'_, PyModule> = py.import("sys")?; + for (stream, text) in &outcome.writes { + let name: &str = match stream { + Stream::Stdout => "stdout", + Stream::Stderr => "stderr", + }; + sys.getattr(name)?.call_method1("write", (text,))?; + } + Ok(()) +} + +/// Parse raw `sqb dbt ` tokens into declared flags and a `--` tail. +pub(crate) fn parsed_flags( + py: Python<'_>, + command: &str, + args: &[String], +) -> PyResult { + let columns: i64 = py + .import("shutil")? + .call_method0("get_terminal_size")? + .getattr("columns")? + .extract()?; + let settings: ArgvSettings = ArgvSettings { + columns, + use_color: false, + }; + let outcome: DbtExecutionOutcome = parse_dbt_execution_arguments(command, args, settings); + write_outputs(py, &outcome)?; + match outcome.parsed { + Ok(flags) => Ok(flags), + Err(DbtExecutionFailure::Exit(code)) => Err(PySystemExit::new_err((code,))), + Err(DbtExecutionFailure::Argument(message)) => { + Err(argument_error(py, message, PARSER_HELP)) + } + Err(DbtExecutionFailure::Unrecognized(unknown)) => Err(argument_error( + py, + format!( + "unrecognized option(s) for sqb dbt {command}: {}", + unknown.join(" ") + ), + UNRECOGNIZED_HELP, + )), + } +} + +fn parsed_args_model<'py>(py: Python<'py>, flags: DbtParsedFlags) -> PyResult> { + call_with( + &imported(py, DBT_MODELS, "DbtInteropParsedArgs")?, + &[ + ("select", PyTuple::new(py, flags.select)?.into_any()), + ("exclude", PyTuple::new(py, flags.exclude)?.into_any()), + ("vars", optional_text(py, flags.vars)), + ("threads", optional_text(py, flags.threads)), + ("full_refresh", flag(py, flags.full_refresh)), + ( + "event_time_start", + optional_text(py, flags.event_time_start), + ), + ("event_time_end", optional_text(py, flags.event_time_end)), + ("project_dir", optional_text(py, flags.project_dir)), + ("profiles_dir", optional_text(py, flags.profiles_dir)), + ("profile", optional_text(py, flags.profile)), + ("target", optional_text(py, flags.target)), + ("target_path", optional_text(py, flags.target_path)), + ("state", optional_text(py, flags.state)), + ( + "indirect_selection", + optional_text(py, flags.indirect_selection), + ), + ("defer", flag(py, flags.defer)), + ("defer_to", optional_text(py, flags.defer_to)), + ("start_cursor_ts", optional_text(py, flags.start_cursor_ts)), + ("end_cursor_ts", optional_text(py, flags.end_cursor_ts)), + ( + "start_cursor_int", + optional_text(py, flags.start_cursor_int), + ), + ("end_cursor_int", optional_text(py, flags.end_cursor_int)), + ("fail_fast", flag(py, flags.fail_fast)), + ("hard_copy", flag(py, flags.hard_copy)), + ( + "dbt_passthrough", + PyTuple::new(py, flags.dbt_passthrough)?.into_any(), + ), + ], + ) +} + +fn texts(values: &Bound<'_, PyAny>) -> PyResult> { + values.try_iter()?.map(|value| value?.extract()).collect() +} + +/// Parse raw `sqb dbt ` tokens into declared flags and a `--` tail. +#[pyfunction] +#[pyo3(signature = (*, command, args))] +pub(crate) fn parse_dbt_execution_args<'py>( + command: &Bound<'py, PyAny>, + args: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = command.py(); + let name: String = command.str()?.extract()?; + parsed_args_model(py, parsed_flags(py, &name, &texts(args)?)?) +} + +/// Parse execution-command args early so `-h` and typos resolve before setup. +pub(crate) fn validate_dbt_execution_args( + command: &Bound<'_, PyAny>, + args: &Bound<'_, PyAny>, +) -> Result<(), PyErr> { + let name: String = command.str()?.extract()?; + if !EXECUTION_COMMANDS.contains(&name.as_str()) { + return Ok(()); + } + let display_free: Vec = texts(args)? + .into_iter() + .filter(|arg| !DISPLAY_FLAGS.contains(&arg.as_str())) + .collect(); + parsed_flags(command.py(), &name, &display_free).map(|_| ()) +} + +/// dbt project, profile, target and target-path overrides from routed dbt args. +pub(crate) fn config_overrides<'py>( + py: Python<'py>, + args: &[String], +) -> PyResult> { + let value = |name: &str| optional_text(py, dbt_flag_value(args, name)); + call_with( + &imported(py, DBT_MODELS, "DbtCliConfigOverrides")?, + &[ + ("project_dir", value("--project-dir")), + ("profiles_dir", value("--profiles-dir")), + ("target", value("--target")), + ("target_path", value("--target-path")), + ], + ) +} + +fn config_error(py: Python<'_>, message: &str) -> PyErr { + raised( + py, + CONFIG_ERROR, + DbtCliError { + message: message.to_owned(), + code: None, + help: None, + }, + ) +} + +fn cli_vars<'py>(py: Python<'py>, raw: Option) -> PyResult> { + let result: Bound<'py, PyDict> = PyDict::new(py); + let Some(raw) = raw else { + return Ok(result); + }; + let parsed: Bound<'py, PyAny> = py.import("yaml")?.call_method1("safe_load", (raw,))?; + if parsed.is_none() { + return Ok(result); + } + let Ok(parsed) = parsed.cast_into::() else { + return Err(config_error(py, "dbt --vars must be a mapping")); + }; + for (key, value) in parsed.iter() { + if !key.is_instance_of::() { + return Err(config_error(py, "dbt --vars keys must be strings")); + } + result.set_item(key, value)?; + } + Ok(result) +} + +/// Merged dbt vars: project config, then local config, then `--vars`. +pub(crate) fn vars_mapping<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, + args: &[String], +) -> PyResult> { + let py: Python<'py> = project_config.py(); + let values: Bound<'py, PyDict> = PyDict::new(py); + values.call_method1("update", (project_config.getattr("vars")?,))?; + values.call_method1("update", (local_config.getattr("vars")?,))?; + values.update(cli_vars(py, dbt_flag_value(args, VARS_FLAG))?.as_mapping())?; + Ok(values) +} + +/// Return merged dbt vars as a structured mapping. +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config, dbt_args))] +fn resolve_dbt_vars_mapping<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, + dbt_args: &Bound<'py, PyAny>, +) -> PyResult> { + vars_mapping(project_config, local_config, &texts(dbt_args)?) +} + +/// Resolved dbt CLI options for `sqb dbt plan|run|build`. +pub(crate) fn plan_options<'py>( + project_dir: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, + args: &[String], +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let project_config: Bound<'py, PyAny> = discovered_inputs + .getattr("project_config")? + .getattr("dbt")?; + let local_config: Bound<'py, PyAny> = + discovered_inputs.getattr("local_config")?.getattr("dbt")?; + let resolved: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_dbt_config")?, + &[ + ("project_root", project_dir.clone()), + ("config", project_config.clone()), + ("local_config", local_config.clone()), + ("overrides", config_overrides(py, args)?), + ("require_project_dir", flag(py, true)), + ], + )?; + let values: Bound<'py, PyDict> = vars_mapping(&project_config, &local_config, args)?; + let vars: Bound<'py, PyAny> = if values.is_empty() { + none(py) + } else { + call_with( + &py.import("json")?.getattr("dumps")?, + &[("obj", values.into_any()), ("sort_keys", flag(py, true))], + )? + }; + call_with( + &imported(py, DBT_MODELS, "DbtCliOptions")?, + &[ + ("project_dir", resolved.getattr("project_dir")?), + ("profiles_dir", resolved.getattr("profiles_dir")?), + ("target", resolved.getattr("target")?), + ("target_path", resolved.getattr("target_path")?), + ("vars", vars), + ( + "state", + resolved_path(project_dir, dbt_flag_value(args, STATE_FLAG))?, + ), + ("defer", flag(py, args.iter().any(|arg| arg == DEFER_FLAG))), + ], + ) +} + +/// The manifest path `dbt compile` writes for these options. +pub(crate) fn manifest_path<'py>(options: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = options.py(); + let project_dir: Bound<'py, PyAny> = options.getattr("project_dir")?; + if project_dir.is_none() { + return Err(config_error(py, "dbt project directory is not configured")); + } + let mut target_path: Bound<'py, PyAny> = options.getattr("target_path")?; + if !target_path.is_truthy()? { + target_path = project_dir.div(DEFAULT_TARGET_DIR)?; + } + target_path.div(MANIFEST_FILENAME) +} + +/// A project adapter of this name, else the built-in one, instantiated for interop planning. +pub(crate) fn interop_adapter<'py>( + adapter_name: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = adapter_name.py(); + let builtin: Bound<'py, PyDict> = + imported(py, BUILTIN_ADAPTERS_MODULE, "builtin_adapter_classes")? + .call0()? + .cast_into()?; + let mut adapter_class: Option> = None; + if !project_dir.is_none() { + let reserved: Bound<'py, PyAny> = py + .import("builtins")? + .getattr("frozenset")? + .call1((builtin.clone(),))?; + let local: Bound<'py, PyDict> = call_with( + &imported(py, PROJECT_ADAPTERS_MODULE, "discover_project_adapters")?, + &[ + ("project_dir", project_dir.clone()), + ("reserved_names", reserved), + ], + )? + .cast_into()?; + adapter_class = local.get_item(adapter_name)?; + } + if adapter_class.is_none() { + adapter_class = builtin.get_item(adapter_name)?; + } + let Some(adapter_class) = adapter_class else { + let mut available: Vec = Vec::new(); + for name in builtin.keys() { + available.push(name.str()?.extract()?); + } + available.sort(); + return Err(config_error( + py, + &format!( + "unknown adapter '{}'. Available built-in adapters: {}.", + adapter_name.str()?, + available.join(", ") + ), + )); + }; + adapter_class.call0() +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(parse_dbt_execution_args, module)?)?; + module.add_function(wrap_pyfunction!(resolve_dbt_vars_mapping, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs index 7860259577..23776640f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -34,7 +34,7 @@ const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const CONTRACT_CONSTANTS_MODULE: &str = "sqlbuild.spec.contracts.constants"; -const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const EXECUTE_MODULE: &str = "sqlbuild.executor.diff.main.execute"; const DIFF_MODELS_MODULE: &str = "sqlbuild.executor.diff.models"; const ADAPTER_ERRORS: (&str, &str) = ("sqlbuild.adapter.contract.exceptions", "AdapterUserError"); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index 2f51f18c81..d9a739d4db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; const PRESENTATION_MODELS: &str = "sqlbuild.presentation.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs index 7997128859..9f2e10cf29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs @@ -12,7 +12,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::query_command::project::{ProjectAdapter, project_adapter}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const PYTHON_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; fn array_length(document: &serde_json::Value, key: &str) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index 5395052fe9..fe9d2e0d21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -30,7 +30,7 @@ const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SCENARIO_CONFIG_MODULE: &str = "sqlbuild._native"; -const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const SCENARIO_MODELS: &str = "sqlbuild.executor.scenario.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 375a2c3dad..729601ba64 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const SEED_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._seed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 9baf8c6a1a..018dbf9435 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -33,7 +33,7 @@ use crate::bindings::_helpers::result_output::integration_values::json_value; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const TEST_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._sql_test"; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 58f3eaa344..5a9059e6dd 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2150,6 +2150,45 @@ def parse_dbt_ls_json_lines(*, stdout: str) -> tuple[DbtLsNode, ...]: ... def route_dbt_interop_args( *, command: DbtInteropCommand | str, parsed: DbtInteropParsedArgs ) -> DbtInteropRoutedArgs: ... +def parse_dbt_execution_args( + *, command: DbtInteropCommand | str, args: Sequence[str] +) -> DbtInteropParsedArgs: ... +def resolve_dbt_vars_mapping( + *, project_config: DbtConfig, local_config: LocalDbtConfig, dbt_args: Sequence[str] +) -> dict[str, object]: ... +def resolve_raw_dbt_profile_connection( + *, + raw_config: dict[str, object], + project_dir: Path, + adapter_name: str, + project_config: ProjectConfig, + cli_vars: dict[str, object] | None = None, +) -> NormalizedDbtProfileConnection | None: ... + +class DbtCompileReferenceResolver: + def __init__(self, *, dbt_manifest: DbtManifestIndex | None) -> None: ... + def validate_model_names(self, *, known_model_names: set[str]) -> None: ... + def extend_sql_test_model_names(self, *, known_model_names: set[str]) -> set[str]: ... + def extend_sql_test_source_names(self, *, known_source_names: set[str]) -> set[str]: ... + def extend_sql_test_seed_names(self, *, known_seed_names: set[str]) -> set[str]: ... + def validate_reference( + self, + *, + ref_kind: str, + ref_name: str, + ref_package: str | None, + owner_relative_sql_path: Path, + ) -> None: ... + def resolve_reference( + self, *, ref_kind: str, ref_name: str, ref_package: str | None + ) -> str | None: ... + +def build_compile_reference_resolver( + *, manifest_contents: str | None +) -> ExternalSqlReferenceResolver | None: ... +def resolve_external_sql_reference_resolver( + *, project_dir: Path, discovered_inputs: DiscoveredProjectInputs +) -> ExternalSqlReferenceResolver | None: ... def resolve_dbt_config( *, project_root: Path, @@ -2331,6 +2370,11 @@ def format_dbt_active_node_status( *, active_nodes: Sequence[tuple[str, float]], now: float ) -> str: ... +class DbtInitProgressReporter: + def __init__(self, *, stream: TextIO, use_color: bool) -> None: ... + def start(self, message: str) -> None: ... + def complete(self, message: str) -> None: ... + class PlanningProgressReporter: def __init__(self, *, stream: TextIO, use_color: bool = False) -> None: ... def on_progress(self, message: str) -> None: ... diff --git a/src/sqlbuild/adapter/discovery/main/builtin_adapter_classes.py b/src/sqlbuild/adapter/discovery/main/_builtin_adapter_classes.py similarity index 100% rename from src/sqlbuild/adapter/discovery/main/builtin_adapter_classes.py rename to src/sqlbuild/adapter/discovery/main/_builtin_adapter_classes.py diff --git a/src/sqlbuild/adapter/discovery/main/builtins.py b/src/sqlbuild/adapter/discovery/main/builtins.py index a9b34e2c2b..cf3e9ba5dd 100644 --- a/src/sqlbuild/adapter/discovery/main/builtins.py +++ b/src/sqlbuild/adapter/discovery/main/builtins.py @@ -1,7 +1,7 @@ """Compatibility entrypoint for built-in adapter class discovery.""" from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.discovery.main.builtin_adapter_classes import ( +from sqlbuild.adapter.discovery.main._builtin_adapter_classes import ( builtin_adapter_classes as _builtin_adapter_classes, ) diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py index 2600991963..dc1b7be2f5 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py @@ -10,14 +10,12 @@ build_compiled_project, build_planner_scope, enforce_model_execution_limit, + resolve_external_sql_reference_resolver, resolve_target_config, run_clone_pipeline_with_options, split_unit_test_selectors, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) from sqlbuild.cli.commands.constants import EXECUTION_LIMIT_BUILD_NOTE from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import ( diff --git a/src/sqlbuild/cli/commands/_helpers/planning/external_refs.py b/src/sqlbuild/cli/commands/_helpers/planning/external_refs.py deleted file mode 100644 index 21657db966..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/planning/external_refs.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Planning support for optional external SQL reference integrations.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.integrations.dbt.main.manifest.build_compile_reference_resolver import ( - build_compile_reference_resolver, -) -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle - - -def resolve_external_sql_reference_resolver( - *, project_dir: Path, discovered_inputs: DiscoveredProjectInputs -) -> ExternalSqlReferenceResolver | None: - """Build the external reference resolver configured for a discovered project.""" - - target_path: str | None = discovered_inputs.project_config.dbt.target_path - if target_path is None: - manifest_contents: str | None = None - else: - with OperationLifecycle( - operation_kind="project", operation_name="external_manifest_discovery" - ): - manifest_contents = _read_dbt_manifest_contents( - project_dir=project_dir, - target_path=target_path, - ) - return build_compile_reference_resolver( - manifest_contents=manifest_contents, - ) - - -def _read_dbt_manifest_contents(*, project_dir: Path, target_path: str | None) -> str | None: - if target_path is None: - return None - raw_target_path: Path = Path(target_path) - resolved_target_path: Path = ( - raw_target_path if raw_target_path.is_absolute() else project_dir / raw_target_path - ) - manifest_path: Path = resolved_target_path / "manifest.json" - if not manifest_path.is_file(): - return None - return manifest_path.read_text(encoding="utf-8") diff --git a/src/sqlbuild/cli/commands/classes/dbt_init_progress_reporter.py b/src/sqlbuild/cli/commands/classes/dbt_init_progress_reporter.py deleted file mode 100644 index ce30fe04a4..0000000000 --- a/src/sqlbuild/cli/commands/classes/dbt_init_progress_reporter.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Progress rendering for `sqb dbt init`.""" - -from __future__ import annotations - -import time -from typing import TextIO - -from sqlbuild._native import TransientStatusReporter - - -class DbtInitProgressReporter: - """Render timed `sqb dbt init` phase progress.""" - - def __init__(self, *, stream: TextIO, use_color: bool) -> None: - self._stream: TextIO = stream - self._status: TransientStatusReporter = TransientStatusReporter( - stream=stream, - use_color=use_color, - ) - self._phase_start: float | None = None - self._started: bool = False - - def start(self, message: str) -> None: - if not self._started: - self._stream.write("\n") - self._stream.flush() - self._started = True - self._phase_start = time.perf_counter() - self._status.start(message) - - def complete(self, message: str) -> None: - phase_start: float | None = self._phase_start - elapsed_seconds: float = 0.0 if phase_start is None else time.perf_counter() - phase_start - self._status.complete(message=f"{message} ({elapsed_seconds:.2f}s)") - self._phase_start = None diff --git a/src/sqlbuild/integrations/dbt/_helpers/cli/arg_parser.py b/src/sqlbuild/integrations/dbt/_helpers/cli/arg_parser.py deleted file mode 100644 index c686b7337e..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/cli/arg_parser.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Declarative argparse front end for `sqb dbt` execution commands.""" - -from __future__ import annotations - -import argparse -from collections.abc import Sequence -from typing import NoReturn - -from sqlbuild.integrations.dbt.exceptions import DbtInteropArgumentError -from sqlbuild.integrations.dbt.models import DbtInteropParsedArgs -from sqlbuild.integrations.dbt.types import DbtInteropCommand - -_DBT_PASSTHROUGH_SEPARATOR: str = "--" - - -def add_dbt_execution_args(parser: argparse.ArgumentParser) -> None: - """Declare category-1 `sqb dbt` flags plus the `--` passthrough positional.""" - - parser.add_argument("--select", "-s", nargs="+", action="extend", default=[]) - parser.add_argument("--exclude", nargs="+", action="extend", default=[]) - parser.add_argument("--vars", default=None) - parser.add_argument("--threads", default=None) - parser.add_argument("--full-refresh", action="store_true", default=False) - parser.add_argument("--event-time-start", default=None) - parser.add_argument("--event-time-end", default=None) - parser.add_argument("--project-dir", default=None) - parser.add_argument("--profiles-dir", default=None) - parser.add_argument("--profile", default=None) - parser.add_argument("--target", default=None) - parser.add_argument("--target-path", default=None) - parser.add_argument("--state", default=None) - parser.add_argument("--indirect-selection", default=None) - parser.add_argument("--defer", action="store_true", default=False) - parser.add_argument("--defer-to", default=None) - parser.add_argument("--start-cursor-ts", default=None) - parser.add_argument("--end-cursor-ts", default=None) - parser.add_argument("--start-cursor-int", default=None) - parser.add_argument("--end-cursor-int", default=None) - parser.add_argument("--fail-fast", action="store_true", default=False) - parser.add_argument("--hard-copy", action="store_true", default=False) - parser.add_argument( - "dbt_passthrough", - nargs="*", - metavar="-- DBT_ARGS", - help="raw dbt flags after `--`, forwarded verbatim", - ) - - -def build_dbt_execution_parser( - *, command: DbtInteropCommand, prog: str | None = None -) -> argparse.ArgumentParser: - """Build the standalone argparse parser declaring flags for a dbt command.""" - - parser: argparse.ArgumentParser = _StrictArgumentParser( - prog=prog or f"sqb dbt {command.value}", - add_help=True, - description=f"Run `dbt {command.value}` with SQLBuild interop.", - epilog="Pass raw dbt operational flags after `--`, e.g. `-- --log-level debug`.", - ) - add_dbt_execution_args(parser) - return parser - - -def parse_dbt_execution_args( - *, command: DbtInteropCommand, args: Sequence[str] -) -> DbtInteropParsedArgs: - """Parse raw `sqb dbt ` tokens into declared flags and a `--` tail.""" - - head, tail = _split_passthrough(args) - parser: argparse.ArgumentParser = build_dbt_execution_parser(command=command) - namespace: argparse.Namespace - unknown: list[str] - namespace, unknown = parser.parse_known_args(head) - if unknown: - raise DbtInteropArgumentError( - f"unrecognized option(s) for sqb dbt {command.value}: {' '.join(unknown)}", - code="C238", - help="SQLBuild flags go before `--`; pass raw dbt flags after `--`.", - ) - leading: tuple[str, ...] = tuple(namespace.dbt_passthrough or ()) - return DbtInteropParsedArgs( - select=tuple(namespace.select), - exclude=tuple(namespace.exclude), - vars=namespace.vars, - threads=namespace.threads, - full_refresh=namespace.full_refresh, - event_time_start=namespace.event_time_start, - event_time_end=namespace.event_time_end, - project_dir=namespace.project_dir, - profiles_dir=namespace.profiles_dir, - profile=namespace.profile, - target=namespace.target, - target_path=namespace.target_path, - state=namespace.state, - indirect_selection=namespace.indirect_selection, - defer=namespace.defer, - defer_to=namespace.defer_to, - start_cursor_ts=namespace.start_cursor_ts, - end_cursor_ts=namespace.end_cursor_ts, - start_cursor_int=namespace.start_cursor_int, - end_cursor_int=namespace.end_cursor_int, - fail_fast=namespace.fail_fast, - hard_copy=namespace.hard_copy, - dbt_passthrough=(*leading, *tail), - ) - - -def _split_passthrough(args: Sequence[str]) -> tuple[tuple[str, ...], tuple[str, ...]]: - tokens: list[str] = list(args) - if _DBT_PASSTHROUGH_SEPARATOR not in tokens: - return tuple(tokens), () - separator: int = tokens.index(_DBT_PASSTHROUGH_SEPARATOR) - return tuple(tokens[:separator]), tuple(tokens[separator + 1 :]) - - -class _StrictArgumentParser(argparse.ArgumentParser): - """argparse parser that raises DbtInteropArgumentError instead of sys.exit.""" - - def error(self, message: str) -> NoReturn: # type: ignore[override] - raise DbtInteropArgumentError( - f"{self.prog}: {message}", - code="C238", - help="Run `-h` to list supported flags; pass raw dbt flags after `--`.", - ) diff --git a/src/sqlbuild/integrations/dbt/_helpers/planning/runtime.py b/src/sqlbuild/integrations/dbt/_helpers/planning/runtime.py deleted file mode 100644 index 18d3aeab87..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/planning/runtime.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Runtime support helpers for dbt interop planning.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import cast - -import yaml - -from sqlbuild._native import resolve_dbt_config -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.adapter.discovery.main.builtin_adapter_classes import builtin_adapter_classes -from sqlbuild.adapter.discovery.main.project_adapters import discover_project_adapters -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt.constants import DBT_DEFER_FLAG -from sqlbuild.integrations.dbt.exceptions import DbtInteropConfigError -from sqlbuild.integrations.dbt.models import DbtCliConfigOverrides, DbtCliOptions, ResolvedDbtConfig -from sqlbuild.spec.contracts.models import DbtConfig, LocalDbtConfig - - -def resolve_dbt_plan_options( - *, project_dir: Path, discovered_inputs: DiscoveredProjectInputs, dbt_args: tuple[str, ...] -) -> DbtCliOptions: - """Resolve dbt CLI options for `sqb dbt plan`.""" - - overrides: DbtCliConfigOverrides = parse_dbt_config_overrides(dbt_args) - resolved: ResolvedDbtConfig = resolve_dbt_config( - project_root=project_dir, - config=discovered_inputs.project_config.dbt, - local_config=discovered_inputs.local_config.dbt, - overrides=overrides, - require_project_dir=True, - ) - return DbtCliOptions( - project_dir=resolved.project_dir, - profiles_dir=resolved.profiles_dir, - target=resolved.target, - target_path=resolved.target_path, - vars=resolve_dbt_vars( - project_config=discovered_inputs.project_config.dbt, - local_config=discovered_inputs.local_config.dbt, - dbt_args=dbt_args, - ), - state=parse_optional_path_flag(args=dbt_args, flag="--state", project_root=project_dir), - defer=DBT_DEFER_FLAG in dbt_args, - ) - - -def resolve_dbt_vars( - *, project_config: DbtConfig, local_config: LocalDbtConfig, dbt_args: tuple[str, ...] -) -> str | None: - """Merge dbt vars from project config, local config, and CLI args.""" - - values: dict[str, object] = resolve_dbt_vars_mapping( - project_config=project_config, - local_config=local_config, - dbt_args=dbt_args, - ) - if not values: - return None - return json.dumps(values, sort_keys=True) - - -def resolve_dbt_vars_mapping( - *, project_config: DbtConfig, local_config: LocalDbtConfig, dbt_args: tuple[str, ...] -) -> dict[str, object]: - """Return merged dbt vars as a structured mapping.""" - - values: dict[str, object] = {} - values.update(project_config.vars) - values.update(local_config.vars) - values.update(_parse_cli_dbt_vars(parse_value_flag(args=dbt_args, flag="--vars"))) - return values - - -def parse_dbt_config_overrides(dbt_args: tuple[str, ...]) -> DbtCliConfigOverrides: - """Parse dbt project/profile/target overrides from routed dbt args.""" - - return DbtCliConfigOverrides( - project_dir=parse_value_flag(args=dbt_args, flag="--project-dir"), - profiles_dir=parse_value_flag(args=dbt_args, flag="--profiles-dir"), - target=parse_value_flag(args=dbt_args, flag="--target"), - target_path=parse_value_flag(args=dbt_args, flag="--target-path"), - ) - - -def parse_value_flag(*, args: tuple[str, ...], flag: str) -> str | None: - """Return the value after a flag, if present.""" - - if flag not in args: - return None - index: int = args.index(flag) - if index + 1 >= len(args): - return None - return args[index + 1] - - -def _parse_cli_dbt_vars(raw_value: str | None) -> dict[str, object]: - if raw_value is None: - return {} - parsed: object = yaml.safe_load(raw_value) - if parsed is None: - return {} - if not isinstance(parsed, dict): - raise DbtInteropConfigError("dbt --vars must be a mapping") - result: dict[str, object] = {} - key: object - value: object - for key, value in parsed.items(): - if not isinstance(key, str): - raise DbtInteropConfigError("dbt --vars keys must be strings") - result[key] = value - return result - - -def parse_optional_path_flag( - *, args: tuple[str, ...], flag: str, project_root: Path -) -> Path | None: - """Return a path flag value resolved relative to the SQLBuild project.""" - - raw_value: str | None = parse_value_flag(args=args, flag=flag) - if raw_value is None: - return None - path: Path = Path(raw_value).expanduser() - if path.is_absolute(): - return path.resolve() - return (project_root.expanduser().resolve() / path).resolve() - - -def resolve_dbt_manifest_path(*, options: DbtCliOptions) -> Path: - """Resolve the manifest path produced by dbt compile.""" - - if options.project_dir is None: - raise DbtInteropConfigError("dbt project directory is not configured") - target_path: Path = options.target_path or (options.project_dir / "target") - return target_path / "manifest.json" - - -def resolve_dbt_interop_adapter( - *, adapter_name: str, project_dir: Path | None = None -) -> BaseAdapter: - """Resolve an adapter for dbt interop runtime planning.""" - - builtin_adapters: dict[str, type[BaseAdapter]] = builtin_adapter_classes() - adapter_class: type[StrictAdapter] | type[BaseAdapter] | None = None - if project_dir is not None: - local_adapters: dict[str, type[StrictAdapter]] = discover_project_adapters( - project_dir=project_dir, - reserved_names=frozenset(builtin_adapters), - ) - adapter_class = local_adapters.get(adapter_name) - if adapter_class is None: - adapter_class = builtin_adapters.get(adapter_name) - if adapter_class is None: - available: tuple[str, ...] = tuple(sorted(builtin_adapters)) - raise DbtInteropConfigError( - f"unknown adapter '{adapter_name}'. Available built-in adapters: " - f"{', '.join(available)}." - ) - return cast(BaseAdapter, adapter_class()) diff --git a/src/sqlbuild/integrations/dbt/_helpers/profile/connection.py b/src/sqlbuild/integrations/dbt/_helpers/profile/connection.py deleted file mode 100644 index 6f47f24aa8..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/profile/connection.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Resolve dbt profile-backed connection dictionaries.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -from sqlbuild._native import normalize_dbt_profile_connection -from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt._helpers.profile.load import ( - default_profiles_dir, - load_dbt_project_metadata, - load_raw_dbt_profile, - select_dbt_profile_output, -) -from sqlbuild.integrations.dbt._helpers.profile.render import render_selected_dbt_profile_output -from sqlbuild.integrations.dbt.exceptions import DbtProfileError -from sqlbuild.integrations.dbt.models import ( - DbtProfileConnectionRequest, - DbtProjectProfileMetadata, - NormalizedDbtProfileConnection, - ResolvedDbtProfileOutput, - SelectedDbtProfileOutput, -) -from sqlbuild.spec.contracts.models import DbtConfig, ProjectConfig - -_DBT_DUCKDB_MEMORY_DATABASE: str = ":memory:" -_DBT_PROFILE_CONNECTION_SOURCE: str = "dbt_profile" -_DBT_PROFILE_CONNECTION_ROUTING_KEYS: frozenset[str] = frozenset( - {"source", "profile", "target", "project_dir", "profiles_dir"} -) - - -def resolve_connection_config( - *, - raw_config: dict[str, object], - project_dir: Path, - adapter_name: str, - discovered_inputs: DiscoveredProjectInputs, -) -> dict[str, object]: - """Resolve dbt interop connection config.""" - - config: dict[str, object] = dict(raw_config) - resolved_dbt_profile: NormalizedDbtProfileConnection | None = ( - resolve_dbt_profile_raw_connection( - raw_config=config, - project_dir=project_dir, - adapter_name=adapter_name, - project_config=discovered_inputs.project_config, - ) - ) - if resolved_dbt_profile is not None: - for warning in resolved_dbt_profile.warnings: - print(f"Warning: {warning}", file=sys.stderr) - user_overrides: dict[str, object] = { - key: value - for key, value in raw_config.items() - if key not in _DBT_PROFILE_CONNECTION_ROUTING_KEYS - } - config = {**resolved_dbt_profile.connection, **user_overrides} - database: object | None = config.get("database") - if ( - adapter_name == BuiltinAdapter.DUCKDB - and isinstance(database, str) - and not Path(database).is_absolute() - and database != _DBT_DUCKDB_MEMORY_DATABASE - ): - config["database"] = str(project_dir / database) - return config - - -def resolve_dbt_profile_connection( - *, request: DbtProfileConnectionRequest -) -> NormalizedDbtProfileConnection: - """Resolve a dbt profile reference into SQLBuild adapter config.""" - - dbt_project_dir: Path = ( - request.dbt_project_dir - if request.dbt_project_dir is not None - else request.sqlbuild_project_dir - ) - profiles_dir: Path = request.profiles_dir or default_profiles_dir() - metadata: DbtProjectProfileMetadata = load_dbt_project_metadata(project_dir=dbt_project_dir) - profile_name: str = request.profile_name or metadata.profile_name - selected: SelectedDbtProfileOutput = select_dbt_profile_output( - profile=load_raw_dbt_profile(profiles_dir=profiles_dir, profile_name=profile_name), - target_name=request.target_name, - ) - resolved: ResolvedDbtProfileOutput = render_selected_dbt_profile_output( - selected=selected, - project_dir=dbt_project_dir, - profiles_dir=profiles_dir, - cli_vars=request.cli_vars, - ) - return normalize_dbt_profile_connection(resolved=resolved) - - -def resolve_dbt_profile_raw_connection( - *, - raw_config: dict[str, object], - project_dir: Path, - adapter_name: str, - project_config: ProjectConfig, - cli_vars: dict[str, object] | None = None, -) -> NormalizedDbtProfileConnection | None: - """Resolve a raw connection config if it references a dbt profile.""" - - if raw_config.get("source") != _DBT_PROFILE_CONNECTION_SOURCE: - return None - dbt_config: DbtConfig = project_config.dbt - dbt_project_dir: Path | None = _resolve_optional_path( - project_root=project_dir, - raw_value=_optional_str(raw_config.get("project_dir")) or dbt_config.project_dir, - ) - profiles_dir: Path | None = _resolve_optional_path( - project_root=project_dir, - raw_value=_optional_str(raw_config.get("profiles_dir")) or dbt_config.profiles_dir, - ) - profile_name: str | None = _optional_str(raw_config.get("profile")) - target_name: str | None = _optional_str(raw_config.get("target")) or dbt_config.target - resolved: NormalizedDbtProfileConnection = resolve_dbt_profile_connection( - request=DbtProfileConnectionRequest( - sqlbuild_project_dir=project_dir, - dbt_project_dir=dbt_project_dir, - profiles_dir=profiles_dir, - profile_name=profile_name, - target_name=target_name, - cli_vars={} if cli_vars is None else cli_vars, - ) - ) - if resolved.adapter != adapter_name: - raise DbtProfileError( - "dbt profile adapter type does not match SQLBuild project adapter: " - f"dbt={resolved.adapter}, sqlbuild={adapter_name}" - ) - return resolved - - -def _optional_str(value: object | None) -> str | None: - if value is None: - return None - if not isinstance(value, str) or not value.strip(): - raise DbtProfileError("dbt_profile connection values must be non-empty strings") - return value.strip() - - -def _resolve_optional_path(*, project_root: Path, raw_value: str | None) -> Path | None: - if raw_value is None: - return None - path: Path = Path(raw_value).expanduser() - if path.is_absolute(): - return path.resolve() - return (project_root.expanduser().resolve() / path).resolve() diff --git a/src/sqlbuild/integrations/dbt/_helpers/runtime/progress.py b/src/sqlbuild/integrations/dbt/_helpers/runtime/progress.py deleted file mode 100644 index 749d3dabb3..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/runtime/progress.py +++ /dev/null @@ -1,12 +0,0 @@ -"""dbt progress reporting callback helper.""" - -from __future__ import annotations - -from collections.abc import Callable - - -def report_progress(*, on_progress: Callable[[str], None] | None, message: str) -> None: - """Invoke the optional progress callback with a message.""" - - if on_progress is not None: - on_progress(message) diff --git a/src/sqlbuild/integrations/dbt/classes/dbt_compile_reference_resolver.py b/src/sqlbuild/integrations/dbt/classes/dbt_compile_reference_resolver.py deleted file mode 100644 index fd2892daf4..0000000000 --- a/src/sqlbuild/integrations/dbt/classes/dbt_compile_reference_resolver.py +++ /dev/null @@ -1,123 +0,0 @@ -"""Compile-time dbt reference resolver.""" - -from pathlib import Path - -from sqlbuild._native import ( - resolve_dbt_reference_relation, - validate_compile_dbt_model_names, - validate_compile_dbt_model_reference, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.integrations.dbt.models import DbtManifestIndex - - -class DbtCompileReferenceResolver: - """External reference resolver backed by a dbt manifest index.""" - - def __init__(self, *, dbt_manifest: DbtManifestIndex | None) -> None: - self._dbt_manifest = dbt_manifest - - def validate_model_names(self, *, known_model_names: set[str]) -> None: - validate_compile_dbt_model_names( - known_model_names=known_model_names, - dbt_manifest=self._dbt_manifest, - ) - - def extend_sql_test_model_names(self, *, known_model_names: set[str]) -> set[str]: - if self._dbt_manifest is None: - return set() - names: set[str] = set() - for model_name, matches in self._dbt_manifest.models_by_name.items(): - if len(matches) == 1: - if model_name in known_model_names: - raise CompileInputError( - f"dbt model fixture '__ref__{model_name}' conflicts with a SQLBuild model", - code="C218", - ) - names.add(model_name) - for package_name, name in self._dbt_manifest.models_by_package_and_name: - qualified_name: str = f"{package_name}__{name}" - if qualified_name in known_model_names: - raise CompileInputError( - f"dbt model fixture '__ref__{qualified_name}' conflicts with a SQLBuild model", - code="C218", - ) - names.add(qualified_name) - return names - - def extend_sql_test_source_names(self, *, known_source_names: set[str]) -> set[str]: - if self._dbt_manifest is None: - return set() - names: set[str] = set() - seen_bare: set[tuple[str, str]] = set() - duplicate_bare: set[tuple[str, str]] = set() - for source in self._dbt_manifest.sources_by_unique_id.values(): - key: tuple[str, str] = (source.source_name, source.name) - if key in seen_bare: - duplicate_bare.add(key) - seen_bare.add(key) - for source in self._dbt_manifest.sources_by_unique_id.values(): - bare_name: str = f"{source.source_name}__{source.name}" - if bare_name in known_source_names: - raise CompileInputError( - f"dbt source fixture '__source__{bare_name}' conflicts with a SQLBuild source", - code="C216", - ) - if (source.source_name, source.name) not in duplicate_bare: - names.add(bare_name) - names.add(f"{source.package_name}__{source.source_name}__{source.name}") - return names - - def extend_sql_test_seed_names(self, *, known_seed_names: set[str]) -> set[str]: - if self._dbt_manifest is None: - return set() - names: set[str] = set() - seen_bare: set[str] = set() - duplicate_bare: set[str] = set() - for seed in self._dbt_manifest.seeds_by_unique_id.values(): - if seed.name in seen_bare: - duplicate_bare.add(seed.name) - seen_bare.add(seed.name) - for seed in self._dbt_manifest.seeds_by_unique_id.values(): - if seed.name in known_seed_names: - raise CompileInputError( - f"dbt seed fixture '__seed__{seed.name}' conflicts with a SQLBuild seed", - code="C217", - ) - if seed.name not in duplicate_bare: - names.add(seed.name) - names.add(f"{seed.package_name}__{seed.name}") - return names - - def validate_reference( - self, - *, - ref_kind: str, - ref_name: str, - ref_package: str | None, - owner_relative_sql_path: Path, - ) -> None: - validate_compile_dbt_model_reference( - reference=CompileSqlReference( - ref_kind=ref_kind, - ref_name=ref_name, - ref_package=ref_package, - ), - model_relative_path=owner_relative_sql_path, - dbt_manifest=self._dbt_manifest, - ) - - def resolve_reference( - self, - *, - ref_kind: str, - ref_name: str, - ref_package: str | None, - ) -> str | None: - return resolve_dbt_reference_relation( - manifest=self._dbt_manifest, - ref_kind=ref_kind, - ref_name=ref_name, - ref_package=ref_package, - ) diff --git a/src/sqlbuild/integrations/dbt/main/cli/__init__.py b/src/sqlbuild/integrations/dbt/main/cli/__init__.py deleted file mode 100644 index 12c74380ea..0000000000 --- a/src/sqlbuild/integrations/dbt/main/cli/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""dbt CLI public operations.""" diff --git a/src/sqlbuild/integrations/dbt/main/cli/_parse_execution_args.py b/src/sqlbuild/integrations/dbt/main/cli/_parse_execution_args.py deleted file mode 100644 index 6456f0c5c0..0000000000 --- a/src/sqlbuild/integrations/dbt/main/cli/_parse_execution_args.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Parse dbt execution arguments.""" - -from collections.abc import Sequence - -from sqlbuild.integrations.dbt._helpers.cli.arg_parser import parse_dbt_execution_args as _parse -from sqlbuild.integrations.dbt.models import DbtInteropParsedArgs -from sqlbuild.integrations.dbt.types import DbtInteropCommand - - -def parse_dbt_execution_args( - *, command: DbtInteropCommand, args: Sequence[str] -) -> DbtInteropParsedArgs: - """Parse declared dbt interop execution arguments.""" - - return _parse(command=command, args=args) diff --git a/src/sqlbuild/integrations/dbt/main/cli/_validate_execution_args.py b/src/sqlbuild/integrations/dbt/main/cli/_validate_execution_args.py deleted file mode 100644 index 4df4470ae2..0000000000 --- a/src/sqlbuild/integrations/dbt/main/cli/_validate_execution_args.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Thin entry to validate `sqb dbt` execution-command arguments early.""" - -from __future__ import annotations - -from collections.abc import Sequence - -from sqlbuild.integrations.dbt._helpers.cli.arg_parser import parse_dbt_execution_args -from sqlbuild.integrations.dbt.constants import ( - DBT_EXECUTION_COMMANDS, - DBT_EXECUTION_DISPLAY_FLAGS, -) -from sqlbuild.integrations.dbt.types import DbtInteropCommand - - -def validate_dbt_execution_args(*, command: DbtInteropCommand, args: Sequence[str]) -> None: - """Parse execution-command args early so `-h` and typos resolve before setup.""" - - if command not in DBT_EXECUTION_COMMANDS: - return - display_free_args: tuple[str, ...] = tuple( - arg for arg in args if arg not in DBT_EXECUTION_DISPLAY_FLAGS - ) - _ = parse_dbt_execution_args(command=command, args=display_free_args) diff --git a/src/sqlbuild/integrations/dbt/main/config/__init__.py b/src/sqlbuild/integrations/dbt/main/config/__init__.py deleted file mode 100644 index 7055afa9b1..0000000000 --- a/src/sqlbuild/integrations/dbt/main/config/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""dbt configuration public operations.""" diff --git a/src/sqlbuild/integrations/dbt/main/config/_parse_config_overrides.py b/src/sqlbuild/integrations/dbt/main/config/_parse_config_overrides.py deleted file mode 100644 index 8b43fa47ad..0000000000 --- a/src/sqlbuild/integrations/dbt/main/config/_parse_config_overrides.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Parse dbt configuration overrides.""" - -from sqlbuild.integrations.dbt._helpers.planning.runtime import parse_dbt_config_overrides as _parse -from sqlbuild.integrations.dbt.models import DbtCliConfigOverrides - - -def parse_dbt_config_overrides(dbt_args: tuple[str, ...]) -> DbtCliConfigOverrides: - """Parse dbt configuration flags.""" - - return _parse(dbt_args) diff --git a/src/sqlbuild/integrations/dbt/main/config/_resolve_plan_options.py b/src/sqlbuild/integrations/dbt/main/config/_resolve_plan_options.py deleted file mode 100644 index df6aa963b3..0000000000 --- a/src/sqlbuild/integrations/dbt/main/config/_resolve_plan_options.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Resolve dbt plan options.""" - -from pathlib import Path - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt._helpers.planning.runtime import resolve_dbt_plan_options as _resolve -from sqlbuild.integrations.dbt.models import DbtCliOptions - - -def resolve_dbt_plan_options( - *, project_dir: Path, discovered_inputs: DiscoveredProjectInputs, dbt_args: tuple[str, ...] -) -> DbtCliOptions: - """Resolve dbt CLI options for interop planning.""" - - return _resolve(project_dir=project_dir, discovered_inputs=discovered_inputs, dbt_args=dbt_args) diff --git a/src/sqlbuild/integrations/dbt/main/config/_resolve_vars_mapping.py b/src/sqlbuild/integrations/dbt/main/config/_resolve_vars_mapping.py deleted file mode 100644 index acdfb30d07..0000000000 --- a/src/sqlbuild/integrations/dbt/main/config/_resolve_vars_mapping.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Resolve dbt variables.""" - -from sqlbuild.integrations.dbt._helpers.planning.runtime import resolve_dbt_vars_mapping as _resolve -from sqlbuild.spec.contracts.models import DbtConfig, LocalDbtConfig - - -def resolve_dbt_vars_mapping( - *, project_config: DbtConfig, local_config: LocalDbtConfig, dbt_args: tuple[str, ...] -) -> dict[str, object]: - """Resolve merged dbt variables as a mapping.""" - - return _resolve(project_config=project_config, local_config=local_config, dbt_args=dbt_args) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/__init__.py b/src/sqlbuild/integrations/dbt/main/manifest/__init__.py deleted file mode 100644 index 677457e3dd..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""dbt manifest public operations.""" diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_manifest_path.py b/src/sqlbuild/integrations/dbt/main/manifest/_resolve_manifest_path.py deleted file mode 100644 index 0247ab773e..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_manifest_path.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Resolve the dbt manifest path.""" - -from pathlib import Path - -from sqlbuild.integrations.dbt._helpers.planning.runtime import ( - resolve_dbt_manifest_path as _resolve, -) -from sqlbuild.integrations.dbt.models import DbtCliOptions - - -def resolve_dbt_manifest_path(*, options: DbtCliOptions) -> Path: - """Resolve the manifest path produced by dbt compile.""" - - return _resolve(options=options) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/build_compile_reference_resolver.py b/src/sqlbuild/integrations/dbt/main/manifest/build_compile_reference_resolver.py deleted file mode 100644 index bb39ded142..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/build_compile_reference_resolver.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Build dbt-backed compile-time external reference resolvers.""" - -from __future__ import annotations - -from sqlbuild._native import build_compile_dbt_manifest_index -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.integrations.dbt.classes.dbt_compile_reference_resolver import ( - DbtCompileReferenceResolver, -) - - -def build_compile_reference_resolver( - *, manifest_contents: str | None -) -> ExternalSqlReferenceResolver | None: - """Build a dbt-backed external reference resolver from manifest contents.""" - - if manifest_contents is None: - return None - return DbtCompileReferenceResolver( - dbt_manifest=build_compile_dbt_manifest_index(manifest_contents=manifest_contents) - ) diff --git a/src/sqlbuild/integrations/dbt/main/profile/__init__.py b/src/sqlbuild/integrations/dbt/main/profile/__init__.py deleted file mode 100644 index 51f0f78dd6..0000000000 --- a/src/sqlbuild/integrations/dbt/main/profile/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""dbt profile public operations.""" diff --git a/src/sqlbuild/integrations/dbt/main/profile/_resolve_connection_config.py b/src/sqlbuild/integrations/dbt/main/profile/_resolve_connection_config.py deleted file mode 100644 index 46fbeaf1b5..0000000000 --- a/src/sqlbuild/integrations/dbt/main/profile/_resolve_connection_config.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Resolve dbt interop connection configuration.""" - -from pathlib import Path - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt._helpers.profile.connection import ( - resolve_connection_config as _resolve, -) - - -def resolve_connection_config( - *, - raw_config: dict[str, object], - project_dir: Path, - adapter_name: str, - discovered_inputs: DiscoveredProjectInputs, -) -> dict[str, object]: - """Resolve connection configuration for dbt interop.""" - - return _resolve( - raw_config=raw_config, - project_dir=project_dir, - adapter_name=adapter_name, - discovered_inputs=discovered_inputs, - ) diff --git a/src/sqlbuild/integrations/dbt/main/profile/profile_connection.py b/src/sqlbuild/integrations/dbt/main/profile/profile_connection.py deleted file mode 100644 index 0b01845391..0000000000 --- a/src/sqlbuild/integrations/dbt/main/profile/profile_connection.py +++ /dev/null @@ -1,28 +0,0 @@ -"""dbt profile connection entrypoint.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.integrations.dbt._helpers.profile.connection import resolve_dbt_profile_raw_connection -from sqlbuild.integrations.dbt.models import NormalizedDbtProfileConnection -from sqlbuild.spec.contracts.models import ProjectConfig - - -def resolve_raw_dbt_profile_connection( - *, - raw_config: dict[str, object], - project_dir: Path, - adapter_name: str, - project_config: ProjectConfig, - cli_vars: dict[str, object] | None = None, -) -> NormalizedDbtProfileConnection | None: - """Resolve a raw connection config if it references a dbt profile.""" - - return resolve_dbt_profile_raw_connection( - raw_config=raw_config, - project_dir=project_dir, - adapter_name=adapter_name, - project_config=project_config, - cli_vars=cli_vars, - ) diff --git a/src/sqlbuild/integrations/dbt/main/profile/profile_init.py b/src/sqlbuild/integrations/dbt/main/profile/profile_init.py deleted file mode 100644 index 634f4efd45..0000000000 --- a/src/sqlbuild/integrations/dbt/main/profile/profile_init.py +++ /dev/null @@ -1,21 +0,0 @@ -"""dbt profile init entrypoint.""" - -from __future__ import annotations - -from sqlbuild.integrations.dbt._helpers.profile.init import ( - build_dbt_init_project, - validate_dbt_init_project, -) -from sqlbuild.integrations.dbt.models import DbtInitRequest, DbtInitResult - - -def run_dbt_profile_init(*, request: DbtInitRequest) -> DbtInitResult: - """Create or render a minimal SQLBuild project from a dbt project.""" - - return build_dbt_init_project(request=request) - - -def _validate_dbt_profile_init_request(*, request: DbtInitRequest) -> None: - """Validate dbt profile init inputs without writing files or prompting.""" - - validate_dbt_init_project(request=request) diff --git a/src/sqlbuild/integrations/dbt/main/runtime/_report_progress.py b/src/sqlbuild/integrations/dbt/main/runtime/_report_progress.py deleted file mode 100644 index 166b6d41db..0000000000 --- a/src/sqlbuild/integrations/dbt/main/runtime/_report_progress.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Report dbt runtime progress.""" - -from collections.abc import Callable - -from sqlbuild.integrations.dbt._helpers.runtime.progress import report_progress as _report - - -def report_progress(*, on_progress: Callable[[str], None] | None, message: str) -> None: - """Report a dbt runtime progress message when configured.""" - - _ = _report(on_progress=on_progress, message=message) diff --git a/src/sqlbuild/integrations/dbt/main/runtime/_resolve_interop_adapter.py b/src/sqlbuild/integrations/dbt/main/runtime/_resolve_interop_adapter.py deleted file mode 100644 index 37a5103182..0000000000 --- a/src/sqlbuild/integrations/dbt/main/runtime/_resolve_interop_adapter.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Resolve a dbt interop adapter.""" - -from pathlib import Path - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.integrations.dbt._helpers.planning.runtime import ( - resolve_dbt_interop_adapter as _resolve, -) - - -def resolve_dbt_interop_adapter( - *, adapter_name: str, project_dir: Path | None = None -) -> BaseAdapter: - """Resolve an adapter for dbt interop runtime planning.""" - - return _resolve(adapter_name=adapter_name, project_dir=project_dir) diff --git a/tests/integration/src/sqlbuild/integrations/dbt/helpers.py b/tests/integration/src/sqlbuild/integrations/dbt/helpers.py index 07151eff97..5bed9a503e 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/helpers.py @@ -2,12 +2,10 @@ from pathlib import Path +from sqlbuild._native import build_compile_reference_resolver from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompileAdapterContext from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.integrations.dbt.main.manifest.build_compile_reference_resolver import ( - build_compile_reference_resolver, -) from sqlbuild.sql_values.types import CollectionRendering DUCKDB_COMPILE_ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py index 6b16cb4a26..5fd1b6e710 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py @@ -4,6 +4,7 @@ import pytest +from sqlbuild import _native as native from sqlbuild._native import ( resolve_connection_config, resolve_project_connection_config, @@ -13,7 +14,6 @@ from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.exceptions import ProjectConfigError from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt.main.profile import profile_connection from sqlbuild.integrations.dbt.models import NormalizedDbtProfileConnection from sqlbuild.spec.contracts.models import ( LocalConfig, @@ -274,7 +274,7 @@ def resolve_raw_dbt_profile_connection(**kwargs: object) -> NormalizedDbtProfile ) monkeypatch.setattr( - profile_connection, + native, "resolve_raw_dbt_profile_connection", resolve_raw_dbt_profile_connection, ) @@ -333,7 +333,7 @@ def resolve_raw_dbt_profile_connection(**kwargs: object) -> NormalizedDbtProfile ) monkeypatch.setattr( - profile_connection, + native, "resolve_raw_dbt_profile_connection", resolve_raw_dbt_profile_connection, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py index 3b34366c44..8e6edb1ba5 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py @@ -289,7 +289,7 @@ def plan_dbt_interop_from_project( return build_empty_dbt_plan() monkeypatch.setattr( - "sqlbuild.integrations.dbt.main.profile.profile_init.run_dbt_profile_init", + "sqlbuild.integrations.dbt._helpers.profile.init.build_dbt_init_project", run_dbt_profile_init, ) monkeypatch.setattr( @@ -378,7 +378,7 @@ def plan_dbt_interop_from_project( return build_empty_dbt_plan() monkeypatch.setattr( - "sqlbuild.integrations.dbt.main.profile.profile_init.run_dbt_profile_init", + "sqlbuild.integrations.dbt._helpers.profile.init.build_dbt_init_project", run_dbt_profile_init, ) monkeypatch.setattr( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py index 8c7f13ab9c..240bb1bba0 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py @@ -6,7 +6,7 @@ from _pytest.capture import CaptureResult import sqlbuild._native as dbt_init_module -import sqlbuild.integrations.dbt.main.profile.profile_init as dbt_profile_init +import sqlbuild.integrations.dbt._helpers.profile.init as dbt_profile_init from sqlbuild.cli.commands.models import DbtInitCommandRequest from sqlbuild.integrations.dbt.exceptions import DbtProfileError from sqlbuild.integrations.dbt.models import DbtInitRequest, DbtInitResult @@ -62,10 +62,10 @@ def run_dbt_profile_init(*, request: DbtInitRequest) -> DbtInitResult: toml='name = "analytics"\n', ) - monkeypatch.setattr(dbt_profile_init, "run_dbt_profile_init", run_dbt_profile_init) + monkeypatch.setattr(dbt_profile_init, "build_dbt_init_project", run_dbt_profile_init) monkeypatch.setattr( dbt_profile_init, - "_validate_dbt_profile_init_request", + "validate_dbt_init_project", lambda *, request: None, ) @@ -136,10 +136,10 @@ def run_dbt_profile_init(*, request: DbtInitRequest) -> DbtInitResult: dry_run=True, ) - monkeypatch.setattr(dbt_profile_init, "run_dbt_profile_init", run_dbt_profile_init) + monkeypatch.setattr(dbt_profile_init, "build_dbt_init_project", run_dbt_profile_init) monkeypatch.setattr( dbt_profile_init, - "_validate_dbt_profile_init_request", + "validate_dbt_init_project", lambda *, request: None, ) @@ -210,10 +210,10 @@ def run_dbt_profile_init(*, request: DbtInitRequest) -> DbtInitResult: ) use_color_terminal(monkeypatch) - monkeypatch.setattr(dbt_profile_init, "run_dbt_profile_init", run_dbt_profile_init) + monkeypatch.setattr(dbt_profile_init, "build_dbt_init_project", run_dbt_profile_init) monkeypatch.setattr( dbt_profile_init, - "_validate_dbt_profile_init_request", + "validate_dbt_init_project", lambda *, request: None, ) @@ -273,10 +273,10 @@ def run_dbt_profile_init(*, request: DbtInitRequest) -> DbtInitResult: monkeypatch.setattr( dbt_profile_init, - "_validate_dbt_profile_init_request", + "validate_dbt_init_project", validate_dbt_profile_init_request, ) - monkeypatch.setattr(dbt_profile_init, "run_dbt_profile_init", run_dbt_profile_init) + monkeypatch.setattr(dbt_profile_init, "build_dbt_init_project", run_dbt_profile_init) exit_code: int = dbt_init_module.run_dbt_init_command( DbtInitCommandRequest( @@ -319,10 +319,10 @@ def run_dbt_profile_init(*, request: DbtInitRequest) -> DbtInitResult: monkeypatch.setattr( dbt_profile_init, - "_validate_dbt_profile_init_request", + "validate_dbt_init_project", validate_dbt_profile_init_request, ) - monkeypatch.setattr(dbt_profile_init, "run_dbt_profile_init", run_dbt_profile_init) + monkeypatch.setattr(dbt_profile_init, "build_dbt_init_project", run_dbt_profile_init) with pytest.raises(DbtProfileError, match=test_case.expected_error_fragment): dbt_init_module.run_dbt_init_command( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py index 0d1820b0af..50ff8083a9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py @@ -11,7 +11,6 @@ from sqlbuild import _native as native from sqlbuild.adapter.discovery.main import resolve_adapter as adapters_module -from sqlbuild.cli.commands._helpers.planning import external_refs as external_refs_module from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlScenario from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlScenarioFile @@ -62,9 +61,7 @@ def configure_native_scenario_command( monkeypatch.setattr(native, "resolve_project_connection_config", lambda **_: {}) monkeypatch.setattr(native, "run_compile_only_pipeline", compile_project) monkeypatch.setattr(adapters_module, "resolve_adapter", lambda **_: Mock()) - monkeypatch.setattr( - external_refs_module, "resolve_external_sql_reference_resolver", lambda **_: None - ) + monkeypatch.setattr(native, "resolve_external_sql_reference_resolver", lambda **_: None) monkeypatch.setattr(executor_run_module, "run_scenario_test_pipeline", lambda **_: ()) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_test_helpers.py index 40bc1481f2..f4dcc0d762 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_test_helpers.py @@ -3,15 +3,13 @@ from types import MappingProxyType from typing import cast +from sqlbuild._native import build_compile_reference_resolver from sqlbuild.compiler.compile.models import ( CompileDirectLogicSqlTestInputPayload, CompileModelSqlTestInputPayload, CompileSqlTestInput, ) from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.integrations.dbt.main.manifest.build_compile_reference_resolver import ( - build_compile_reference_resolver, -) def base_repo_files() -> dict[str, str]: diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py index 6fe28897c9..95886abb32 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild._native import build_dbt_manifest_index +from sqlbuild._native import DbtCompileReferenceResolver, build_dbt_manifest_index from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -20,9 +20,6 @@ resolve_udf_references, ) from sqlbuild.compiler.planner.models import CursorBounds -from sqlbuild.integrations.dbt.classes.dbt_compile_reference_resolver import ( - DbtCompileReferenceResolver, -) from sqlbuild.integrations.dbt.models import DbtManifestIndex from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( ApplyDeferredTargetsTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/profile/_test_types.py b/tests/unit/src/sqlbuild/integrations/dbt/profile/_test_types.py index 30dcb07d8c..61893566e4 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/profile/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/profile/_test_types.py @@ -47,3 +47,19 @@ class DbtProfileNormalizeErrorTestCase: description: str output: dict[str, object] expected_error_fragment: str + + +@dataclass(frozen=True) +class DbtProfileConnectionTestCase: + description: str + raw_config: dict[str, object] + adapter_name: str + expected_connection: dict[str, object] | None + + +@dataclass(frozen=True) +class DbtProfileConnectionErrorTestCase: + description: str + raw_config: dict[str, object] + adapter_name: str + expected_error_fragment: str diff --git a/tests/unit/src/sqlbuild/integrations/dbt/profile/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/profile/helpers.py new file mode 100644 index 0000000000..b7bddb52b5 --- /dev/null +++ b/tests/unit/src/sqlbuild/integrations/dbt/profile/helpers.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.spec.contracts.models import DbtConfig, ProjectConfig + + +def write_duckdb_dbt_workspace(root: Path) -> None: + """Write a dbt project and a DuckDB profile under `root`.""" + + (root / "dbt_project").mkdir() + (root / "profiles").mkdir() + (root / "dbt_project" / "dbt_project.yml").write_text( + "name: shop\nprofile: shop\n", encoding="utf-8" + ) + (root / "profiles" / "profiles.yml").write_text( + "shop:\n target: dev\n outputs:\n dev:\n type: duckdb\n" + " path: warehouse.duckdb\n schema: main\n", + encoding="utf-8", + ) + + +def dbt_workspace_project_config(*, adapter_name: str) -> ProjectConfig: + """A project config pointing at the workspace's dbt project and profiles.""" + + return ProjectConfig( + name="shop", + adapter=adapter_name, + dbt=DbtConfig(project_dir="dbt_project", profiles_dir="profiles"), + ) diff --git a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_connection.py b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_connection.py new file mode 100644 index 0000000000..0453d04311 --- /dev/null +++ b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_connection.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import resolve_raw_dbt_profile_connection +from sqlbuild.integrations.dbt.exceptions import DbtProfileError +from sqlbuild.integrations.dbt.models import NormalizedDbtProfileConnection +from tests.unit.src.sqlbuild.integrations.dbt.profile._test_types import ( + DbtProfileConnectionErrorTestCase, + DbtProfileConnectionTestCase, +) +from tests.unit.src.sqlbuild.integrations.dbt.profile.helpers import ( + dbt_workspace_project_config, + write_duckdb_dbt_workspace, +) + + +@pytest.mark.parametrize( + "test_case", + [ + DbtProfileConnectionTestCase( + description="a connection without the dbt_profile source is not resolved", + raw_config={"database": "orders.duckdb"}, + adapter_name="duckdb", + expected_connection=None, + ), + DbtProfileConnectionTestCase( + description="the project's dbt profile resolves to adapter config", + raw_config={"source": "dbt_profile", "target": " dev "}, + adapter_name="duckdb", + expected_connection={"database": "warehouse.duckdb"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_raw_connection_when_resolving_dbt_profile_then_returns_expected_connection( + test_case: DbtProfileConnectionTestCase, tmp_path: Path +) -> None: + write_duckdb_dbt_workspace(tmp_path) + + resolved: NormalizedDbtProfileConnection | None = resolve_raw_dbt_profile_connection( + raw_config=test_case.raw_config, + project_dir=tmp_path, + adapter_name=test_case.adapter_name, + project_config=dbt_workspace_project_config(adapter_name=test_case.adapter_name), + ) + + assert (None if resolved is None else resolved.connection) == test_case.expected_connection + + +@pytest.mark.parametrize( + "test_case", + [ + DbtProfileConnectionErrorTestCase( + description="a blank profile name is rejected", + raw_config={"source": "dbt_profile", "profile": " "}, + adapter_name="duckdb", + expected_error_fragment="dbt_profile connection values must be non-empty strings", + ), + DbtProfileConnectionErrorTestCase( + description="a profile for another adapter is rejected", + raw_config={"source": "dbt_profile"}, + adapter_name="snowflake", + expected_error_fragment="dbt=duckdb, sqlbuild=snowflake", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_dbt_profile_reference_when_resolving_then_raises_profile_error( + test_case: DbtProfileConnectionErrorTestCase, tmp_path: Path +) -> None: + write_duckdb_dbt_workspace(tmp_path) + + with pytest.raises(DbtProfileError, match=test_case.expected_error_fragment) as captured: + resolve_raw_dbt_profile_connection( + raw_config=test_case.raw_config, + project_dir=tmp_path, + adapter_name=test_case.adapter_name, + project_config=dbt_workspace_project_config(adapter_name=test_case.adapter_name), + ) + + assert captured.value.code == "C242" diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_args.py b/tests/unit/src/sqlbuild/integrations/dbt/test_args.py index ed503a1a62..20cd87212c 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_args.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_args.py @@ -2,8 +2,7 @@ import pytest -from sqlbuild._native import route_dbt_interop_args -from sqlbuild.integrations.dbt._helpers.cli.arg_parser import parse_dbt_execution_args +from sqlbuild._native import parse_dbt_execution_args, route_dbt_interop_args from sqlbuild.integrations.dbt.exceptions import DbtInteropArgumentError from sqlbuild.integrations.dbt.models import DbtInteropParsedArgs, DbtInteropRoutedArgs from sqlbuild.integrations.dbt.types import DbtInteropCommand diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_config.py b/tests/unit/src/sqlbuild/integrations/dbt/test_config.py index 39f04923f5..083a9795a0 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_config.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_config.py @@ -1,12 +1,10 @@ from __future__ import annotations -import json from pathlib import Path import pytest -from sqlbuild._native import resolve_dbt_config -from sqlbuild.integrations.dbt._helpers.planning.runtime import resolve_dbt_vars +from sqlbuild._native import resolve_dbt_config, resolve_dbt_vars_mapping from sqlbuild.integrations.dbt.exceptions import DbtInteropConfigError from sqlbuild.integrations.dbt.models import ResolvedDbtConfig from sqlbuild.spec.contracts.models import DbtConfig, LocalDbtConfig @@ -136,14 +134,13 @@ def test_given_dbt_config_when_resolving_then_returns_expected_values( def test_given_dbt_vars_sources_when_resolving_then_vars_are_merged( test_case: DbtVarsResolutionTestCase, ) -> None: - resolved: str | None = resolve_dbt_vars( + resolved: dict[str, object] = resolve_dbt_vars_mapping( project_config=test_case.project_config, local_config=test_case.local_config, dbt_args=test_case.dbt_args, ) - assert resolved is not None - assert json.loads(resolved) == test_case.expected_vars + assert resolved == test_case.expected_vars @pytest.mark.parametrize( From c17c854a43ff9b92a50a0d7d6cde2fe17d5377b9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 03:22:04 +0100 Subject: [PATCH 06/11] refactor: host the CLI namespace, launch record state and compile entry natively CliNamespace becomes a native class whose defaults come from a Rust table, the offered launch record lives in native context variables read from the leading launch record option, and run_compile builds the reuse request and replays or runs the compile in Rust. The replaced Python modules are deleted. --- crates/sqlbuild-cli/src/argv/constants.rs | 183 +++++++++ .../src/argv/main/cli_namespace_defaults.rs | 10 + crates/sqlbuild-cli/src/argv/main/mod.rs | 1 + crates/sqlbuild-cli/src/argv/models.rs | 17 + crates/sqlbuild-cli/src/dispatch/constants.rs | 2 +- .../src/bindings/_helpers/cli/argv.rs | 4 +- .../src/bindings/_helpers/cli/dispatch.rs | 4 +- .../src/bindings/_helpers/cli/entry.rs | 23 ++ .../src/bindings/_helpers/cli/launch_state.rs | 66 ++++ .../src/bindings/_helpers/cli/mod.rs | 2 + .../src/bindings/_helpers/cli/namespace.rs | 72 ++++ .../bindings/_helpers/cli/observability.rs | 4 +- .../src/bindings/_helpers/cli/registration.rs | 5 +- .../_helpers/compile_command/reuse_attempt.rs | 62 +++- fensu.toml | 5 + src/sqlbuild/_native.pyi | 170 ++++++++- .../cli/commands/classes/cli_namespace.py | 346 ------------------ .../entrypoint/_dispatch_with_compute_logs.py | 3 +- src/sqlbuild/cli/compile/main/run.py | 45 --- .../cli/entry/_helpers/launch_state.py | 25 -- .../cli/entry/main/_launch_record_path.py | 11 - .../cli/entry/main/_withhold_launch_record.py | 9 - src/sqlbuild/cli/entry/main/launched.py | 12 +- src/sqlbuild/cli/entry/models.py | 2 +- .../test_cli_concurrent_history.py | 3 +- .../cli/commands/classes}/__init__.py | 0 .../cli/commands/classes/_test_types.py | 34 ++ .../commands/classes/test_cli_namespace.py | 147 ++++++++ .../main/compile/test_compile_command.py | 2 +- .../cli/commands/main/entry/test_main.py | 3 - .../main/entry/test_observability_cleanup.py | 2 +- .../dbt/profile/test_connection.py | 34 +- 32 files changed, 834 insertions(+), 474 deletions(-) create mode 100644 crates/sqlbuild-cli/src/argv/main/cli_namespace_defaults.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs delete mode 100644 src/sqlbuild/cli/commands/classes/cli_namespace.py delete mode 100644 src/sqlbuild/cli/compile/main/run.py delete mode 100644 src/sqlbuild/cli/entry/_helpers/launch_state.py delete mode 100644 src/sqlbuild/cli/entry/main/_launch_record_path.py delete mode 100644 src/sqlbuild/cli/entry/main/_withhold_launch_record.py rename {src/sqlbuild/cli/compile/main => tests/unit/src/sqlbuild/cli/commands/classes}/__init__.py (100%) create mode 100644 tests/unit/src/sqlbuild/cli/commands/classes/_test_types.py create mode 100644 tests/unit/src/sqlbuild/cli/commands/classes/test_cli_namespace.py diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 4ddca6dcee..2a653e1255 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -1,5 +1,7 @@ //! The `sqb` command table, `argparse` vocabulary and the post-parse rules' names. +use crate::argv::models::NamespaceDefault; + /// The exported `sqb` parser tree: every parser, group and action in `add_argument` order. pub(crate) const COMMAND_TABLE: &str = include_str!("command_table.json"); /// The exported `sqb dbt plan|run|build` execution parser, its command name a placeholder. @@ -59,3 +61,184 @@ pub(crate) const SCOPE_KIND_DEST: &str = "scope_kind"; pub(crate) const INT_CONVERTER: &str = "int"; /// The `type=pathlib.Path` converter, which never rejects a string. pub(crate) const PATH_CONVERTER: &str = "Path"; + +/// Every `CliNamespace` attribute with a default, in declaration order. +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 162] = [ + ("command", NamespaceDefault::None), + ("project_dir", NamespaceDefault::None), + ("dbt_project_dir", NamespaceDefault::None), + ("dbt_profiles_dir", NamespaceDefault::None), + ("dbt_target", NamespaceDefault::None), + ("dbt_profile", NamespaceDefault::None), + ("sqb_output_dir", NamespaceDefault::None), + ("dry_run", NamespaceDefault::Bool(false)), + ("overwrite", NamespaceDefault::Bool(false)), + ("skip_dbt_debug", NamespaceDefault::Bool(false)), + ("no_sql_validation", NamespaceDefault::Bool(false)), + ("no_cache", NamespaceDefault::Bool(false)), + ("defer_to", NamespaceDefault::None), + ("defer_clone_from", NamespaceDefault::None), + ("defer_sources_to", NamespaceDefault::None), + ("target", NamespaceDefault::None), + ("warehouse", NamespaceDefault::None), + ("case", NamespaceDefault::None), + ("inspect", NamespaceDefault::Bool(false)), + ("target_range", NamespaceDefault::None), + ("from_target", NamespaceDefault::None), + ("to_target", NamespaceDefault::None), + ("hard_copy", NamespaceDefault::Bool(false)), + ("json", NamespaceDefault::Bool(false)), + ("json_output", NamespaceDefault::None), + ("manifest", NamespaceDefault::Bool(false)), + ("dag", NamespaceDefault::None), + ("compile_lineage_mode", NamespaceDefault::CompileLineageFast), + ( + "profile_skip_discovery_sql_analysis", + NamespaceDefault::Bool(false), + ), + ( + "profile_skip_column_inference", + NamespaceDefault::Bool(false), + ), + ("profile_skip_contracts", NamespaceDefault::Bool(false)), + ("profile_skip_write", NamespaceDefault::Bool(false)), + ("start_cursor", NamespaceDefault::None), + ("end_cursor", NamespaceDefault::None), + ("start_cursor_ts", NamespaceDefault::None), + ("max_microbatches", NamespaceDefault::None), + ("end_cursor_ts", NamespaceDefault::None), + ("start_cursor_int", NamespaceDefault::None), + ("end_cursor_int", NamespaceDefault::None), + ("no_color", NamespaceDefault::Bool(false)), + ("fail_fast", NamespaceDefault::Bool(false)), + ("full_refresh", NamespaceDefault::Bool(false)), + ("as_target", NamespaceDefault::None), + ("include_python", NamespaceDefault::Bool(true)), + ("load_sources", NamespaceDefault::None), + ("reload", NamespaceDefault::Bool(false)), + ("run_tests", NamespaceDefault::Bool(true)), + ("run_audits", NamespaceDefault::Bool(true)), + ("allow_snapshot_full_refresh", NamespaceDefault::Bool(false)), + ("allow_table_type_downgrade", NamespaceDefault::Bool(false)), + ("allow_retention_decrease", NamespaceDefault::Bool(false)), + ( + "allow_missing_migration_origin", + NamespaceDefault::Bool(false), + ), + ( + "allow_snapshot_schema_change", + NamespaceDefault::Bool(false), + ), + ("concurrency", NamespaceDefault::None), + ("verbose", NamespaceDefault::Bool(false)), + ("debug", NamespaceDefault::Bool(false)), + ("auto_approve", NamespaceDefault::Bool(false)), + ("retention_days", NamespaceDefault::None), + ("direct_state_history_versions", NamespaceDefault::None), + ("drop_old_name_views", NamespaceDefault::EmptyList), + ("bounded", NamespaceDefault::None), + ("max_column_examples", NamespaceDefault::None), + ("max_row_only_examples", NamespaceDefault::None), + ("sample_rows", NamespaceDefault::None), + ("sample_seed", NamespaceDefault::None), + ("exhaustive", NamespaceDefault::Bool(false)), + ("max_models", NamespaceDefault::None), + ("max_columns", NamespaceDefault::None), + ("query_sql", NamespaceDefault::None), + ("query_file", NamespaceDefault::None), + ("query_format", NamespaceDefault::Text("long")), + ("query_limit", NamespaceDefault::Int(20)), + ("query_no_limit", NamespaceDefault::Bool(false)), + ("cost_selector", NamespaceDefault::Text("latest")), + ("cost_limit", NamespaceDefault::None), + ("cost_no_limit", NamespaceDefault::Bool(false)), + ("cost_sort", NamespaceDefault::None), + ("cost_order", NamespaceDefault::None), + ("cost_since", NamespaceDefault::None), + ("cost_until", NamespaceDefault::None), + ("lineage_targets", NamespaceDefault::EmptyList), + ("lineage_format", NamespaceDefault::Text("tree")), + ("lineage_direction", NamespaceDefault::None), + ("lineage_depth", NamespaceDefault::Text("all")), + ("lineage_mode", NamespaceDefault::ColumnLineageRich), + ("lineage_include_uses", NamespaceDefault::Bool(false)), + ("no_connection", NamespaceDefault::Bool(false)), + ("fail_on_error", NamespaceDefault::Bool(false)), + ("fail_on_stale", NamespaceDefault::Bool(false)), + ("state", NamespaceDefault::Bool(false)), + ("full", NamespaceDefault::Bool(false)), + ("schema_only", NamespaceDefault::Bool(false)), + ("left_query", NamespaceDefault::None), + ("left_query_file", NamespaceDefault::None), + ("right_query", NamespaceDefault::None), + ("right_query_file", NamespaceDefault::None), + ("left_label", NamespaceDefault::None), + ("right_label", NamespaceDefault::None), + ("key", NamespaceDefault::EmptyList), + ("unkeyed", NamespaceDefault::Bool(false)), + ("exclude_column", NamespaceDefault::EmptyList), + ("tolerance", NamespaceDefault::EmptyList), + ("max_value_length", NamespaceDefault::None), + ("no_example_values", NamespaceDefault::Bool(false)), + ("full_example_values", NamespaceDefault::Bool(false)), + ( + "playground_path", + NamespaceDefault::Text("sqlbuild-playground"), + ), + ("playground_template", NamespaceDefault::Text("waffle_shop")), + ("scenario_command", NamespaceDefault::None), + ("scenario_selector", NamespaceDefault::EmptyList), + ("scenario_retain", NamespaceDefault::Bool(false)), + ("scenario_namespace", NamespaceDefault::None), + ("scenario_local", NamespaceDefault::Bool(false)), + ("scenario_strict", NamespaceDefault::Bool(false)), + ("scenario_sync_snapshots", NamespaceDefault::Bool(false)), + ("scenario_refresh", NamespaceDefault::Bool(false)), + ("scenario_force", NamespaceDefault::Bool(false)), + ("scenario_max_snapshot_rows", NamespaceDefault::None), + ("scenario_max_snapshot_total_rows", NamespaceDefault::None), + ("scenario_max_snapshot_bytes", NamespaceDefault::None), + ("scenario_max_snapshot_total_bytes", NamespaceDefault::None), + ("skills_global", NamespaceDefault::Bool(false)), + ("skills_target", NamespaceDefault::EmptyList), + ("skills_force", NamespaceDefault::Bool(false)), + ("select", NamespaceDefault::EmptyList), + ("positional_select", NamespaceDefault::EmptyList), + ("select_file", NamespaceDefault::EmptyList), + ("exclude", NamespaceDefault::EmptyList), + ("dbt_command", NamespaceDefault::None), + ("dbt_args", NamespaceDefault::EmptyList), + ("vars", NamespaceDefault::EmptyDict), + ("rules_command", NamespaceDefault::None), + ("rules_rule_selector", NamespaceDefault::None), + ("rules_skills_check", NamespaceDefault::Bool(false)), + ("format_check", NamespaceDefault::Bool(false)), + ("format_diff", NamespaceDefault::Bool(false)), + ("format_fixtures_only", NamespaceDefault::Bool(false)), + ("format_fix", NamespaceDefault::Bool(false)), + ("format_paths", NamespaceDefault::EmptyList), + ("scope_target", NamespaceDefault::None), + ("scope_at", NamespaceDefault::None), + ("scope_as_path", NamespaceDefault::None), + ("scope_browse", NamespaceDefault::None), + ("scope_list", NamespaceDefault::None), + ("contract_command", NamespaceDefault::None), + ("contract_from", NamespaceDefault::None), + ("contract_write", NamespaceDefault::Bool(false)), + ("scope_defined_under", NamespaceDefault::None), + ("scope_kind", NamespaceDefault::EmptyList), + ("scope_match", NamespaceDefault::None), + ("scope_used_only", NamespaceDefault::Bool(false)), + ("scope_include_nearby", NamespaceDefault::Bool(false)), + ("scope_nearby_depth", NamespaceDefault::Int(1)), + ("scope_dependency_depth", NamespaceDefault::Int(0)), + ("scope_explain", NamespaceDefault::None), + ("scope_globals", NamespaceDefault::Text("summary")), + ("scope_page_size", NamespaceDefault::Int(100)), + ("scope_after", NamespaceDefault::None), + ("scope_paths", NamespaceDefault::Text("relative")), + ("refactor_target", NamespaceDefault::None), + ("refactor_new_name", NamespaceDefault::None), + ("refactor_destination", NamespaceDefault::None), + ("refactor_cascade", NamespaceDefault::Bool(false)), +]; diff --git a/crates/sqlbuild-cli/src/argv/main/cli_namespace_defaults.rs b/crates/sqlbuild-cli/src/argv/main/cli_namespace_defaults.rs new file mode 100644 index 0000000000..ce0b46b489 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/main/cli_namespace_defaults.rs @@ -0,0 +1,10 @@ +//! The attribute defaults every new `CliNamespace` starts from. + +use crate::argv::constants::CLI_NAMESPACE_DEFAULTS; +use crate::argv::models::NamespaceDefault; + +/// Every defaulted `CliNamespace` attribute and its default, in declaration order. +#[must_use] +pub fn cli_namespace_defaults() -> &'static [(&'static str, NamespaceDefault)] { + &CLI_NAMESPACE_DEFAULTS +} diff --git a/crates/sqlbuild-cli/src/argv/main/mod.rs b/crates/sqlbuild-cli/src/argv/main/mod.rs index 42e5fe2b00..0dabda8f2b 100644 --- a/crates/sqlbuild-cli/src/argv/main/mod.rs +++ b/crates/sqlbuild-cli/src/argv/main/mod.rs @@ -1,2 +1,3 @@ +pub mod cli_namespace_defaults; pub mod parse_cli_arguments; pub mod parse_dbt_execution_arguments; diff --git a/crates/sqlbuild-cli/src/argv/models.rs b/crates/sqlbuild-cli/src/argv/models.rs index f7da7be220..72f814054e 100644 --- a/crates/sqlbuild-cli/src/argv/models.rs +++ b/crates/sqlbuild-cli/src/argv/models.rs @@ -180,3 +180,20 @@ pub struct DbtExecutionOutcome { pub writes: Vec<(Stream, String)>, pub parsed: Result, } + +/// One `CliNamespace` attribute default, rebuilt fresh for every namespace. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NamespaceDefault { + None, + Bool(bool), + Int(i64), + Text(&'static str), + /// A new empty `list`. + EmptyList, + /// A new empty `dict`. + EmptyDict, + /// `CompileLineageMode.FAST`. + CompileLineageFast, + /// `ColumnLineageMode.RICH`. + ColumnLineageRich, +} diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index ad78844d34..47223516b4 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -618,7 +618,7 @@ const COMPILE_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const COMPILE_ROUTE: Route = routed( "run_compile", - ("sqlbuild.cli.compile.main.run", "run_compile"), + ("sqlbuild._native", "run_compile"), &COMPILE_ARGUMENTS, REQUEST_SHAPE, ); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs index 3a9e4d51d1..a4be411f9d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs @@ -11,10 +11,10 @@ use sqlbuild_cli::argv::models::{ArgvOutcome, ArgvSettings, Stream, TypeFailure, use sqlbuild_cli::argv::types::ArgvHost; use sqlbuild_core::text::main::python_str_repr::python_str_repr; +use crate::bindings::_helpers::cli::namespace::new_cli_namespace; use crate::bindings::_helpers::cli::surrogates::{python_text, rust_text, uses_placeholders}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -const NAMESPACE_MODULE: &str = "sqlbuild.cli.commands.classes.cli_namespace"; const ENTRY_MODELS_MODULE: &str = "sqlbuild.cli.entry.models"; const INT_CONVERTER: &str = "int"; const POSITIVE_INTEGER_CONVERTER: &str = "_positive_integer"; @@ -243,7 +243,7 @@ pub(crate) fn parse_cli_invocation<'py>( argv: Vec>, use_color: bool, ) -> PyResult> { - let args: Bound<'py, PyAny> = imported(py, NAMESPACE_MODULE, "CliNamespace")?.call0()?; + let args: Bound<'py, PyAny> = new_cli_namespace(py)?; let defaults: Bound<'py, PyList> = py .get_type::() .call1((args.getattr("__dict__")?.call_method0("items")?,))? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index b20ec28c02..3c6625fcf7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -2,6 +2,7 @@ use std::path::PathBuf; +use crate::bindings::_helpers::cli::launch_state::withhold_launch_record; use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehouse_scope; use crate::bindings::_helpers::diff_report::support::cli_error; use pyo3::IntoPyObjectExt; @@ -22,7 +23,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const ENTRY_MODELS_MODULE: &str = "sqlbuild.cli.entry.models"; const WAREHOUSES_MODULE: &str = "sqlbuild._native"; -const LAUNCH_RECORD_MODULE: &str = "sqlbuild.cli.entry.main._withhold_launch_record"; const PATHLIB_MODULE: &str = "pathlib"; const COMMENT_PREFIX: &str = "#"; const UTF8: &str = "utf-8"; @@ -298,7 +298,7 @@ fn routed_command<'py>( }; let (select, selector_files) = selector_inputs(args)?; if args.getattr("select_file")?.is_truthy()? { - imported(py, LAUNCH_RECORD_MODULE, "withhold_launch_record")?.call0()?; + withhold_launch_record(py)?; } let invocation = Invocation { args: args.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index d57da3604a..b5e2c3f20d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -14,9 +14,11 @@ use sqlbuild_cli::dispatch::models::{CommandFacts, CommandFailure, InvocationPol use crate::bindings::_helpers::cli::argv::parse_cli_invocation; use crate::bindings::_helpers::cli::dispatch::{command_facts, dispatch_command}; +use crate::bindings::_helpers::cli::launch_state::offer_launch_record; use crate::bindings::_helpers::cli::observability::cli_observability_scope; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +use sqlbuild_cli::launcher::constants::LAUNCH_RECORD_OPTION; use sqlbuild_core::cli_text::main::format_coded_error::format_coded_error; use sqlbuild_core::cli_text::models::CliStyle; @@ -570,6 +572,26 @@ fn parsed_invocation<'py>( ) } +/// `argv` without a leading launch record option, after offering the record it names. +fn offered_launch_record<'py>(argv: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = argv.py(); + let mut first: Option = None; + if argv.len()? > 0 { + let item: Bound<'py, PyAny> = argv.get_item(0)?; + if item.is_instance_of::() { + first = Some(item.extract()?); + } + } + let Some(path) = + first.and_then(|first| first.strip_prefix(LAUNCH_RECORD_OPTION).map(str::to_owned)) + else { + offer_launch_record(py, None)?; + return Ok(argv.clone()); + }; + offer_launch_record(py, Some(imported(py, "pathlib", "Path")?.call1((path,))?))?; + argv.get_item(pyo3::types::PySlice::new(py, 1, isize::MAX, 1)) +} + /// Run the CLI for `argv` (default `sys.argv[1:]`) and return its exit code. #[pyfunction] #[pyo3(signature = (argv=None, handlers=None))] @@ -585,6 +607,7 @@ fn run_cli<'py>( .getattr("argv")? .get_item(pyo3::types::PySlice::new(py, 1, isize::MAX, 1))?, }; + let argv: Bound<'py, PyAny> = offered_launch_record(&argv)?; let external: Bound<'py, PyAny> = external_context(py)?; let invocation_scope: Bound<'py, PyAny> = imported(py, INVOCATION_SCOPE_MODULE, "invocation_scope")?.call0()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs new file mode 100644 index 0000000000..be0bb290ed --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs @@ -0,0 +1,66 @@ +//! The launch record the native `sqb` executable offered this invocation. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +/// The module attributes holding the offered path's and the withheld flag's `ContextVar`s. +const OFFERED_ATTRIBUTE: &str = "_launch_record_offered"; +const WITHHELD_ATTRIBUTE: &str = "_launch_record_withheld"; +const OFFERED_NAME: &str = "sqlbuild_launch_record"; +const WITHHELD_NAME: &str = "sqlbuild_launch_record_withheld"; + +/// Remember where `sqb` looks for this invocation's launch record, or that it offered none. +pub(crate) fn offer_launch_record(py: Python<'_>, path: Option>) -> PyResult<()> { + let path: Bound<'_, PyAny> = path.unwrap_or_else(|| py.None().into_bound(py)); + imported(py, NATIVE_MODULE, OFFERED_ATTRIBUTE)?.call_method1("set", (path,))?; + imported(py, NATIVE_MODULE, WITHHELD_ATTRIBUTE)?.call_method1("set", (false,))?; + Ok(()) +} + +/// Withhold the launch record: `sqb` cannot replay what this invocation also depends on. +pub(crate) fn withhold_launch_record(py: Python<'_>) -> PyResult<()> { + imported(py, NATIVE_MODULE, WITHHELD_ATTRIBUTE)?.call_method1("set", (true,))?; + Ok(()) +} + +/// The offered launch record path, unless an input outside its key was read. +pub(crate) fn launch_record_path(py: Python<'_>) -> PyResult> { + if imported(py, NATIVE_MODULE, WITHHELD_ATTRIBUTE)? + .call_method0("get")? + .is_truthy()? + { + return Ok(py.None().into_bound(py)); + } + imported(py, NATIVE_MODULE, OFFERED_ATTRIBUTE)?.call_method0("get") +} + +fn context_variable<'py>( + py: Python<'py>, + name: &str, + default: Bound<'py, PyAny>, +) -> PyResult> { + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("default", default)?; + py.import("contextvars")? + .getattr("ContextVar")? + .call((name,), Some(&keywords)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py: Python<'_> = module.py(); + module.add( + OFFERED_ATTRIBUTE, + context_variable(py, OFFERED_NAME, py.None().into_bound(py))?, + )?; + module.add( + WITHHELD_ATTRIBUTE, + context_variable( + py, + WITHHELD_NAME, + PyBool::new(py, false).to_owned().into_any(), + )?, + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs index 101172f2be..118be0fbed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs @@ -1,6 +1,8 @@ pub(crate) mod argv; pub(crate) mod dispatch; pub(crate) mod entry; +pub(crate) mod launch_state; +pub(crate) mod namespace; pub(crate) mod observability; pub(crate) mod progress; pub(crate) mod progress_reporters; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs new file mode 100644 index 0000000000..50472bef31 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs @@ -0,0 +1,72 @@ +//! `CliNamespace`: the mutable attribute namespace every `sqb` command reads its options from. + +use pyo3::exceptions::PyAttributeError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyInt, PyList, PyString}; +use pyo3::{IntoPyObjectExt, pyclass, pymethods}; +use sqlbuild_cli::argv::main::cli_namespace_defaults::cli_namespace_defaults; +use sqlbuild_cli::argv::models::NamespaceDefault; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const COMPILE_LINEAGE_MODULE: &str = "sqlbuild.cli.commands.types"; +const COLUMN_LINEAGE_MODULE: &str = "sqlbuild.compiler.lineage.types"; + +/// Typed namespace for all CLI arguments across all commands. +#[pyclass(module = "sqlbuild._native", dict, frozen)] +pub(crate) struct CliNamespace; + +fn default_object(py: Python<'_>, default: NamespaceDefault) -> PyResult> { + match default { + NamespaceDefault::None => Ok(py.None().into_bound(py)), + NamespaceDefault::Bool(value) => Ok(PyBool::new(py, value).to_owned().into_any()), + NamespaceDefault::Int(value) => Ok(PyInt::new(py, value).into_any()), + NamespaceDefault::Text(value) => Ok(PyString::new(py, value).into_any()), + NamespaceDefault::EmptyList => Ok(PyList::empty(py).into_any()), + NamespaceDefault::EmptyDict => Ok(PyDict::new(py).into_any()), + NamespaceDefault::CompileLineageFast => { + imported(py, COMPILE_LINEAGE_MODULE, "CompileLineageMode")?.getattr("FAST") + } + NamespaceDefault::ColumnLineageRich => { + imported(py, COLUMN_LINEAGE_MODULE, "ColumnLineageMode")?.getattr("RICH") + } + } +} + +/// A namespace holding a fresh copy of every default as an instance attribute. +pub(crate) fn new_cli_namespace(py: Python<'_>) -> PyResult> { + let namespace: Bound<'_, PyAny> = Bound::new(py, CliNamespace)?.into_any(); + for (name, default) in cli_namespace_defaults() { + namespace.setattr(*name, default_object(py, *default)?)?; + } + Ok(namespace) +} + +#[pymethods] +impl CliNamespace { + #[new] + fn new() -> Self { + Self + } + + /// A default not yet stored on this namespace, stored on first read. + fn __getattr__<'py>(slf: &Bound<'py, Self>, name: &str) -> PyResult> { + let py: Python<'py> = slf.py(); + let Some((_, default)) = cli_namespace_defaults() + .iter() + .find(|(attribute, _)| *attribute == name) + else { + return Err(PyAttributeError::new_err(format!( + "'CliNamespace' object has no attribute {}", + name.into_bound_py_any(py)?.repr()? + ))); + }; + let value: Bound<'py, PyAny> = default_object(py, *default)?; + slf.as_any().setattr(name, &value)?; + Ok(value) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs index c51f83098a..6cb27ba298 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -12,6 +12,7 @@ use pyo3::{IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_cli::dispatch::main::event_export_warning::event_export_warning; use sqlbuild_cli::dispatch::models::{ExportSummary, ExporterLoss}; +use crate::bindings::_helpers::cli::launch_state::withhold_launch_record; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; @@ -20,7 +21,6 @@ const EVENT_DISPATCHER_MODULE: &str = "sqlbuild.runtime.observability.classes.ev const DISPATCHER_SCOPE_MODULE: &str = "sqlbuild.runtime.observability.main.dispatcher_scope"; const IDENTITY_MODULE: &str = "sqlbuild.runtime.observability.main.current_execution_identity"; const EXTENSIONS_MODULE: &str = "sqlbuild.compiler.discovery.main.runtime_extensions"; -const LAUNCH_RECORD_MODULE: &str = "sqlbuild.cli.entry.main._withhold_launch_record"; const EXPORTER_DISPATCHER_MODULE: &str = "sqlbuild.runtime.event_exporting.classes.dispatcher"; const EXPORTER_SCOPE_CLASS_MODULE: &str = "sqlbuild.runtime.event_exporting.classes.command_scope"; const EXPORTER_SCOPE_MODULE: &str = @@ -340,7 +340,7 @@ impl Observed { let sinks: Bound<'_, PyAny> = extensions.getattr("command_output_sinks")?; command_output_sinks = sinks.is_truthy()?; if event_exporters.is_truthy()? || command_output_sinks { - imported(py, LAUNCH_RECORD_MODULE, "withhold_launch_record")?.call0()?; + withhold_launch_record(py)?; let timeout: Bound<'_, PyAny> = extensions.getattr("lifecycle_shutdown_timeout_seconds")?; let timeout: Bound<'_, PyAny> = if timeout.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 3bf2d20a93..b49ba9c7b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -3,7 +3,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::cli::{ - argv, entry, observability, progress, progress_reporters, progress_text, + argv, entry, launch_state, namespace, observability, progress, progress_reporters, + progress_text, }; use crate::bindings::_helpers::{ audit_command, build_command, build_progress, check, clone, command_runtime, contract_command, @@ -16,6 +17,8 @@ use crate::bindings::_helpers::{ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { argv::register(module)?; entry::register(module)?; + launch_state::register(module)?; + namespace::register(module)?; observability::register(module)?; progress::register(module)?; progress_reporters::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index 2d8b81a517..30ba913b70 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -10,6 +10,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyString}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_cli::replay::main::write_launch_record::write_launch_record as write_record_file; +use crate::bindings::_helpers::cli::launch_state::launch_record_path; use crate::bindings::_helpers::compile_driver::phases::{elapsed_ms, monotonic}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::compile_outputs::json_report::plain_json; @@ -23,7 +24,6 @@ const COMPILE_CONSTANTS: &str = "sqlbuild.compiler.compile.constants"; const RUNTIME_IDENTITY_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.runtime_identity"; const STARTUP_INPUTS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.startup_inputs"; const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; -const LAUNCH_RECORD_PATH_MODULE: &str = "sqlbuild.cli.entry.main._launch_record_path"; const COMPUTE_LOG_TEE_MODULE: &str = "sqlbuild.runtime.compute_logs.classes.text_tee"; const OUTPUT_CAPTURE_CONSTANTS: &str = "sqlbuild.runtime.output_capture.constants"; const SECONDS_FIELD: &str = "{seconds:.1f}"; @@ -199,8 +199,7 @@ fn write_record( /// Record this reusable compile for `sqb` when nothing it cannot replay took part. pub(crate) fn write_launch_record(attempt: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = attempt.py(); - let path: Bound<'_, PyAny> = - imported(py, LAUNCH_RECORD_PATH_MODULE, "launch_record_path")?.call0()?; + let path: Bound<'_, PyAny> = launch_record_path(py)?; let native: Bound<'_, PyAny> = attempt.getattr("native")?; if path.is_none() || native.is_none() { return Ok(()); @@ -449,6 +448,61 @@ fn attempt_compile_reuse<'py>(request: &Bound<'py, PyAny>) -> PyResult(request: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = request.py(); + let flags: Bound<'py, PyAny> = request.getattr("profile_flags")?; + let mut profiling: bool = false; + for name in [ + "skip_discovery_sql_analysis", + "skip_column_inference", + "skip_contracts", + "skip_write", + ] { + profiling = profiling || flags.getattr(name)?.is_truthy()?; + } + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = Vec::new(); + for name in [ + "project_dir", + "selected_target", + "json_output", + "no_color", + "manifest", + "dag_path", + "defer_to", + "no_cache", + "no_sql_validation", + ] { + fields.push((name, request.getattr(name)?)); + } + fields.push(( + "lineage_mode", + request + .getattr("lineage_mode")? + .getattr("value")? + .str()? + .into_any(), + )); + for name in ["select", "exclude", "cli_vars", "debug"] { + fields.push((name, request.getattr(name)?)); + } + fields.push(("profiling", profiling.into_bound_py_any(py)?)); + call_with(&imported(py, REUSE_MODELS, "CompileReuseRequest")?, &fields) +} + +/// Run one compile command request, replaying the previous compile when nothing changed. +#[pyfunction] +fn run_compile<'py>(request: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = request.py(); + let attempt: Bound<'py, PyAny> = attempt_compile_reuse(&reuse_request(request)?)?; + let exit_code: Bound<'py, PyAny> = attempt.getattr("exit_code")?; + if attempt.getattr("outcome")?.is(&outcome(py, HIT)?) && !exit_code.is_none() { + return Ok(exit_code); + } + imported(py, NATIVE_MODULE, "run_compile_command")?.call1((request, attempt)) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(attempt_compile_reuse, module)?) + module.add_function(wrap_pyfunction!(attempt_compile_reuse, module)?)?; + module.add_function(wrap_pyfunction!(run_compile, module)?) } diff --git a/fensu.toml b/fensu.toml index 5c8cc82a29..c3c999eea3 100644 --- a/fensu.toml +++ b/fensu.toml @@ -38,6 +38,11 @@ rule = "FFR309" path = "src/sqlbuild/sql_values/constants.py" reason = "Typed SQL values are normalized and state literals rendered natively; the namespace package keeps the value models, types, limits and errors the native bindings build and raise." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/cli/compile/__init__.py" +reason = "Native bindings own the compile command, its reuse attempt and its analysis; this package only declares the compile analysis models the native code builds." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/cli/compile_reuse/__init__.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5a9059e6dd..8e0637723b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -22,7 +22,6 @@ from sqlbuild.adapter.contract.types import ( TypeDialect, ) from sqlbuild.adapter.relations.models import InspectionQueryRecord -from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.cli.commands.models import ( AuditCommandRequest, BuildCommandRequest, @@ -1634,6 +1633,7 @@ class NativeReuseAttempt: def record(self, record: dict[str, object]) -> tuple[str, int]: ... def check_compile_reuse(check: dict[str, object]) -> NativeReuseAttempt: ... +def run_compile(request: CompileCommandRequest) -> int: ... def attempt_compile_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: ... class CompileArtifactWrites: @@ -2370,6 +2370,174 @@ def format_dbt_active_node_status( *, active_nodes: Sequence[tuple[str, float]], now: float ) -> str: ... +class CliNamespace: + command: str | None + project_dir: str | None + dbt_project_dir: str | None + dbt_profiles_dir: str | None + dbt_target: str | None + dbt_profile: str | None + sqb_output_dir: str | None + dry_run: bool + overwrite: bool + skip_dbt_debug: bool + no_sql_validation: bool + no_cache: bool + defer_to: str | None + defer_clone_from: str | None + defer_sources_to: str | None + target: str | None + warehouse: str | None + case: str | None + inspect: bool + target_range: str | None + from_target: str | None + to_target: str | None + hard_copy: bool + json: bool + json_output: Path | None + event_output: Path | None + manifest: bool + dag: str | None + compile_lineage_mode: CompileLineageMode + profile_skip_discovery_sql_analysis: bool + profile_skip_column_inference: bool + profile_skip_contracts: bool + profile_skip_write: bool + start_cursor: str | None + end_cursor: str | None + start_cursor_ts: str | None + max_microbatches: int | None + end_cursor_ts: str | None + start_cursor_int: str | None + end_cursor_int: str | None + no_color: bool + fail_fast: bool + full_refresh: bool + as_target: str | None + selection_diagnostics: bool + include_python: bool + load_sources: bool | None + reload: bool + run_tests: bool + run_audits: bool + allow_snapshot_full_refresh: bool + allow_table_type_downgrade: bool + allow_retention_decrease: bool + allow_missing_migration_origin: bool + allow_snapshot_schema_change: bool + concurrency: int | None + verbose: bool + debug: bool + auto_approve: bool + retention_days: int | None + direct_state_history_versions: int | None + drop_old_name_views: list[str] + bounded: str | None + max_column_examples: int | None + max_row_only_examples: int | None + sample_rows: int | None + sample_seed: int | None + exhaustive: bool + max_models: int | None + max_columns: int | None + query_sql: str | None + query_file: str | None + query_format: str + query_limit: int | None + query_no_limit: bool + cost_selector: str + refactor_target: str | None + refactor_new_name: str | None + refactor_destination: str | None + refactor_cascade: bool + cost_limit: int | None + cost_no_limit: bool + cost_sort: str | None + cost_order: str | None + cost_since: str | None + cost_until: str | None + lineage_targets: list[str] + lineage_format: str + lineage_direction: str | None + lineage_depth: str + lineage_mode: ColumnLineageMode + lineage_include_uses: bool + no_connection: bool + fail_on_error: bool + fail_on_stale: bool + state: bool + full: bool + schema_only: bool + left_query: str | None + left_query_file: Path | None + left_label: str | None + right_query: str | None + right_query_file: Path | None + right_label: str | None + key: list[str] + unkeyed: bool + exclude_column: list[str] + tolerance: list[str] + max_value_length: int | None + no_example_values: bool + full_example_values: bool + playground_path: str + playground_template: str + scenario_command: str | None + scenario_selector: list[str] + scenario_retain: bool + scenario_namespace: str | None + scenario_local: bool + scenario_strict: bool + scenario_sync_snapshots: bool + scenario_refresh: bool + scenario_force: bool + scenario_max_snapshot_rows: int | None + scenario_max_snapshot_total_rows: int | None + scenario_max_snapshot_bytes: int | None + scenario_max_snapshot_total_bytes: int | None + skills_global: bool + skills_target: list[str] + skills_force: bool + select: list[str] + positional_select: list[str] + select_file: list[str] + exclude: list[str] + dbt_command: str | None + dbt_args: list[str] + vars: dict[str, object] + rules_command: str | None + rules_rule_selector: str | None + rules_skills_check: bool + format_check: bool + format_diff: bool + format_fixtures_only: bool + format_fix: bool + format_paths: list[str] + scope_target: str | None + scope_at: str | None + scope_as_path: str | None + scope_browse: str | None + scope_list: str | None + contract_command: str | None + contract_from: str | None + contract_write: bool + scope_defined_under: str | None + scope_kind: list[str] + scope_match: str | None + scope_used_only: bool + scope_include_nearby: bool + scope_nearby_depth: int + scope_dependency_depth: int + scope_explain: str | None + scope_globals: str + scope_page_size: int + scope_after: str | None + scope_paths: str + + def __init__(self) -> None: ... + class DbtInitProgressReporter: def __init__(self, *, stream: TextIO, use_color: bool) -> None: ... def start(self, message: str) -> None: ... diff --git a/src/sqlbuild/cli/commands/classes/cli_namespace.py b/src/sqlbuild/cli/commands/classes/cli_namespace.py deleted file mode 100644 index b07cb18c1c..0000000000 --- a/src/sqlbuild/cli/commands/classes/cli_namespace.py +++ /dev/null @@ -1,346 +0,0 @@ -"""Mutable CLI argparse namespace.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.commands.types import CompileLineageMode -from sqlbuild.compiler.lineage.types import ColumnLineageMode - -_DEFAULT_VALUES: dict[str, object] = { - "command": None, - "project_dir": None, - "dbt_project_dir": None, - "dbt_profiles_dir": None, - "dbt_target": None, - "dbt_profile": None, - "sqb_output_dir": None, - "dry_run": False, - "overwrite": False, - "skip_dbt_debug": False, - "no_sql_validation": False, - "no_cache": False, - "defer_to": None, - "defer_clone_from": None, - "defer_sources_to": None, - "target": None, - "warehouse": None, - "case": None, - "inspect": False, - "target_range": None, - "from_target": None, - "to_target": None, - "hard_copy": False, - "json": False, - "json_output": None, - "manifest": False, - "dag": None, - "compile_lineage_mode": CompileLineageMode.FAST, - "profile_skip_discovery_sql_analysis": False, - "profile_skip_column_inference": False, - "profile_skip_contracts": False, - "profile_skip_write": False, - "start_cursor": None, - "end_cursor": None, - "start_cursor_ts": None, - "max_microbatches": None, - "end_cursor_ts": None, - "start_cursor_int": None, - "end_cursor_int": None, - "no_color": False, - "fail_fast": False, - "full_refresh": False, - "as_target": None, - "include_python": True, - "load_sources": None, - "reload": False, - "run_tests": True, - "run_audits": True, - "allow_snapshot_full_refresh": False, - "allow_table_type_downgrade": False, - "allow_retention_decrease": False, - "allow_missing_migration_origin": False, - "allow_snapshot_schema_change": False, - "concurrency": None, - "verbose": False, - "debug": False, - "auto_approve": False, - "retention_days": None, - "direct_state_history_versions": None, - "drop_old_name_views": [], - "bounded": None, - "max_column_examples": None, - "max_row_only_examples": None, - "sample_rows": None, - "sample_seed": None, - "exhaustive": False, - "max_models": None, - "max_columns": None, - "query_sql": None, - "query_file": None, - "query_format": "long", - "query_limit": 20, - "query_no_limit": False, - "cost_selector": "latest", - "cost_limit": None, - "cost_no_limit": False, - "cost_sort": None, - "cost_order": None, - "cost_since": None, - "cost_until": None, - "lineage_targets": [], - "lineage_format": "tree", - "lineage_direction": None, - "lineage_depth": "all", - "lineage_mode": ColumnLineageMode.RICH, - "lineage_include_uses": False, - "no_connection": False, - "fail_on_error": False, - "fail_on_stale": False, - "state": False, - "full": False, - "schema_only": False, - "left_query": None, - "left_query_file": None, - "right_query": None, - "right_query_file": None, - "left_label": None, - "right_label": None, - "key": [], - "unkeyed": False, - "exclude_column": [], - "tolerance": [], - "max_value_length": None, - "no_example_values": False, - "full_example_values": False, - "playground_path": "sqlbuild-playground", - "playground_template": "waffle_shop", - "scenario_command": None, - "scenario_selector": [], - "scenario_retain": False, - "scenario_namespace": None, - "scenario_local": False, - "scenario_strict": False, - "scenario_sync_snapshots": False, - "scenario_refresh": False, - "scenario_force": False, - "scenario_max_snapshot_rows": None, - "scenario_max_snapshot_total_rows": None, - "scenario_max_snapshot_bytes": None, - "scenario_max_snapshot_total_bytes": None, - "skills_global": False, - "skills_target": [], - "skills_force": False, - "select": [], - "positional_select": [], - "select_file": [], - "exclude": [], - "dbt_command": None, - "dbt_args": [], - "vars": {}, - "rules_command": None, - "rules_rule_selector": None, - "rules_skills_check": False, - "format_check": False, - "format_diff": False, - "format_fixtures_only": False, - "format_fix": False, - "format_paths": [], - "scope_target": None, - "scope_at": None, - "scope_as_path": None, - "scope_browse": None, - "scope_list": None, - "contract_command": None, - "contract_from": None, - "contract_write": False, - "scope_defined_under": None, - "scope_kind": [], - "scope_match": None, - "scope_used_only": False, - "scope_include_nearby": False, - "scope_nearby_depth": 1, - "scope_dependency_depth": 0, - "scope_explain": None, - "scope_globals": "summary", - "scope_page_size": 100, - "scope_after": None, - "scope_paths": "relative", - "refactor_target": None, - "refactor_new_name": None, - "refactor_destination": None, - "refactor_cascade": False, -} - - -class CliNamespace: - """Typed namespace for all CLI arguments across all commands.""" - - command: str | None - project_dir: str | None - dbt_project_dir: str | None - dbt_profiles_dir: str | None - dbt_target: str | None - dbt_profile: str | None - sqb_output_dir: str | None - dry_run: bool - overwrite: bool - skip_dbt_debug: bool - no_sql_validation: bool - no_cache: bool - defer_to: str | None - defer_clone_from: str | None - defer_sources_to: str | None - target: str | None - warehouse: str | None - case: str | None - inspect: bool - target_range: str | None - from_target: str | None - to_target: str | None - hard_copy: bool - json: bool - json_output: Path | None - event_output: Path | None - manifest: bool - dag: str | None - compile_lineage_mode: CompileLineageMode - profile_skip_discovery_sql_analysis: bool - profile_skip_column_inference: bool - profile_skip_contracts: bool - profile_skip_write: bool - start_cursor: str | None - end_cursor: str | None - start_cursor_ts: str | None - max_microbatches: int | None - end_cursor_ts: str | None - start_cursor_int: str | None - end_cursor_int: str | None - no_color: bool - fail_fast: bool - full_refresh: bool - as_target: str | None - selection_diagnostics: bool - include_python: bool - load_sources: bool | None - reload: bool - run_tests: bool - run_audits: bool - allow_snapshot_full_refresh: bool - allow_table_type_downgrade: bool - allow_retention_decrease: bool - allow_missing_migration_origin: bool - allow_snapshot_schema_change: bool - concurrency: int | None - verbose: bool - debug: bool - auto_approve: bool - retention_days: int | None - direct_state_history_versions: int | None - drop_old_name_views: list[str] - bounded: str | None - max_column_examples: int | None - max_row_only_examples: int | None - sample_rows: int | None - sample_seed: int | None - exhaustive: bool - max_models: int | None - max_columns: int | None - query_sql: str | None - query_file: str | None - query_format: str - query_limit: int | None - query_no_limit: bool - cost_selector: str - refactor_target: str | None - refactor_new_name: str | None - refactor_destination: str | None - refactor_cascade: bool - cost_limit: int | None - cost_no_limit: bool - cost_sort: str | None - cost_order: str | None - cost_since: str | None - cost_until: str | None - lineage_targets: list[str] - lineage_format: str - lineage_direction: str | None - lineage_depth: str - lineage_mode: ColumnLineageMode - lineage_include_uses: bool - no_connection: bool - fail_on_error: bool - fail_on_stale: bool - state: bool - full: bool - schema_only: bool - left_query: str | None - left_query_file: Path | None - left_label: str | None - right_query: str | None - right_query_file: Path | None - right_label: str | None - key: list[str] - unkeyed: bool - exclude_column: list[str] - tolerance: list[str] - max_value_length: int | None - no_example_values: bool - full_example_values: bool - playground_path: str - playground_template: str - scenario_command: str | None - scenario_selector: list[str] - scenario_retain: bool - scenario_namespace: str | None - scenario_local: bool - scenario_strict: bool - scenario_sync_snapshots: bool - scenario_refresh: bool - scenario_force: bool - scenario_max_snapshot_rows: int | None - scenario_max_snapshot_total_rows: int | None - scenario_max_snapshot_bytes: int | None - scenario_max_snapshot_total_bytes: int | None - skills_global: bool - skills_target: list[str] - skills_force: bool - select: list[str] - positional_select: list[str] - select_file: list[str] - exclude: list[str] - dbt_command: str | None - dbt_args: list[str] - vars: dict[str, object] - rules_command: str | None - rules_rule_selector: str | None - rules_skills_check: bool - format_check: bool - format_diff: bool - format_fixtures_only: bool - format_fix: bool - format_paths: list[str] - scope_target: str | None - scope_at: str | None - scope_as_path: str | None - scope_browse: str | None - scope_list: str | None - contract_command: str | None - contract_from: str | None - contract_write: bool - scope_defined_under: str | None - scope_kind: list[str] - scope_match: str | None - scope_used_only: bool - scope_include_nearby: bool - scope_nearby_depth: int - scope_dependency_depth: int - scope_explain: str | None - scope_globals: str - scope_page_size: int - scope_after: str | None - scope_paths: str - - def __init__(self) -> None: - for name, value in _DEFAULT_VALUES.items(): - setattr(self, name, value.copy() if isinstance(value, (dict, list)) else value) diff --git a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py index 29671a69ed..93f9a1c0ad 100644 --- a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py +++ b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py @@ -6,11 +6,10 @@ from datetime import UTC, datetime from pathlib import Path -from sqlbuild._native import supports_color +from sqlbuild._native import CliNamespace, supports_color from sqlbuild.cli.commands._helpers.entry.compute_log_diagnostics import ( log_compute_capture_failure, ) -from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.diagnostics.classes.invocation_diagnostic_routing import InvocationDiagnosticRouting from sqlbuild.diagnostics.main.configure import configure_diagnostics from sqlbuild.diagnostics.models import DiagnosticRoutingOptions diff --git a/src/sqlbuild/cli/compile/main/run.py b/src/sqlbuild/cli/compile/main/run.py deleted file mode 100644 index 52f9307154..0000000000 --- a/src/sqlbuild/cli/compile/main/run.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Public compile command entrypoint.""" - -from __future__ import annotations - -from sqlbuild._native import attempt_compile_reuse -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt, CompileReuseRequest -from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome -from sqlbuild.cli.entry.models import CompileCommandRequest, CompileProfileFlags - - -def run_compile(request: CompileCommandRequest) -> int: - """Run one compile command request, replaying the previous compile when nothing changed.""" - - attempt: CompileReuseAttempt = attempt_compile_reuse(request=_reuse_request(request=request)) - if attempt.outcome is CompileReuseOutcome.HIT and attempt.exit_code is not None: - return attempt.exit_code - from sqlbuild._native import run_compile_command - - return run_compile_command(request, attempt) - - -def _reuse_request(*, request: CompileCommandRequest) -> CompileReuseRequest: - flags: CompileProfileFlags = request.profile_flags - return CompileReuseRequest( - project_dir=request.project_dir, - selected_target=request.selected_target, - json_output=request.json_output, - no_color=request.no_color, - manifest=request.manifest, - dag_path=request.dag_path, - defer_to=request.defer_to, - no_cache=request.no_cache, - no_sql_validation=request.no_sql_validation, - lineage_mode=str(request.lineage_mode.value), - select=request.select, - exclude=request.exclude, - cli_vars=request.cli_vars, - debug=request.debug, - profiling=( - flags.skip_discovery_sql_analysis - or flags.skip_column_inference - or flags.skip_contracts - or flags.skip_write - ), - ) diff --git a/src/sqlbuild/cli/entry/_helpers/launch_state.py b/src/sqlbuild/cli/entry/_helpers/launch_state.py deleted file mode 100644 index d0bfd1330e..0000000000 --- a/src/sqlbuild/cli/entry/_helpers/launch_state.py +++ /dev/null @@ -1,25 +0,0 @@ -"""The launch record the native `sqb` executable offered this invocation.""" - -from contextvars import ContextVar, Token -from pathlib import Path - -_OFFERED: ContextVar[Path | None] = ContextVar("sqlbuild_launch_record", default=None) -_WITHHELD: ContextVar[bool] = ContextVar("sqlbuild_launch_record_withheld", default=False) - - -def offer_launch_record(path: Path) -> Token[Path | None]: - """Remember where `sqb` looks for this invocation's launch record.""" - - return _OFFERED.set(path) - - -def withhold_offered_launch_record() -> Token[bool]: - """Forget the offered launch record for the rest of this invocation.""" - - return _WITHHELD.set(True) - - -def offered_launch_record() -> Path | None: - """The offered launch record path, unless it was withheld.""" - - return None if _WITHHELD.get() else _OFFERED.get() diff --git a/src/sqlbuild/cli/entry/main/_launch_record_path.py b/src/sqlbuild/cli/entry/main/_launch_record_path.py deleted file mode 100644 index 9ba485c42a..0000000000 --- a/src/sqlbuild/cli/entry/main/_launch_record_path.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Where this invocation may leave a launch record for the native `sqb` executable.""" - -from pathlib import Path - -from sqlbuild.cli.entry._helpers.launch_state import offered_launch_record - - -def launch_record_path() -> Path | None: - """The launch record path `sqb` offered, unless an input outside its key was read.""" - - return offered_launch_record() diff --git a/src/sqlbuild/cli/entry/main/_withhold_launch_record.py b/src/sqlbuild/cli/entry/main/_withhold_launch_record.py deleted file mode 100644 index fb5e4decf7..0000000000 --- a/src/sqlbuild/cli/entry/main/_withhold_launch_record.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Stop this invocation from leaving a launch record.""" - -from sqlbuild.cli.entry._helpers.launch_state import withhold_offered_launch_record - - -def withhold_launch_record() -> None: - """Withhold the launch record: `sqb` cannot replay what this invocation also depends on.""" - - _ = withhold_offered_launch_record() diff --git a/src/sqlbuild/cli/entry/main/launched.py b/src/sqlbuild/cli/entry/main/launched.py index 60b51f56dc..69729dc592 100644 --- a/src/sqlbuild/cli/entry/main/launched.py +++ b/src/sqlbuild/cli/entry/main/launched.py @@ -1,19 +1,15 @@ """Python CLI entrypoint the native `sqb` executable hands every invocation it does not serve.""" import sys -from pathlib import Path -from sqlbuild.cli.entry._helpers.launch_state import offer_launch_record from sqlbuild.cli.entry.constants import LAUNCH_RECORD_OPTION from sqlbuild.cli.entry.main.entry import main as run_cli def main() -> int: - """Run the CLI with the arguments `sqb` received, remembering where its launch record goes.""" + """Run the CLI with the arguments `sqb` received; the CLI reads a leading launch record.""" arguments: list[str] = sys.argv[1:] - if arguments and arguments[0].startswith(LAUNCH_RECORD_OPTION): - _ = offer_launch_record(Path(arguments[0].removeprefix(LAUNCH_RECORD_OPTION))) - arguments = arguments[1:] - sys.argv = [sys.argv[0], *arguments] - return run_cli() + offered: bool = bool(arguments) and arguments[0].startswith(LAUNCH_RECORD_OPTION) + sys.argv = [sys.argv[0], *arguments[int(offered) :]] + return run_cli(arguments) diff --git a/src/sqlbuild/cli/entry/models.py b/src/sqlbuild/cli/entry/models.py index 0f8ce77795..eb9ca9c7ec 100644 --- a/src/sqlbuild/cli/entry/models.py +++ b/src/sqlbuild/cli/entry/models.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace +from sqlbuild._native import CliNamespace from sqlbuild.cli.commands.types import ( CompileLineageMode, DagCommandHandler, diff --git a/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py b/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py index dd1ac19762..120ae60668 100644 --- a/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py +++ b/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py @@ -12,8 +12,7 @@ import pytest -from sqlbuild._native import cli_observability_scope -from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace +from sqlbuild._native import CliNamespace, cli_observability_scope from sqlbuild.execution_history import ( CanonicalLifecycleEvent, EventFilter, diff --git a/src/sqlbuild/cli/compile/main/__init__.py b/tests/unit/src/sqlbuild/cli/commands/classes/__init__.py similarity index 100% rename from src/sqlbuild/cli/compile/main/__init__.py rename to tests/unit/src/sqlbuild/cli/commands/classes/__init__.py diff --git a/tests/unit/src/sqlbuild/cli/commands/classes/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/classes/_test_types.py new file mode 100644 index 0000000000..07b6e6a8fd --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/commands/classes/_test_types.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class CliNamespaceDefaultTestCase: + description: str + attribute: str + expected_value: object + + +@dataclass(frozen=True) +class CliNamespaceListIsolationTestCase: + description: str + attribute: str + appended_value: str + expected_first: list[str] + expected_second: list[str] + + +@dataclass(frozen=True) +class CliNamespaceAssignmentTestCase: + description: str + attribute: str + value: object + expected_value: object + + +@dataclass(frozen=True) +class CliNamespaceMissingTestCase: + description: str + attribute: str + expected_error_fragment: str diff --git a/tests/unit/src/sqlbuild/cli/commands/classes/test_cli_namespace.py b/tests/unit/src/sqlbuild/cli/commands/classes/test_cli_namespace.py new file mode 100644 index 0000000000..d30b53075d --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/commands/classes/test_cli_namespace.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import pytest + +from sqlbuild._native import CliNamespace +from sqlbuild.cli.commands.types import CompileLineageMode +from sqlbuild.compiler.lineage.types import ColumnLineageMode +from tests.unit.src.sqlbuild.cli.commands.classes._test_types import ( + CliNamespaceAssignmentTestCase, + CliNamespaceDefaultTestCase, + CliNamespaceListIsolationTestCase, + CliNamespaceMissingTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + CliNamespaceDefaultTestCase( + description="an unset option is None", + attribute="project_dir", + expected_value=None, + ), + CliNamespaceDefaultTestCase( + description="a flag defaults to False", + attribute="dry_run", + expected_value=False, + ), + CliNamespaceDefaultTestCase( + description="an enabled-by-default flag is True", + attribute="run_tests", + expected_value=True, + ), + CliNamespaceDefaultTestCase( + description="an integer option keeps its default", + attribute="query_limit", + expected_value=20, + ), + CliNamespaceDefaultTestCase( + description="a text option keeps its default", + attribute="playground_template", + expected_value="waffle_shop", + ), + CliNamespaceDefaultTestCase( + description="compile lineage defaults to the fast mode", + attribute="compile_lineage_mode", + expected_value=CompileLineageMode.FAST, + ), + CliNamespaceDefaultTestCase( + description="column lineage defaults to the rich mode", + attribute="lineage_mode", + expected_value=ColumnLineageMode.RICH, + ), + CliNamespaceDefaultTestCase( + description="CLI vars default to an empty mapping", + attribute="vars", + expected_value={}, + ), + ], + ids=lambda case: case.description, +) +def test_given_new_namespace_when_reading_option_then_returns_its_default( + test_case: CliNamespaceDefaultTestCase, +) -> None: + namespace: CliNamespace = CliNamespace() + + assert getattr(namespace, test_case.attribute) == test_case.expected_value + + +@pytest.mark.parametrize( + "test_case", + [ + CliNamespaceListIsolationTestCase( + description="selectors appended to one namespace stay out of another", + attribute="select", + appended_value="orders", + expected_first=["orders"], + expected_second=[], + ), + CliNamespaceListIsolationTestCase( + description="excludes appended to one namespace stay out of another", + attribute="exclude", + appended_value="customers", + expected_first=["customers"], + expected_second=[], + ), + ], + ids=lambda case: case.description, +) +def test_given_two_namespaces_when_extending_a_list_default_then_the_other_is_unchanged( + test_case: CliNamespaceListIsolationTestCase, +) -> None: + first: CliNamespace = CliNamespace() + second: CliNamespace = CliNamespace() + + getattr(first, test_case.attribute).append(test_case.appended_value) + + assert getattr(first, test_case.attribute) == test_case.expected_first + assert getattr(second, test_case.attribute) == test_case.expected_second + + +@pytest.mark.parametrize( + "test_case", + [ + CliNamespaceAssignmentTestCase( + description="an assigned option overrides its default", + attribute="command", + value="build", + expected_value="build", + ), + CliNamespaceAssignmentTestCase( + description="an option without a default can be assigned", + attribute="event_output", + value=None, + expected_value=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_namespace_when_assigning_then_attribute_holds_the_value( + test_case: CliNamespaceAssignmentTestCase, +) -> None: + namespace: CliNamespace = CliNamespace() + + setattr(namespace, test_case.attribute, test_case.value) + + assert getattr(namespace, test_case.attribute) == test_case.expected_value + + +@pytest.mark.parametrize( + "test_case", + [ + CliNamespaceMissingTestCase( + description="an option without a default is missing until assigned", + attribute="event_output", + expected_error_fragment="'CliNamespace' object has no attribute 'event_output'", + ), + ], + ids=lambda case: case.description, +) +def test_given_namespace_when_reading_undeclared_default_then_raises_attribute_error( + test_case: CliNamespaceMissingTestCase, +) -> None: + namespace: CliNamespace = CliNamespace() + + with pytest.raises(AttributeError, match=test_case.expected_error_fragment): + getattr(namespace, test_case.attribute) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 394f533461..94e79a7794 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -11,9 +11,9 @@ import pytest import sqlbuild._native as discovery_entry +from sqlbuild._native import run_compile from sqlbuild.adapter.discovery.main import resolve_adapter as runtime_adapters from sqlbuild.cli.commands.types import CompileLineageMode -from sqlbuild.cli.compile.main.run import run_compile from sqlbuild.cli.entry.models import CompileCommandRequest from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.compiler.planner.main.execution import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index ae59421a72..2bda854f13 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -159,12 +159,9 @@ def test_given_root_help_arguments_when_running_main_then_it_returns_expected_ex test_case: MainTestCase, monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.delitem(sys.modules, "sqlbuild.cli.compile.main.run", raising=False) - exit_code: int = main(test_case.argv) assert exit_code == test_case.expected_exit_code - assert "sqlbuild.cli.compile.main.run" not in sys.modules @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py index 10a3861cee..62c9577aa9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py @@ -10,11 +10,11 @@ from _pytest.capture import CaptureResult from sqlbuild._native import ( + CliNamespace, NativeProgressProjector, cli_observability_scope, current_native_progress_projector, ) -from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.cli.output.constants import INTEGRATION_RESULT_PATH_ENV from sqlbuild.observability import OperationLifecycle from tests.unit.src.sqlbuild.cli.commands.main.entry._test_types import ( diff --git a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_connection.py b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_connection.py index 0453d04311..7bafffcf43 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_connection.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_connection.py @@ -20,12 +20,6 @@ @pytest.mark.parametrize( "test_case", [ - DbtProfileConnectionTestCase( - description="a connection without the dbt_profile source is not resolved", - raw_config={"database": "orders.duckdb"}, - adapter_name="duckdb", - expected_connection=None, - ), DbtProfileConnectionTestCase( description="the project's dbt profile resolves to adapter config", raw_config={"source": "dbt_profile", "target": " dev "}, @@ -47,7 +41,33 @@ def test_given_raw_connection_when_resolving_dbt_profile_then_returns_expected_c project_config=dbt_workspace_project_config(adapter_name=test_case.adapter_name), ) - assert (None if resolved is None else resolved.connection) == test_case.expected_connection + assert resolved is not None + assert resolved.connection == test_case.expected_connection + + +@pytest.mark.parametrize( + "test_case", + [ + DbtProfileConnectionTestCase( + description="a connection without the dbt_profile source is not resolved", + raw_config={"database": "orders.duckdb"}, + adapter_name="duckdb", + expected_connection=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_non_profile_connection_when_resolving_dbt_profile_then_returns_none( + test_case: DbtProfileConnectionTestCase, tmp_path: Path +) -> None: + resolved: NormalizedDbtProfileConnection | None = resolve_raw_dbt_profile_connection( + raw_config=test_case.raw_config, + project_dir=tmp_path, + adapter_name=test_case.adapter_name, + project_config=dbt_workspace_project_config(adapter_name=test_case.adapter_name), + ) + + assert resolved == test_case.expected_connection @pytest.mark.parametrize( From 7cb01c8b05470315f3cbbc4d46c51292710fb06d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 03:40:50 +0100 Subject: [PATCH 07/11] refactor: host build cost, check planning, refactor compiles and defer-clone natively Move build cost finalization, Python check run-state recording, dependency relation targets, relevant-check selection and read-side preflight, the offline refactor compile, and the build defer-clone prephase into the native build, check and refactor owners. Executor, adapter and planner traversal callbacks stay in Python; the clone executor entry modules only Rust imports gain a leading underscore. The replaced Python modules and their dead models are deleted. --- .../_helpers/build_command/completion.rs | 15 +- .../_helpers/build_command/defer_clone.rs | 664 ++++++++++++++++++ .../_helpers/build_command/execution.rs | 57 +- .../bindings/_helpers/build_command/mod.rs | 1 + .../_helpers/build_command/planning.rs | 16 +- .../_helpers/build_command/registration.rs | 3 +- .../bindings/_helpers/check/check_command.rs | 68 +- .../src/bindings/_helpers/check/check_core.rs | 545 ++++++++++++++ .../src/bindings/_helpers/check/mod.rs | 1 + .../bindings/_helpers/check/registration.rs | 3 +- .../_helpers/clone/clone_execution.rs | 6 +- .../_helpers/cost_report/degradation.rs | 202 +++++- .../src/bindings/_helpers/refactor/mod.rs | 1 + .../_helpers/refactor/offline_compile.rs | 197 ++++++ .../_helpers/refactor/refactor_command.rs | 17 +- python-boundary.toml | 2 + src/sqlbuild/_native.pyi | 17 + .../_helpers/build_planning/__init__.py | 0 .../_helpers/build_planning/defer_clone.py | 374 ---------- .../cli/commands/_helpers/check/core.py | 335 --------- .../cli/commands/_helpers/cost/collection.py | 144 ---- .../commands/_helpers/refactor/__init__.py | 1 - .../_helpers/refactor/compile_facts.py | 79 --- src/sqlbuild/cli/commands/models.py | 48 -- ...=> _build_destination_transient_models.py} | 0 ...quests.py => _build_retention_requests.py} | 0 .../{fingerprinting.py => _fingerprinting.py} | 0 ...tream.py => _run_prephase_clone_stream.py} | 0 .../_helpers/test_module_paths.py | 1 + .../commands/_helpers/check/test_planning.py | 34 +- .../commands/_helpers/cost/test_collection.py | 3 +- .../commands/main/build/test_defer_clone.py | 18 +- 32 files changed, 1719 insertions(+), 1133 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs delete mode 100644 src/sqlbuild/cli/commands/_helpers/build_planning/__init__.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/check/core.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/cost/collection.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/refactor/__init__.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py rename src/sqlbuild/executor/clone/main/{build_destination_transient_models.py => _build_destination_transient_models.py} (100%) rename src/sqlbuild/executor/clone/main/{build_retention_requests.py => _build_retention_requests.py} (100%) rename src/sqlbuild/executor/clone/main/{fingerprinting.py => _fingerprinting.py} (100%) rename src/sqlbuild/executor/clone/main/{run_prephase_clone_stream.py => _run_prephase_clone_stream.py} (100%) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index 34b0529700..78c78c97db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -21,10 +21,12 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::compile_outputs::runtime_target::{ write_python_check_runtime_target, write_runtime_target, }; +use crate::bindings::_helpers::cost_report::degradation::{ + finalize_build_cost as finalize_cost, render_build_cost, +}; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const COST_COLLECTION_MODULE: &str = "sqlbuild.cli.commands._helpers.cost.collection"; const COST_TYPES: &str = "sqlbuild.cost.types"; const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const BUILD_TYPES: &str = "sqlbuild.executor.build.types"; @@ -77,7 +79,7 @@ pub(crate) fn finalize_build_cost<'py>( &imported(py, COMMAND_MODELS, "BuildCostFinalization")?, &fields, )?; - imported(py, COST_COLLECTION_MODULE, "finalize_build_cost")?.call1((finalization,)) + finalize_cost(&finalization) } fn cost_status<'py>(py: Python<'py>, name: &str) -> PyResult> { @@ -370,14 +372,7 @@ pub(crate) fn complete_direct_build<'py>( ], )?; write_json(context, payload)?; - call_with( - &imported(py, COST_COLLECTION_MODULE, "render_build_cost")?, - &[ - ("record", record), - ("output_stream", context.stream.clone()), - ("use_color", boolean(py, context.use_color)), - ], - )?; + render_build_cost(&record, &context.stream, context.use_color)?; if context.verbose { let timings: Bound<'py, PyAny> = result.getattr("timings")?; let seconds = |name: &str| -> PyResult> { timings.getattr(name)?.extract() }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs new file mode 100644 index 0000000000..5a42ecf164 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -0,0 +1,664 @@ +//! `sqb build --defer-clone-from`: clone out-of-selection boundaries before build planning. + +use std::time::Instant; + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyCFunction, PyDict, PyFrozenSet, PyString, PyTuple, PyTupleMethods}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::build_command::invocation::BuildContext; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const CONSTANTS_MODULE: &str = "sqlbuild.cli.commands.constants"; +const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; +const CLONE_MODELS: &str = "sqlbuild.executor.clone.models"; +const CLONE_TYPES: &str = "sqlbuild.executor.clone.types"; +const SPEC_TYPES: &str = "sqlbuild.spec.contracts.types"; +const CLONE_BOUNDARY_MODULE: &str = "sqlbuild.compiler.planner.main.clone.resolve_clone_boundary"; +const VIEW_CHAIN_MODULE: &str = "sqlbuild.compiler.planner.main.clone.resolve_skipped_view_chain"; +const TRANSIENT_MODELS_MODULE: &str = + "sqlbuild.executor.clone.main._build_destination_transient_models"; +const RETENTION_REQUESTS_MODULE: &str = "sqlbuild.executor.clone.main._build_retention_requests"; +const EXECUTE_CLONE_MODULE: &str = "sqlbuild.executor.clone.main.execute"; +const FINGERPRINTING_MODULE: &str = "sqlbuild.executor.clone.main._fingerprinting"; +const CLONE_STREAM_MODULE: &str = "sqlbuild.executor.clone.main._run_prephase_clone_stream"; +const VIEW_MATERIALIZATION: &str = "view"; +const RECREATED_TYPES: [&str; 2] = ["udf", "table_fn"]; +const CLONABLE_TYPES: [&str; 4] = ["model", "seed", "udf", "table_fn"]; +const MODEL_TYPE: &str = "model"; + +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +fn user_error(py: Python<'_>, message: String, code: &str) -> PyErr { + let created = || -> PyResult { + let error: Bound<'_, PyAny> = imported(py, ERRORS_MODULE, "CliUserError")?.call( + (message,), + Some(&keywords(py, &[("code", text(py, code))])?), + )?; + Ok(PyErr::from_value(error)) + }; + created().unwrap_or_else(|failure| failure) +} + +fn resource_type(key: &Bound<'_, PyAny>) -> PyResult { + key.getattr("resource_type")?.getattr("value")?.extract() +} + +/// Functions and view models are recreated over cloned boundaries instead of cloned. +fn recreated_over_clones(scope: &Bound<'_, PyAny>, key: &Bound<'_, PyAny>) -> PyResult { + if RECREATED_TYPES.contains(&resource_type(key)?.as_str()) { + return Ok(true); + } + let model: Bound<'_, PyAny> = scope + .getattr("models_by_name")? + .call_method1("get", (key.getattr("name")?,))?; + if model.is_none() { + return Ok(false); + } + let materialized: Bound<'_, PyAny> = model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("materialized", VIEW_MATERIALIZATION))?; + let lowered: String = materialized.str()?.call_method0("lower")?.extract()?; + Ok(lowered == VIEW_MATERIALIZATION) +} + +/// Every key the scope's traversal can visit: the selection and the upstream graph. +fn scope_keys<'py>(scope: &Bound<'py, PyAny>) -> PyResult>> { + let mut keys: Vec> = scope + .getattr("selected_keys")? + .try_iter()? + .collect::>()?; + for item in scope + .getattr("upstream_deps")? + .call_method0("items")? + .try_iter()? + { + let (key, upstream): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + keys.push(key); + for upstream_key in upstream.try_iter()? { + keys.push(upstream_key?); + } + } + Ok(keys) +} + +/// `frozenset.__contains__` over the keys a predicate accepts, as the traversal callback. +fn membership<'py>( + py: Python<'py>, + keys: &[Bound<'py, PyAny>], + accepted: impl Fn(&Bound<'py, PyAny>) -> PyResult, +) -> PyResult> { + let mut members: Vec> = Vec::new(); + for key in keys { + if accepted(key)? { + members.push(key.clone()); + } + } + PyFrozenSet::new(py, &members)?.getattr("__contains__") +} + +fn traversed_names<'py>( + scope: &Bound<'py, PyAny>, + (module, function): (&str, &str), + (is_clonable, is_view): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = scope.py(); + let keys: Bound<'py, PyAny> = call_with( + &imported(py, module, function)?, + &[ + ("selected", scope.getattr("selected_keys")?), + ("upstream", scope.getattr("upstream_deps")?), + ("is_clonable", is_clonable.clone()), + ("is_view", is_view.clone()), + ], + )?; + let mut names: Vec = Vec::new(); + for key in keys.try_iter()? { + names.push(key?.getattr("name")?.extract()?); + } + names.sort(); + Ok(names) +} + +/// Selected models except those explicitly disabled. +fn selected_executable_model_count(scope: &Bound<'_, PyAny>) -> PyResult { + let models: Bound<'_, PyAny> = scope.getattr("models_by_name")?; + let mut count: usize = 0; + for key in scope.getattr("selected_keys")?.try_iter()? { + let key: Bound<'_, PyAny> = key?; + if resource_type(&key)? != MODEL_TYPE { + continue; + } + let model: Bound<'_, PyAny> = models.call_method1("get", (key.getattr("name")?,))?; + if !model.is_none() + && model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("enabled",))? + .is(false.into_pyobject(scope.py())?.to_owned().into_any()) + { + continue; + } + count += 1; + } + Ok(count) +} + +/// Boundary clone selectors, rebuilt view-chain selectors and the selected model count. +fn defer_clone_selection(scope: &Bound<'_, PyAny>) -> PyResult<(Vec, Vec, usize)> { + let py: Python<'_> = scope.py(); + let keys: Vec> = scope_keys(scope)?; + let is_clonable: Bound<'_, PyAny> = membership(py, &keys, |key| { + Ok(CLONABLE_TYPES.contains(&resource_type(key)?.as_str())) + })?; + let is_view: Bound<'_, PyAny> = membership(py, &keys, |key| recreated_over_clones(scope, key))?; + let callbacks: (&Bound<'_, PyAny>, &Bound<'_, PyAny>) = (&is_clonable, &is_view); + Ok(( + traversed_names( + scope, + (CLONE_BOUNDARY_MODULE, "resolve_clone_boundary"), + callbacks, + )?, + traversed_names( + scope, + (VIEW_CHAIN_MODULE, "resolve_skipped_view_chain"), + callbacks, + )?, + selected_executable_model_count(scope)?, + )) +} + +/// Boundary selectors, view-chain selectors and selected executable model count of a scope. +#[pyfunction] +#[pyo3(signature = (*, scope))] +fn resolve_defer_clone_selection( + scope: &Bound<'_, PyAny>, +) -> PyResult<(Vec, Vec, usize)> { + defer_clone_selection(scope) +} + +fn native<'py>( + py: Python<'py>, + name: &str, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with(&imported(py, NATIVE_MODULE, name)?, fields) +} + +fn external_resolver<'py>( + project_dir: &Bound<'py, PyAny>, + discovered: &Bound<'py, PyAny>, +) -> PyResult> { + native( + project_dir.py(), + "resolve_external_sql_reference_resolver", + &[ + ("project_dir", project_dir.clone()), + ("discovered_inputs", discovered.clone()), + ], + ) +} + +/// The resolved defer-clone prephase: what to clone, from where, into which target. +struct Prephase<'py> { + discovered: Bound<'py, PyAny>, + adapter: Bound<'py, PyAny>, + origin: Bound<'py, PyAny>, + destination: Bound<'py, PyAny>, + select: Vec, + caused_by: Bound<'py, PyAny>, + project_dir: Bound<'py, PyAny>, +} + +fn progress(on_progress: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { + if !on_progress.is_none() { + on_progress.call1((message,))?; + } + Ok(()) +} + +fn clone_input<'py>( + context: &BuildContext<'py>, + prephase: &Prephase<'py>, + (pipeline, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + on_item: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = context.py(); + let project: Bound<'py, PyAny> = pipeline.getattr("destination_project")?; + let destination_models: Bound<'py, PyAny> = pipeline.getattr("destination_model_entries")?; + let target: Bound<'py, PyAny> = native( + py, + "resolve_target_config", + &[ + ( + "project_config", + prephase.discovered.getattr("project_config")?, + ), + ("local_config", prephase.discovered.getattr("local_config")?), + ("target_name", prephase.destination.clone()), + ], + )?; + let mut selected: Vec> = Vec::new(); + for entry in destination_models.try_iter()? { + selected.push(entry?.getattr("name")?); + } + let retention: Bound<'py, PyAny> = call_with( + &imported( + py, + RETENTION_REQUESTS_MODULE, + "build_destination_retention_requests", + )?, + &[ + ("project", project.clone()), + ("adapter_name", prephase.adapter.getattr("adapter_name")?), + ( + "selected_model_names", + PyFrozenSet::new(py, &selected)?.into_any(), + ), + ( + "namespace_owned", + target.getattr("owns_time_travel_retention_namespace")?, + ), + ], + )?; + let allow_decrease: bool = target + .getattr("time_travel_retention_decrease")? + .eq(imported(py, SPEC_TYPES, "RetentionDecreasePolicy")?.getattr("ALLOW")?)?; + let sources: Bound<'py, PyAny> = call_with( + &imported(py, CLONE_MODELS, "CloneSourceEntries")?, + &[ + ("origin", pipeline.getattr("origin_source_entries")?), + ( + "destination", + pipeline.getattr("destination_source_entries")?, + ), + ], + )?; + call_with( + &imported(py, CLONE_MODELS, "CloneExecutionInput")?, + &[ + ("source_entries", sources), + ( + "origin_model_entries", + pipeline.getattr("origin_model_entries")?, + ), + ("destination_model_entries", destination_models), + ( + "origin_seed_entries", + pipeline.getattr("origin_seed_entries")?, + ), + ( + "destination_seed_entries", + pipeline.getattr("destination_seed_entries")?, + ), + ( + "destination_function_entries", + pipeline.getattr("destination_function_entries")?, + ), + ( + "execution_order", + pipeline.getattr("clone_plan")?.getattr("execution_order")?, + ), + ("adapter", prephase.adapter.clone()), + ("destination_connection", connection.clone()), + ("hard_copy", false.into_pyobject(py)?.to_owned().into_any()), + ("run_id", project.getattr("run_id")?), + ( + "query_change_tracking", + project + .getattr("settings")? + .getattr("query_change_tracking")?, + ), + ( + "destination_transient_models", + call_with( + &imported( + py, + TRANSIENT_MODELS_MODULE, + "build_destination_transient_models", + )?, + &[("project", project.clone())], + )?, + ), + ("destination_retention_requests", retention), + ( + "allow_namespace_retention_decrease", + allow_decrease.into_pyobject(py)?.to_owned().into_any(), + ), + ("on_item", on_item), + ], + ) +} + +/// Execute the clone, through the prephase progress stream when there is one. +fn executed_clone<'py>( + context: &BuildContext<'py>, + prephase: &Prephase<'py>, + (pipeline, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = context.py(); + let progress_stream: Bound<'py, PyAny> = context.invocation.getattr("progress_stream")?; + let execute: Bound<'py, PyAny> = imported(py, EXECUTE_CLONE_MODULE, "execute_clone")?; + if progress_stream.is_none() { + let inputs: Bound<'py, PyAny> = clone_input( + context, + prephase, + (pipeline, connection), + py.None().into_bound(py), + )?; + return call_with(&execute, &[("inputs", inputs)]); + } + let prepared: (Py, Py) = ( + clone_input( + context, + prephase, + (pipeline, connection), + py.None().into_bound(py), + )? + .unbind(), + execute.unbind(), + ); + let run_clone: Bound<'py, PyCFunction> = PyCFunction::new_closure( + py, + None, + None, + move |arguments: &Bound<'_, PyTuple>, + _: Option<&Bound<'_, PyDict>>| + -> PyResult> { + let py: Python<'_> = arguments.py(); + let inputs: Bound<'_, PyAny> = imported(py, "dataclasses", "replace")?.call( + (prepared.0.bind(py),), + Some(&keywords(py, &[("on_item", arguments.get_item(0)?)])?), + )?; + Ok(prepared + .1 + .bind(py) + .call((), Some(&keywords(py, &[("inputs", inputs)])?))? + .unbind()) + }, + )?; + call_with( + &imported(py, CLONE_STREAM_MODULE, "run_prephase_clone_stream")?, + &[ + ("stream", progress_stream), + ("title", text(py, "defer clone")), + ("caused_by_names", prephase.caused_by.clone()), + ("use_color", context.invocation.getattr("use_color")?), + ("run_clone", run_clone.into_any()), + ], + ) +} + +fn failed_items(result: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = result.py(); + let status: Bound<'_, PyAny> = imported(py, CLONE_TYPES, "CloneStatus")?; + let (failed, warning) = (status.getattr("FAILED")?, status.getattr("WARNING")?); + let mut items: Vec = Vec::new(); + for item in result.getattr("item_results")?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + let item_status: Bound<'_, PyAny> = item.getattr("status")?; + if !(item_status.eq(&failed)? || item_status.eq(&warning)?) { + continue; + } + let message: Bound<'_, PyAny> = item.getattr("message")?; + let detail: Bound<'_, PyAny> = if message.is_truthy()? { + message + } else { + item.getattr("action")?.getattr("value")? + }; + items.push(format!( + "{}: {}", + item.getattr("name")?.str()?, + detail.str()? + )); + } + Ok(items) +} + +/// Clone the boundary relations from the origin target, then copy their fingerprints. +fn clone_boundaries<'py>( + context: &BuildContext<'py>, + prephase: &Prephase<'py>, + connection: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py: Python<'py> = context.py(); + let on_progress: Bound<'py, PyAny> = context + .invocation + .getattr("planning_progress")? + .getattr("on_progress")?; + let start: Instant = Instant::now(); + let request: &Bound<'py, PyAny> = &context.request; + let pipeline: Bound<'py, PyAny> = native( + py, + "run_clone_pipeline_with_options", + &[ + ("discovered_inputs", prephase.discovered.clone()), + ("adapter", prephase.adapter.clone()), + ("origin_target_name", prephase.origin.clone()), + ("destination_target_name", prephase.destination.clone()), + ( + "destination_connection", + call_with( + &imported(py, PIPELINE_MODELS, "ClonePipelineConnection")?, + &[ + ("config", context.invocation.getattr("connection_config")?), + ("handle", connection.clone()), + ], + )?, + ), + ( + "options", + call_with( + &imported(py, PIPELINE_MODELS, "ClonePipelineOptions")?, + &[ + ("no_sql_validation", request.getattr("no_sql_validation")?), + ("no_cache", request.getattr("no_cache")?), + ("select", PyTuple::new(py, &prephase.select)?.into_any()), + ("cli_vars", request.getattr("cli_vars")?), + ], + )?, + ), + ( + "external_sql_reference_resolver", + external_resolver(&prephase.project_dir, &prephase.discovered)?, + ), + ], + )?; + progress( + &on_progress, + &format!( + "Prepared defer clone plan. ({:.2}s)", + start.elapsed().as_secs_f64() + ), + )?; + let result: Bound<'py, PyAny> = executed_clone(context, prephase, (&pipeline, connection))?; + let failed: Vec = failed_items(&result)?; + if !failed.is_empty() { + return Err(user_error( + py, + format!( + "failed to clone one or more deferred boundary relations: {}", + failed.join("; ") + ), + "C411", + )); + } + let project: Bound<'py, PyAny> = pipeline.getattr("destination_project")?; + call_with( + &imported(py, FINGERPRINTING_MODULE, "copy_clone_fingerprints")?, + &[ + ("result", result), + ( + "origin_model_entries", + pipeline.getattr("origin_model_entries")?, + ), + ( + "destination_model_entries", + pipeline.getattr("destination_model_entries")?, + ), + ( + "origin_seed_entries", + pipeline.getattr("origin_seed_entries")?, + ), + ( + "destination_seed_entries", + pipeline.getattr("destination_seed_entries")?, + ), + ("adapter", prephase.adapter.clone()), + ("destination_connection", connection.clone()), + ("run_id", project.getattr("run_id")?), + ( + "query_change_tracking", + project + .getattr("settings")? + .getattr("query_change_tracking")?, + ), + ], + )?; + Ok(()) +} + +/// Clone the selected boundary relations from the origin before build planning. +fn run_prephase(context: &BuildContext<'_>, prephase: &Prephase<'_>) -> PyResult<()> { + let py: Python<'_> = context.py(); + if prephase.select.is_empty() { + return Ok(()); + } + if prephase.destination.is_none() { + return Err(user_error( + py, + "--defer-clone-from requires an active target".to_owned(), + "C409", + )); + } + if prephase.origin.eq(&prephase.destination)? { + return Err(user_error( + py, + format!( + "Cannot defer-clone from the current target '{}'", + prephase.origin.str()? + ), + "C410", + )); + } + progress( + &context + .invocation + .getattr("planning_progress")? + .getattr("on_progress")?, + "Preparing defer clone plan...", + )?; + let connection: Bound<'_, PyAny> = prephase.adapter.call_method1( + "connect", + (context.invocation.getattr("connection_config")?,), + )?; + let cloned: PyResult<()> = clone_boundaries(context, prephase, &connection); + let closed: PyResult> = prephase.adapter.call_method1("close", (connection,)); + cloned?; + closed?; + Ok(()) +} + +/// Resolve defer-clone boundaries and clone them before build planning. +pub(crate) fn run_defer_clone_boundary_prephase( + context: &BuildContext<'_>, + origin: Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = context.py(); + let (request, invocation) = (&context.request, &context.invocation); + let discovered: Bound<'_, PyAny> = invocation.getattr("discovered_inputs")?; + let split: Bound<'_, PyAny> = native( + py, + "split_unit_test_selectors", + &[ + ("select", request.getattr("select")?), + ("exclude", request.getattr("exclude")?), + ("discovered_inputs", discovered.clone()), + ], + )?; + if !split + .getattr("sql_test_selection")? + .getattr("models_selected")? + .is_truthy()? + { + return Ok(()); + } + let adapter: Bound<'_, PyAny> = invocation.getattr("adapter")?; + let project_dir: Bound<'_, PyAny> = invocation.getattr("effective_project_dir")?; + let project: Bound<'_, PyAny> = native( + py, + "build_compiled_project", + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ("selected_target", request.getattr("selected_target")?), + ("no_sql_validation", request.getattr("no_sql_validation")?), + ( + "analysis_selection", + call_with( + &imported(py, COMPILE_MODELS, "CompileAnalysisSelection")?, + &[("no_cache", request.getattr("no_cache")?)], + )?, + ), + ("cli_vars", request.getattr("cli_vars")?), + ( + "external_sql_reference_resolver", + external_resolver(&project_dir, &discovered)?, + ), + ], + )?; + let scope: Bound<'_, PyAny> = native( + py, + "build_planner_scope", + &[ + ("project", project.clone()), + ("select", split.getattr("select")?), + ("exclude", split.getattr("exclude")?), + ( + "auto_load_sources", + invocation.getattr("should_load_sources")?, + ), + ], + )?; + let (boundary, view_chain, model_count): (Vec, Vec, usize) = + defer_clone_selection(&scope)?; + native( + py, + "enforce_model_execution_limit", + &[ + ("model_count", model_count.into_pyobject(py)?.into_any()), + ("target_name", invocation.getattr("effective_target_name")?), + ("limits", invocation.getattr("execution_limits")?), + ( + "refusal_note", + imported(py, CONSTANTS_MODULE, "EXECUTION_LIMIT_BUILD_NOTE")?, + ), + ], + )?; + let mut select: Vec = boundary; + select.extend(view_chain); + run_prephase( + context, + &Prephase { + discovered, + adapter, + origin, + destination: project.getattr("effective_target_name")?, + select, + caused_by: split.getattr("select")?, + project_dir, + }, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_defer_clone_selection, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index eaad8e76b9..865ea135f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -14,6 +14,10 @@ use crate::bindings::_helpers::build_command::lifecycle::{ use crate::bindings::_helpers::build_command::support::{ boolean, display_text, emit, frozenset, joined, keyword_relay, line_writer, none, text, tuple, }; +use crate::bindings::_helpers::check::check_core::{ + build_check_relation_targets, load_results_by_loader_name, record_python_run_state_results, + relevant_check_functions, +}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; @@ -25,7 +29,6 @@ const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const PIPELINE_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const FRESHNESS_MODULE: &str = "sqlbuild.cli.commands._helpers.freshness.source_freshness"; const COST_CONTEXT_MODULE: &str = "sqlbuild.cost.classes.cost_context"; -const CHECK_CORE_MODULE: &str = "sqlbuild.cli.commands._helpers.check.core"; const GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const CHECKS_MODULE: &str = "sqlbuild.executor.python_nodes.main._checks"; const EXECUTOR_PYTHON_MODELS: &str = "sqlbuild.executor.python_nodes.models"; @@ -452,15 +455,7 @@ fn check_runtime<'py>( ("default_schema", adapter.call_method0("default_schema")?), ( "relation_targets", - call_with( - &imported(py, CHECK_CORE_MODULE, "build_check_relation_targets")?, - &[ - ("adapter", adapter.clone()), - ("pipeline_result", pipeline.clone()), - ("python_graph", graph.clone()), - ("selected_python_names", frozenset(py, &names)?), - ], - )?, + build_check_relation_targets(&adapter, pipeline, graph, frozenset(py, &names)?)?, ), ( "project_relations", @@ -503,14 +498,8 @@ pub(crate) fn run_post_build_python_checks<'py>( &imported(py, GRAPH_MODULE, "build_discovered_python_node_graph")?, &[("discovered_inputs", discovered.clone())], )?; - let by_loader = || { - call_with( - &imported(py, CHECK_CORE_MODULE, "load_results_by_loader_name")?, - &[ - ("source_map", plan.getattr("source_map")?), - ("load_results", load_results.clone()), - ], - ) + let by_loader = || -> PyResult> { + Ok(load_results_by_loader_name(&plan.getattr("source_map")?, &load_results)?.into_any()) }; let mut dependencies: Vec> = names_of(&outcome.python_results, "node_name")?; for name in by_loader()?.try_iter()? { @@ -520,18 +509,12 @@ pub(crate) fn run_post_build_python_checks<'py>( names_of(&result.getattr("model_results")?, "model_name")?; completed.extend(names_of(&result.getattr("seed_results")?, "seed_name")?); completed.extend(names_of(&load_results, "source_name")?); - let checks: Vec> = call_with( - &imported(py, CHECK_CORE_MODULE, "relevant_check_functions")?, - &[ - ("discovered_inputs", discovered.clone()), - ("python_graph", graph.clone()), - ("exclude", context.request.getattr("exclude")?), - ("selected_dependency_names", frozenset(py, &dependencies)?), - ("completed_sql_names", frozenset(py, &completed)?), - ], - )? - .try_iter()? - .collect::>()?; + let checks: Vec> = relevant_check_functions( + &discovered, + &graph, + context.request.getattr("exclude")?, + (&frozenset(py, &dependencies)?, &frozenset(py, &completed)?), + )?; if checks.is_empty() { return tuple(py, &[]); } @@ -541,15 +524,11 @@ pub(crate) fn run_post_build_python_checks<'py>( let executed: PyResult> = (|| { let run_state: Bound<'py, PyAny> = imported(py, EXECUTOR_PYTHON_MODELS, "PythonNodeRunState")?.call0()?; - call_with( - &imported(py, CHECK_CORE_MODULE, "record_python_run_state_results")?, - &[ - ("discovered_inputs", discovered.clone()), - ("run_state", run_state.clone()), - ("python_results", outcome.python_results.clone()), - ("load_results", load_results.clone()), - ("source_map", plan.getattr("source_map")?), - ], + record_python_run_state_results( + &discovered, + &run_state, + (&outcome.python_results, Some(&load_results)), + Some(&plan.getattr("source_map")?), )?; let callbacks: Bound<'py, PyAny> = call_with( &imported(py, EXECUTOR_PYTHON_MODELS, "PythonCheckCallbacks")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/mod.rs index dc7093648e..643665def4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod command; pub(crate) mod completion; +pub(crate) mod defer_clone; pub(crate) mod execution; pub(crate) mod invocation; pub(crate) mod lifecycle; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index da72502d25..64be5880e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -3,6 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use sqlbuild_core::build_command::models::PlanSafetyGates; +use crate::bindings::_helpers::build_command::defer_clone::run_defer_clone_boundary_prephase; use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::build_command::policies::{enforce_plan_policies, model_limit}; use crate::bindings::_helpers::build_command::support::{ @@ -13,7 +14,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const CONSTANTS_MODULE: &str = "sqlbuild.cli.commands.constants"; -const DEFER_CLONE_MODULE: &str = "sqlbuild.cli.commands._helpers.build_planning.defer_clone"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; @@ -25,19 +25,7 @@ pub(crate) fn run_defer_clone_prephase(context: &BuildContext<'_>) -> PyResult<( if origin.is_none() { return Ok(()); } - call_with( - &imported( - context.py(), - DEFER_CLONE_MODULE, - "run_defer_clone_boundary_prephase", - )?, - &[ - ("request", context.request.clone()), - ("invocation", context.invocation.clone()), - ("origin_target_name", origin), - ], - )?; - Ok(()) + run_defer_clone_boundary_prephase(context, origin) } fn request_options<'py>(context: &BuildContext<'py>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/registration.rs index 4263b665d3..82ba906d4a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::build_command::{command, policies}; +use crate::bindings::_helpers::build_command::{command, defer_clone, policies}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + defer_clone::register(module)?; policies::register(module)?; command::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 6638a667b3..b40ea8a85c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -12,6 +12,10 @@ use sqlbuild_core::check_command::main::validate_check_selection::validate_check use sqlbuild_core::check_command::models::CheckResultText; use sqlbuild_core::cli_text::models::CliStyle; +use crate::bindings::_helpers::check::check_core::{ + ReadSideInputs, load_results_by_loader_name, record_python_run_state_results, + run_read_side_dependencies, +}; use crate::bindings::_helpers::check::check_results::{ check_result_text, results_json, write_results, }; @@ -23,7 +27,6 @@ use crate::bindings::_helpers::graph::python_selection::python_node_names; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const CHECK_CORE_MODULE: &str = "sqlbuild.cli.commands._helpers.check.core"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; @@ -424,24 +427,12 @@ fn record_run_state<'py>( ), source_map: &Bound<'py, PyAny>, ) -> PyResult<()> { - let py: Python<'py> = invocation.py(); - let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ - ( - "discovered_inputs", - invocation.getattr("discovered_inputs")?, - ), - ("run_state", run_state), - ("python_results", python_results), - ]; - if let Some(load_results) = load_results { - fields.push(("load_results", load_results)); - } - fields.push(("source_map", source_map.clone())); - call_with( - &imported(py, CHECK_CORE_MODULE, "record_python_run_state_results")?, - &fields, - )?; - Ok(()) + record_python_run_state_results( + &invocation.getattr("discovered_inputs")?, + &run_state, + (&python_results, load_results.as_ref()), + Some(source_map), + ) } /// Run ingress loaders and read-side dependencies, then the checks themselves. @@ -497,24 +488,18 @@ fn run_checks<'py>( ), &source_map, )?; - let read_side: Bound<'py, PyAny> = call_with( - &imported(py, CHECK_CORE_MODULE, "run_check_read_side_dependencies")?, - &[ - ("adapter", invocation.getattr("adapter")?), - ( - "connection_config", - invocation.getattr("connection_config")?, - ), - ("connection", connection.clone()), - ("pipeline_result", prepared.pipeline.clone()), - ("python_graph", prepared.python_graph.clone()), - ("lifecycle_plan", prepared.lifecycle_plan.clone()), - ("relation_targets", prepared.relation_targets.clone()), - ("validation_refs", prepared.relation_refs.clone()), - ("project_relations", prepared.project_relations.clone()), - ("providers", providers.clone()), - ], - )?; + let read_side: Bound<'py, PyAny> = run_read_side_dependencies(&ReadSideInputs { + adapter: invocation.getattr("adapter")?, + connection_config: invocation.getattr("connection_config")?, + connection: connection.clone(), + pipeline_result: prepared.pipeline.clone(), + python_graph: prepared.python_graph.clone(), + lifecycle_plan: prepared.lifecycle_plan.clone(), + relation_targets: prepared.relation_targets.clone(), + validation_refs: prepared.relation_refs.clone(), + project_relations: prepared.project_relations.clone(), + providers: providers.clone(), + })?; record_run_state( invocation, (run_state.clone(), read_side.clone(), None), @@ -524,13 +509,8 @@ fn run_checks<'py>( .try_iter()? .chain(read_side.try_iter()?) .collect::>()?; - let by_loader: Bound<'py, PyAny> = call_with( - &imported(py, CHECK_CORE_MODULE, "load_results_by_loader_name")?, - &[ - ("source_map", source_map), - ("load_results", load_results.clone()), - ], - )?; + let by_loader: Bound<'py, PyAny> = + load_results_by_loader_name(&source_map, &load_results)?.into_any(); let on_complete: Bound<'py, PyAny> = Bound::new( py, CheckCompletionRelay { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs new file mode 100644 index 0000000000..6e87586e50 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs @@ -0,0 +1,545 @@ +//! Python check planning: run-state recording, dependency relations and read-side preflight. + +use pyo3::IntoPyObject; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const BUILD_TYPES: &str = "sqlbuild.executor.build.types"; +const PYTHON_NODE_TYPES: &str = "sqlbuild.compiler.python_nodes.types"; +const EXECUTOR_MODELS: &str = "sqlbuild.executor.python_nodes.models"; +const READ_SIDE_MODULE: &str = "sqlbuild.executor.python_nodes.main.read_side"; +const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; +const QUALIFIED_NAME_MODULE: &str = + "sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name"; +const MISSING_RELATION_CODE: &str = "C682"; +const MODEL_KIND: &str = "model"; +const SEED_KIND: &str = "seed"; +const SOURCE_KIND: &str = "source"; +const READ_SIDE_OPTIONS: (&[&str], &[&str]) = ( + &[ + "adapter", + "connection_config", + "connection", + "pipeline_result", + "python_graph", + "lifecycle_plan", + "relation_targets", + ], + &["validation_refs", "project_relations", "providers"], +); + +fn user_error(py: Python<'_>, message: String) -> PyErr { + let created = || -> PyResult { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("code", MISSING_RELATION_CODE)?; + let error: Bound<'_, PyAny> = + imported(py, ERRORS_MODULE, "CliUserError")?.call((message,), Some(&keywords))?; + Ok(PyErr::from_value(error)) + }; + created().unwrap_or_else(|failure| failure) +} + +/// The loader node result a load result stands for. +fn load_result_as_python_result<'py>( + name: &Bound<'py, PyAny>, + result: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = name.py(); + let execution: Bound<'py, PyAny> = imported(py, BUILD_TYPES, "ExecutionStatus")?; + let node_status: Bound<'py, PyAny> = imported(py, PYTHON_NODE_TYPES, "PythonNodeStatus")?; + let status: Bound<'py, PyAny> = result.getattr("status")?; + let mapped: Bound<'py, PyAny> = if status.eq(execution.getattr("SUCCESS")?)? { + node_status.getattr("SUCCESS")? + } else if status.eq(execution.getattr("SKIPPED")?)? { + node_status.getattr("SKIPPED")? + } else { + node_status.getattr("FAILED")? + }; + call_with( + &imported(py, EXECUTOR_MODELS, "PythonNodeExecutionResult")?, + &[ + ("node_name", name.clone()), + ( + "kind", + imported(py, PYTHON_NODE_TYPES, "PythonNodeKind")?.getattr("LOADER")?, + ), + ("status", mapped), + ("payload", result.clone()), + ("error_message", result.getattr("error_message")?), + ], + ) +} + +/// Load results keyed by loader name for check dependency lookup. +pub(crate) fn load_results_by_loader_name<'py>( + source_map: &Bound<'py, PyAny>, + load_results: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = source_map.py(); + let loader_by_source: Bound<'py, PyDict> = PyDict::new(py); + for item in source_map.call_method0("items")?.try_iter()? { + let (name, source): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let loader: Bound<'py, PyAny> = source.getattr("loader")?; + if !loader.is_none() { + loader_by_source.set_item(name, loader)?; + } + } + let by_loader: Bound<'py, PyDict> = PyDict::new(py); + for result in load_results.try_iter()? { + let result: Bound<'py, PyAny> = result?; + if let Some(loader) = loader_by_source.get_item(result.getattr("source_name")?)? + && !loader.is_none() + { + by_loader.set_item(loader, result)?; + } + } + Ok(by_loader) +} + +/// Record Python and loader results in the Python run-state. +pub(crate) fn record_python_run_state_results<'py>( + discovered_inputs: &Bound<'py, PyAny>, + run_state: &Bound<'py, PyAny>, + (python_results, load_results): (&Bound<'py, PyAny>, Option<&Bound<'py, PyAny>>), + source_map: Option<&Bound<'py, PyAny>>, +) -> PyResult<()> { + let py: Python<'py> = discovered_inputs.py(); + let record = |function: Bound<'py, PyAny>, result: Bound<'py, PyAny>| -> PyResult<()> { + call_with( + &run_state.getattr("record_result")?, + &[("node_function", function), ("result", result)], + )?; + Ok(()) + }; + let result_by_name: Bound<'py, PyDict> = PyDict::new(py); + for result in python_results.try_iter()? { + let result: Bound<'py, PyAny> = result?; + result_by_name.set_item(result.getattr("node_name")?, result)?; + } + let node_by_name: Bound<'py, PyDict> = PyDict::new(py); + for attribute in ["task_functions", "asset_functions"] { + for node in discovered_inputs.getattr(attribute)?.try_iter()? { + let node: Bound<'py, PyAny> = node?; + node_by_name.set_item(node.getattr("name")?, node.getattr("function")?)?; + } + } + for (name, function) in node_by_name.iter() { + if let Some(result) = result_by_name.get_item(name)? { + record(function, result)?; + } + } + let by_loader: Bound<'py, PyDict> = match (source_map, load_results) { + (Some(source_map), Some(load_results)) => { + load_results_by_loader_name(source_map, load_results)? + } + (Some(source_map), None) => { + load_results_by_loader_name(source_map, &PyTuple::empty(py).into_any())? + } + _ => PyDict::new(py), + }; + for loader in discovered_inputs.getattr("loader_functions")?.try_iter()? { + let loader: Bound<'py, PyAny> = loader?; + let name: Bound<'py, PyAny> = loader.getattr("name")?; + let Some(result) = by_loader.get_item(&name)? else { + continue; + }; + record( + loader.getattr("function")?, + load_result_as_python_result(&name, &result)?, + )?; + } + Ok(()) +} + +/// SQL relation locations available to Python check dependencies. +pub(crate) fn build_check_relation_targets<'py>( + adapter: &Bound<'py, PyAny>, + pipeline_result: &Bound<'py, PyAny>, + python_graph: &Bound<'py, PyAny>, + selected_python_names: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let required: Bound<'py, PyAny> = call_with( + &python_graph.getattr("selected_sql_refs")?, + &[("selected_names", selected_python_names)], + )?; + call_with( + &imported(py, NATIVE_MODULE, "build_python_relation_targets")?, + &[ + ("adapter", adapter.clone()), + ("project", pipeline_result.getattr("project")?), + ("plan_output", pipeline_result.getattr("plan_output")?), + ("required_refs", required), + ], + ) +} + +/// Non-excluded checks whose Python and SQL dependencies all ran in this build. +pub(crate) fn relevant_check_functions<'py>( + discovered_inputs: &Bound<'py, PyAny>, + python_graph: &Bound<'py, PyAny>, + exclude: Bound<'py, PyAny>, + (selected_dependency_names, completed_sql_names): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult>> { + let py: Python<'py> = discovered_inputs.py(); + let excluded: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_python_check_excludes")?, + &[("graph", python_graph.clone()), ("exclude", exclude)], + )?; + let check_kind: Bound<'py, PyAny> = + imported(py, PYTHON_NODE_TYPES, "PythonNodeKind")?.getattr("CHECK")?; + let upstream_deps: Bound<'py, PyAny> = python_graph.getattr("upstream_deps")?; + let mut check_names: Vec> = Vec::new(); + for node in python_graph.getattr("nodes")?.try_iter()? { + let node: Bound<'py, PyAny> = node?; + let name: Bound<'py, PyAny> = node.getattr("name")?; + if !node.getattr("kind")?.eq(&check_kind)? || excluded.contains(&name)? { + continue; + } + let upstream: Vec> = upstream_deps + .call_method1("get", (&name, PyTuple::empty(py)))? + .try_iter()? + .collect::>()?; + let mut sql_names: Vec> = Vec::new(); + for reference in node.getattr("sql_deps")?.try_iter()? { + sql_names.push(reference?.getattr("name")?); + } + if upstream.is_empty() && sql_names.is_empty() { + continue; + } + let mut ready: bool = true; + for upstream_name in &upstream { + ready = ready && selected_dependency_names.contains(upstream_name)?; + } + for sql_name in &sql_names { + ready = ready && completed_sql_names.contains(sql_name)?; + } + if ready { + check_names.push(name); + } + } + let check_names: Bound<'py, PyFrozenSet> = PyFrozenSet::new(py, &check_names)?; + let mut checks: Vec> = Vec::new(); + for check in discovered_inputs.getattr("check_functions")?.try_iter()? { + let check: Bound<'py, PyAny> = check?; + if check_names.contains(check.getattr("name")?)? { + checks.push(check); + } + } + Ok(checks) +} + +/// Inputs of the read-side preflight `sqb check` runs before the checks themselves. +pub(crate) struct ReadSideInputs<'py> { + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) connection_config: Bound<'py, PyAny>, + pub(crate) connection: Bound<'py, PyAny>, + pub(crate) pipeline_result: Bound<'py, PyAny>, + pub(crate) python_graph: Bound<'py, PyAny>, + pub(crate) lifecycle_plan: Bound<'py, PyAny>, + pub(crate) relation_targets: Bound<'py, PyAny>, + pub(crate) validation_refs: Bound<'py, PyAny>, + pub(crate) project_relations: Bound<'py, PyAny>, + pub(crate) providers: Bound<'py, PyAny>, +} + +fn reference_kind(reference: &Bound<'_, PyAny>) -> PyResult { + reference.getattr("kind")?.getattr("value")?.extract() +} + +/// A plan source the check can look up as a table: not an expression, with a table name. +fn table_source<'py>( + plan_output: &Bound<'py, PyAny>, + name: &Bound<'py, PyAny>, +) -> PyResult>> { + let source: Bound<'py, PyAny> = plan_output + .getattr("python_source_entries")? + .call_method1("get", (name,))?; + if source.is_none() + || !source.getattr("expression")?.is_none() + || source.getattr("table")?.is_none() + { + return Ok(None); + } + Ok(Some(source)) +} + +fn located<'py>( + plan_output: &Bound<'py, PyAny>, + reference: &Bound<'py, PyAny>, +) -> PyResult>> { + let attribute: &str = match reference_kind(reference)?.as_str() { + MODEL_KIND => "model_locations", + SEED_KIND => "seed_locations", + _ => return Ok(None), + }; + let location: Bound<'py, PyAny> = plan_output + .getattr(attribute)? + .call_method1("get", (reference.getattr("name")?,))?; + Ok((!location.is_none()).then_some(location)) +} + +/// `(database, schema, name)` of the relation a reference reads, when it has one. +fn lookup_location<'py>( + plan_output: &Bound<'py, PyAny>, + reference: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = plan_output.py(); + let parts: (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>) = + if reference_kind(reference)? == SOURCE_KIND { + let Some(source) = table_source(plan_output, &reference.getattr("name")?)? else { + return Ok(None); + }; + ( + source.getattr("database")?, + source.getattr("schema")?, + source.getattr("table")?, + ) + } else { + let Some(location) = located(plan_output, reference)? else { + return Ok(None); + }; + ( + location.getattr("database")?, + location.getattr("schema")?, + location.getattr("name")?, + ) + }; + Ok(Some( + PyTuple::new(py, [parts.0, parts.1, parts.2])?.into_any(), + )) +} + +fn relation_exists( + lookup: &Bound<'_, PyAny>, + (database, schema, name): (Bound<'_, PyAny>, Bound<'_, PyAny>, Bound<'_, PyAny>), +) -> PyResult { + call_with( + &lookup.getattr("exists")?, + &[("database", database), ("schema", schema), ("name", name)], + )? + .is_truthy() +} + +/// Reject a check SQL dependency whose relation does not exist yet. +fn validate_reference_exists( + inputs: &ReadSideInputs<'_>, + lookup: &Bound<'_, PyAny>, + reference: &Bound<'_, PyAny>, +) -> Result<(), PyErr> { + let py: Python<'_> = reference.py(); + let plan_output: Bound<'_, PyAny> = inputs.pipeline_result.getattr("plan_output")?; + let kind: String = reference_kind(reference)?; + let name: Bound<'_, PyAny> = reference.getattr("name")?; + let (exists, relation): (bool, Bound<'_, PyAny>) = match kind.as_str() { + MODEL_KIND | SEED_KIND => { + let Some(location) = located(&plan_output, reference)? else { + let noun: &str = if kind == MODEL_KIND { + "SQL model" + } else { + "seed" + }; + return Err(user_error( + py, + format!( + "Python check dependency requires unknown {noun} '{}'", + name.str()? + ), + )); + }; + let exists: bool = relation_exists( + lookup, + ( + location.getattr("database")?, + location.getattr("schema")?, + location.getattr("name")?, + ), + )?; + let relation: Bound<'_, PyAny> = call_with( + &imported( + py, + QUALIFIED_NAME_MODULE, + "resolve_relation_location_qualified_name", + )?, + &[("adapter", inputs.adapter.clone()), ("location", location)], + )?; + (exists, relation) + } + SOURCE_KIND => { + let Some(source) = table_source(&plan_output, &name)? else { + return Ok(()); + }; + let exists: bool = relation_exists( + lookup, + ( + source.getattr("database")?, + source.getattr("schema")?, + source.getattr("table")?, + ), + )?; + (exists, inputs.relation_targets.get_item(reference)?) + } + _ => return Ok(()), + }; + if exists { + return Ok(()); + } + Err(user_error( + py, + format!( + "Python check dependency requires existing SQL relation '{}' for {kind} '{}'; run sqb build first", + relation.str()?, + name.str()? + ), + )) +} + +fn sorted_references<'py>(references: &Bound<'py, PyAny>) -> PyResult>> { + let mut keyed: Vec<((String, String), Bound<'py, PyAny>)> = Vec::new(); + for reference in references.try_iter()? { + let reference: Bound<'py, PyAny> = reference?; + let key: (String, String) = ( + reference_kind(&reference)?, + reference.getattr("name")?.extract()?, + ); + keyed.push((key, reference)); + } + keyed.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(keyed.into_iter().map(|(_, reference)| reference).collect()) +} + +fn read_side_references<'py>( + python_graph: &Bound<'py, PyAny>, + read_side_names: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = python_graph.py(); + let nodes: Bound<'py, PyAny> = python_graph.getattr("nodes_by_name")?; + let mut references: Vec> = Vec::new(); + for name in read_side_names.try_iter()? { + for reference in nodes.get_item(name?)?.getattr("sql_deps")?.try_iter()? { + references.push(reference?); + } + } + Ok(PyFrozenSet::new(py, &references)?.into_any()) +} + +fn node_runtime<'py>(inputs: &ReadSideInputs<'py>) -> PyResult> { + let py: Python<'py> = inputs.adapter.py(); + let project: Bound<'py, PyAny> = inputs.pipeline_result.getattr("project")?; + call_with( + &imported(py, EXECUTOR_MODELS, "PythonNodeRuntime")?, + &[ + ("adapter", inputs.adapter.clone()), + ("connection_config", inputs.connection_config.clone()), + ("connection", inputs.connection.clone()), + ("run_id", project.getattr("run_id")?), + ("target", project.getattr("effective_target_name")?), + ("vars", project.getattr("effective_vars")?), + ("is_reload", false.into_pyobject(py)?.to_owned().into_any()), + ( + "default_database", + inputs.adapter.call_method0("default_database")?, + ), + ( + "default_schema", + inputs.adapter.call_method0("default_schema")?, + ), + ("relation_targets", inputs.relation_targets.clone()), + ("project_relations", inputs.project_relations.clone()), + ("providers", inputs.providers.clone()), + ], + ) +} + +/// Run read-side Python dependencies for checks against existing SQL relations. +pub(crate) fn run_read_side_dependencies<'py>( + inputs: &ReadSideInputs<'py>, +) -> PyResult> { + let py: Python<'py> = inputs.adapter.py(); + let read_side_names: Bound<'py, PyAny> = inputs + .lifecycle_plan + .getattr("read_side_python_node_names")?; + let to_validate: Bound<'py, PyAny> = if inputs.validation_refs.is_none() { + read_side_references(&inputs.python_graph, &read_side_names)? + } else { + inputs.validation_refs.clone() + }; + let references: Vec> = sorted_references(&to_validate)?; + let plan_output: Bound<'py, PyAny> = inputs.pipeline_result.getattr("plan_output")?; + let mut locations: Vec> = Vec::new(); + for reference in &references { + if let Some(location) = lookup_location(&plan_output, reference)? { + locations.push(location); + } + } + let lookup: Bound<'py, PyAny> = call_with( + &imported(py, RELATION_LOOKUP_MODULE, "build_relation_lookup")?, + &[ + ("adapter", inputs.adapter.clone()), + ("connection", inputs.connection.clone()), + ("locations", PyTuple::new(py, locations)?.into_any()), + ], + )?; + for reference in &references { + validate_reference_exists(inputs, &lookup, reference)?; + } + if !read_side_names.is_truthy()? { + return Ok(PyTuple::empty(py).into_any()); + } + let tracker: Bound<'py, PyAny> = call_with( + &imported( + py, + READ_SIDE_MODULE, + "create_read_side_python_execution_tracker", + )?, + &[ + ("python_graph", inputs.python_graph.clone()), + ("selected_python_names", read_side_names), + ("scheduled_sql_names", PyFrozenSet::empty(py)?.into_any()), + ("runtime", node_runtime(inputs)?), + ], + )?; + tracker.call_method0("dispatch_ready_python_nodes")?; + tracker.call_method0("finalize_unrun_python_nodes")?; + tracker.getattr("results") +} + +/// Run read-side Python dependencies for checks against existing SQL relations. +#[pyfunction] +#[pyo3(signature = (**options))] +fn run_check_read_side_dependencies<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "run_check_read_side_dependencies", + options, + READ_SIDE_OPTIONS, + )?; + run_read_side_dependencies(&ReadSideInputs { + adapter: arguments.value("adapter")?, + connection_config: arguments.value("connection_config")?, + connection: arguments.value("connection")?, + pipeline_result: arguments.value("pipeline_result")?, + python_graph: arguments.value("python_graph")?, + lifecycle_plan: arguments.value("lifecycle_plan")?, + relation_targets: arguments.value("relation_targets")?, + validation_refs: arguments.value("validation_refs")?, + project_relations: arguments.value("project_relations")?, + providers: arguments.value("providers")?, + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_check_read_side_dependencies, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/mod.rs index 3bde938874..d9bb6ff627 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod check_command; +pub(crate) mod check_core; pub(crate) mod check_results; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/registration.rs index 5baf04cec2..f1df493414 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::check::{check_command, check_results}; +use crate::bindings::_helpers::check::{check_command, check_core, check_results}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { check_command::register(module)?; + check_core::register(module)?; check_results::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs index c5f6e01005..fa12657b63 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs @@ -18,10 +18,10 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const CLONE_MODELS: &str = "sqlbuild.executor.clone.models"; -const RETENTION_REQUESTS_MODULE: &str = "sqlbuild.executor.clone.main.build_retention_requests"; +const RETENTION_REQUESTS_MODULE: &str = "sqlbuild.executor.clone.main._build_retention_requests"; const TRANSIENT_MODELS_MODULE: &str = - "sqlbuild.executor.clone.main.build_destination_transient_models"; -const FINGERPRINTING_MODULE: &str = "sqlbuild.executor.clone.main.fingerprinting"; + "sqlbuild.executor.clone.main._build_destination_transient_models"; +const FINGERPRINTING_MODULE: &str = "sqlbuild.executor.clone.main._fingerprinting"; const EXECUTE_MODULE: &str = "sqlbuild.executor.clone.main.execute"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs index 60a366e763..aabd7e53ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs @@ -6,23 +6,34 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyTypeMethods}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTypeMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cost_report::main::format_cost_breakdown::format_cost_breakdown; use sqlbuild_core::cost_report::models::BreakdownOptions; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::cost_report::records::cost_run; use crate::bindings::_helpers::cost_report::store; const LOGGER_NAME: &str = "sqlbuild.cli.commands._helpers.cost.collection"; +const COLLECTION_LOGGER_NAME: &str = "sqlbuild.cost"; +const COST_TYPES: &str = "sqlbuild.cost.types"; +const COST_MODELS: &str = "sqlbuild.cost.models"; +const COST_CONSTANTS: &str = "sqlbuild.cost.constants"; +const CONFIGURED_RATE_SOURCE: &str = "configured"; +const STATEMENT_LEDGER_PATH: [&str; 4] = ["target", "executions", "", "statements.jsonl"]; const BUILD_BREAKDOWN_LIMIT: usize = 5; const FALLBACK_TERMINAL_COLUMNS: usize = 100; /// Log the failure with its traceback, as `_LOGGER.exception` inside the handler did. fn log_failure(py: Python<'_>, message: &str, error: &PyErr) -> PyResult<()> { + log_to(py, LOGGER_NAME, message, error) +} + +fn log_to(py: Python<'_>, logger_name: &str, message: &str, error: &PyErr) -> PyResult<()> { let logger: Bound<'_, PyAny> = py .import("logging")? - .call_method1("getLogger", (LOGGER_NAME,))?; + .call_method1("getLogger", (logger_name,))?; let keywords: Bound<'_, PyDict> = PyDict::new(py); keywords.set_item("exc_info", error.value(py))?; logger.call_method("exception", (message,), Some(&keywords))?; @@ -118,7 +129,194 @@ fn write_build_cost_breakdown( ) } +/// One build's cost record with `cost` as its summary. +fn cost_record<'py>( + finalization: &Bound<'py, PyAny>, + summary: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = finalization.py(); + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = Vec::new(); + for name in [ + "run_id", + "adapter_name", + "target_name", + "build_status", + "started_at", + "completed_at", + ] { + fields.push((name, finalization.getattr(name)?)); + } + fields.push(("cost", summary)); + fields.push(( + "had_executable_work", + finalization.getattr("had_executable_work")?, + )); + call_with(&imported(py, COST_MODELS, "CostRunRecord")?, &fields) +} + +fn summary<'py>( + py: Python<'py>, + status: Bound<'py, PyAny>, + (config, rate_source): (&Bound<'py, PyAny>, &str), + message: Bound<'py, PyAny>, +) -> PyResult> { + call_with( + &imported(py, COST_MODELS, "RunCostSummary")?, + &[ + ("status", status), + ("usd_per_credit", config.getattr("usd_per_credit")?), + ("rate_source", PyString::new(py, rate_source).into_any()), + ("message", message), + ], + ) +} + +fn persist<'py>( + finalization: &Bound<'py, PyAny>, + record: &Bound<'py, PyAny>, + stage: &str, +) -> PyResult<()> { + persist_cost_run( + finalization.getattr("project_dir")?.extract()?, + record, + &finalization.getattr("output_stream")?, + stage, + ) +} + +/// `_persist_collection_failure`: log, warn and persist a failed-collection record. +fn collection_failure<'py>( + finalization: &Bound<'py, PyAny>, + (config, rate_source): (&Bound<'py, PyAny>, &str), + stage: &str, + error: &PyErr, +) -> PyResult> { + let py: Python<'py> = finalization.py(); + log_to( + py, + COLLECTION_LOGGER_NAME, + &format!("automatic run-cost {stage} failed"), + error, + )?; + let error_type: String = error_type(py, error)?; + warn( + &finalization.getattr("output_stream")?, + &format!("Cost telemetry failed during {stage} ({error_type}); build result unchanged."), + )?; + let status: Bound<'py, PyAny> = + imported(py, COST_TYPES, "CostStatus")?.getattr("COLLECTION_FAILED")?; + let message: Bound<'py, PyAny> = PyString::new( + py, + &format!("Cost telemetry failed during {stage} ({error_type})."), + ) + .into_any(); + let record: Bound<'py, PyAny> = cost_record( + finalization, + summary(py, status, (config, rate_source), message)?, + )?; + persist(finalization, &record, "failure artifact persistence")?; + Ok(record) +} + +fn collected_summary<'py>( + finalization: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + (config, rate_source): (&Bound<'py, PyAny>, &str), +) -> PyResult> { + let py: Python<'py> = finalization.py(); + let run_id: Bound<'py, PyAny> = finalization.getattr("run_id")?; + let mut ledger: Bound<'py, PyAny> = finalization.getattr("project_dir")?; + for part in STATEMENT_LEDGER_PATH { + ledger = if part.is_empty() { + ledger.div(&run_id)? + } else { + ledger.div(part)? + }; + } + call_with( + &adapter.getattr("collect_run_cost")?, + &[ + ( + "connection_config", + finalization.getattr("connection_config")?, + ), + ("target_database", finalization.getattr("target_database")?), + ("run_id", run_id), + ("started_at", finalization.getattr("started_at")?), + ("completed_at", finalization.getattr("completed_at")?), + ("statement_ledger_path", ledger), + ("usd_per_credit", config.getattr("usd_per_credit")?), + ("rate_source", PyString::new(py, rate_source).into_any()), + ], + ) +} + +/// Collect, persist, and render without allowing cost failures to affect a build. +#[pyfunction] +pub(crate) fn finalize_build_cost<'py>( + finalization: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = finalization.py(); + let adapter: Bound<'py, PyAny> = finalization.getattr("adapter")?; + if !adapter.is_instance(&imported(py, COST_TYPES, "CostAwareAdapter")?)? { + return Ok(py.None().into_bound(py)); + } + let config: Bound<'py, PyAny> = finalization.getattr("config")?; + let rate_source: String = if config.getattr("usd_per_credit_is_default")?.is_truthy()? { + imported(py, COST_CONSTANTS, "DEFAULT_RATE_SOURCE")?.extract()? + } else { + CONFIGURED_RATE_SOURCE.to_owned() + }; + let rate: (&Bound<'py, PyAny>, &str) = (&config, &rate_source); + let capability: Bound<'py, PyAny> = match adapter.call_method0("cost_capability") { + Ok(capability) => capability, + Err(error) => { + return collection_failure(finalization, rate, "capability detection", &error); + } + }; + if capability.eq(imported(py, COST_TYPES, "CostCapability")?.getattr("NONE")?)? { + return Ok(py.None().into_bound(py)); + } + let cost: Bound<'py, PyAny> = if finalization.getattr("collect")?.is_truthy()? { + match collected_summary(finalization, &adapter, rate) { + Ok(summary) => summary, + Err(error) => return collection_failure(finalization, rate, "collection", &error), + } + } else { + summary( + py, + finalization.getattr("cost_status")?, + rate, + finalization.getattr("cost_message")?, + )? + }; + let record: Bound<'py, PyAny> = cost_record(finalization, cost)?; + persist(finalization, &record, "artifact persistence")?; + if !finalization.getattr("render")?.is_truthy()? { + return Ok(record); + } + write_build_cost_breakdown( + &record, + &finalization.getattr("output_stream")?, + finalization.getattr("use_color")?.is_truthy()?, + )?; + Ok(record) +} + +/// Render a previously finalized record without affecting native build output. +pub(crate) fn render_build_cost( + record: &Bound<'_, PyAny>, + output_stream: &Bound<'_, PyAny>, + use_color: bool, +) -> PyResult<()> { + if record.is_none() { + return Ok(()); + } + write_build_cost_breakdown(record, output_stream, use_color) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(finalize_build_cost, module)?)?; module.add_function(wrap_pyfunction!(persist_cost_run, module)?)?; module.add_function(wrap_pyfunction!(write_build_cost_breakdown, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs index 8e8a05ddf7..218f3a4910 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod contract_compare; pub(crate) mod contract_edits; +pub(crate) mod offline_compile; pub(crate) mod python_strings; pub(crate) mod refactor_command; pub(crate) mod refactor_facts; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs new file mode 100644 index 0000000000..2a914f63f2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs @@ -0,0 +1,197 @@ +//! Offline compiles that feed and verify a refactoring. + +use std::path::Path; + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyBool, PyString}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; +const COMPILE_ERRORS: &str = "sqlbuild.compiler.compile.exceptions"; +const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; +const RULES_ERRORS: &str = "sqlbuild.rule_engine.exceptions"; +const SPEC_ERRORS: &str = "sqlbuild.spec.contracts.exceptions"; +const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; +const COMMAND_TYPES: &str = "sqlbuild.cli.commands.types"; +const REFACTOR_MODELS: &str = "sqlbuild.compiler.refactoring.models"; +const DEFAULT_ERROR_CODE: &str = "E001"; + +/// One offline compile's project facts, or `None`, and every error that stopped or followed it. +pub(crate) struct OfflineCompile<'py> { + pub(crate) project: Bound<'py, PyAny>, + pub(crate) errors: Vec>, +} + +fn enum_member<'py>(py: Python<'py>, class: &str, member: &str) -> PyResult> { + imported(py, COMPILE_TYPES, class)?.getattr(member) +} + +fn matches_any_class(py: Python<'_>, error: &PyErr, classes: &[(&str, &str)]) -> PyResult { + for (module, name) in classes { + if error.value(py).is_instance(&imported(py, module, name)?)? { + return Ok(true); + } + } + Ok(false) +} + +/// A compile failure as the one error diagnostic it reports. +fn compile_error_diagnostic<'py>(py: Python<'py>, error: &PyErr) -> PyResult> { + let value: Bound<'py, PyAny> = error.value(py).clone().into_any(); + let code: Bound<'py, PyAny> = if value.hasattr("code")? { + value.getattr("code")?.str()?.into_any() + } else { + PyString::new(py, DEFAULT_ERROR_CODE).into_any() + }; + let message: Bound<'py, PyAny> = if value.hasattr("message")? { + value.getattr("message")?.str()?.into_any() + } else { + value.str()?.into_any() + }; + call_with( + &imported(py, COMPILE_MODELS, "CompilerDiagnostic")?, + &[ + ("phase", enum_member(py, "DiagnosticPhase", "COMPILE")?), + ("severity", enum_member(py, "DiagnosticSeverity", "ERROR")?), + ("code", code), + ("message", message), + ], + ) +} + +/// A SQL-test planning failure as the test-phase error diagnostic it reports. +fn planning_error_diagnostic<'py>(py: Python<'py>, error: &PyErr) -> PyResult> { + let value: Bound<'py, PyAny> = error.value(py).clone().into_any(); + let location: Bound<'py, PyAny> = + if value.is_instance(&imported(py, COMPILE_ERRORS, "SqlTestReferenceError")?)? { + value.getattr("location")? + } else { + py.None().into_bound(py) + }; + call_with( + &imported(py, COMPILE_MODELS, "CompilerDiagnostic")?, + &[ + ("phase", enum_member(py, "DiagnosticPhase", "TEST")?), + ("severity", enum_member(py, "DiagnosticSeverity", "ERROR")?), + ("code", value.getattr("code")?), + ("message", value.str()?.into_any()), + ("help", value.getattr("help")?), + ("location", location), + ], + ) +} + +fn error_diagnostics<'py>(diagnostics: Bound<'py, PyAny>) -> PyResult>> { + let mut errors: Vec> = Vec::new(); + for diagnostic in diagnostics.try_iter()? { + let diagnostic: Bound<'py, PyAny> = diagnostic?; + if diagnostic.getattr("is_error")?.is_truthy()? { + errors.push(diagnostic); + } + } + Ok(errors) +} + +fn analyzed<'py>( + py: Python<'py>, + project_dir: &Path, + no_cache: bool, +) -> PyResult> { + let selection: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS, "CompileAnalysisSelection")?, + &[("no_cache", PyBool::new(py, no_cache).to_owned().into_any())], + )?; + call_with( + &imported(py, NATIVE_MODULE, "analyze_compile_project")?, + &[ + ("project_dir", project_dir.into_pyobject(py)?.into_any()), + ( + "no_sql_validation", + PyBool::new(py, false).to_owned().into_any(), + ), + ("selected_target", py.None().into_bound(py)), + ( + "lineage_mode", + imported(py, COMMAND_TYPES, "CompileLineageMode")?.getattr("NONE")?, + ), + ("cli_vars", py.None().into_bound(py)), + ( + "profile_flags", + imported(py, ENTRY_MODELS, "CompileProfileFlags")?.call0()?, + ), + ("analysis_selection", selection), + ("status", py.None().into_bound(py)), + ], + ) +} + +/// Compile a project the way `sqb compile` does and collect every error. +pub(crate) fn compile_for_refactor<'py>( + py: Python<'py>, + project_dir: &Path, + no_cache: bool, +) -> PyResult> { + let analysis: Bound<'py, PyAny> = match analyzed(py, project_dir, no_cache) { + Ok(analysis) => analysis, + Err(error) + if matches_any_class( + py, + &error, + &[ + ("builtins", "ValueError"), + (RULES_ERRORS, "RulesError"), + (SPEC_ERRORS, "SpecConfigError"), + ], + )? => + { + return Ok(OfflineCompile { + project: py.None().into_bound(py), + errors: vec![compile_error_diagnostic(py, &error)?], + }); + } + Err(error) => return Err(error), + }; + let mut errors: Vec> = error_diagnostics(analysis.getattr("diagnostics")?)?; + if errors.is_empty() { + let planned: PyResult> = call_with( + &imported(py, NATIVE_MODULE, "static_sql_test_planning_diagnostics")?, + &[ + ( + "target_dir", + project_dir.join("target").into_pyobject(py)?.into_any(), + ), + ("adapter", analysis.getattr("adapter")?), + ("project", analysis.getattr("graph")?.getattr("project")?), + ], + ); + match planned { + Ok(diagnostics) => errors.extend(error_diagnostics(diagnostics)?), + Err(error) + if matches_any_class( + py, + &error, + &[ + (COMPILE_ERRORS, "CompileInputError"), + (PLANNER_ERRORS, "PlannerInputError"), + ], + )? => + { + errors.push(planning_error_diagnostic(py, &error)?); + } + Err(error) => return Err(error), + } + } + let project: Bound<'py, PyAny> = call_with( + &imported(py, REFACTOR_MODELS, "RefactorProject")?, + &[ + ("project_dir", project_dir.into_pyobject(py)?.into_any()), + ("graph", analysis.getattr("graph")?), + ("discovered", analysis.getattr("discovered_inputs")?), + ], + )?; + Ok(OfflineCompile { project, errors }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs index 9a3490c70d..1f2117bf89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs @@ -36,13 +36,13 @@ use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_ use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::lineage::lineage_index::lineage_key; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +use crate::bindings::_helpers::refactor::offline_compile::{OfflineCompile, compile_for_refactor}; use crate::bindings::_helpers::refactor::refactor_facts::{ declaration_placement, model_facts, refactor_facts, }; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const REFACTOR_ERRORS_MODULE: &str = "sqlbuild.compiler.refactoring.exceptions"; -const COMPILE_MODULE: &str = "sqlbuild.cli.commands._helpers.refactor.compile_facts"; const STAGING_PREFIX: &str = "sqb-refactor-"; const MOVE_COMMAND: &str = "mv"; const EDIT_ERROR_CODE: &str = "C957"; @@ -86,24 +86,17 @@ fn refactor_error(py: Python<'_>, error: RefactorError) -> PyErr { } } -/// One offline compile: the Python `RefactorCompile`. +/// One offline compile: its project facts and errors. struct Compiled<'py> { project: Bound<'py, PyAny>, errors: Vec>, } fn compile<'py>(py: Python<'py>, project_dir: &Path, no_cache: bool) -> PyResult> { - let options: Bound<'py, PyDict> = PyDict::new(py); - options.set_item("project_dir", project_dir)?; - options.set_item("no_cache", no_cache)?; - let compiled: Bound<'py, PyAny> = - imported(py, COMPILE_MODULE, "compile_for_refactor")?.call((), Some(&options))?; + let compiled: OfflineCompile<'py> = compile_for_refactor(py, project_dir, no_cache)?; Ok(Compiled { - project: compiled.getattr("project")?, - errors: compiled - .getattr("errors")? - .try_iter()? - .collect::>()?, + project: compiled.project, + errors: compiled.errors, }) } diff --git a/python-boundary.toml b/python-boundary.toml index ba895ed983..fe686b58c3 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -616,3 +616,5 @@ note = "A bound that is not integer text is parsed as a timestamp instead; Curso [[handled_native_errors]] location = "cli/commands/_helpers/refactor/compile_facts.py::compile_for_refactor" note = "Reports the Python stage errors the native compile driver propagates as refactor diagnostics; a native compiler failure is a RuntimeError and is not caught." +location = "compiler/planner/_helpers/warehouse/snapshot.py::_parse_warehouse_cursor" +note = "A warehouse cursor value that is not an integer is parsed as a timestamp instead; CursorAlgebraError is an input error, and native compiler failures are not caught." diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8e0637723b..e3ca62302f 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -25,6 +25,7 @@ from sqlbuild.adapter.relations.models import InspectionQueryRecord from sqlbuild.cli.commands.models import ( AuditCommandRequest, BuildCommandRequest, + BuildCostFinalization, CheckCommandRequest, CloneCommandRequest, ContractCommandRequest, @@ -424,6 +425,7 @@ from sqlbuild.lint.types import RelationKeyRow, RelationKeys from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.models import DisplayOptions from sqlbuild.presentation.types import CompletionState +from sqlbuild.provider.classes.container import ProviderContainer from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.rule_engine.models import ( Finding, @@ -1752,6 +1754,21 @@ def read_cost_run(*, project_dir: Path, run_id: str) -> CostRunRecord | None: .. def list_cost_runs(*, project_dir: Path) -> tuple[CostRunRecord, ...]: ... def resolve_cost_run(*, project_dir: Path, selector: str) -> CostRunRecord | None: ... def format_cost_json(record: CostRunRecord) -> str: ... +def resolve_defer_clone_selection(*, scope: PlannerScope) -> tuple[list[str], list[str], int]: ... +def run_check_read_side_dependencies( + *, + adapter: BaseAdapter, + connection_config: dict[str, object], + connection: Any, + pipeline_result: CompilePipelineResult, + python_graph: PythonNodeGraph, + lifecycle_plan: PythonSqlRunLifecyclePlan, + relation_targets: dict[SqlResourceRef, str], + validation_refs: frozenset[SqlResourceRef] | None = None, + project_relations: dict[SqlResourceRef, str] | None = None, + providers: ProviderContainer | None = None, +) -> tuple[PythonNodeExecutionResult, ...]: ... +def finalize_build_cost(finalization: BuildCostFinalization) -> CostRunRecord | None: ... def persist_cost_run( *, project_dir: Path, record: CostRunRecord, output_stream: TextIO, stage: str ) -> None: ... diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/__init__.py b/src/sqlbuild/cli/commands/_helpers/build_planning/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py deleted file mode 100644 index dc1b7be2f5..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py +++ /dev/null @@ -1,374 +0,0 @@ -"""Helpers for direct build defer-clone prephase.""" - -from __future__ import annotations - -import time -from pathlib import Path -from typing import Any - -from sqlbuild._native import ( - build_compiled_project, - build_planner_scope, - enforce_model_execution_limit, - resolve_external_sql_reference_resolver, - resolve_target_config, - run_clone_pipeline_with_options, - split_unit_test_selectors, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands.constants import EXECUTION_LIMIT_BUILD_NOTE -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ( - BuildCommandRequest, - BuildInvocation, - DeferClonePrephaseInputs, - DeferClonePrephaseOutcome, - DeferClonePrephaseOutputContext, -) -from sqlbuild.compiler.compile.models import ( - CompileAnalysisSelection, - CompiledObjectKey, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ( - ClonePipelineConnection, - ClonePipelineOptions, - ClonePipelineResult, -) -from sqlbuild.compiler.planner.main.clone.resolve_clone_boundary import ( - resolve_clone_boundary, -) -from sqlbuild.compiler.planner.main.clone.resolve_skipped_view_chain import ( - resolve_skipped_view_chain, -) -from sqlbuild.compiler.planner.models import PlannerScope, UnitTestSelectorSplit -from sqlbuild.compiler.planner.types import MaterializationType -from sqlbuild.executor.clone.main.build_destination_transient_models import ( - build_destination_transient_models, -) -from sqlbuild.executor.clone.main.build_retention_requests import ( - build_destination_retention_requests, -) -from sqlbuild.executor.clone.main.execute import execute_clone -from sqlbuild.executor.clone.main.fingerprinting import copy_clone_fingerprints -from sqlbuild.executor.clone.main.run_prephase_clone_stream import run_prephase_clone_stream -from sqlbuild.executor.clone.models import ( - CloneExecutionInput, - CloneExecutionResult, - CloneSourceEntries, -) -from sqlbuild.executor.clone.types import CloneStatus -from sqlbuild.spec.contracts.types import RetentionDecreasePolicy - - -def build_defer_clone_boundary_selectors( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - selected_target: str | None, - no_sql_validation: bool, - no_cache: bool, - select: tuple[str, ...], - exclude: tuple[str, ...], - cli_vars: dict[str, object] | None, - project_dir: Path, - auto_load_sources: bool, -) -> tuple[CompiledProjectView, tuple[str, ...], tuple[str, ...], int]: - """Resolve out-of-selection clone boundaries and the view chain to rebuild over them.""" - - project: CompiledProjectView = build_compiled_project( - discovered_inputs=discovered_inputs, - adapter=adapter, - selected_target=selected_target, - no_sql_validation=no_sql_validation, - analysis_selection=CompileAnalysisSelection(no_cache=no_cache), - cli_vars=cli_vars, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - ), - ) - scope: PlannerScope = build_planner_scope( - project=project, - select=select, - exclude=exclude, - auto_load_sources=auto_load_sources, - ) - return ( - project, - defer_clone_boundary_selectors(scope=scope), - defer_clone_view_chain_selectors(scope=scope), - selected_executable_model_count(scope=scope), - ) - - -def _scope_is_recreated(*, scope: PlannerScope, key: CompiledObjectKey) -> bool: - if key.resource_type in {CompiledResourceType.UDF, CompiledResourceType.TABLE_FN}: - return True - model: CompiledModelView | None = scope.models_by_name.get(key.name) - if model is None: - return False - return ( - str(model.config.values.get("materialized", MaterializationType.VIEW)).lower() - == MaterializationType.VIEW - ) - - -def _scope_is_clonable(key: CompiledObjectKey) -> bool: - return key.resource_type in { - CompiledResourceType.MODEL, - CompiledResourceType.SEED, - CompiledResourceType.UDF, - CompiledResourceType.TABLE_FN, - } - - -def defer_clone_boundary_selectors(*, scope: PlannerScope) -> tuple[str, ...]: - """Return clone selectors for the first non-view model/seed upstreams outside selection.""" - - boundary_keys: frozenset[CompiledObjectKey] = resolve_clone_boundary( - selected=scope.selected_keys, - upstream=scope.upstream_deps, - is_clonable=_scope_is_clonable, - is_view=lambda key: _scope_is_recreated(scope=scope, key=key), - ) - return tuple(sorted(key.name for key in boundary_keys)) - - -def defer_clone_view_chain_selectors(*, scope: PlannerScope) -> tuple[str, ...]: - """Return out-of-selection views/functions rebuilt over cloned boundaries.""" - - view_keys: frozenset[CompiledObjectKey] = resolve_skipped_view_chain( - selected=scope.selected_keys, - upstream=scope.upstream_deps, - is_clonable=_scope_is_clonable, - is_view=lambda key: _scope_is_recreated(scope=scope, key=key), - ) - return tuple(sorted(key.name for key in view_keys)) - - -def selected_executable_model_count(*, scope: PlannerScope) -> int: - """Count selected models except those explicitly disabled.""" - - count: int = 0 - for key in scope.selected_keys: - if key.resource_type != CompiledResourceType.MODEL: - continue - model: CompiledModelView | None = scope.models_by_name.get(key.name) - if model is not None and model.config.values.get("enabled") is False: - continue - count += 1 - return count - - -def run_defer_clone_boundary_prephase( - *, - request: BuildCommandRequest, - invocation: BuildInvocation, - origin_target_name: str, -) -> DeferClonePrephaseOutcome: - """Resolve defer-clone boundaries and clone them before build planning.""" - - split: UnitTestSelectorSplit = split_unit_test_selectors( - select=request.select, - exclude=request.exclude, - discovered_inputs=invocation.discovered_inputs, - ) - if not split.sql_test_selection.models_selected: - return DeferClonePrephaseOutcome( - destination_target_name=invocation.effective_target_name, - boundary_selectors=(), - view_chain_selectors=(), - ) - cloned_project: CompiledProjectView - boundary_selectors: tuple[str, ...] - view_chain_selectors: tuple[str, ...] - ( - cloned_project, - boundary_selectors, - view_chain_selectors, - selected_model_count, - ) = build_defer_clone_boundary_selectors( - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - selected_target=request.selected_target, - no_sql_validation=request.no_sql_validation, - no_cache=request.no_cache, - select=split.select, - exclude=split.exclude, - cli_vars=request.cli_vars, - project_dir=invocation.effective_project_dir, - auto_load_sources=invocation.should_load_sources, - ) - enforce_model_execution_limit( - model_count=selected_model_count, - target_name=invocation.effective_target_name, - limits=invocation.execution_limits, - refusal_note=EXECUTION_LIMIT_BUILD_NOTE, - ) - run_defer_clone_prephase( - inputs=DeferClonePrephaseInputs( - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - origin_target_name=origin_target_name, - destination_target_name=cloned_project.effective_target_name, - no_sql_validation=request.no_sql_validation, - no_cache=request.no_cache, - select=(*boundary_selectors, *view_chain_selectors), - caused_by_names=split.select, - cli_vars=request.cli_vars, - connection_config=invocation.connection_config, - project_dir=invocation.effective_project_dir, - ), - output_context=DeferClonePrephaseOutputContext( - on_progress=invocation.planning_progress.on_progress, - progress_stream=invocation.progress_stream, - use_color=invocation.use_color, - ), - ) - return DeferClonePrephaseOutcome( - destination_target_name=cloned_project.effective_target_name, - boundary_selectors=boundary_selectors, - view_chain_selectors=view_chain_selectors, - ) - - -def run_defer_clone_prephase( - *, - inputs: DeferClonePrephaseInputs, - output_context: DeferClonePrephaseOutputContext | None = None, -) -> None: - """Clone selected boundary relations from origin before build planning.""" - - context: DeferClonePrephaseOutputContext = ( - output_context if output_context is not None else DeferClonePrephaseOutputContext() - ) - discovered_inputs: DiscoveredProjectInputs = inputs.discovered_inputs - adapter: BaseAdapter = inputs.adapter - origin_target_name: str = inputs.origin_target_name - destination_target_name: str | None = inputs.destination_target_name - cli_vars: dict[str, object] | None = inputs.cli_vars - project_dir: Path = inputs.project_dir - on_progress: Any = context.on_progress - if not inputs.select: - return - if destination_target_name is None: - raise CliUserError("--defer-clone-from requires an active target", code="C409") - if origin_target_name == destination_target_name: - raise CliUserError( - f"Cannot defer-clone from the current target '{origin_target_name}'", - code="C410", - ) - if on_progress is not None: - on_progress("Preparing defer clone plan...") - start: float = time.monotonic() - destination_connection: Any = adapter.connect(inputs.connection_config) - try: - clone_pipeline: ClonePipelineResult = run_clone_pipeline_with_options( - discovered_inputs=discovered_inputs, - adapter=adapter, - origin_target_name=origin_target_name, - destination_target_name=destination_target_name, - destination_connection=ClonePipelineConnection( - config=inputs.connection_config, - handle=destination_connection, - ), - options=ClonePipelineOptions( - no_sql_validation=inputs.no_sql_validation, - no_cache=inputs.no_cache, - select=inputs.select, - cli_vars=cli_vars, - ), - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - ), - ) - if on_progress is not None: - on_progress(f"Prepared defer clone plan. ({time.monotonic() - start:.2f}s)") - - def run_clone(on_item: Any) -> CloneExecutionResult: - return execute_clone( - inputs=CloneExecutionInput( - source_entries=CloneSourceEntries( - origin=clone_pipeline.origin_source_entries, - destination=clone_pipeline.destination_source_entries, - ), - origin_model_entries=clone_pipeline.origin_model_entries, - destination_model_entries=clone_pipeline.destination_model_entries, - origin_seed_entries=clone_pipeline.origin_seed_entries, - destination_seed_entries=clone_pipeline.destination_seed_entries, - destination_function_entries=clone_pipeline.destination_function_entries, - execution_order=clone_pipeline.clone_plan.execution_order, - adapter=adapter, - destination_connection=destination_connection, - hard_copy=False, - run_id=clone_pipeline.destination_project.run_id, - query_change_tracking=clone_pipeline.destination_project.settings.query_change_tracking, - destination_transient_models=build_destination_transient_models( - project=clone_pipeline.destination_project - ), - destination_retention_requests=build_destination_retention_requests( - project=clone_pipeline.destination_project, - adapter_name=adapter.adapter_name, - selected_model_names=frozenset( - entry.name for entry in clone_pipeline.destination_model_entries - ), - namespace_owned=resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=destination_target_name, - ).owns_time_travel_retention_namespace, - ), - allow_namespace_retention_decrease=( - resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=destination_target_name, - ).time_travel_retention_decrease - == RetentionDecreasePolicy.ALLOW - ), - on_item=on_item, - ) - ) - - result: CloneExecutionResult = ( - run_clone(None) - if context.progress_stream is None - else run_prephase_clone_stream( - stream=context.progress_stream, - title="defer clone", - caused_by_names=inputs.caused_by_names, - use_color=context.use_color, - run_clone=run_clone, - ) - ) - failed_or_warning_items: tuple[str, ...] = tuple( - f"{item.name}: {item.message or item.action.value}" - for item in result.item_results - if item.status in {CloneStatus.FAILED, CloneStatus.WARNING} - ) - if failed_or_warning_items: - raise CliUserError( - "failed to clone one or more deferred boundary relations: " - + "; ".join(failed_or_warning_items), - code="C411", - ) - copy_clone_fingerprints( - result=result, - origin_model_entries=clone_pipeline.origin_model_entries, - destination_model_entries=clone_pipeline.destination_model_entries, - origin_seed_entries=clone_pipeline.origin_seed_entries, - destination_seed_entries=clone_pipeline.destination_seed_entries, - adapter=adapter, - destination_connection=destination_connection, - run_id=clone_pipeline.destination_project.run_id, - query_change_tracking=clone_pipeline.destination_project.settings.query_change_tracking, - ) - finally: - adapter.close(destination_connection) diff --git a/src/sqlbuild/cli/commands/_helpers/check/core.py b/src/sqlbuild/cli/commands/_helpers/check/core.py deleted file mode 100644 index 994fcfb4f9..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/check/core.py +++ /dev/null @@ -1,335 +0,0 @@ -"""Helpers for the sqb check command.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild._native import ( - build_python_relation_targets, - resolve_python_check_excludes, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationLookup -from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( - resolve_relation_location_qualified_name, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction, DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.python_nodes.models import ( - PythonNodeGraph, - PythonSqlRunLifecyclePlan, -) -from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus -from sqlbuild.executor.build.types import ExecutionStatus -from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.python_nodes.main.read_side import create_read_side_python_execution_tracker -from sqlbuild.executor.python_nodes.models import ( - PythonCheckExecutionResult, - PythonNodeExecutionResult, - PythonNodeRunState, - PythonNodeRuntime, -) -from sqlbuild.provider.main.runtime import ProviderContainer -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.spec.contracts.models import SourceEntry - - -def record_python_run_state_results( - *, - discovered_inputs: DiscoveredProjectInputs, - run_state: PythonNodeRunState, - python_results: tuple[PythonNodeExecutionResult, ...], - load_results: tuple[LoadExecutionResult, ...] = (), - source_map: dict[str, SourceEntry] | None = None, -) -> None: - """Record Python and loader results in the Python run-state.""" - - result_by_name: dict[str, PythonNodeExecutionResult] = { - result.node_name: result for result in python_results - } - node_by_name: dict[str, Callable[..., object]] = { - **{task.name: task.function for task in discovered_inputs.task_functions}, - **{asset.name: asset.function for asset in discovered_inputs.asset_functions}, - } - for node_name, node_function in node_by_name.items(): - result: PythonNodeExecutionResult | None = result_by_name.get(node_name) - if result is not None: - run_state.record_result(node_function=node_function, result=result) - if source_map is not None: - result_by_loader_name: dict[str, LoadExecutionResult] = load_results_by_loader_name( - source_map=source_map, - load_results=load_results, - ) - else: - result_by_loader_name = {} - for loader in discovered_inputs.loader_functions: - load_result: LoadExecutionResult | None = result_by_loader_name.get(loader.name) - if load_result is None: - continue - run_state.record_result( - node_function=loader.function, - result=_load_result_to_python_result(node_name=loader.name, result=load_result), - ) - - -def build_check_relation_targets( - *, - adapter: BaseAdapter, - pipeline_result: CompilePipelineResult, - python_graph: PythonNodeGraph, - selected_python_names: frozenset[str], -) -> dict[SqlResourceRef, str]: - """Return SQL relation locations available to Python check dependencies.""" - - return build_python_relation_targets( - adapter=adapter, - project=pipeline_result.project, - plan_output=pipeline_result.plan_output, - required_refs=python_graph.selected_sql_refs( - selected_names=selected_python_names, - ), - ) - - -def run_check_read_side_dependencies( - *, - adapter: BaseAdapter, - connection_config: dict[str, object], - connection: Any, - pipeline_result: CompilePipelineResult, - python_graph: PythonNodeGraph, - lifecycle_plan: PythonSqlRunLifecyclePlan, - relation_targets: dict[SqlResourceRef, str], - validation_refs: frozenset[SqlResourceRef] | None = None, - project_relations: dict[SqlResourceRef, str] | None = None, - providers: ProviderContainer | None = None, -) -> tuple[PythonNodeExecutionResult, ...]: - """Run read-side Python dependencies for checks against existing SQL relations.""" - - read_side_names: frozenset[str] = lifecycle_plan.read_side_python_node_names - read_side_refs: frozenset[SqlResourceRef] = _read_side_sql_refs( - read_side_names=read_side_names, python_graph=python_graph - ) - refs_to_validate: frozenset[SqlResourceRef] = ( - validation_refs if validation_refs is not None else read_side_refs - ) - sql_refs: tuple[SqlResourceRef, ...] = tuple( - sorted( - refs_to_validate, - key=_sql_ref_sort_key, - ) - ) - relation_lookup: RelationLookup = build_relation_lookup( - adapter=adapter, - connection=connection, - locations=tuple( - location - for sql_ref in sql_refs - if (location := _check_sql_ref_location(pipeline_result=pipeline_result, ref=sql_ref)) - is not None - ), - ) - sql_ref: SqlResourceRef - for sql_ref in sql_refs: - _validate_check_sql_ref_exists( - adapter=adapter, - pipeline_result=pipeline_result, - relation_targets=relation_targets, - relation_lookup=relation_lookup, - ref=sql_ref, - ) - if not read_side_names: - return () - tracker: Any = create_read_side_python_execution_tracker( - python_graph=python_graph, - selected_python_names=read_side_names, - scheduled_sql_names=frozenset(), - runtime=PythonNodeRuntime( - adapter=adapter, - connection_config=connection_config, - connection=connection, - run_id=pipeline_result.project.run_id, - target=pipeline_result.project.effective_target_name, - vars=pipeline_result.project.effective_vars, - is_reload=False, - default_database=adapter.default_database(), - default_schema=adapter.default_schema(), - relation_targets=relation_targets, - project_relations=project_relations, - providers=providers, - ), - ) - tracker.dispatch_ready_python_nodes() - tracker.finalize_unrun_python_nodes() - return tracker.results - - -def load_results_by_loader_name( - *, source_map: dict[str, SourceEntry], load_results: tuple[LoadExecutionResult, ...] -) -> dict[str, LoadExecutionResult]: - """Return load results keyed by loader name for check dependency lookup.""" - - loader_by_source_name: dict[str, str] = { - name: source.loader for name, source in source_map.items() if source.loader is not None - } - return { - loader_name: result - for result in load_results - if (loader_name := loader_by_source_name.get(result.source_name)) is not None - } - - -def check_results_failed(results: tuple[PythonCheckExecutionResult, ...]) -> bool: - """Return whether any Python check produced an error-severity failure.""" - - return any(result.failed for result in results) - - -def relevant_check_functions( - *, - discovered_inputs: DiscoveredProjectInputs, - python_graph: PythonNodeGraph, - exclude: tuple[str, ...], - selected_dependency_names: frozenset[str], - completed_sql_names: frozenset[str] = frozenset(), -) -> tuple[DiscoveredCheckFunction, ...]: - """Return non-excluded checks whose Python and SQL dependencies all ran in this build.""" - - excluded_names: frozenset[str] = resolve_python_check_excludes( - graph=python_graph, exclude=exclude - ) - check_names: set[str] = set() - for node in python_graph.nodes: - if node.kind != PythonNodeKind.CHECK: - continue - if node.name in excluded_names: - continue - upstream_names: tuple[str, ...] = python_graph.upstream_deps.get(node.name, ()) - sql_names: tuple[str, ...] = tuple(sql_ref.name for sql_ref in node.sql_deps) - if ( - (upstream_names or sql_names) - and all(name in selected_dependency_names for name in upstream_names) - and all(name in completed_sql_names for name in sql_names) - ): - check_names.add(node.name) - return tuple(check for check in discovered_inputs.check_functions if check.name in check_names) - - -def _load_result_to_python_result( - *, node_name: str, result: LoadExecutionResult -) -> PythonNodeExecutionResult: - status: PythonNodeStatus = ( - PythonNodeStatus.SUCCESS - if result.status == ExecutionStatus.SUCCESS - else PythonNodeStatus.SKIPPED - if result.status == ExecutionStatus.SKIPPED - else PythonNodeStatus.FAILED - ) - return PythonNodeExecutionResult( - node_name=node_name, - kind=PythonNodeKind.LOADER, - status=status, - payload=result, - error_message=result.error_message, - ) - - -def _read_side_sql_refs( - *, read_side_names: frozenset[str], python_graph: PythonNodeGraph -) -> frozenset[SqlResourceRef]: - refs: set[SqlResourceRef] = set() - node_name: str - for node_name in read_side_names: - refs.update(python_graph.nodes_by_name[node_name].sql_deps) - return frozenset(refs) - - -def _validate_check_sql_ref_exists( - *, - adapter: BaseAdapter, - pipeline_result: CompilePipelineResult, - relation_targets: dict[SqlResourceRef, str], - relation_lookup: RelationLookup, - ref: SqlResourceRef, -) -> None: - if ref.kind == SqlResourceRefKind.MODEL: - target: CompiledRelationLocation | None = pipeline_result.plan_output.model_locations.get( - ref.name - ) - if target is None: - raise CliUserError( - f"Python check dependency requires unknown SQL model '{ref.name}'", - code="C682", - ) - exists: bool = relation_lookup.exists( - database=target.database, schema=target.schema, name=target.name - ) - relation: str = resolve_relation_location_qualified_name(adapter=adapter, location=target) - elif ref.kind == SqlResourceRefKind.SEED: - seed_target: CompiledRelationLocation | None = ( - pipeline_result.plan_output.seed_locations.get(ref.name) - ) - if seed_target is None: - raise CliUserError( - f"Python check dependency requires unknown seed '{ref.name}'", - code="C682", - ) - exists = relation_lookup.exists( - database=seed_target.database, schema=seed_target.schema, name=seed_target.name - ) - relation = resolve_relation_location_qualified_name(adapter=adapter, location=seed_target) - elif ref.kind == SqlResourceRefKind.SOURCE: - source: SourceEntry | None = (pipeline_result.plan_output.python_source_entries).get( - ref.name - ) - if source is None or source.expression is not None or source.table is None: - return - exists = relation_lookup.exists( - database=source.database, schema=source.schema, name=source.table - ) - relation = relation_targets[ref] - else: - return - if exists: - return - raise CliUserError( - "Python check dependency requires existing SQL relation " - f"'{relation}' for {ref.kind.value} '{ref.name}'; run sqb build first", - code="C682", - ) - - -def _check_sql_ref_location( - *, pipeline_result: CompilePipelineResult, ref: SqlResourceRef -) -> tuple[str | None, str | None, str] | None: - if ref.kind == SqlResourceRefKind.MODEL: - target: CompiledRelationLocation | None = pipeline_result.plan_output.model_locations.get( - ref.name - ) - if target is None: - return None - return (target.database, target.schema, target.name) - if ref.kind == SqlResourceRefKind.SEED: - seed_target: CompiledRelationLocation | None = ( - pipeline_result.plan_output.seed_locations.get(ref.name) - ) - if seed_target is None: - return None - return (seed_target.database, seed_target.schema, seed_target.name) - if ref.kind == SqlResourceRefKind.SOURCE: - source: SourceEntry | None = (pipeline_result.plan_output.python_source_entries).get( - ref.name - ) - if source is None or source.expression is not None or source.table is None: - return None - return (source.database, source.schema, source.table) - return None - - -def _sql_ref_sort_key(ref: SqlResourceRef) -> tuple[str, str]: - return (ref.kind.value, ref.name) diff --git a/src/sqlbuild/cli/commands/_helpers/cost/collection.py b/src/sqlbuild/cli/commands/_helpers/cost/collection.py deleted file mode 100644 index 60f6c3ca12..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/cost/collection.py +++ /dev/null @@ -1,144 +0,0 @@ -"""Nonfatal automatic cost collection after a native build.""" - -from __future__ import annotations - -import logging -from typing import TextIO - -from sqlbuild._native import persist_cost_run, write_build_cost_breakdown -from sqlbuild.cli.commands.models import BuildCostFinalization -from sqlbuild.cost.constants import DEFAULT_RATE_SOURCE -from sqlbuild.cost.models import CostRunRecord, RunCostSummary -from sqlbuild.cost.types import CostAwareAdapter, CostCapability, CostStatus -from sqlbuild.spec.contracts.models import CostConfig - -_LOGGER: logging.Logger = logging.getLogger("sqlbuild.cost") - - -def finalize_build_cost(finalization: BuildCostFinalization) -> CostRunRecord | None: - """Collect, persist, and render without allowing cost failures to affect a build.""" - - if not isinstance(finalization.adapter, CostAwareAdapter): - return None - config: CostConfig = finalization.config - rate_source: str = DEFAULT_RATE_SOURCE if config.usd_per_credit_is_default else "configured" - try: - capability: CostCapability = finalization.adapter.cost_capability() - except BaseException as error: - return _persist_collection_failure( - finalization=finalization, - rate_source=rate_source, - stage="capability detection", - error=error, - ) - if capability == CostCapability.NONE: - return None - if finalization.collect: - try: - summary: RunCostSummary = finalization.adapter.collect_run_cost( - connection_config=finalization.connection_config, - target_database=finalization.target_database, - run_id=finalization.run_id, - started_at=finalization.started_at, - completed_at=finalization.completed_at, - statement_ledger_path=( - finalization.project_dir - / "target" - / "executions" - / finalization.run_id - / "statements.jsonl" - ), - usd_per_credit=config.usd_per_credit, - rate_source=rate_source, - ) - except BaseException as error: - return _persist_collection_failure( - finalization=finalization, - rate_source=rate_source, - stage="collection", - error=error, - ) - else: - summary = RunCostSummary( - status=finalization.cost_status, - usd_per_credit=config.usd_per_credit, - rate_source=rate_source, - message=finalization.cost_message, - ) - record: CostRunRecord = CostRunRecord( - run_id=finalization.run_id, - adapter_name=finalization.adapter_name, - target_name=finalization.target_name, - build_status=finalization.build_status, - started_at=finalization.started_at, - completed_at=finalization.completed_at, - cost=summary, - had_executable_work=finalization.had_executable_work, - ) - persist_cost_run( - project_dir=finalization.project_dir, - record=record, - output_stream=finalization.output_stream, - stage="artifact persistence", - ) - if not finalization.render: - return record - write_build_cost_breakdown( - record=record, output_stream=finalization.output_stream, use_color=finalization.use_color - ) - return record - - -def render_build_cost( - *, record: CostRunRecord | None, output_stream: TextIO, use_color: bool -) -> None: - """Render a previously finalized record without affecting native build output.""" - - if record is None: - return - write_build_cost_breakdown(record=record, output_stream=output_stream, use_color=use_color) - - -def _write_warning(*, output_stream: TextIO, message: str) -> None: - try: - output_stream.write(f"\nWARNING: {message}\n") - output_stream.flush() - except BaseException: - _LOGGER.exception("automatic run-cost warning output failed") - - -def _persist_collection_failure( - *, - finalization: BuildCostFinalization, - rate_source: str, - stage: str, - error: BaseException, -) -> CostRunRecord: - _LOGGER.exception("automatic run-cost %s failed", stage) - error_type: str = type(error).__name__ - _write_warning( - output_stream=finalization.output_stream, - message=(f"Cost telemetry failed during {stage} ({error_type}); build result unchanged."), - ) - record: CostRunRecord = CostRunRecord( - run_id=finalization.run_id, - adapter_name=finalization.adapter_name, - target_name=finalization.target_name, - build_status=finalization.build_status, - started_at=finalization.started_at, - completed_at=finalization.completed_at, - cost=RunCostSummary( - status=CostStatus.COLLECTION_FAILED, - usd_per_credit=finalization.config.usd_per_credit, - rate_source=rate_source, - message=f"Cost telemetry failed during {stage} ({error_type}).", - ), - had_executable_work=finalization.had_executable_work, - ) - persist_cost_run( - project_dir=finalization.project_dir, - record=record, - output_stream=finalization.output_stream, - stage="failure artifact persistence", - ) - return record diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/__init__.py b/src/sqlbuild/cli/commands/_helpers/refactor/__init__.py deleted file mode 100644 index d3461aba0d..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/refactor/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Helpers for the rename and mv commands.""" diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py b/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py deleted file mode 100644 index ca61bd42d4..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Offline compiles that feed and verify a refactoring.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild._native import analyze_compile_project, static_sql_test_planning_diagnostics -from sqlbuild.cli.commands.models import RefactorCompile -from sqlbuild.cli.commands.types import CompileLineageMode -from sqlbuild.cli.compile.models import CompileAnalysis -from sqlbuild.cli.entry.models import CompileProfileFlags -from sqlbuild.compiler.compile.exceptions import CompileInputError, SqlTestReferenceError -from sqlbuild.compiler.compile.models import CompileAnalysisSelection, CompilerDiagnostic -from sqlbuild.compiler.compile.types import DiagnosticPhase, DiagnosticSeverity -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.refactoring.models import RefactorProject -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.spec.contracts.exceptions import SpecConfigError - - -def compile_for_refactor(*, project_dir: Path, no_cache: bool = False) -> RefactorCompile: - """Compile a project the way `sqb compile` does and collect every error.""" - - try: - analysis: CompileAnalysis = analyze_compile_project( - project_dir=project_dir, - no_sql_validation=False, - selected_target=None, - lineage_mode=CompileLineageMode.NONE, - cli_vars=None, - profile_flags=CompileProfileFlags(), - analysis_selection=CompileAnalysisSelection(no_cache=no_cache), - status=None, - ) - except (ValueError, RulesError, SpecConfigError) as error: - return RefactorCompile( - project=None, - errors=( - CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=str(getattr(error, "code", "E001")), - message=str(getattr(error, "message", str(error))), - ), - ), - ) - errors: list[CompilerDiagnostic] = [ - diagnostic for diagnostic in analysis.diagnostics if diagnostic.is_error - ] - if not errors: - try: - errors.extend( - diagnostic - for diagnostic in static_sql_test_planning_diagnostics( - target_dir=project_dir / "target", - adapter=analysis.adapter, - project=analysis.graph.project, - ) - if diagnostic.is_error - ) - except (CompileInputError, PlannerInputError) as error: - errors.append( - CompilerDiagnostic( - phase=DiagnosticPhase.TEST, - severity=DiagnosticSeverity.ERROR, - code=error.code, - message=str(error), - help=error.help, - location=error.location if isinstance(error, SqlTestReferenceError) else None, - ) - ) - return RefactorCompile( - project=RefactorProject( - project_dir=project_dir, - graph=analysis.graph, - discovered=analysis.discovered_inputs, - ), - errors=tuple(errors), - ) diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 08aab9280d..9fbfa50fa6 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -2,7 +2,6 @@ from __future__ import annotations -from collections.abc import Callable from dataclasses import dataclass, field from datetime import datetime from pathlib import Path @@ -43,12 +42,8 @@ PlanningProgressReporter, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter - from sqlbuild.compiler.compile.models import ( - CompilerDiagnostic, - ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.models import CursorOverrides, PlanOutput - from sqlbuild.compiler.refactoring.models import RefactorProject @dataclass(frozen=True) @@ -152,41 +147,6 @@ class BuildInvocation: execution_limits: ExecutionLimitsConfig = field(default_factory=ExecutionLimitsConfig) -@dataclass(frozen=True) -class DeferClonePrephaseOutcome: - """Selectors and destination resolved by the build defer-clone prephase.""" - - destination_target_name: str | None - boundary_selectors: tuple[str, ...] - view_chain_selectors: tuple[str, ...] - - -@dataclass(frozen=True) -class DeferClonePrephaseInputs: - """Resolved project, targets, and selection for the defer-clone prephase.""" - - discovered_inputs: DiscoveredProjectInputs - adapter: BaseAdapter - origin_target_name: str - destination_target_name: str | None - no_sql_validation: bool - select: tuple[str, ...] - caused_by_names: tuple[str, ...] - cli_vars: dict[str, object] | None - connection_config: dict[str, object] - project_dir: Path - no_cache: bool = False - - -@dataclass(frozen=True) -class DeferClonePrephaseOutputContext: - """Progress reporting context for the defer-clone prephase.""" - - on_progress: Callable[[str], None] | None = None - progress_stream: TextIO | None = None - use_color: bool = False - - @dataclass(frozen=True) class CheckCommandRequest: """CLI inputs for one check command invocation.""" @@ -560,14 +520,6 @@ class RefactorCommandRequest: no_color: bool = False -@dataclass(frozen=True) -class RefactorCompile: - """One offline compile for a refactoring: its facts, or the errors that stopped it.""" - - project: RefactorProject | None - errors: tuple[CompilerDiagnostic, ...] - - @dataclass(frozen=True) class ScopeCommandRequest: """CLI inputs for one offline declaration-scope inspection.""" diff --git a/src/sqlbuild/executor/clone/main/build_destination_transient_models.py b/src/sqlbuild/executor/clone/main/_build_destination_transient_models.py similarity index 100% rename from src/sqlbuild/executor/clone/main/build_destination_transient_models.py rename to src/sqlbuild/executor/clone/main/_build_destination_transient_models.py diff --git a/src/sqlbuild/executor/clone/main/build_retention_requests.py b/src/sqlbuild/executor/clone/main/_build_retention_requests.py similarity index 100% rename from src/sqlbuild/executor/clone/main/build_retention_requests.py rename to src/sqlbuild/executor/clone/main/_build_retention_requests.py diff --git a/src/sqlbuild/executor/clone/main/fingerprinting.py b/src/sqlbuild/executor/clone/main/_fingerprinting.py similarity index 100% rename from src/sqlbuild/executor/clone/main/fingerprinting.py rename to src/sqlbuild/executor/clone/main/_fingerprinting.py diff --git a/src/sqlbuild/executor/clone/main/run_prephase_clone_stream.py b/src/sqlbuild/executor/clone/main/_run_prephase_clone_stream.py similarity index 100% rename from src/sqlbuild/executor/clone/main/run_prephase_clone_stream.py rename to src/sqlbuild/executor/clone/main/_run_prephase_clone_stream.py diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py index fe0ec8fa83..468c2c0081 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -114,6 +114,7 @@ def test_given_native_source_when_checking_module_paths_then_reports_errors( "sqlbuild.cli.commands.classes.prepared_compile_artifacts", "sqlbuild.cli.observability", "sqlbuild.compile", + "sqlbuild.cost", "sqlbuild.executor.runtime", "sqlbuild.fingerprints", "sqlbuild.lineage", diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py index f8245e03e2..6f0764caff 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py @@ -6,13 +6,15 @@ import pytest +from sqlbuild._native import run_check_read_side_dependencies from sqlbuild.adapter.contract.models import RelationLookup -from sqlbuild.cli.commands._helpers.check import core as check_core +from sqlbuild.adapter.relations.main import relation_lookup as relation_lookup_module from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.compiler.compile.models import CompiledProject, CompiledRelationLocation from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.python_nodes.models import PythonSqlRunLifecyclePlan +from sqlbuild.executor.python_nodes.main import read_side as read_side_module from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind @@ -27,7 +29,7 @@ ( CheckPlanningTestCase( description="direct model and source refs are validated without dependency execution", - expected_names=frozenset({"orders", "raw_orders"}), + expected_names=frozenset({"analytics.orders", "raw.orders"}), ), ), ids=lambda case: case.description, @@ -39,15 +41,17 @@ def test_given_direct_check_sql_refs_when_preflighting_then_refs_are_validated_w model_ref: SqlResourceRef = SqlResourceRef(SqlResourceRefKind.MODEL, "orders") source_ref: SqlResourceRef = SqlResourceRef(SqlResourceRefKind.SOURCE, "raw_orders") refs: frozenset[SqlResourceRef] = frozenset({model_ref, source_ref}) - validated: list[SqlResourceRef] = [] - monkeypatch.setattr(check_core, "build_relation_lookup", lambda **_: Mock()) - monkeypatch.setattr( - check_core, - "_validate_check_sql_ref_exists", - lambda *, ref, **_: validated.append(ref), - ) + looked_up: list[tuple[str | None, str | None, str]] = [] + + def build_relation_lookup( + *, locations: tuple[tuple[str | None, str | None, str], ...], **_: object + ) -> Mock: + looked_up.extend(locations) + return Mock() + + monkeypatch.setattr(relation_lookup_module, "build_relation_lookup", build_relation_lookup) monkeypatch.setattr( - check_core, + read_side_module, "create_read_side_python_execution_tracker", lambda **_: pytest.fail("direct check refs must not execute as dependencies"), ) @@ -77,7 +81,7 @@ def test_given_direct_check_sql_refs_when_preflighting_then_refs_are_validated_w ), ) - results: tuple[PythonNodeExecutionResult, ...] = check_core.run_check_read_side_dependencies( + results: tuple[PythonNodeExecutionResult, ...] = run_check_read_side_dependencies( adapter=Mock(), connection_config={}, connection=object(), @@ -93,7 +97,9 @@ def test_given_direct_check_sql_refs_when_preflighting_then_refs_are_validated_w ) assert results == () - assert frozenset(ref.name for ref in validated) == test_case.expected_names + assert frozenset(f"{schema}.{name}" for _, schema, name in looked_up) == ( + test_case.expected_names + ) @pytest.mark.parametrize( @@ -125,7 +131,7 @@ def test_given_missing_direct_check_sql_ref_when_preflighting_then_existing_rela next(iter(test_case.expected_names)), ) monkeypatch.setattr( - check_core, + relation_lookup_module, "build_relation_lookup", lambda **_: RelationLookup(relations_by_key={}), ) @@ -158,7 +164,7 @@ def test_given_missing_direct_check_sql_ref_when_preflighting_then_existing_rela ) with pytest.raises(CliUserError, match=test_case.expected_error_fragment): - check_core.run_check_read_side_dependencies( + run_check_read_side_dependencies( adapter=adapter, connection_config={}, connection=object(), diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_collection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_collection.py index b2d4fc9d97..7576891f3d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_collection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_collection.py @@ -7,8 +7,7 @@ import pytest -from sqlbuild._native import read_cost_run -from sqlbuild.cli.commands._helpers.cost.collection import finalize_build_cost +from sqlbuild._native import finalize_build_cost, read_cost_run from sqlbuild.cost.models import CostRunRecord, RunCostSummary from sqlbuild.cost.types import CostCapability, CostStatus from tests.unit.src.sqlbuild.cli.commands._helpers.cost._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py b/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py index a2d3ebc54b..546fdddd4f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py @@ -4,11 +4,7 @@ import pytest -from sqlbuild.cli.commands._helpers.build_planning.defer_clone import ( - defer_clone_boundary_selectors, - defer_clone_view_chain_selectors, - selected_executable_model_count, -) +from sqlbuild._native import resolve_defer_clone_selection from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -78,9 +74,9 @@ def test_given_scope_when_resolving_defer_clone_boundary_then_returns_expected_s execution_order=(), ) - result: tuple[str, ...] = defer_clone_boundary_selectors(scope=scope) + boundary, _, _ = resolve_defer_clone_selection(scope=scope) - assert result == test_case.expected_selectors + assert tuple(boundary) == test_case.expected_selectors @pytest.mark.parametrize( @@ -110,8 +106,10 @@ def test_given_function_between_model_and_seed_when_resolving_then_recreates_fun execution_order=(), ) - assert defer_clone_boundary_selectors(scope=scope) == test_case.expected_boundary_selectors - assert defer_clone_view_chain_selectors(scope=scope) == test_case.expected_view_chain_selectors + boundary, view_chain, _ = resolve_defer_clone_selection(scope=scope) + + assert tuple(boundary) == test_case.expected_boundary_selectors + assert tuple(view_chain) == test_case.expected_view_chain_selectors @pytest.mark.parametrize( @@ -158,6 +156,6 @@ def test_given_disabled_model_in_defer_clone_scope_when_counting_then_only_enabl execution_order=tuple(keys_by_name.values()), ) - result: int = selected_executable_model_count(scope=scope) + _, _, result = resolve_defer_clone_selection(scope=scope) assert result == test_case.expected_model_count From 84e0c32bb10b8cfa8d4e3c1525605861e76a149f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 03:58:50 +0100 Subject: [PATCH 08/11] refactor: share the build command error constructor across native CLI owners --- .../_helpers/build_command/defer_clone.rs | 20 ++++++++++--------- .../src/bindings/_helpers/check/check_core.rs | 20 ++++++++++--------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index 5a42ecf164..7360ddb8cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -8,11 +8,13 @@ use pyo3::prelude::{ use pyo3::types::{PyCFunction, PyDict, PyFrozenSet, PyString, PyTuple, PyTupleMethods}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_core::build_command::errors::BuildCommandError; + use crate::bindings::_helpers::build_command::invocation::BuildContext; +use crate::bindings::_helpers::build_command::policies::build_command_error; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const CONSTANTS_MODULE: &str = "sqlbuild.cli.commands.constants"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; @@ -37,14 +39,14 @@ fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { } fn user_error(py: Python<'_>, message: String, code: &str) -> PyErr { - let created = || -> PyResult { - let error: Bound<'_, PyAny> = imported(py, ERRORS_MODULE, "CliUserError")?.call( - (message,), - Some(&keywords(py, &[("code", text(py, code))])?), - )?; - Ok(PyErr::from_value(error)) - }; - created().unwrap_or_else(|failure| failure) + build_command_error( + py, + BuildCommandError { + code: Some(code.to_owned()), + message, + help: None, + }, + ) } fn resource_type(key: &Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs index 6e87586e50..b9ccb8007e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs @@ -7,12 +7,14 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::build_command::errors::BuildCommandError; + +use crate::bindings::_helpers::build_command::policies::build_command_error; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const BUILD_TYPES: &str = "sqlbuild.executor.build.types"; const PYTHON_NODE_TYPES: &str = "sqlbuild.compiler.python_nodes.types"; const EXECUTOR_MODELS: &str = "sqlbuild.executor.python_nodes.models"; @@ -38,14 +40,14 @@ const READ_SIDE_OPTIONS: (&[&str], &[&str]) = ( ); fn user_error(py: Python<'_>, message: String) -> PyErr { - let created = || -> PyResult { - let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("code", MISSING_RELATION_CODE)?; - let error: Bound<'_, PyAny> = - imported(py, ERRORS_MODULE, "CliUserError")?.call((message,), Some(&keywords))?; - Ok(PyErr::from_value(error)) - }; - created().unwrap_or_else(|failure| failure) + build_command_error( + py, + BuildCommandError { + code: Some(MISSING_RELATION_CODE.to_owned()), + message, + help: None, + }, + ) } /// The loader node result a load result stands for. From ed7b108bfc1514b0b94cc8802a8e5e607b97b00c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 04:08:48 +0100 Subject: [PATCH 09/11] chore: reconcile integrated Python boundary --- python-boundary.toml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/python-boundary.toml b/python-boundary.toml index fe686b58c3..924fcc11ea 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 23650 +remaining_lines = 21360 [[stays]] path = "executor" @@ -612,9 +612,3 @@ note = "Public facade." [[handled_native_errors]] location = "compiler/planner/models.py::CursorBounds._parse_inferred" note = "A bound that is not integer text is parsed as a timestamp instead; CursorAlgebraError is an input error, and native compiler failures are not caught." - -[[handled_native_errors]] -location = "cli/commands/_helpers/refactor/compile_facts.py::compile_for_refactor" -note = "Reports the Python stage errors the native compile driver propagates as refactor diagnostics; a native compiler failure is a RuntimeError and is not caught." -location = "compiler/planner/_helpers/warehouse/snapshot.py::_parse_warehouse_cursor" -note = "A warehouse cursor value that is not an integer is parsed as a timestamp instead; CursorAlgebraError is an input error, and native compiler failures are not caught." From dbf4e7bec09c51ade4d73f5d439b40a063c753b4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 04:56:09 +0100 Subject: [PATCH 10/11] fix: keep cost warning logger and defer-clone close error parity --- .../bindings/_helpers/build_command/defer_clone.rs | 6 ++---- .../bindings/_helpers/cost_report/degradation.rs | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index 7360ddb8cf..0bb5ab41d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -563,10 +563,8 @@ fn run_prephase(context: &BuildContext<'_>, prephase: &Prephase<'_>) -> PyResult (context.invocation.getattr("connection_config")?,), )?; let cloned: PyResult<()> = clone_boundaries(context, prephase, &connection); - let closed: PyResult> = prephase.adapter.call_method1("close", (connection,)); - cloned?; - closed?; - Ok(()) + prephase.adapter.call_method1("close", (connection,))?; + cloned } /// Resolve defer-clone boundaries and clone them before build planning. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs index aabd7e53ec..15fce9daa5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs @@ -46,14 +46,23 @@ fn error_type(py: Python<'_>, error: &PyErr) -> PyResult { /// `_write_warning`: a warning line on the build's stream; its own failure is only logged. fn warn(output_stream: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { + warn_logging_to(output_stream, message, LOGGER_NAME) +} + +fn warn_logging_to( + output_stream: &Bound<'_, PyAny>, + message: &str, + logger_name: &str, +) -> PyResult<()> { let written: PyResult<()> = output_stream .call_method1("write", (format!("\nWARNING: {message}\n"),)) .and_then(|_| output_stream.call_method0("flush")) .map(|_| ()); match written { Ok(()) => Ok(()), - Err(error) => log_failure( + Err(error) => log_to( output_stream.py(), + logger_name, "automatic run-cost warning output failed", &error, ), @@ -199,9 +208,10 @@ fn collection_failure<'py>( error, )?; let error_type: String = error_type(py, error)?; - warn( + warn_logging_to( &finalization.getattr("output_stream")?, &format!("Cost telemetry failed during {stage} ({error_type}); build result unchanged."), + COLLECTION_LOGGER_NAME, )?; let status: Bound<'py, PyAny> = imported(py, COST_TYPES, "CostStatus")?.getattr("COLLECTION_FAILED")?; From 9dee6ec861c701f0346b36d35c9c91f5907e28b6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:22:02 +0100 Subject: [PATCH 11/11] fix: capture the native dbt reference resolver as its Python model --- crates/sqlbuild-cache/src/stage_capture/constants.rs | 12 ++++++++++-- .../_helpers/dbt_interop/reference_resolver.rs | 8 ++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/crates/sqlbuild-cache/src/stage_capture/constants.rs b/crates/sqlbuild-cache/src/stage_capture/constants.rs index c54fce5156..a3877a527a 100644 --- a/crates/sqlbuild-cache/src/stage_capture/constants.rs +++ b/crates/sqlbuild-cache/src/stage_capture/constants.rs @@ -28,7 +28,11 @@ pub const UNORDERED_ATTRIBUTES: [(&str, &[&str]); 1] = [( &["analysis_shapes", "expression_shapes", "schemas"], )]; /// The attributes captured, in order, for each native type. -pub const PROJECTED_ATTRIBUTES: [(&str, &[&str]); 14] = [ +pub const PROJECTED_ATTRIBUTES: [(&str, &[&str]); 15] = [ + ( + "sqlbuild._native:DbtCompileReferenceResolver", + &["_dbt_manifest"], + ), ( "sqlbuild._native:BindingCatalog", &[ @@ -330,7 +334,11 @@ pub const OMITTED_ATTRIBUTES: [(&str, &[&str]); 6] = [ ), ]; /// The Python model type a native type is captured as. -pub const CANONICAL_TYPES: [(&str, &str); 10] = [ +pub const CANONICAL_TYPES: [(&str, &str); 11] = [ + ( + "sqlbuild._native:DbtCompileReferenceResolver", + "sqlbuild.integrations.dbt.classes.dbt_compile_reference_resolver:DbtCompileReferenceResolver", + ), ( "sqlbuild._native:BindingCatalog", "sqlbuild.compiler.sql_analysis.classes.binding_catalog:BindingCatalog", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs index 1eb5d3c1a8..e9ef7b9675 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs @@ -92,6 +92,14 @@ impl DbtCompileReferenceResolver { Self::new(dbt_manifest) } + #[getter(_dbt_manifest)] + fn dbt_manifest<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { + self.dbt_manifest.as_ref().map_or_else( + || py.None().into_bound(py), + |manifest| manifest.bind(py).clone(), + ) + } + #[pyo3(signature = (*, known_model_names))] fn validate_model_names( &self,