diff --git a/crates/sqlbuild-analysis/src/janitor_planning/_helpers/mod.rs b/crates/sqlbuild-analysis/src/janitor_planning/_helpers/mod.rs new file mode 100644 index 0000000000..7fb30a095a --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod name_parts; diff --git a/crates/sqlbuild-analysis/src/janitor_planning/_helpers/name_parts.rs b/crates/sqlbuild-analysis/src/janitor_planning/_helpers/name_parts.rs new file mode 100644 index 0000000000..7a3e2d2bfc --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/_helpers/name_parts.rs @@ -0,0 +1,30 @@ +//! Fixed-width ASCII segments of reserved relation names. + +/// The rest of `name` after an ASCII case-insensitive `prefix`. +pub(crate) fn strip_prefix_folded<'a>(name: &'a str, prefix: &str) -> Option<&'a str> { + let head: &str = name.get(..prefix.len())?; + head.eq_ignore_ascii_case(prefix) + .then(|| name.get(prefix.len()..)) + .flatten() +} + +/// `width` leading ASCII digits as a number, and the rest of `text`. +pub(crate) fn leading_number(text: &str, width: usize) -> Option<(u32, &str)> { + let digits: &str = text.get(..width)?; + if !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + let value: u32 = digits + .bytes() + .fold(0, |total, byte| total * 10 + u32::from(byte - b'0')); + Some((value, text.get(width..)?)) +} + +/// The rest of `text` after one ASCII case-insensitive `marker` character. +pub(crate) fn after_marker(text: &str, marker: char) -> Option<&str> { + let mut characters = text.chars(); + characters + .next() + .filter(|character| character.eq_ignore_ascii_case(&marker)) + .map(|_| characters.as_str()) +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs new file mode 100644 index 0000000000..9a1087f740 --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs @@ -0,0 +1,69 @@ +//! Janitor planning names, prefixes and skip reasons. + +/// SQLBuild state tables the janitor never acts on. +pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 9] = [ + "_sqlbuild_fingerprints", + "_sqlbuild_microbatches", + "_sqlbuild_source_freshness", + "_sqlbuild_node_results", + "_sqlbuild_audit_results", + "_sqlbuild_janitor_events", + "_sqlbuild_migrations", + "_sqlbuild_column_migrations", + "_sqlbuild_old_name_views", +]; +/// Prefix of every strict janitor archive name. +pub const ARCHIVE_NAME_PREFIX: &str = "_SQB_ARCHIVE__"; +/// Separator between archive name segments. +pub const ARCHIVE_NAME_SEPARATOR: &str = "__"; +/// Folded prefix reserving a name for janitor archive handling. +pub const ARCHIVE_LOOKALIKE_PREFIX: &str = "_sqb_archive"; +/// Label used in archive name fitting errors. +pub const ARCHIVE_ARTIFACT_LABEL: &str = "Janitor archive"; +/// Reason for names that may require quoting. +pub const UNQUOTED_ADDRESSING_REASON: &str = "relation name is not a plain lowercase identifier and may require quoting; janitor does not act on it"; +/// Reason for names that collide case-insensitively. +pub const CASE_COLLISION_REASON: &str = + "relation name collides case-insensitively with another relation; janitor does not act on it"; +/// Reason for archive lookalikes outside the strict grammar. +pub const MALFORMED_ARCHIVE_REASON: &str = + "name resembles a janitor archive but does not match the strict archive grammar"; +/// Reason for query-diff artifacts left to fingerprint expiry cleanup. +pub const QUERY_DIFF_ARTIFACT_REASON: &str = "query-diff artifacts use fingerprint expiry cleanup"; +/// Reason for reserved query-diff names without ownership evidence. +pub const UNTRACKED_QUERY_DIFF_REASON: &str = + "reserved query-diff name lacks matching ownership evidence"; +/// Default reason for protected relations. +pub const PROTECTED_REASON: &str = "relation is protected from cleanup"; +/// Reason for relations without a SQLBuild fingerprint. +pub const UNTRACKED_REASON: &str = "relation is not tracked by SQLBuild"; +/// Reason when the adapter has no relation age metadata. +pub const AGE_UNSUPPORTED_REASON: &str = "adapter does not expose relation age metadata"; +/// Reason when a relation's age is unknown. +pub const AGE_UNAVAILABLE_REASON: &str = "relation age is unavailable"; +/// Reason protecting compatibility views at old model names. +pub const OLD_NAME_VIEW_REASON: &str = "compatibility view at an old model name"; +/// Parts of a fully qualified migration origin. +pub const QUALIFIED_ORIGIN_PARTS: usize = 3; +/// Default janitor relation retention in days. +pub const DEFAULT_RETENTION_DAYS: i64 = 14; +/// Prefix of reserved query-diff artifact names. +pub const QUERY_DIFF_ARTIFACT_PREFIX: &str = "__sqlbuild_query_diff_"; +/// Sides a query-diff artifact may hold. +pub const QUERY_DIFF_SIDES: [&str; 2] = ["left", "right"]; +/// Hex digits ending a query-diff run id. +pub const ARTIFACT_RUN_HEX_DIGITS: usize = 12; +/// Digits of a year in a compact UTC timestamp. +pub const YEAR_DIGITS: usize = 4; +/// Digits of a month or day in a compact UTC timestamp. +pub const DATE_DIGITS: usize = 2; +/// Digits of an hour, minute or second in a compact UTC timestamp. +pub const TIME_DIGITS: usize = 2; +/// Largest month number. +pub const LAST_MONTH: u32 = 12; +/// Largest hour. +pub const LAST_HOUR: u32 = 23; +/// Largest minute. +pub const LAST_MINUTE: u32 = 59; +/// Largest second; leap seconds are rejected as Python's `datetime` rejects them. +pub const LAST_SECOND: u32 = 59; diff --git a/crates/sqlbuild-analysis/src/janitor_planning/main/build_archive_name.rs b/crates/sqlbuild-analysis/src/janitor_planning/main/build_archive_name.rs new file mode 100644 index 0000000000..94c76bf7b8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/main/build_archive_name.rs @@ -0,0 +1,33 @@ +//! Fitted, case-folded janitor archive names. + +use crate::identifier_fitting::main::fit_artifact_logical_name::fit_artifact_logical_name; +use crate::janitor_planning::constants::{ + ARCHIVE_ARTIFACT_LABEL, ARCHIVE_NAME_PREFIX, ARCHIVE_NAME_SEPARATOR, +}; +use crate::janitor_planning::models::ArchiveTimestamp; + +/// The archive name keeping prefix, timestamp and kind intact; `Err` carries the fitting error. +pub fn build_archive_name( + (original_name, archived_at): (&str, ArchiveTimestamp), + identifier_limit: i64, + kind: Option<&str>, +) -> Result { + let kind_segment: String = kind.map_or_else(String::new, |kind| { + format!("{kind}{ARCHIVE_NAME_SEPARATOR}") + }); + let fixed_prefix: String = format!( + "{ARCHIVE_NAME_PREFIX}{:04}{:02}{:02}T{:02}{:02}{:02}Z{ARCHIVE_NAME_SEPARATOR}{kind_segment}", + archived_at.year, + archived_at.month, + archived_at.day, + archived_at.hour, + archived_at.minute, + archived_at.second, + ); + let logical_part: String = fit_artifact_logical_name( + (original_name, &fixed_prefix), + identifier_limit, + ARCHIVE_ARTIFACT_LABEL, + )?; + Ok(format!("{fixed_prefix}{logical_part}").to_lowercase()) +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/main/case_colliding_names.rs b/crates/sqlbuild-analysis/src/janitor_planning/main/case_colliding_names.rs new file mode 100644 index 0000000000..044804760d --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/main/case_colliding_names.rs @@ -0,0 +1,16 @@ +//! Folded names shared by several relations in one listing. + +use std::collections::{HashMap, HashSet}; + +/// Case-folded names that occur more than once in `names`. +pub fn case_colliding_names<'a>(names: impl IntoIterator) -> HashSet { + let mut counts: HashMap = HashMap::new(); + for name in names { + *counts.entry(name.to_lowercase()).or_insert(0) += 1; + } + counts + .into_iter() + .filter(|(_, count)| *count > 1) + .map(|(name, _)| name) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/main/case_collision_reason.rs b/crates/sqlbuild-analysis/src/janitor_planning/main/case_collision_reason.rs new file mode 100644 index 0000000000..9ec054a6e3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/main/case_collision_reason.rs @@ -0,0 +1,16 @@ +//! Relations whose folded name is shared in their listing. + +use std::collections::HashSet; + +use crate::janitor_planning::constants::CASE_COLLISION_REASON; + +/// Why a relation must not be dropped because its folded name collides, if it does. +#[must_use] +pub fn case_collision_reason( + name: &str, + colliding_names: &HashSet, +) -> Option<&'static str> { + colliding_names + .contains(&name.to_lowercase()) + .then_some(CASE_COLLISION_REASON) +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/main/is_archive_lookalike_name.rs b/crates/sqlbuild-analysis/src/janitor_planning/main/is_archive_lookalike_name.rs new file mode 100644 index 0000000000..00a6d8636b --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/main/is_archive_lookalike_name.rs @@ -0,0 +1,9 @@ +//! Names reserved for janitor archive handling. + +use crate::janitor_planning::constants::ARCHIVE_LOOKALIKE_PREFIX; + +/// Whether a relation name starts, ignoring case, with the archive prefix. +#[must_use] +pub fn is_archive_lookalike_name(name: &str) -> bool { + name.to_lowercase().starts_with(ARCHIVE_LOOKALIKE_PREFIX) +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/main/matching_exclude_pattern.rs b/crates/sqlbuild-analysis/src/janitor_planning/main/matching_exclude_pattern.rs new file mode 100644 index 0000000000..b4b33a21f0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/main/matching_exclude_pattern.rs @@ -0,0 +1,20 @@ +//! Janitor exclude patterns matched against relation and qualified names. + +use sqlbuild_core::text::main::glob_match::glob_match; + +/// The first pattern matching the name or the qualified display name, ignoring case. +#[must_use] +pub fn matching_exclude_pattern<'a>( + (name, display_name): (&str, &str), + patterns: &'a [String], +) -> Option<&'a str> { + let name: String = name.to_lowercase(); + let display_name: String = display_name.to_lowercase(); + patterns + .iter() + .find(|pattern| { + let folded: String = pattern.to_lowercase(); + glob_match(&name, &folded) || glob_match(&display_name, &folded) + }) + .map(String::as_str) +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/main/migration_origin_keys.rs b/crates/sqlbuild-analysis/src/janitor_planning/main/migration_origin_keys.rs new file mode 100644 index 0000000000..44df97e4b8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/main/migration_origin_keys.rs @@ -0,0 +1,41 @@ +//! Relations a `migrate_from` value may name. + +use crate::janitor_planning::constants::QUALIFIED_ORIGIN_PARTS; +use crate::janitor_planning::models::OriginKey; + +/// Origin keys for `raw`: a bare name in each sorted schema, else the one qualified relation. +#[must_use] +pub fn migration_origin_keys( + raw: &str, + destination_database: Option<&str>, + sorted_schemas: &[(Option, Option)], +) -> Vec { + let parts: Vec = raw + .split('.') + .map(|part| { + part.trim() + .trim_matches(|c| c == '"' || c == '`') + .to_owned() + }) + .collect(); + if let [name] = parts.as_slice() { + return sorted_schemas + .iter() + .map(|(database, schema)| OriginKey { + database: database.clone(), + schema: schema.clone(), + name: name.clone(), + }) + .collect(); + } + let count: usize = parts.len(); + vec![OriginKey { + database: if count == QUALIFIED_ORIGIN_PARTS { + Some(parts[0].clone()) + } else { + destination_database.map(str::to_owned) + }, + schema: Some(parts[count - 2].clone()), + name: parts[count - 1].clone(), + }] +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/main/mod.rs b/crates/sqlbuild-analysis/src/janitor_planning/main/mod.rs new file mode 100644 index 0000000000..69c555cdcd --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/main/mod.rs @@ -0,0 +1,11 @@ +pub mod build_archive_name; +pub mod case_colliding_names; +pub mod case_collision_reason; +pub mod is_archive_lookalike_name; +pub mod matching_exclude_pattern; +pub mod migration_origin_keys; +pub mod old_name_request_matches; +pub mod parse_archive_name; +pub mod query_diff_artifact_parts; +pub mod resolve_retention_days; +pub mod unaddressable_relation_reason; diff --git a/crates/sqlbuild-analysis/src/janitor_planning/main/old_name_request_matches.rs b/crates/sqlbuild-analysis/src/janitor_planning/main/old_name_request_matches.rs new file mode 100644 index 0000000000..aab3b97246 --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/main/old_name_request_matches.rs @@ -0,0 +1,19 @@ +//! `--drop-old-name-view` requests matched against recorded old names. + +/// Whether the dotted request names the trailing parts of the recorded old name, ignoring case. +#[must_use] +pub fn old_name_request_matches( + request: &str, + (database, schema, name): (Option<&str>, Option<&str>, &str), +) -> bool { + let parts: Vec = request + .split('.') + .map(|part| part.trim().trim_matches('"').to_lowercase()) + .collect(); + let spelled: Vec = [database, schema, Some(name)] + .into_iter() + .flatten() + .map(str::to_lowercase) + .collect(); + spelled.len() >= parts.len() && spelled[spelled.len() - parts.len()..] == parts[..] +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/main/parse_archive_name.rs b/crates/sqlbuild-analysis/src/janitor_planning/main/parse_archive_name.rs new file mode 100644 index 0000000000..ff846d0006 --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/main/parse_archive_name.rs @@ -0,0 +1,61 @@ +//! The strict janitor archive name grammar. + +use crate::janitor_planning::_helpers::name_parts::{ + after_marker, leading_number, strip_prefix_folded, +}; +use crate::janitor_planning::constants::{ + ARCHIVE_NAME_PREFIX, ARCHIVE_NAME_SEPARATOR, DATE_DIGITS, LAST_HOUR, LAST_MINUTE, LAST_MONTH, + LAST_SECOND, TIME_DIGITS, YEAR_DIGITS, +}; +use crate::janitor_planning::models::{ArchiveTimestamp, ParsedArchiveName}; + +/// The archive timestamp and logical name, or `None` for anything outside the strict grammar. +#[must_use] +pub fn parse_archive_name(name: &str) -> Option { + let rest: &str = strip_prefix_folded(name, ARCHIVE_NAME_PREFIX)?; + let (year, rest) = leading_number(rest, YEAR_DIGITS)?; + let (month, rest) = leading_number(rest, DATE_DIGITS)?; + let (day, rest) = leading_number(rest, DATE_DIGITS)?; + let rest: &str = after_marker(rest, 'T')?; + let (hour, rest) = leading_number(rest, TIME_DIGITS)?; + let (minute, rest) = leading_number(rest, TIME_DIGITS)?; + let (second, rest) = leading_number(rest, TIME_DIGITS)?; + let rest: &str = after_marker(rest, 'Z')?; + let logical_name: &str = rest.strip_prefix(ARCHIVE_NAME_SEPARATOR)?; + if logical_name.is_empty() || logical_name.contains('\n') { + return None; + } + let archived_at = ArchiveTimestamp { + year, + month, + day, + hour, + minute, + second, + }; + valid(archived_at).then(|| ParsedArchiveName { + archived_at, + logical_name: logical_name.to_owned(), + }) +} + +fn valid(value: ArchiveTimestamp) -> bool { + value.year >= 1 + && (1..=LAST_MONTH).contains(&value.month) + && value.day >= 1 + && value.day <= days_in_month(value.year, value.month) + && value.hour <= LAST_HOUR + && value.minute <= LAST_MINUTE + && value.second <= LAST_SECOND +} + +fn days_in_month(year: u32, month: u32) -> u32 { + let leap: bool = + year.is_multiple_of(4) && (!year.is_multiple_of(100) || year.is_multiple_of(400)); + match month { + 2 if leap => 29, + 2 => 28, + 4 | 6 | 9 | 11 => 30, + _ => 31, + } +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/main/query_diff_artifact_parts.rs b/crates/sqlbuild-analysis/src/janitor_planning/main/query_diff_artifact_parts.rs new file mode 100644 index 0000000000..82e46a1b67 --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/main/query_diff_artifact_parts.rs @@ -0,0 +1,31 @@ +//! The strict reserved query-diff artifact name format. + +use crate::janitor_planning::_helpers::name_parts::{ + after_marker, leading_number, strip_prefix_folded, +}; +use crate::janitor_planning::constants::{ + ARTIFACT_RUN_HEX_DIGITS, DATE_DIGITS, QUERY_DIFF_ARTIFACT_PREFIX, QUERY_DIFF_SIDES, + TIME_DIGITS, YEAR_DIGITS, +}; + +/// The artifact's run id and folded side, or `None` when the name is not a query-diff artifact. +#[must_use] +pub fn query_diff_artifact_parts(name: &str) -> Option<(String, String)> { + let run_and_side: &str = strip_prefix_folded(name, QUERY_DIFF_ARTIFACT_PREFIX)?; + let (_, rest) = leading_number(run_and_side, YEAR_DIGITS + 2 * DATE_DIGITS)?; + let rest: &str = after_marker(rest, 'T')?; + let (_, rest) = leading_number(rest, 3 * TIME_DIGITS)?; + let rest: &str = after_marker(rest, 'Z')?; + let rest: &str = rest.strip_prefix('_')?; + let hex: &str = rest.get(..ARTIFACT_RUN_HEX_DIGITS)?; + if !hex.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return None; + } + let side: &str = rest.get(ARTIFACT_RUN_HEX_DIGITS..)?.strip_prefix('_')?; + let folded: String = side.to_ascii_lowercase(); + if !QUERY_DIFF_SIDES.contains(&folded.as_str()) { + return None; + } + let run_id: &str = run_and_side.get(..run_and_side.len() - side.len() - 1)?; + Some((run_id.to_owned(), folded)) +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/main/resolve_retention_days.rs b/crates/sqlbuild-analysis/src/janitor_planning/main/resolve_retention_days.rs new file mode 100644 index 0000000000..e9eb1a2ea3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/main/resolve_retention_days.rs @@ -0,0 +1,11 @@ +//! Effective janitor relation retention. + +use crate::janitor_planning::constants::DEFAULT_RETENTION_DAYS; + +/// The CLI override, else the configured days, else the default. +#[must_use] +pub fn resolve_retention_days(override_days: Option, configured: Option) -> i64 { + override_days + .or(configured) + .unwrap_or(DEFAULT_RETENTION_DAYS) +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/main/unaddressable_relation_reason.rs b/crates/sqlbuild-analysis/src/janitor_planning/main/unaddressable_relation_reason.rs new file mode 100644 index 0000000000..ed975f7b35 --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/main/unaddressable_relation_reason.rs @@ -0,0 +1,31 @@ +//! Relation names the janitor can address without quoting. + +use std::collections::HashSet; + +use crate::janitor_planning::constants::UNQUOTED_ADDRESSING_REASON; +use crate::janitor_planning::main::case_collision_reason::case_collision_reason; + +/// Why the janitor must not rename or drop a relation by its catalog name, if it must not. +#[must_use] +pub fn unaddressable_relation_reason( + name: &str, + colliding_names: &HashSet, +) -> Option<&'static str> { + if !plain_lowercase_identifier(name) { + return Some(UNQUOTED_ADDRESSING_REASON); + } + case_collision_reason(name, colliding_names) +} + +fn plain_lowercase_identifier(name: &str) -> bool { + let mut characters = name.chars(); + characters + .next() + .is_some_and(|first| first.is_ascii_lowercase() || first == '_') + && characters.all(|character| { + character.is_ascii_lowercase() + || character.is_ascii_digit() + || character == '_' + || character == '$' + }) +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/mod.rs b/crates/sqlbuild-analysis/src/janitor_planning/mod.rs new file mode 100644 index 0000000000..29eb3e7cae --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/mod.rs @@ -0,0 +1,8 @@ +//! Pure janitor planning decisions: archive names, addressing safety, exclusions and origins. + +mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/janitor_planning/models.rs b/crates/sqlbuild-analysis/src/janitor_planning/models.rs new file mode 100644 index 0000000000..5b8e290634 --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/models.rs @@ -0,0 +1,27 @@ +//! Janitor planning values. + +/// UTC calendar time at second resolution, as embedded in archive names. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ArchiveTimestamp { + pub year: u32, + pub month: u32, + pub day: u32, + pub hour: u32, + pub minute: u32, + pub second: u32, +} + +/// Structured parts of one strict janitor archive relation name. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParsedArchiveName { + pub archived_at: ArchiveTimestamp, + pub logical_name: String, +} + +/// One migration origin as `(database, schema, name)`; `None` database inherits the destination's. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OriginKey { + pub database: Option, + pub schema: Option, + pub name: String, +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/tests.rs b/crates/sqlbuild-analysis/src/janitor_planning/tests.rs new file mode 100644 index 0000000000..e07593c327 --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_archive_names.rs"] +mod test_archive_names; +#[path = "tests/test_relation_rules.rs"] +mod test_relation_rules; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/janitor_planning/tests/helpers.rs b/crates/sqlbuild-analysis/src/janitor_planning/tests/helpers.rs new file mode 100644 index 0000000000..dd295c99ff --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/tests/helpers.rs @@ -0,0 +1,12 @@ +//! Shared builders for janitor planning tests. + +use crate::janitor_planning::models::OriginKey; + +/// An origin key in one schema. +pub(super) fn key(database: Option<&str>, schema: &str, name: &str) -> OriginKey { + OriginKey { + database: database.map(str::to_owned), + schema: Some(schema.to_owned()), + name: name.to_owned(), + } +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/tests/test_archive_names.rs b/crates/sqlbuild-analysis/src/janitor_planning/tests/test_archive_names.rs new file mode 100644 index 0000000000..a322336757 --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/tests/test_archive_names.rs @@ -0,0 +1,150 @@ +//! Archive names follow the strict grammar and fit identifier limits. + +use crate::janitor_planning::main::build_archive_name::build_archive_name; +use crate::janitor_planning::main::parse_archive_name::parse_archive_name; +use crate::janitor_planning::main::query_diff_artifact_parts::query_diff_artifact_parts; +use crate::janitor_planning::models::ArchiveTimestamp; +use crate::janitor_planning::tests::test_types::{ + ArtifactNameTestCase, BuildArchiveNameTestCase, ParseArchiveNameTestCase, +}; + +const ARCHIVED_AT: ArchiveTimestamp = ArchiveTimestamp { + year: 2026, + month: 1, + day: 15, + hour: 12, + minute: 30, + second: 5, +}; + +#[test] +fn given_relation_name_when_parsing_archive_name_then_returns_strict_parts() { + let test_cases = [ + ParseArchiveNameTestCase { + description: "lowercase archive name", + name: "_sqb_archive__20260115t123005z__orders", + expected_parts: Some((ARCHIVED_AT, "orders")), + }, + ParseArchiveNameTestCase { + description: "uppercase archive name keeps logical spelling", + name: "_SQB_ARCHIVE__20260115T123005Z__Orders__v2", + expected_parts: Some((ARCHIVED_AT, "Orders__v2")), + }, + ParseArchiveNameTestCase { + description: "impossible calendar day", + name: "_sqb_archive__20260230t000000z__orders", + expected_parts: None, + }, + ParseArchiveNameTestCase { + description: "leap second", + name: "_sqb_archive__20260115t000060z__orders", + expected_parts: None, + }, + ParseArchiveNameTestCase { + description: "impossible month", + name: "_SQB_ARCHIVE__20261324T101500Z__orders", + expected_parts: None, + }, + ParseArchiveNameTestCase { + description: "missing timezone marker", + name: "_SQB_ARCHIVE__20260924T101500__orders", + expected_parts: None, + }, + ParseArchiveNameTestCase { + description: "ordinary relation", + name: "orders_archive", + expected_parts: None, + }, + ParseArchiveNameTestCase { + description: "single separator", + name: "_sqb_archive_20260115t000000z__orders", + expected_parts: None, + }, + ParseArchiveNameTestCase { + description: "missing logical name", + name: "_sqb_archive__20260115t000000z__", + expected_parts: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + parse_archive_name(test_case.name) + .map(|parsed| (parsed.archived_at, parsed.logical_name)), + test_case + .expected_parts + .map(|(archived_at, logical)| (archived_at, logical.to_owned())), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_original_name_when_building_archive_name_then_fits_and_folds() { + let test_cases = [ + BuildArchiveNameTestCase { + description: "short name is kept and folded", + original_name: "Orders", + identifier_limit: 255, + kind: None, + expected_name: Ok("_sqb_archive__20260115t123005z__orders"), + }, + BuildArchiveNameTestCase { + description: "kind segment follows the timestamp", + original_name: "orders", + identifier_limit: 255, + kind: Some("migrated"), + expected_name: Ok("_sqb_archive__20260115t123005z__migrated__orders"), + }, + BuildArchiveNameTestCase { + description: "prefix exceeding the limit fails", + original_name: "orders", + identifier_limit: 10, + kind: None, + expected_name: Err( + "Janitor archive prefix '_SQB_ARCHIVE__20260115T123005Z__' does not fit within identifier limit 10", + ), + }, + ]; + for test_case in test_cases { + assert_eq!( + build_archive_name( + (test_case.original_name, ARCHIVED_AT), + test_case.identifier_limit, + test_case.kind, + ), + test_case + .expected_name + .map(str::to_owned) + .map_err(str::to_owned), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_relation_name_when_checking_query_diff_artifact_then_returns_run_and_side() { + let test_cases = [ + ArtifactNameTestCase { + description: "strict artifact name", + name: "__sqlbuild_query_diff_20260115T123005Z_0123456789ab_LEFT", + expected_parts: Some(("20260115T123005Z_0123456789ab", "left")), + }, + ArtifactNameTestCase { + description: "short run suffix", + name: "__sqlbuild_query_diff_20260115T123005Z_0123_left", + expected_parts: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + query_diff_artifact_parts(test_case.name), + test_case + .expected_parts + .map(|(run_id, side)| (run_id.to_owned(), side.to_owned())), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/tests/test_relation_rules.rs b/crates/sqlbuild-analysis/src/janitor_planning/tests/test_relation_rules.rs new file mode 100644 index 0000000000..9aef7f943b --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/tests/test_relation_rules.rs @@ -0,0 +1,222 @@ +//! Addressing, exclusion, origin and request rules decide which relations the janitor touches. + +use crate::janitor_planning::constants::{CASE_COLLISION_REASON, UNQUOTED_ADDRESSING_REASON}; +use crate::janitor_planning::main::case_colliding_names::case_colliding_names; +use crate::janitor_planning::main::matching_exclude_pattern::matching_exclude_pattern; +use crate::janitor_planning::main::migration_origin_keys::migration_origin_keys; +use crate::janitor_planning::main::old_name_request_matches::old_name_request_matches; +use crate::janitor_planning::main::resolve_retention_days::resolve_retention_days; +use crate::janitor_planning::main::unaddressable_relation_reason::unaddressable_relation_reason; +use crate::janitor_planning::tests::helpers::key; +use crate::janitor_planning::tests::test_types::{ + AddressingTestCase, ExcludePatternTestCase, OriginKeysTestCase, RequestMatchTestCase, + RetentionTestCase, +}; + +#[test] +fn given_listing_when_checking_addressing_then_returns_reason() { + let test_cases = [ + AddressingTestCase { + description: "plain lowercase name", + names: vec!["orders", "customers"], + name: "orders", + expected_reason: None, + }, + AddressingTestCase { + description: "mixed case name needs quoting", + names: vec!["Orders"], + name: "Orders", + expected_reason: Some(UNQUOTED_ADDRESSING_REASON), + }, + AddressingTestCase { + description: "digits dollar and leading underscore", + names: vec!["_orders$2026"], + name: "_orders$2026", + expected_reason: None, + }, + AddressingTestCase { + description: "punctuation needs quoting", + names: vec!["old-orders;drop"], + name: "old-orders;drop", + expected_reason: Some(UNQUOTED_ADDRESSING_REASON), + }, + AddressingTestCase { + description: "leading digit needs quoting", + names: vec!["2026_orders"], + name: "2026_orders", + expected_reason: Some(UNQUOTED_ADDRESSING_REASON), + }, + AddressingTestCase { + description: "uppercase side of a collision needs quoting", + names: vec!["old_orders", "OLD_ORDERS"], + name: "OLD_ORDERS", + expected_reason: Some(UNQUOTED_ADDRESSING_REASON), + }, + AddressingTestCase { + description: "folded collision", + names: vec!["orders", "ORDERS"], + name: "orders", + expected_reason: Some(CASE_COLLISION_REASON), + }, + ]; + for test_case in test_cases { + assert_eq!( + unaddressable_relation_reason( + test_case.name, + &case_colliding_names(test_case.names.iter().copied()) + ), + test_case.expected_reason, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_patterns_when_matching_relation_then_returns_first_folded_match() { + let test_cases = [ + ExcludePatternTestCase { + description: "name pattern ignores case", + name: "PARTITION_state", + display_name: "main.PARTITION_state", + patterns: vec!["other".to_owned(), "partition_*".to_owned()], + expected_pattern: Some("partition_*"), + }, + ExcludePatternTestCase { + description: "qualified pattern", + name: "orders", + display_name: "analytics.orders", + patterns: vec!["ANALYTICS.*".to_owned()], + expected_pattern: Some("ANALYTICS.*"), + }, + ExcludePatternTestCase { + description: "no match", + name: "orders", + display_name: "main.orders", + patterns: vec!["customers".to_owned()], + expected_pattern: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + matching_exclude_pattern( + (test_case.name, test_case.display_name), + &test_case.patterns + ), + test_case.expected_pattern, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_migrate_from_value_when_resolving_origins_then_returns_keys() { + let schemas: Vec<(Option, Option)> = vec![ + (None, Some("main".to_owned())), + (None, Some("staging".to_owned())), + ]; + let test_cases = [ + OriginKeysTestCase { + description: "bare name in every schema", + raw: "old_orders", + schemas: schemas.clone(), + expected_keys: vec![ + key(None, "main", "old_orders"), + key(None, "staging", "old_orders"), + ], + }, + OriginKeysTestCase { + description: "schema-qualified name inherits destination database", + raw: " legacy . \"old_orders\" ", + schemas: schemas.clone(), + expected_keys: vec![key(Some("warehouse"), "legacy", "old_orders")], + }, + OriginKeysTestCase { + description: "fully qualified name", + raw: "archive.legacy.`old_orders`", + schemas, + expected_keys: vec![key(Some("archive"), "legacy", "old_orders")], + }, + ]; + for test_case in test_cases { + assert_eq!( + migration_origin_keys(test_case.raw, Some("warehouse"), &test_case.schemas), + test_case.expected_keys, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_drop_request_when_matching_old_name_then_compares_trailing_parts() { + let test_cases = [ + RequestMatchTestCase { + description: "bare name", + request: "OLD_ORDERS", + expected_matches: true, + }, + RequestMatchTestCase { + description: "schema-qualified quoted name", + request: "main.\"old_orders\"", + expected_matches: true, + }, + RequestMatchTestCase { + description: "longer than the recorded name", + request: "db.extra.main.old_orders", + expected_matches: false, + }, + RequestMatchTestCase { + description: "other schema", + request: "staging.old_orders", + expected_matches: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + old_name_request_matches(test_case.request, (None, Some("main"), "old_orders")), + test_case.expected_matches, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_retention_inputs_when_resolving_then_override_wins_over_config_and_default() { + let test_cases = [ + RetentionTestCase { + description: "unset retention defaults to 14", + override_days: None, + configured: None, + expected_days: 14, + }, + RetentionTestCase { + description: "configured retention applies", + override_days: None, + configured: Some(7), + expected_days: 7, + }, + RetentionTestCase { + description: "override wins over configuration", + override_days: Some(3), + configured: Some(7), + expected_days: 3, + }, + RetentionTestCase { + description: "zero override wins over the default", + override_days: Some(0), + configured: None, + expected_days: 0, + }, + ]; + for test_case in test_cases { + assert_eq!( + resolve_retention_days(test_case.override_days, test_case.configured), + test_case.expected_days, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/tests/test_types.rs b/crates/sqlbuild-analysis/src/janitor_planning/tests/test_types.rs new file mode 100644 index 0000000000..1dde854e76 --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/tests/test_types.rs @@ -0,0 +1,58 @@ +//! Cases for janitor planning decisions. + +use crate::janitor_planning::models::{ArchiveTimestamp, OriginKey}; + +pub(super) struct ParseArchiveNameTestCase { + pub description: &'static str, + pub name: &'static str, + pub expected_parts: Option<(ArchiveTimestamp, &'static str)>, +} + +pub(super) struct BuildArchiveNameTestCase { + pub description: &'static str, + pub original_name: &'static str, + pub identifier_limit: i64, + pub kind: Option<&'static str>, + pub expected_name: Result<&'static str, &'static str>, +} + +pub(super) struct AddressingTestCase { + pub description: &'static str, + pub names: Vec<&'static str>, + pub name: &'static str, + pub expected_reason: Option<&'static str>, +} + +pub(super) struct ExcludePatternTestCase { + pub description: &'static str, + pub name: &'static str, + pub display_name: &'static str, + pub patterns: Vec, + pub expected_pattern: Option<&'static str>, +} + +pub(super) struct OriginKeysTestCase { + pub description: &'static str, + pub raw: &'static str, + pub schemas: Vec<(Option, Option)>, + pub expected_keys: Vec, +} + +pub(super) struct RequestMatchTestCase { + pub description: &'static str, + pub request: &'static str, + pub expected_matches: bool, +} + +pub(super) struct ArtifactNameTestCase { + pub description: &'static str, + pub name: &'static str, + pub expected_parts: Option<(&'static str, &'static str)>, +} + +pub(super) struct RetentionTestCase { + pub description: &'static str, + pub override_days: Option, + pub configured: Option, + pub expected_days: i64, +} diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 7e636a4508..53e6d5545c 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -16,6 +16,7 @@ pub mod explicit_references; pub mod fingerprint_storage; pub mod graph; pub mod identifier_fitting; +pub mod janitor_planning; pub mod lineage; pub mod lineage_command; pub mod lineage_graph; diff --git a/crates/sqlbuild-cache/src/stage_capture/constants.rs b/crates/sqlbuild-cache/src/stage_capture/constants.rs index a3877a527a..f9d5b74f87 100644 --- a/crates/sqlbuild-cache/src/stage_capture/constants.rs +++ b/crates/sqlbuild-cache/src/stage_capture/constants.rs @@ -382,3 +382,852 @@ pub const CANONICAL_TYPES: [(&str, &str); 11] = [ ]; /// Sequence types captured as their items. pub const DECODED_SEQUENCES: [&str; 1] = ["sqlbuild.compiler.compile.models:CompactLineageFacts"]; +/// The Python model type a natively hosted compiler shape is captured as. +pub const HOSTED_SHAPE_TYPES: [(&str, &str); 213] = [ + ( + "sqlbuild._native:AnalysisCacheContext", + "sqlbuild.compiler.compile.models:AnalysisCacheContext", + ), + ( + "sqlbuild._native:AttachedAuditGateEdge", + "sqlbuild.compiler.graph.models:AttachedAuditGateEdge", + ), + ( + "sqlbuild._native:ClonePipelineConnection", + "sqlbuild.compiler.pipeline.models:ClonePipelineConnection", + ), + ( + "sqlbuild._native:ClonePipelineOptions", + "sqlbuild.compiler.pipeline.models:ClonePipelineOptions", + ), + ( + "sqlbuild._native:ClonePipelineResult", + "sqlbuild.compiler.pipeline.models:ClonePipelineResult", + ), + ( + "sqlbuild._native:CollectionRendering", + "sqlbuild.sql_values.types:CollectionRendering", + ), + ( + "sqlbuild._native:ColumnLineage", + "sqlbuild.compiler.lineage.models:ColumnLineage", + ), + ( + "sqlbuild._native:ColumnLineageConfidence", + "sqlbuild.compiler.lineage.types:ColumnLineageConfidence", + ), + ( + "sqlbuild._native:ColumnLineageEdge", + "sqlbuild.compiler.lineage.models:ColumnLineageEdge", + ), + ( + "sqlbuild._native:ColumnLineageMode", + "sqlbuild.compiler.lineage.types:ColumnLineageMode", + ), + ( + "sqlbuild._native:ColumnLineageSource", + "sqlbuild.compiler.lineage.models:ColumnLineageSource", + ), + ( + "sqlbuild._native:ColumnTransformKind", + "sqlbuild.compiler.lineage.types:ColumnTransformKind", + ), + ( + "sqlbuild._native:CompileAdapterContext", + "sqlbuild.compiler.compile.models:CompileAdapterContext", + ), + ( + "sqlbuild._native:CompileAnalysisSelection", + "sqlbuild.compiler.compile.models:CompileAnalysisSelection", + ), + ( + "sqlbuild._native:CompileDirectLogicSqlTestInputPayload", + "sqlbuild.compiler.compile.models:CompileDirectLogicSqlTestInputPayload", + ), + ( + "sqlbuild._native:CompileFacts", + "sqlbuild.lint.models:CompileFacts", + ), + ( + "sqlbuild._native:CompileModelConfig", + "sqlbuild.compiler.compile.models:CompileModelConfig", + ), + ( + "sqlbuild._native:CompileModelSqlTestInputPayload", + "sqlbuild.compiler.compile.models:CompileModelSqlTestInputPayload", + ), + ( + "sqlbuild._native:CompilePipelineOptions", + "sqlbuild.compiler.pipeline.models:CompilePipelineOptions", + ), + ( + "sqlbuild._native:CompileProjectInputs", + "sqlbuild.compiler.compile.models:CompileProjectInputs", + ), + ( + "sqlbuild._native:CompileSqlScenarioCte", + "sqlbuild.compiler.compile.models:CompileSqlScenarioCte", + ), + ( + "sqlbuild._native:CompileSqlScenarioInput", + "sqlbuild.compiler.compile.models:CompileSqlScenarioInput", + ), + ( + "sqlbuild._native:CompileSqlTestCte", + "sqlbuild.compiler.compile.models:CompileSqlTestCte", + ), + ( + "sqlbuild._native:CompileSqlTestInput", + "sqlbuild.compiler.compile.models:CompileSqlTestInput", + ), + ( + "sqlbuild._native:CompiledDirectLogicSqlTestPayload", + "sqlbuild.compiler.compile.models:CompiledDirectLogicSqlTestPayload", + ), + ( + "sqlbuild._native:CompiledFunction", + "sqlbuild.compiler.compile.models:CompiledFunction", + ), + ( + "sqlbuild._native:CompiledLineageColumnFact", + "sqlbuild.compiler.compile.models:CompiledLineageColumnFact", + ), + ( + "sqlbuild._native:CompiledModelSqlTestPayload", + "sqlbuild.compiler.compile.models:CompiledModelSqlTestPayload", + ), + ( + "sqlbuild._native:CompiledProjectPhaseResult", + "sqlbuild.compiler.pipeline.models:CompiledProjectPhaseResult", + ), + ( + "sqlbuild._native:CompiledSeed", + "sqlbuild.compiler.compile.models:CompiledSeed", + ), + ( + "sqlbuild._native:CompiledSource", + "sqlbuild.compiler.compile.models:CompiledSource", + ), + ( + "sqlbuild._native:CompiledSqlExpansion", + "sqlbuild.compiler.compile.models:CompiledSqlExpansion", + ), + ( + "sqlbuild._native:CompiledSqlTestResource", + "sqlbuild.compiler.compile.models:CompiledSqlTestResource", + ), + ( + "sqlbuild._native:CompilerStage", + "sqlbuild.compiler.frontier.types:CompilerStage", + ), + ( + "sqlbuild._native:ConfigValueTypeError", + "sqlbuild.spec.contracts.exceptions:ConfigValueTypeError", + ), + ( + "sqlbuild._native:ContractAdoptionError", + "sqlbuild.compiler.contract_adoption.exceptions:ContractAdoptionError", + ), + ( + "sqlbuild._native:ContractAdoptionResult", + "sqlbuild.compiler.contract_adoption.models:ContractAdoptionResult", + ), + ( + "sqlbuild._native:ContractEvidence", + "sqlbuild.compiler.contract_adoption.models:ContractEvidence", + ), + ( + "sqlbuild._native:ContractFinding", + "sqlbuild.compiler.contract_adoption.models:ContractFinding", + ), + ( + "sqlbuild._native:ContractFindingKind", + "sqlbuild.compiler.contract_adoption.types:ContractFindingKind", + ), + ( + "sqlbuild._native:ContractValidationResult", + "sqlbuild.compiler.contracts.models:ContractValidationResult", + ), + ( + "sqlbuild._native:CursorInputRoles", + "sqlbuild.compiler.compile.models:CursorInputRoles", + ), + ( + "sqlbuild._native:CustomRulesOutcome", + "sqlbuild.rule_engine.models:CustomRulesOutcome", + ), + ( + "sqlbuild._native:DbtCliOptions", + "sqlbuild.integrations.dbt.models:DbtCliOptions", + ), + ( + "sqlbuild._native:DbtCommandResult", + "sqlbuild.integrations.dbt.models:DbtCommandResult", + ), + ( + "sqlbuild._native:DbtInitProgressCallbacks", + "sqlbuild.integrations.dbt.models:DbtInitProgressCallbacks", + ), + ( + "sqlbuild._native:DbtInitRequest", + "sqlbuild.integrations.dbt.models:DbtInitRequest", + ), + ( + "sqlbuild._native:DbtInitResult", + "sqlbuild.integrations.dbt.models:DbtInitResult", + ), + ( + "sqlbuild._native:DbtInteropArgumentError", + "sqlbuild.integrations.dbt.exceptions:DbtInteropArgumentError", + ), + ( + "sqlbuild._native:DbtInteropConfigError", + "sqlbuild.integrations.dbt.exceptions:DbtInteropConfigError", + ), + ( + "sqlbuild._native:DbtInteropError", + "sqlbuild.integrations.dbt.exceptions:DbtInteropError", + ), + ( + "sqlbuild._native:DbtInteropRuntimeError", + "sqlbuild.integrations.dbt.exceptions:DbtInteropRuntimeError", + ), + ( + "sqlbuild._native:DbtLsResult", + "sqlbuild.integrations.dbt.models:DbtLsResult", + ), + ( + "sqlbuild._native:DbtNodeExecutionResult", + "sqlbuild.integrations.dbt.models:DbtNodeExecutionResult", + ), + ( + "sqlbuild._native:DeclarationExpansionContext", + "sqlbuild.compiler.compile.models:DeclarationExpansionContext", + ), + ( + "sqlbuild._native:DeclarationParseError", + "sqlbuild.compiler.discovery.exceptions:DeclarationParseError", + ), + ( + "sqlbuild._native:DeclarationRuntimeProjection", + "sqlbuild.compiler.compile.models:DeclarationRuntimeProjection", + ), + ( + "sqlbuild._native:DeclarationScopeBuild", + "sqlbuild.compiler.compile.models:DeclarationScopeBuild", + ), + ( + "sqlbuild._native:DeclarationScopeResolver", + "sqlbuild.compiler.compile.models:DeclarationScopeResolver", + ), + ( + "sqlbuild._native:DeferralInputs", + "sqlbuild.compiler.planner.models:DeferralInputs", + ), + ( + "sqlbuild._native:DiagnosticPhase", + "sqlbuild.compiler.compile.types:DiagnosticPhase", + ), + ( + "sqlbuild._native:DiagnosticSeverity", + "sqlbuild.compiler.compile.types:DiagnosticSeverity", + ), + ( + "sqlbuild._native:DirectSemanticColumnUse", + "sqlbuild.compiler.lineage.models:DirectSemanticColumnUse", + ), + ( + "sqlbuild._native:DirectSourceFreshnessPlanningResult", + "sqlbuild.compiler.source_freshness.models:DirectSourceFreshnessPlanningResult", + ), + ( + "sqlbuild._native:DiscoveredAdapterFile", + "sqlbuild.compiler.discovery.models:DiscoveredAdapterFile", + ), + ( + "sqlbuild._native:DiscoveredAuditBlock", + "sqlbuild.compiler.discovery.models:DiscoveredAuditBlock", + ), + ( + "sqlbuild._native:DiscoveredAuditFile", + "sqlbuild.compiler.discovery.models:DiscoveredAuditFile", + ), + ( + "sqlbuild._native:DiscoveredConstantFile", + "sqlbuild.compiler.discovery.models:DiscoveredConstantFile", + ), + ( + "sqlbuild._native:DiscoveredEnumFile", + "sqlbuild.compiler.discovery.models:DiscoveredEnumFile", + ), + ( + "sqlbuild._native:DiscoveredModelSchemaFile", + "sqlbuild.compiler.discovery.models:DiscoveredModelSchemaFile", + ), + ( + "sqlbuild._native:DiscoveredPythonAssetMetadata", + "sqlbuild.compiler.python_nodes.models:DiscoveredPythonAssetMetadata", + ), + ( + "sqlbuild._native:DiscoveredPythonCheckMetadata", + "sqlbuild.compiler.python_nodes.models:DiscoveredPythonCheckMetadata", + ), + ( + "sqlbuild._native:DiscoveredPythonFunctionFile", + "sqlbuild.compiler.discovery.models:DiscoveredPythonFunctionFile", + ), + ( + "sqlbuild._native:DiscoveredPythonLoaderMetadata", + "sqlbuild.compiler.python_nodes.models:DiscoveredPythonLoaderMetadata", + ), + ( + "sqlbuild._native:DiscoveredPythonTaskMetadata", + "sqlbuild.compiler.python_nodes.models:DiscoveredPythonTaskMetadata", + ), + ( + "sqlbuild._native:DiscoveredSchemaFile", + "sqlbuild.compiler.discovery.models:DiscoveredSchemaFile", + ), + ( + "sqlbuild._native:DiscoveredSeedFile", + "sqlbuild.compiler.discovery.models:DiscoveredSeedFile", + ), + ( + "sqlbuild._native:DiscoveredSqlFunctionFile", + "sqlbuild.compiler.discovery.models:DiscoveredSqlFunctionFile", + ), + ( + "sqlbuild._native:DiscoveredSqlHookFile", + "sqlbuild.compiler.discovery.models:DiscoveredSqlHookFile", + ), + ( + "sqlbuild._native:DiscoveredSqlTestBlock", + "sqlbuild.compiler.discovery.models:DiscoveredSqlTestBlock", + ), + ( + "sqlbuild._native:DiscoveredSqlTestCase", + "sqlbuild.compiler.discovery.models:DiscoveredSqlTestCase", + ), + ( + "sqlbuild._native:DiscoveryConflictError", + "sqlbuild.compiler.discovery.exceptions:DiscoveryConflictError", + ), + ( + "sqlbuild._native:DiscoveryError", + "sqlbuild.compiler.discovery.exceptions:DiscoveryError", + ), + ( + "sqlbuild._native:DiscoveryFileFault", + "sqlbuild.compiler.discovery.models:DiscoveryFileFault", + ), + ( + "sqlbuild._native:DisplayOptions", + "sqlbuild.presentation.models:DisplayOptions", + ), + ( + "sqlbuild._native:DynamicColumnContractProof", + "sqlbuild.compiler.compile.models:DynamicColumnContractProof", + ), + ( + "sqlbuild._native:DynamicColumnFamilyProof", + "sqlbuild.compiler.compile.models:DynamicColumnFamilyProof", + ), + ( + "sqlbuild._native:EnumMember", + "sqlbuild.compiler.discovery.models:EnumMember", + ), + ( + "sqlbuild._native:EventExporterDiscoveryError", + "sqlbuild.compiler.discovery.exceptions:EventExporterDiscoveryError", + ), + ( + "sqlbuild._native:ExpandedLintFile", + "sqlbuild.lint.models:ExpandedLintFile", + ), + ( + "sqlbuild._native:ExpansionSpan", + "sqlbuild.compiler.compile.models:ExpansionSpan", + ), + ( + "sqlbuild._native:FingerprintInputError", + "sqlbuild.compiler.fingerprints.exceptions:FingerprintInputError", + ), + ( + "sqlbuild._native:FixVerdict", + "sqlbuild.lint.models:FixVerdict", + ), + ( + "sqlbuild._native:FixtureColumnInference", + "sqlbuild.compiler.compile.models:FixtureColumnInference", + ), + ( + "sqlbuild._native:FunctionArgument", + "sqlbuild.compiler.compile.models:FunctionArgument", + ), + ( + "sqlbuild._native:FunctionReturnColumn", + "sqlbuild.compiler.compile.models:FunctionReturnColumn", + ), + ( + "sqlbuild._native:GraphEdgeExclusion", + "sqlbuild.rule_engine.models:GraphEdgeExclusion", + ), + ( + "sqlbuild._native:HandBuiltAnalysis", + "sqlbuild.compiler.compile.models:HandBuiltAnalysis", + ), + ( + "sqlbuild._native:HeaderSpan", + "sqlbuild.lint.models:HeaderSpan", + ), + ( + "sqlbuild._native:HookReadEdge", + "sqlbuild.compiler.graph.models:HookReadEdge", + ), + ( + "sqlbuild._native:HookReadType", + "sqlbuild.compiler.graph.types:HookReadType", + ), + ( + "sqlbuild._native:InterpolationSite", + "sqlbuild.lint.models:InterpolationSite", + ), + ( + "sqlbuild._native:InvalidQualifiedIdentityError", + "sqlbuild.compiler.scopes.exceptions:InvalidQualifiedIdentityError", + ), + ( + "sqlbuild._native:InvalidScopePathError", + "sqlbuild.compiler.scopes.exceptions:InvalidScopePathError", + ), + ( + "sqlbuild._native:InvalidSqlReferenceCall", + "sqlbuild.compiler.compile.models:InvalidSqlReferenceCall", + ), + ( + "sqlbuild._native:LineageGraphViews", + "sqlbuild.compiler.graph.models:LineageGraphViews", + ), + ("sqlbuild._native:LintBody", "sqlbuild.lint.models:LintBody"), + ( + "sqlbuild._native:LintConfig", + "sqlbuild.lint.models:LintConfig", + ), + ("sqlbuild._native:LintEdit", "sqlbuild.lint.models:LintEdit"), + ( + "sqlbuild._native:LintError", + "sqlbuild.lint.exceptions:LintError", + ), + ( + "sqlbuild._native:LintFileRole", + "sqlbuild.lint.models:LintFileRole", + ), + ( + "sqlbuild._native:LintRunResult", + "sqlbuild.lint.models:LintRunResult", + ), + ( + "sqlbuild._native:LintSeverity", + "sqlbuild.lint.types:LintSeverity", + ), + ( + "sqlbuild._native:LintStoppedError", + "sqlbuild.lint.exceptions:LintStoppedError", + ), + ( + "sqlbuild._native:LintViolation", + "sqlbuild.lint.models:LintViolation", + ), + ( + "sqlbuild._native:LiteralSqlRelation", + "sqlbuild.compiler.references.models:LiteralSqlRelation", + ), + ( + "sqlbuild._native:MacroExpansionResult", + "sqlbuild.compiler.compile.models:MacroExpansionResult", + ), + ( + "sqlbuild._native:ManualLocation", + "sqlbuild.compiler.refactoring.models:ManualLocation", + ), + ( + "sqlbuild._native:MappedOffset", + "sqlbuild.compiler.compile.models:MappedOffset", + ), + ( + "sqlbuild._native:MeasurementAuditError", + "sqlbuild.compiler.auditing.exceptions:MeasurementAuditError", + ), + ( + "sqlbuild._native:MigrationCompatibility", + "sqlbuild.compiler.migrations.types:MigrationCompatibility", + ), + ( + "sqlbuild._native:MigrationDiscovery", + "sqlbuild.compiler.migrations.types:MigrationDiscovery", + ), + ( + "sqlbuild._native:ModelColumnLineage", + "sqlbuild.compiler.lineage.models:ModelColumnLineage", + ), + ( + "sqlbuild._native:ModelSchemaDeclaration", + "sqlbuild.compiler.discovery.models:ModelSchemaDeclaration", + ), + ( + "sqlbuild._native:ModelSqlAnalysis", + "sqlbuild.compiler.compile.models:ModelSqlAnalysis", + ), + ( + "sqlbuild._native:ModelSqlParseError", + "sqlbuild.compiler.discovery.exceptions:ModelSqlParseError", + ), + ( + "sqlbuild._native:ModelValidationRequest", + "sqlbuild.compiler.compile.models:ModelValidationRequest", + ), + ( + "sqlbuild._native:NamedSqlHookEntry", + "sqlbuild.compiler.discovery.models:NamedSqlHookEntry", + ), + ( + "sqlbuild._native:NativeColumnQuery", + "sqlbuild.compiler.analysis_session.models:NativeColumnQuery", + ), + ( + "sqlbuild._native:NativeLintError", + "sqlbuild.lint.exceptions:NativeLintError", + ), + ( + "sqlbuild._native:NativeModelAnalyses", + "sqlbuild.compiler.analysis_session.models:NativeModelAnalyses", + ), + ( + "sqlbuild._native:NativeModelAnalysisRequest", + "sqlbuild.compiler.analysis_session.models:NativeModelAnalysisRequest", + ), + ( + "sqlbuild._native:NativeModelConfigInputs", + "sqlbuild.compiler.compile.models:NativeModelConfigInputs", + ), + ( + "sqlbuild._native:NativeModelConfigSession", + "sqlbuild.compiler.compile.models:NativeModelConfigSession", + ), + ( + "sqlbuild._native:NativePivotTables", + "sqlbuild.compiler.analysis_session.models:NativePivotTables", + ), + ( + "sqlbuild._native:NativeQueryColumns", + "sqlbuild.compiler.analysis_session.models:NativeQueryColumns", + ), + ( + "sqlbuild._native:NativeRulesEvaluation", + "sqlbuild.rule_engine.models:NativeRulesEvaluation", + ), + ( + "sqlbuild._native:NativeSqlTestAssembly", + "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestAssembly", + ), + ( + "sqlbuild._native:NativeSqlTestChainRequest", + "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestChainRequest", + ), + ( + "sqlbuild._native:NativeSqlTestFailure", + "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestFailure", + ), + ( + "sqlbuild._native:NativeSqlTestPlanningRequest", + "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestPlanningRequest", + ), + ( + "sqlbuild._native:NativeStageMismatchError", + "sqlbuild.compiler.frontier.exceptions:NativeStageMismatchError", + ), + ( + "sqlbuild._native:NormalizedDbtProfileConnection", + "sqlbuild.integrations.dbt.models:NormalizedDbtProfileConnection", + ), + ( + "sqlbuild._native:OldNameViewAction", + "sqlbuild.compiler.migrations.types:OldNameViewAction", + ), + ( + "sqlbuild._native:ParsedSelector", + "sqlbuild.compiler.planner.models:ParsedSelector", + ), + ( + "sqlbuild._native:PathSelector", + "sqlbuild.compiler.planner.models:PathSelector", + ), + ( + "sqlbuild._native:PendingLintBody", + "sqlbuild.lint.models:PendingLintBody", + ), + ( + "sqlbuild._native:PlannerOverrides", + "sqlbuild.compiler.planner.models:PlannerOverrides", + ), + ( + "sqlbuild._native:PlannerPolicies", + "sqlbuild.compiler.planner.models:PlannerPolicies", + ), + ( + "sqlbuild._native:PlannerRelationsContext", + "sqlbuild.compiler.planner.models:PlannerRelationsContext", + ), + ( + "sqlbuild._native:PlannerScope", + "sqlbuild.compiler.planner.models:PlannerScope", + ), + ( + "sqlbuild._native:PlannerSelection", + "sqlbuild.compiler.planner.models:PlannerSelection", + ), + ( + "sqlbuild._native:PolyglotAnalysisDialect", + "sqlbuild.compiler.lineage.types:PolyglotAnalysisDialect", + ), + ( + "sqlbuild._native:PolyglotAnalysisResult", + "sqlbuild.compiler.compile.models:PolyglotAnalysisResult", + ), + ( + "sqlbuild._native:PreparedLintFiles", + "sqlbuild.lint.models:PreparedLintFiles", + ), + ( + "sqlbuild._native:PreparedSqlLint", + "sqlbuild.rule_engine.models:PreparedSqlLint", + ), + ( + "sqlbuild._native:PreparedSqlLintResult", + "sqlbuild.rule_engine.models:PreparedSqlLintResult", + ), + ( + "sqlbuild._native:ProjectCompileError", + "sqlbuild.lint.exceptions:ProjectCompileError", + ), + ( + "sqlbuild._native:ProjectConfigError", + "sqlbuild.compiler.discovery.exceptions:ProjectConfigError", + ), + ( + "sqlbuild._native:ProjectPathError", + "sqlbuild.compiler.discovery.exceptions:ProjectPathError", + ), + ( + "sqlbuild._native:ProjectPythonPathError", + "sqlbuild.compiler.discovery.exceptions:ProjectPythonPathError", + ), + ( + "sqlbuild._native:ProviderDiscoveryError", + "sqlbuild.compiler.discovery.exceptions:ProviderDiscoveryError", + ), + ( + "sqlbuild._native:PythonIdentityStatus", + "sqlbuild.compiler.python_nodes.types:PythonIdentityStatus", + ), + ( + "sqlbuild._native:PythonNodeDependencyEdge", + "sqlbuild.compiler.python_nodes.models:PythonNodeDependencyEdge", + ), + ( + "sqlbuild._native:PythonNodeDiscoveryError", + "sqlbuild.compiler.discovery.exceptions:PythonNodeDiscoveryError", + ), + ( + "sqlbuild._native:PythonPlanEntry", + "sqlbuild.compiler.pipeline.models:PythonPlanEntry", + ), + ( + "sqlbuild._native:PythonRunPhase", + "sqlbuild.compiler.python_nodes.types:PythonRunPhase", + ), + ( + "sqlbuild._native:PythonRunPlanOutputs", + "sqlbuild.compiler.pipeline.models:PythonRunPlanOutputs", + ), + ( + "sqlbuild._native:PythonSqlReferenceReport", + "sqlbuild.compiler.compile.models:PythonSqlReferenceReport", + ), + ( + "sqlbuild._native:PythonSqlRunSelection", + "sqlbuild.compiler.python_nodes.models:PythonSqlRunSelection", + ), + ( + "sqlbuild._native:QualifiedLineageColumn", + "sqlbuild.compiler.lineage.models:QualifiedLineageColumn", + ), + ( + "sqlbuild._native:QueryFingerprintError", + "sqlbuild.compiler.fingerprints.exceptions:QueryFingerprintError", + ), + ( + "sqlbuild._native:RefactorIOError", + "sqlbuild.compiler.refactoring.exceptions:RefactorIOError", + ), + ( + "sqlbuild._native:RefactorProject", + "sqlbuild.compiler.refactoring.models:RefactorProject", + ), + ( + "sqlbuild._native:RefactorValueError", + "sqlbuild.compiler.refactoring.exceptions:RefactorValueError", + ), + ( + "sqlbuild._native:RelatedLocation", + "sqlbuild.compiler.compile.models:RelatedLocation", + ), + ( + "sqlbuild._native:ResourceIdentityError", + "sqlbuild.compiler.resource_names.exceptions:ResourceIdentityError", + ), + ( + "sqlbuild._native:RuleExemption", + "sqlbuild.rule_engine.models:RuleExemption", + ), + ( + "sqlbuild._native:RuleFixResult", + "sqlbuild.lint.models:RuleFixResult", + ), + ( + "sqlbuild._native:RuleGuidance", + "sqlbuild.rule_engine.models:RuleGuidance", + ), + ( + "sqlbuild._native:RuleIgnore", + "sqlbuild.rule_engine.models:RuleIgnore", + ), + ( + "sqlbuild._native:RuleRunPlan", + "sqlbuild.rule_engine.models:RuleRunPlan", + ), + ( + "sqlbuild._native:RulesCacheConfig", + "sqlbuild.rule_engine.models:RulesCacheConfig", + ), + ( + "sqlbuild._native:RulesRunResult", + "sqlbuild.rule_engine.models:RulesRunResult", + ), + ( + "sqlbuild._native:SchemaParseError", + "sqlbuild.compiler.discovery.exceptions:SchemaParseError", + ), + ( + "sqlbuild._native:ScopeError", + "sqlbuild.compiler.scopes.exceptions:ScopeError", + ), + ( + "sqlbuild._native:ScopeValidationError", + "sqlbuild.compiler.scopes.exceptions:ScopeValidationError", + ), + ( + "sqlbuild._native:SeedDiscoveryError", + "sqlbuild.compiler.discovery.exceptions:SeedDiscoveryError", + ), + ( + "sqlbuild._native:SelectStarAllow", + "sqlbuild.rule_engine.models:SelectStarAllow", + ), + ( + "sqlbuild._native:SelectorKind", + "sqlbuild.compiler.planner.types:SelectorKind", + ), + ( + "sqlbuild._native:SourceParseError", + "sqlbuild.compiler.discovery.exceptions:SourceParseError", + ), + ( + "sqlbuild._native:SpecConfigError", + "sqlbuild.spec.contracts.exceptions:SpecConfigError", + ), + ( + "sqlbuild._native:SqlAuditParseError", + "sqlbuild.compiler.discovery.exceptions:SqlAuditParseError", + ), + ( + "sqlbuild._native:SqlBindingDiagnostic", + "sqlbuild.compiler.sql_analysis.models:SqlBindingDiagnostic", + ), + ( + "sqlbuild._native:SqlBindingResult", + "sqlbuild.compiler.sql_analysis.models:SqlBindingResult", + ), + ( + "sqlbuild._native:SqlExpansionContext", + "sqlbuild.compiler.compile.models:SqlExpansionContext", + ), + ( + "sqlbuild._native:SqlExpansionReuse", + "sqlbuild.rule_engine.models:SqlExpansionReuse", + ), + ( + "sqlbuild._native:SqlHookParseError", + "sqlbuild.compiler.discovery.exceptions:SqlHookParseError", + ), + ( + "sqlbuild._native:SqlReferenceScan", + "sqlbuild.compiler.compile.models:SqlReferenceScan", + ), + ( + "sqlbuild._native:SqlRulesEvaluation", + "sqlbuild.rule_engine.models:SqlRulesEvaluation", + ), + ( + "sqlbuild._native:SqlScenarioParseError", + "sqlbuild.compiler.discovery.exceptions:SqlScenarioParseError", + ), + ( + "sqlbuild._native:SqlSchemaValidationRequest", + "sqlbuild.compiler.sql_analysis.models:SqlSchemaValidationRequest", + ), + ( + "sqlbuild._native:SqlTestAssertionStep", + "sqlbuild.compiler.planner.models:SqlTestAssertionStep", + ), + ( + "sqlbuild._native:SqlTestParseError", + "sqlbuild.compiler.discovery.exceptions:SqlTestParseError", + ), + ( + "sqlbuild._native:SqlTestRulesConfig", + "sqlbuild.rule_engine.models:SqlTestRulesConfig", + ), + ( + "sqlbuild._native:SqlTestSelection", + "sqlbuild.compiler.planner.models:SqlTestSelection", + ), + ( + "sqlbuild._native:StaticCommandContext", + "sqlbuild.compiler.pipeline.models:StaticCommandContext", + ), + ( + "sqlbuild._native:ThresholdOverride", + "sqlbuild.rule_engine.models:ThresholdOverride", + ), + ( + "sqlbuild._native:TolerantScopeDiscovery", + "sqlbuild.compiler.discovery.models:TolerantScopeDiscovery", + ), + ( + "sqlbuild._native:UnindexableScopeError", + "sqlbuild.compiler.scopes.exceptions:UnindexableScopeError", + ), + ( + "sqlbuild._native:UnknownLineageDialectError", + "sqlbuild.compiler.lineage.exceptions:UnknownLineageDialectError", + ), + ( + "sqlbuild._native:UnsupportedLexicalSyntaxError", + "sqlbuild.compiler.sql_analysis.exceptions:UnsupportedLexicalSyntaxError", + ), + ( + "sqlbuild._native:VisibleModelDeclarations", + "sqlbuild.compiler.compile.models:VisibleModelDeclarations", + ), +]; diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 1847867028..d61be461e5 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -349,7 +349,7 @@ const LINEAGE_FIELDS: [Field; 11] = [ field( "lineage_mode", Source::Member( - class("sqlbuild.compiler.lineage.types", "ColumnLineageMode"), + class("sqlbuild._native", "ColumnLineageMode"), "lineage_mode", ), ), diff --git a/crates/sqlbuild-core/src/dagster_invocation/_helpers/graph.rs b/crates/sqlbuild-core/src/dagster_invocation/_helpers/graph.rs new file mode 100644 index 0000000000..62602ba23f --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/_helpers/graph.rs @@ -0,0 +1,72 @@ +//! Edge sets of a DAG artifact keyed by node position. + +use std::collections::{BTreeSet, HashMap, VecDeque}; + +/// Upstream positions and ordered, distinct downstream positions of each node. +pub(crate) struct EdgeSets { + pub(crate) incoming: HashMap>, + pub(crate) outgoing: HashMap>, +} + +/// Edge sets over known nodes; edges naming unknown nodes are ignored. +pub(crate) fn edge_sets( + unique: &[usize], + (positions, edges): (&HashMap<&str, usize>, &[(String, String)]), +) -> EdgeSets { + let mut incoming: HashMap> = unique + .iter() + .map(|position| (*position, BTreeSet::new())) + .collect(); + let mut outgoing: HashMap> = unique + .iter() + .map(|position| (*position, Vec::new())) + .collect(); + for (from_id, to_id) in edges { + let (Some(from), Some(to)) = ( + positions.get(from_id.as_str()), + positions.get(to_id.as_str()), + ) else { + continue; + }; + incoming.entry(*to).or_default().insert(*from); + let downstream: &mut Vec = outgoing.entry(*from).or_default(); + if !downstream.contains(to) { + downstream.push(*to); + } + } + EdgeSets { incoming, outgoing } +} + +/// The last position of each distinct id, in first-seen id order, as a dict keyed by id keeps. +pub(crate) fn unique_positions(ids: &[String], positions: &HashMap<&str, usize>) -> Vec { + let mut seen: BTreeSet<&str> = BTreeSet::new(); + let mut unique: Vec = Vec::new(); + for id in ids { + if seen.insert(id.as_str()) + && let Some(position) = positions.get(id.as_str()) + { + unique.push(*position); + } + } + unique +} + +/// The last position of each id. +pub(crate) fn id_positions(ids: &[String]) -> HashMap<&str, usize> { + ids.iter() + .enumerate() + .map(|(position, id)| (id.as_str(), position)) + .collect() +} + +/// Nodes without upstream nodes, in node order. +pub(crate) fn initially_ready( + unique: &[usize], + incoming: &HashMap>, +) -> VecDeque { + unique + .iter() + .copied() + .filter(|position| incoming.get(position).is_none_or(BTreeSet::is_empty)) + .collect() +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/_helpers/mod.rs b/crates/sqlbuild-core/src/dagster_invocation/_helpers/mod.rs new file mode 100644 index 0000000000..07ebd31772 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod graph; diff --git a/crates/sqlbuild-core/src/dagster_invocation/constants.rs b/crates/sqlbuild-core/src/dagster_invocation/constants.rs new file mode 100644 index 0000000000..ede32be2a3 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/constants.rs @@ -0,0 +1,23 @@ +//! Dagster invocation flags and limits. + +/// The flag that writes the execution JSON document to a path. +pub const JSON_OUTPUT_FLAG: &str = "--json-output"; +/// Flags that already request JSON output. +pub const JSON_OUTPUT_FLAGS: [&str; 2] = ["--json", "--json-output"]; +/// Scenario test flags that take a value. +pub const SCENARIO_VALUE_FLAGS: [&str; 4] = [ + "--max-snapshot-rows", + "--max-snapshot-total-rows", + "--max-snapshot-bytes", + "--max-snapshot-total-bytes", +]; +/// Arguments of `scenario test` before its selectors. +pub const SCENARIO_TEST_ARGUMENT_COUNT: usize = 2; +/// Separator replacing characters a Dagster check name cannot hold. +pub const CHECK_NAME_SEPARATOR: char = '_'; +/// Check name used when nothing printable remains. +pub const DEFAULT_CHECK_NAME: &str = "check"; +/// Characters of process output kept in failure diagnostics. +pub const MAX_DIAGNOSTIC_CHARS: usize = 4_000; +/// Indentation of SQL in failure text. +pub const SQL_INDENT: &str = " "; diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/asset_failure_text.rs b/crates/sqlbuild-core/src/dagster_invocation/main/asset_failure_text.rs new file mode 100644 index 0000000000..f20886a878 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/main/asset_failure_text.rs @@ -0,0 +1,47 @@ +//! Failure text of one incomplete asset. + +use crate::dagster_invocation::constants::SQL_INDENT; +use crate::dagster_invocation::models::AssetFailure; + +/// `kind:name (status) [during phase]` with the error, staging relation and SQL indented below. +#[must_use] +pub fn asset_failure_text(failure: &AssetFailure) -> String { + let mut first: String = format!("{}:{} ({})", failure.kind, failure.name, failure.status); + if let Some(phase) = &failure.failed_phase { + first.push_str(&format!(" during {phase}")); + } + let mut lines: Vec = vec![first]; + if let Some(message) = &failure.error_message { + let prefix: String = failure + .error_code + .as_ref() + .map_or_else(String::new, |code| format!("[{code}] ")); + lines.push(format!(" {prefix}{message}")); + } + if let Some(relation) = &failure.staging_relation { + lines.push(format!(" staging relation: {relation}")); + } + for (label, sql) in [ + (" failed SQL:", &failure.failed_sql), + (" last recorded SQL:", &failure.last_recorded_sql), + ] { + if let Some(sql) = sql.as_ref().filter(|sql| !sql.is_empty()) { + lines.push(label.to_owned()); + lines.push(indented(sql)); + } + } + lines.join("\n") +} + +/// `textwrap.indent(text, SQL_INDENT)`: prefix every line that is not only whitespace. +fn indented(text: &str) -> String { + text.split_inclusive('\n') + .map(|line| { + if line.trim().is_empty() { + line.to_owned() + } else { + format!("{SQL_INDENT}{line}") + } + }) + .collect() +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/caller_json_output_path.rs b/crates/sqlbuild-core/src/dagster_invocation/main/caller_json_output_path.rs new file mode 100644 index 0000000000..960b8461f3 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/main/caller_json_output_path.rs @@ -0,0 +1,22 @@ +//! The JSON output path a caller passed. + +use crate::dagster_invocation::constants::JSON_OUTPUT_FLAG; + +/// The first `--json-output PATH` or `--json-output=PATH` value. +#[must_use] +pub fn caller_json_output_path(args: &[String]) -> Option { + let prefix: String = format!("{JSON_OUTPUT_FLAG}="); + for (index, argument) in args.iter().enumerate() { + if argument == JSON_OUTPUT_FLAG + && let Some(path) = args.get(index + 1) + { + return Some(path.clone()); + } + if let Some(path) = argument.strip_prefix(&prefix) + && !path.is_empty() + { + return Some(path.to_owned()); + } + } + None +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/diagnostic_tail.rs b/crates/sqlbuild-core/src/dagster_invocation/main/diagnostic_tail.rs new file mode 100644 index 0000000000..7c3bbed924 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/main/diagnostic_tail.rs @@ -0,0 +1,13 @@ +//! Bounded process output for failure diagnostics. + +use crate::dagster_invocation::constants::MAX_DIAGNOSTIC_CHARS; + +/// The last characters of `value` kept in failure metadata. +#[must_use] +pub fn diagnostic_tail(value: &str) -> String { + let count: usize = value.chars().count(); + value + .chars() + .skip(count.saturating_sub(MAX_DIAGNOSTIC_CHARS)) + .collect() +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/has_explicit_scenario_selector.rs b/crates/sqlbuild-core/src/dagster_invocation/main/has_explicit_scenario_selector.rs new file mode 100644 index 0000000000..e50ff1a0d6 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/main/has_explicit_scenario_selector.rs @@ -0,0 +1,24 @@ +//! Explicit selectors after `scenario test`. + +use crate::dagster_invocation::constants::{SCENARIO_TEST_ARGUMENT_COUNT, SCENARIO_VALUE_FLAGS}; + +/// Whether `scenario test` arguments name a scenario beyond flags and their values. +#[must_use] +pub fn has_explicit_scenario_selector(args: &[String]) -> bool { + let mut skip_next: bool = false; + for argument in args.iter().skip(SCENARIO_TEST_ARGUMENT_COUNT) { + if skip_next { + skip_next = false; + continue; + } + if SCENARIO_VALUE_FLAGS.contains(&argument.as_str()) { + skip_next = true; + continue; + } + if argument.starts_with('-') { + continue; + } + return true; + } + false +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/json_output_requested.rs b/crates/sqlbuild-core/src/dagster_invocation/main/json_output_requested.rs new file mode 100644 index 0000000000..e969ac47b8 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/main/json_output_requested.rs @@ -0,0 +1,12 @@ +//! Caller-requested JSON output. + +use crate::dagster_invocation::constants::{JSON_OUTPUT_FLAG, JSON_OUTPUT_FLAGS}; + +/// Whether the arguments already request JSON output. +#[must_use] +pub fn json_output_requested(args: &[String]) -> bool { + let prefix: String = format!("{JSON_OUTPUT_FLAG}="); + args.iter().any(|argument| { + JSON_OUTPUT_FLAGS.contains(&argument.as_str()) || argument.starts_with(&prefix) + }) +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/mod.rs b/crates/sqlbuild-core/src/dagster_invocation/main/mod.rs new file mode 100644 index 0000000000..38e4f2ce5e --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/main/mod.rs @@ -0,0 +1,8 @@ +pub mod asset_failure_text; +pub mod caller_json_output_path; +pub mod diagnostic_tail; +pub mod has_explicit_scenario_selector; +pub mod json_output_requested; +pub mod normalize_check_name; +pub mod topological_order; +pub mod wrap_selectors; diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/normalize_check_name.rs b/crates/sqlbuild-core/src/dagster_invocation/main/normalize_check_name.rs new file mode 100644 index 0000000000..2ebe69f43b --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/main/normalize_check_name.rs @@ -0,0 +1,24 @@ +//! Dagster-safe check name segments. + +use crate::dagster_invocation::constants::{CHECK_NAME_SEPARATOR, DEFAULT_CHECK_NAME}; + +/// `value` with every non-alphanumeric character replaced and outer separators trimmed. +#[must_use] +pub fn normalize_check_name(value: &str) -> String { + let normalized: String = value + .chars() + .map(|character| { + if character.is_alphanumeric() || character == CHECK_NAME_SEPARATOR { + character + } else { + CHECK_NAME_SEPARATOR + } + }) + .collect(); + let trimmed: &str = normalized.trim_matches(CHECK_NAME_SEPARATOR); + if trimmed.is_empty() { + DEFAULT_CHECK_NAME.to_owned() + } else { + trimmed.to_owned() + } +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/topological_order.rs b/crates/sqlbuild-core/src/dagster_invocation/main/topological_order.rs new file mode 100644 index 0000000000..7b8e5b12fd --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/main/topological_order.rs @@ -0,0 +1,31 @@ +//! DAG node order for emitting Dagster events. + +use std::collections::{BTreeSet, HashMap, VecDeque}; + +use crate::dagster_invocation::_helpers::graph::{ + EdgeSets, edge_sets, id_positions, initially_ready, unique_positions, +}; + +/// Node positions in Kahn order over known edges, or `None` when a cycle leaves nodes out. +#[must_use] +pub fn topological_order(ids: &[String], edges: &[(String, String)]) -> Option> { + let positions: HashMap<&str, usize> = id_positions(ids); + let unique: Vec = unique_positions(ids, &positions); + let EdgeSets { + mut incoming, + outgoing, + } = edge_sets(&unique, (&positions, edges)); + let mut ready: VecDeque = initially_ready(&unique, &incoming); + let mut ordered: Vec = Vec::new(); + while let Some(position) = ready.pop_front() { + ordered.push(position); + for downstream in outgoing.get(&position).cloned().unwrap_or_default() { + let waiting: &mut BTreeSet = incoming.entry(downstream).or_default(); + waiting.remove(&position); + if waiting.is_empty() { + ready.push_back(downstream); + } + } + } + (ordered.len() == ids.len()).then_some(ordered) +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/wrap_selectors.rs b/crates/sqlbuild-core/src/dagster_invocation/main/wrap_selectors.rs new file mode 100644 index 0000000000..1d63b4439c --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/main/wrap_selectors.rs @@ -0,0 +1,27 @@ +//! Selectors packed into log lines. + +/// Selectors joined by spaces into lines of at most `width` characters; longer ones stand alone. +#[must_use] +pub fn wrap_selectors(selectors: &[String], width: usize) -> Vec { + let mut lines: Vec = Vec::new(); + let mut current: String = String::new(); + for selector in selectors { + let candidate: String = if current.is_empty() { + selector.clone() + } else { + format!("{current} {selector}") + }; + if candidate.chars().count() <= width { + current = candidate; + continue; + } + if !current.is_empty() { + lines.push(current); + } + current = selector.clone(); + } + if !current.is_empty() { + lines.push(current); + } + lines +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/mod.rs b/crates/sqlbuild-core/src/dagster_invocation/mod.rs new file mode 100644 index 0000000000..4e8248503e --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/mod.rs @@ -0,0 +1,8 @@ +//! Pure Dagster invocation assembly: CLI arguments, check names, DAG order and failure text. + +mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/dagster_invocation/models.rs b/crates/sqlbuild-core/src/dagster_invocation/models.rs new file mode 100644 index 0000000000..d182c8d052 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/models.rs @@ -0,0 +1,15 @@ +//! Values of Dagster invocation assembly. + +/// The display facts of one incomplete asset in an execution document. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AssetFailure { + pub kind: String, + pub name: String, + pub status: String, + pub failed_phase: Option, + pub error_message: Option, + pub error_code: Option, + pub staging_relation: Option, + pub failed_sql: Option, + pub last_recorded_sql: Option, +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/tests.rs b/crates/sqlbuild-core/src/dagster_invocation/tests.rs new file mode 100644 index 0000000000..969cb00f66 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_arguments.rs"] +mod test_arguments; +#[path = "tests/test_texts.rs"] +mod test_texts; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/dagster_invocation/tests/helpers.rs b/crates/sqlbuild-core/src/dagster_invocation/tests/helpers.rs new file mode 100644 index 0000000000..87dddaf7fc --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/tests/helpers.rs @@ -0,0 +1,6 @@ +//! Shared builders for Dagster invocation tests. + +/// Owned copies of string literals. +pub(super) fn owned(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/tests/test_arguments.rs b/crates/sqlbuild-core/src/dagster_invocation/tests/test_arguments.rs new file mode 100644 index 0000000000..90d04c86d7 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/tests/test_arguments.rs @@ -0,0 +1,96 @@ +//! Invocation arguments and DAG order follow the CLI contract. + +use crate::dagster_invocation::main::caller_json_output_path::caller_json_output_path; +use crate::dagster_invocation::main::has_explicit_scenario_selector::has_explicit_scenario_selector; +use crate::dagster_invocation::main::json_output_requested::json_output_requested; +use crate::dagster_invocation::main::topological_order::topological_order; +use crate::dagster_invocation::tests::helpers::owned; +use crate::dagster_invocation::tests::test_types::{ArgumentsTestCase, OrderTestCase}; + +#[test] +fn given_cli_arguments_when_reading_flags_then_reports_selectors_and_json_output() { + let test_cases = [ + ArgumentsTestCase { + description: "flag values are not selectors", + args: vec!["scenario", "test", "--max-snapshot-rows", "10", "--json"], + expected_explicit_scenario: false, + expected_json_requested: true, + expected_json_path: None, + }, + ArgumentsTestCase { + description: "a bare argument is a selector", + args: vec![ + "scenario", + "test", + "orders_scenario", + "--json-output", + "out.json", + ], + expected_explicit_scenario: true, + expected_json_requested: true, + expected_json_path: Some("out.json"), + }, + ArgumentsTestCase { + description: "equals form with an empty path is not a path", + args: vec!["build", "--json-output=", "--json-output=run.json"], + expected_explicit_scenario: false, + expected_json_requested: true, + expected_json_path: Some("run.json"), + }, + ArgumentsTestCase { + description: "no JSON flags", + args: vec!["build", "--select", "orders"], + expected_explicit_scenario: true, + expected_json_requested: false, + expected_json_path: None, + }, + ]; + for test_case in test_cases { + let args: Vec = owned(&test_case.args); + assert_eq!( + ( + has_explicit_scenario_selector(&args), + json_output_requested(&args), + caller_json_output_path(&args), + ), + ( + test_case.expected_explicit_scenario, + test_case.expected_json_requested, + test_case.expected_json_path.map(str::to_owned), + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_dag_edges_when_ordering_then_returns_kahn_order_or_none_on_cycles() { + let test_cases = [ + OrderTestCase { + description: "upstream before downstream, ties in node order", + ids: vec!["orders", "customers", "raw"], + edges: vec![("raw", "orders"), ("raw", "customers"), ("raw", "missing")], + expected_order: Some(vec![2, 0, 1]), + }, + OrderTestCase { + description: "cycle keeps original order", + ids: vec!["a", "b"], + edges: vec![("a", "b"), ("b", "a")], + expected_order: None, + }, + ]; + for test_case in test_cases { + let edges: Vec<(String, String)> = test_case + .edges + .iter() + .map(|(from, to)| ((*from).to_owned(), (*to).to_owned())) + .collect(); + assert_eq!( + topological_order(&owned(&test_case.ids), &edges), + test_case.expected_order, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/tests/test_texts.rs b/crates/sqlbuild-core/src/dagster_invocation/tests/test_texts.rs new file mode 100644 index 0000000000..8ee21180a9 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/tests/test_texts.rs @@ -0,0 +1,83 @@ +//! Check names, log lines and failure text match the Dagster integration's output. + +use crate::dagster_invocation::main::asset_failure_text::asset_failure_text; +use crate::dagster_invocation::main::normalize_check_name::normalize_check_name; +use crate::dagster_invocation::main::wrap_selectors::wrap_selectors; +use crate::dagster_invocation::models::AssetFailure; +use crate::dagster_invocation::tests::test_types::{ + CheckNameTestCase, FailureTextTestCase, WrapTestCase, +}; + +#[test] +fn given_raw_check_name_when_normalizing_then_replaces_punctuation() { + let test_cases = [ + CheckNameTestCase { + description: "punctuation becomes separators", + value: "not_null(order-id)", + expected_name: "not_null_order_id", + }, + CheckNameTestCase { + description: "only punctuation falls back", + value: "--", + expected_name: "check", + }, + ]; + for test_case in test_cases { + assert_eq!( + normalize_check_name(test_case.value), + test_case.expected_name, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_selectors_when_wrapping_then_packs_lines_within_width() { + let test_cases = [WrapTestCase { + description: "long selector stands alone", + selectors: vec!["orders", "customers", "a_very_long_selector_name"], + width: 16, + expected_lines: vec!["orders customers", "a_very_long_selector_name"], + }]; + for test_case in test_cases { + let selectors: Vec = test_case + .selectors + .iter() + .map(|selector| (*selector).to_owned()) + .collect(); + assert_eq!( + wrap_selectors(&selectors, test_case.width), + test_case.expected_lines, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_failed_asset_when_formatting_then_lists_error_relation_and_sql() { + let test_cases = [FailureTextTestCase { + description: "every detail", + failure: AssetFailure { + kind: "model".to_owned(), + name: "orders".to_owned(), + status: "failed".to_owned(), + failed_phase: Some("execute".to_owned()), + error_message: Some("boom".to_owned()), + error_code: Some("X100".to_owned()), + staging_relation: Some("main.orders__staging".to_owned()), + failed_sql: Some("SELECT 1\n\nFROM t".to_owned()), + last_recorded_sql: None, + }, + expected_text: "model:orders (failed) during execute\n [X100] boom\n staging relation: main.orders__staging\n failed SQL:\n SELECT 1\n\n FROM t", + }]; + for test_case in test_cases { + assert_eq!( + asset_failure_text(&test_case.failure), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/tests/test_types.rs b/crates/sqlbuild-core/src/dagster_invocation/tests/test_types.rs new file mode 100644 index 0000000000..1f7b24722c --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/tests/test_types.rs @@ -0,0 +1,37 @@ +//! Cases for Dagster invocation assembly. + +use crate::dagster_invocation::models::AssetFailure; + +pub(super) struct ArgumentsTestCase { + pub description: &'static str, + pub args: Vec<&'static str>, + pub expected_explicit_scenario: bool, + pub expected_json_requested: bool, + pub expected_json_path: Option<&'static str>, +} + +pub(super) struct OrderTestCase { + pub description: &'static str, + pub ids: Vec<&'static str>, + pub edges: Vec<(&'static str, &'static str)>, + pub expected_order: Option>, +} + +pub(super) struct CheckNameTestCase { + pub description: &'static str, + pub value: &'static str, + pub expected_name: &'static str, +} + +pub(super) struct WrapTestCase { + pub description: &'static str, + pub selectors: Vec<&'static str>, + pub width: usize, + pub expected_lines: Vec<&'static str>, +} + +pub(super) struct FailureTextTestCase { + pub description: &'static str, + pub failure: AssetFailure, + pub expected_text: &'static str, +} diff --git a/crates/sqlbuild-core/src/event_export/constants.rs b/crates/sqlbuild-core/src/event_export/constants.rs new file mode 100644 index 0000000000..c3bc6e2626 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/constants.rs @@ -0,0 +1,34 @@ +//! Destination-neutral lifecycle-event export defaults and policy vocabulary. + +pub const DEFAULT_EVENT_EXPORT_QUEUE_CAPACITY: i64 = 1024; +pub const DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS: f64 = 2.0; +pub const DEFAULT_EVENT_EXPORT_INVOCATION_TIMEOUT_SECONDS: f64 = 1.0; +pub const DEFAULT_EVENT_EXPORT_NOTIFICATION_QUEUE_CAPACITY: i64 = 256; +pub const DEFAULT_EVENT_EXPORT_HEALTH_INTERVAL_SECONDS: f64 = 30.0; +/// Every lifecycle-event kind a sink can filter on, sorted. +pub const EVENT_EXPORT_KINDS: [&str; 8] = [ + "audit", + "invocation", + "microbatch", + "operation", + "resource", + "retry", + "run", + "statement", +]; +/// Invocation terminals, delivered only after every other queued event. +pub const INVOCATION_TERMINAL_EVENT_TYPES: [&str; 2] = + ["invocation_completed", "invocation_failed"]; +pub const RETRY_EVENT_TYPE: &str = "retry_scheduled"; +pub const RETRY_EVENT_KIND: &str = "retry"; +pub const RESOURCE_EVENT_PREFIX: &str = "resource"; +pub const FAILED_EVENT_SUFFIX: &str = "_failed"; +pub const FAILED_EVENT_PRIORITY: u8 = 3; +pub const TERMINAL_EVENT_PRIORITY: u8 = 2; +pub const RETRY_EVENT_PRIORITY: u8 = 1; +pub const PROGRESS_EVENT_PRIORITY: u8 = 0; +/// Longest exception type name a failure diagnostic carries. +pub const MAX_EXCEPTION_TYPE_LENGTH: usize = 100; +pub const FALLBACK_EXCEPTION_TYPE: &str = "Exception"; +/// The failure type reported when an invocation outlives its deadline. +pub const TIMEOUT_ERROR_TYPE: &str = "TimeoutError"; diff --git a/crates/sqlbuild-core/src/event_export/main/drop_lifecycle_attempts.rs b/crates/sqlbuild-core/src/event_export/main/drop_lifecycle_attempts.rs new file mode 100644 index 0000000000..ebb57bc124 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/main/drop_lifecycle_attempts.rs @@ -0,0 +1,17 @@ +//! Account undelivered exporter attempts as dropped. + +use crate::event_export::models::ExporterCounts; + +/// The counts with one dropped attempt for each exporter index in `indices`. +#[must_use] +pub fn drop_lifecycle_attempts( + mut counts: Vec, + indices: &[usize], +) -> Vec { + for index in indices { + if let Some(counts) = counts.get_mut(*index) { + counts.dropped += 1; + } + } + counts +} diff --git a/crates/sqlbuild-core/src/event_export/main/export_snapshot.rs b/crates/sqlbuild-core/src/event_export/main/export_snapshot.rs new file mode 100644 index 0000000000..1ccc4ee011 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/main/export_snapshot.rs @@ -0,0 +1,26 @@ +//! Summarise per-exporter accounting into one immutable snapshot. + +use crate::event_export::models::{ExportSnapshot, ExporterCounts, SnapshotInputs}; + +/// The aggregate and per-exporter counts; complete only when drained and every attempt is settled. +#[must_use] +pub fn export_snapshot(inputs: SnapshotInputs<'_>) -> ExportSnapshot { + let mut aggregate: ExporterCounts = ExporterCounts::default(); + for (_, counts) in inputs.exporters { + aggregate.accepted += counts.accepted; + aggregate.filtered += counts.filtered; + aggregate.delivered += counts.delivered; + aggregate.dropped += counts.dropped; + aggregate.failed += counts.failed; + } + let flush_complete: bool = inputs.drained + && inputs.queue_depth == 0 + && aggregate.accepted == aggregate.delivered + aggregate.failed + aggregate.dropped; + ExportSnapshot { + aggregate, + per_exporter: inputs.exporters.to_vec(), + queue_depth: inputs.queue_depth, + queue_capacity: inputs.queue_capacity, + flush_complete, + } +} diff --git a/crates/sqlbuild-core/src/event_export/main/lifecycle_export_policy.rs b/crates/sqlbuild-core/src/event_export/main/lifecycle_export_policy.rs new file mode 100644 index 0000000000..7d7afe7de0 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/main/lifecycle_export_policy.rs @@ -0,0 +1,33 @@ +//! Destination-neutral export dimensions of one catalogued lifecycle event type. + +use crate::event_export::constants::{ + FAILED_EVENT_PRIORITY, FAILED_EVENT_SUFFIX, PROGRESS_EVENT_PRIORITY, RETRY_EVENT_KIND, + RETRY_EVENT_PRIORITY, RETRY_EVENT_TYPE, TERMINAL_EVENT_PRIORITY, +}; +use crate::event_export::models::{ExportPolicy, ExportSeverity}; + +/// The kind, severity and queue priority of `event_type`, whose catalogue entry is `terminal`. +#[must_use] +pub fn lifecycle_export_policy(event_type: &str, terminal: bool) -> ExportPolicy { + if event_type == RETRY_EVENT_TYPE { + return ExportPolicy { + kind: RETRY_EVENT_KIND.to_owned(), + severity: ExportSeverity::Warning, + priority: RETRY_EVENT_PRIORITY, + }; + } + let kind: &str = event_type.split('_').next().unwrap_or(event_type); + let failed: bool = event_type.ends_with(FAILED_EVENT_SUFFIX); + let (severity, priority) = if failed { + (ExportSeverity::Error, FAILED_EVENT_PRIORITY) + } else if terminal { + (ExportSeverity::Info, TERMINAL_EVENT_PRIORITY) + } else { + (ExportSeverity::Debug, PROGRESS_EVENT_PRIORITY) + }; + ExportPolicy { + kind: kind.to_owned(), + severity, + priority, + } +} diff --git a/crates/sqlbuild-core/src/event_export/main/mod.rs b/crates/sqlbuild-core/src/event_export/main/mod.rs new file mode 100644 index 0000000000..59d4f3ce19 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/main/mod.rs @@ -0,0 +1,9 @@ +pub mod drop_lifecycle_attempts; +pub mod export_snapshot; +pub mod lifecycle_export_policy; +pub mod queue_lifecycle_event; +pub mod route_lifecycle_event; +pub mod sanitized_exception_type; +pub mod severity_at_least; +pub mod stricter_severity; +pub mod take_lifecycle_event; diff --git a/crates/sqlbuild-core/src/event_export/main/queue_lifecycle_event.rs b/crates/sqlbuild-core/src/event_export/main/queue_lifecycle_event.rs new file mode 100644 index 0000000000..1a585982c3 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/main/queue_lifecycle_event.rs @@ -0,0 +1,37 @@ +//! Insert into the bounded priority queue, displacing the oldest lower-priority event when full. + +use crate::event_export::models::{EventQueue, QueueOutcome, QueuedEvent}; + +/// Queue `event`, or reject it when every queued event has at least its priority. +pub fn queue_lifecycle_event( + mut queue: EventQueue, + event: QueuedEvent, +) -> QueueOutcome { + let mut displaced: Option> = None; + if queue.items.len() >= queue.capacity { + let lowest: Option = queue.items.iter().map(|queued| queued.priority).min(); + let Some(lowest) = lowest.filter(|lowest| *lowest < event.priority) else { + return QueueOutcome { + queue, + inserted: false, + displaced: None, + rejected: Some(event), + }; + }; + let position: Option = queue + .items + .iter() + .enumerate() + .filter(|(_, queued)| queued.priority == lowest) + .min_by_key(|(_, queued)| queued.sequence) + .map(|(position, _)| position); + displaced = position.map(|position| queue.items.remove(position)); + } + queue.items.push(event); + QueueOutcome { + queue, + inserted: true, + displaced, + rejected: None, + } +} diff --git a/crates/sqlbuild-core/src/event_export/main/route_lifecycle_event.rs b/crates/sqlbuild-core/src/event_export/main/route_lifecycle_event.rs new file mode 100644 index 0000000000..5ca58061e2 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/main/route_lifecycle_event.rs @@ -0,0 +1,25 @@ +//! Filter one event across bound sinks before it is queued. + +use crate::event_export::main::severity_at_least::severity_at_least; +use crate::event_export::models::{ExportPolicy, ExporterCounts, ExporterFilter}; + +/// The counts with each sink accepted or filtered, and the indices of sinks accepting `policy`. +#[must_use] +pub fn route_lifecycle_event( + policy: &ExportPolicy, + filters: &[ExporterFilter], + mut counts: Vec, +) -> (Vec, Vec) { + let mut eligible: Vec = Vec::new(); + for (index, (filter, counts)) in filters.iter().zip(counts.iter_mut()).enumerate() { + if filter.event_kinds.contains(&policy.kind) + && severity_at_least(policy.severity, filter.min_severity) + { + counts.accepted += 1; + eligible.push(index); + } else { + counts.filtered += 1; + } + } + (counts, eligible) +} diff --git a/crates/sqlbuild-core/src/event_export/main/sanitized_exception_type.rs b/crates/sqlbuild-core/src/event_export/main/sanitized_exception_type.rs new file mode 100644 index 0000000000..0b7b5dd988 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/main/sanitized_exception_type.rs @@ -0,0 +1,24 @@ +//! Bounded, low-cardinality exception class names safe for exporter health diagnostics. + +use crate::event_export::constants::{FALLBACK_EXCEPTION_TYPE, MAX_EXCEPTION_TYPE_LENGTH}; + +/// `name` with every character outside `[A-Za-z0-9_]` replaced by `_`, truncated to 100 characters. +#[must_use] +pub fn sanitized_exception_type(name: &str) -> String { + let sanitized: String = name + .chars() + .take(MAX_EXCEPTION_TYPE_LENGTH) + .map(|character| { + if character.is_ascii_alphanumeric() || character == '_' { + character + } else { + '_' + } + }) + .collect(); + if sanitized.is_empty() { + FALLBACK_EXCEPTION_TYPE.to_owned() + } else { + sanitized + } +} diff --git a/crates/sqlbuild-core/src/event_export/main/severity_at_least.rs b/crates/sqlbuild-core/src/event_export/main/severity_at_least.rs new file mode 100644 index 0000000000..0a9b3fcf78 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/main/severity_at_least.rs @@ -0,0 +1,9 @@ +//! Compare a catalogued severity with a sink's minimum. + +use crate::event_export::models::ExportSeverity; + +/// Whether `severity` meets the configured `minimum`. +#[must_use] +pub fn severity_at_least(severity: ExportSeverity, minimum: ExportSeverity) -> bool { + severity >= minimum +} diff --git a/crates/sqlbuild-core/src/event_export/main/stricter_severity.rs b/crates/sqlbuild-core/src/event_export/main/stricter_severity.rs new file mode 100644 index 0000000000..b496ab706e --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/main/stricter_severity.rs @@ -0,0 +1,12 @@ +//! Narrow a sink's minimum severity by an optional configured minimum. + +use crate::event_export::models::ExportSeverity; + +/// The stricter of a required minimum and an optional one. +#[must_use] +pub fn stricter_severity(first: ExportSeverity, second: Option) -> ExportSeverity { + match second { + Some(second) if second > first => second, + _ => first, + } +} diff --git a/crates/sqlbuild-core/src/event_export/main/take_lifecycle_event.rs b/crates/sqlbuild-core/src/event_export/main/take_lifecycle_event.rs new file mode 100644 index 0000000000..aec19201b6 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/main/take_lifecycle_event.rs @@ -0,0 +1,26 @@ +//! Remove the next event to deliver from the bounded priority queue. + +use crate::event_export::models::{EventQueue, QueuedEvent}; + +/// The queue without its next event: the oldest at the highest priority, invocation terminals last. +pub fn take_lifecycle_event( + mut queue: EventQueue, +) -> (EventQueue, Option>) { + let only_terminals: bool = queue.items.iter().all(|queued| queued.invocation_terminal); + let selectable = |queued: &QueuedEvent| only_terminals || !queued.invocation_terminal; + let highest: Option = queue + .items + .iter() + .filter(|queued| selectable(queued)) + .map(|queued| queued.priority) + .max(); + let position: Option = queue + .items + .iter() + .enumerate() + .filter(|(_, queued)| selectable(queued) && Some(queued.priority) == highest) + .min_by_key(|(_, queued)| queued.sequence) + .map(|(position, _)| position); + let next: Option> = position.map(|position| queue.items.remove(position)); + (queue, next) +} diff --git a/crates/sqlbuild-core/src/event_export/mod.rs b/crates/sqlbuild-core/src/event_export/mod.rs new file mode 100644 index 0000000000..a4ff771957 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/mod.rs @@ -0,0 +1,8 @@ +//! Lifecycle-event export: policy dimensions, severity filters, the bounded priority queue and accounting. + +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/event_export/models.rs b/crates/sqlbuild-core/src/event_export/models.rs new file mode 100644 index 0000000000..66a5e2159c --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/models.rs @@ -0,0 +1,137 @@ +//! Export severities, policies, sink filters, attempt counters and the bounded priority queue. + +use std::collections::BTreeSet; + +/// `EventExportSeverity`, ordered from least to most severe. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum ExportSeverity { + Debug, + Info, + Warning, + Error, +} + +/// Catalogued export dimensions of one lifecycle event type. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct ExportPolicy { + pub kind: String, + pub severity: ExportSeverity, + pub priority: u8, +} + +/// The event kinds and minimum severity one bound sink accepts. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExporterFilter { + pub event_kinds: BTreeSet, + pub min_severity: ExportSeverity, +} + +/// Attempt counters for one exporter or their aggregate. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct ExporterCounts { + pub accepted: u64, + pub filtered: u64, + pub delivered: u64, + pub failed: u64, + pub dropped: u64, +} + +/// One queued event and the exact exporters still owed an attempt. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct QueuedEvent { + pub sequence: u64, + pub item: T, + pub priority: u8, + /// Whether the event ends the invocation and so waits behind every other queued event. + pub invocation_terminal: bool, + pub eligible_exporters: Vec, +} + +/// Bounded queue: highest priority first, FIFO within a priority. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EventQueue { + pub capacity: usize, + pub items: Vec>, +} + +/// The queue after an insert, whether the event was queued, and the event it displaced. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct QueueOutcome { + pub queue: EventQueue, + pub inserted: bool, + pub displaced: Option>, + /// The incoming event, handed back when it was not queued. + pub rejected: Option>, +} + +/// Final or point-in-time delivery accounting of every exporter. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct ExportSnapshot { + pub aggregate: ExporterCounts, + pub per_exporter: Vec<(String, ExporterCounts)>, + pub queue_depth: usize, + pub queue_capacity: usize, + pub flush_complete: bool, +} + +/// What a snapshot is taken of: exporters, the queue and whether delivery has fully stopped. +#[derive(Clone, Copy, Debug)] +pub struct SnapshotInputs<'a> { + pub exporters: &'a [(String, ExporterCounts)], + pub queue_depth: usize, + pub queue_capacity: usize, + /// A final snapshot taken after the dispatcher and every invocation finished. + pub drained: bool, +} + +impl ExportSeverity { + /// The severity named `value`, as `EventExportSeverity(value)`. + #[must_use] + pub fn from_value(value: &str) -> Option { + match value { + "debug" => Some(Self::Debug), + "info" => Some(Self::Info), + "warning" => Some(Self::Warning), + "error" => Some(Self::Error), + _ => None, + } + } + + /// The severity's `EventExportSeverity` value. + #[must_use] + pub fn value(self) -> &'static str { + match self { + Self::Debug => "debug", + Self::Info => "info", + Self::Warning => "warning", + Self::Error => "error", + } + } +} + +impl EventQueue { + /// An empty queue holding at most `capacity` events. + #[must_use] + pub fn new(capacity: usize) -> Self { + Self { + capacity, + items: Vec::new(), + } + } +} + +impl ExportSnapshot { + /// Counters and queue fields compared to skip unchanged periodic health summaries. + #[must_use] + pub fn health_signature(&self) -> [u64; 7] { + [ + self.aggregate.accepted, + self.aggregate.filtered, + self.aggregate.delivered, + self.aggregate.dropped, + self.aggregate.failed, + u64::try_from(self.queue_depth).unwrap_or(u64::MAX), + u64::try_from(self.queue_capacity).unwrap_or(u64::MAX), + ] + } +} diff --git a/crates/sqlbuild-core/src/event_export/tests.rs b/crates/sqlbuild-core/src/event_export/tests.rs new file mode 100644 index 0000000000..cef0bfeb55 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/tests.rs @@ -0,0 +1,10 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_policy.rs"] +mod policy; +#[path = "tests/test_queue.rs"] +mod queue; +#[path = "tests/test_routing.rs"] +mod routing; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/event_export/tests/helpers.rs b/crates/sqlbuild-core/src/event_export/tests/helpers.rs new file mode 100644 index 0000000000..03087a70d4 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/tests/helpers.rs @@ -0,0 +1,15 @@ +use crate::event_export::models::QueuedEvent; + +/// A queued unit event with `sequence`, `(priority, invocation terminal)` and one eligible exporter. +pub(crate) fn queued( + sequence: u64, + (priority, invocation_terminal): (u8, bool), +) -> QueuedEvent<()> { + QueuedEvent { + sequence, + item: (), + priority, + invocation_terminal, + eligible_exporters: vec![0], + } +} diff --git a/crates/sqlbuild-core/src/event_export/tests/test_policy.rs b/crates/sqlbuild-core/src/event_export/tests/test_policy.rs new file mode 100644 index 0000000000..51d386ec81 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/tests/test_policy.rs @@ -0,0 +1,125 @@ +use crate::event_export::main::lifecycle_export_policy::lifecycle_export_policy; +use crate::event_export::main::sanitized_exception_type::sanitized_exception_type; +use crate::event_export::main::stricter_severity::stricter_severity; +use crate::event_export::models::ExportSeverity; +use crate::event_export::tests::test_types::{ + PolicyTestCase, SanitizationTestCase, StricterSeverityTestCase, +}; + +#[test] +fn given_catalogued_event_types_when_deriving_policy_then_kind_severity_and_priority_match() { + let test_cases = [ + PolicyTestCase { + description: "progress events are debug at the lowest priority", + event_type: "statement_submitted", + terminal: false, + expected_kind: "statement", + expected_severity: ExportSeverity::Debug, + expected_priority: 0, + }, + PolicyTestCase { + description: "terminal successes are info", + event_type: "resource_attempt_completed", + terminal: true, + expected_kind: "resource", + expected_severity: ExportSeverity::Info, + expected_priority: 2, + }, + PolicyTestCase { + description: "failures are errors at the highest priority", + event_type: "invocation_failed", + terminal: true, + expected_kind: "invocation", + expected_severity: ExportSeverity::Error, + expected_priority: 3, + }, + PolicyTestCase { + description: "retries are warnings of their own kind", + event_type: "retry_scheduled", + terminal: false, + expected_kind: "retry", + expected_severity: ExportSeverity::Warning, + expected_priority: 1, + }, + ]; + + for test_case in test_cases { + let policy = lifecycle_export_policy(test_case.event_type, test_case.terminal); + assert_eq!( + (policy.kind.as_str(), policy.severity, policy.priority), + ( + test_case.expected_kind, + test_case.expected_severity, + test_case.expected_priority + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_exception_names_when_sanitizing_then_only_bounded_identifier_characters_remain() { + let test_cases = [ + SanitizationTestCase { + description: "plain names are unchanged", + name: "RuntimeError".to_owned(), + expected_name: "RuntimeError".to_owned(), + }, + SanitizationTestCase { + description: "each non-identifier character becomes one underscore", + name: "Bad-Name!\u{e9}".to_owned(), + expected_name: "Bad_Name__".to_owned(), + }, + SanitizationTestCase { + description: "long names are truncated to 100 characters", + name: "E".repeat(150), + expected_name: "E".repeat(100), + }, + SanitizationTestCase { + description: "empty names fall back to Exception", + name: String::new(), + expected_name: "Exception".to_owned(), + }, + ]; + + for test_case in test_cases { + assert_eq!( + sanitized_exception_type(&test_case.name), + test_case.expected_name, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_required_and_optional_minimums_when_narrowing_then_stricter_severity_wins() { + let test_cases = [ + StricterSeverityTestCase { + description: "absent optional minimum keeps the required one", + severities: (ExportSeverity::Info, None), + expected_severity: ExportSeverity::Info, + }, + StricterSeverityTestCase { + description: "stricter optional minimum wins", + severities: (ExportSeverity::Info, Some(ExportSeverity::Error)), + expected_severity: ExportSeverity::Error, + }, + StricterSeverityTestCase { + description: "looser optional minimum is ignored", + severities: (ExportSeverity::Warning, Some(ExportSeverity::Debug)), + expected_severity: ExportSeverity::Warning, + }, + ]; + + for test_case in test_cases { + let (first, second) = test_case.severities; + assert_eq!( + stricter_severity(first, second), + test_case.expected_severity, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/event_export/tests/test_queue.rs b/crates/sqlbuild-core/src/event_export/tests/test_queue.rs new file mode 100644 index 0000000000..95aa2ce728 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/tests/test_queue.rs @@ -0,0 +1,89 @@ +use crate::event_export::main::queue_lifecycle_event::queue_lifecycle_event; +use crate::event_export::main::take_lifecycle_event::take_lifecycle_event; +use crate::event_export::models::EventQueue; +use crate::event_export::tests::helpers::queued; +use crate::event_export::tests::test_types::{DeliveryOrderTestCase, DisplacementTestCase}; + +#[test] +fn given_full_queue_when_priority_pair_arrives_then_displacement_policy_is_exact() { + let test_cases = [ + DisplacementTestCase { + description: "equal priority is rejected", + priorities: (2, 2), + expected_inserted: false, + expected_displaced: None, + expected_retained: 1, + }, + DisplacementTestCase { + description: "lower priority is rejected", + priorities: (3, 0), + expected_inserted: false, + expected_displaced: None, + expected_retained: 1, + }, + DisplacementTestCase { + description: "higher priority displaces the queued event", + priorities: (0, 1), + expected_inserted: true, + expected_displaced: Some(1), + expected_retained: 2, + }, + ]; + + for test_case in test_cases { + let (queued_priority, incoming_priority) = test_case.priorities; + let first = queue_lifecycle_event(EventQueue::new(1), queued(1, (queued_priority, false))); + let outcome = queue_lifecycle_event(first.queue, queued(2, (incoming_priority, false))); + let (_, retained) = take_lifecycle_event(outcome.queue); + let retained = retained.map(|event| event.sequence); + assert!(first.inserted, "{}", test_case.description); + assert_eq!( + ( + outcome.inserted, + outcome.displaced.map(|event| event.sequence), + retained + ), + ( + test_case.expected_inserted, + test_case.expected_displaced, + Some(test_case.expected_retained) + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_queued_events_when_taking_then_priority_then_fifo_with_invocation_terminals_last() { + let test_cases = [ + DeliveryOrderTestCase { + description: "highest priority first and FIFO within a priority", + events: &[(0, false), (3, false), (2, false), (3, false), (0, false)], + expected_sequences: &[1, 3, 2, 0, 4], + }, + DeliveryOrderTestCase { + description: "invocation terminals wait behind lower-priority events", + events: &[(3, true), (0, false), (1, false)], + expected_sequences: &[2, 1, 0], + }, + ]; + + for test_case in test_cases { + let mut queue: EventQueue<()> = EventQueue::new(16); + for (sequence, event) in (0_u64..).zip(test_case.events) { + queue = queue_lifecycle_event(queue, queued(sequence, *event)).queue; + } + let mut order: Vec = Vec::new(); + for _ in test_case.events { + let (rest, next) = take_lifecycle_event(queue); + queue = rest; + order.extend(next.map(|event| event.sequence)); + } + assert_eq!( + order, test_case.expected_sequences, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/event_export/tests/test_routing.rs b/crates/sqlbuild-core/src/event_export/tests/test_routing.rs new file mode 100644 index 0000000000..8500e3c78f --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/tests/test_routing.rs @@ -0,0 +1,126 @@ +use crate::event_export::main::export_snapshot::export_snapshot; +use crate::event_export::main::route_lifecycle_event::route_lifecycle_event; +use crate::event_export::models::{ + ExportPolicy, ExportSeverity, ExporterCounts, ExporterFilter, SnapshotInputs, +}; +use crate::event_export::tests::test_types::{RoutingTestCase, SnapshotTestCase}; + +#[test] +fn given_sink_filters_when_routing_then_kind_and_severity_select_eligible_sinks() { + let test_cases = [ + RoutingTestCase { + description: "kind and minimum severity both apply", + policy: ("run", ExportSeverity::Info), + filters: &[ + (&["run"], ExportSeverity::Debug), + (&["run"], ExportSeverity::Error), + (&["statement"], ExportSeverity::Debug), + ], + expected_eligible: &[0], + expected_filtered: 2, + }, + RoutingTestCase { + description: "every matching sink is eligible in order", + policy: ("invocation", ExportSeverity::Error), + filters: &[ + (&["invocation"], ExportSeverity::Warning), + (&["run", "invocation"], ExportSeverity::Error), + ], + expected_eligible: &[0, 1], + expected_filtered: 0, + }, + ]; + + for test_case in test_cases { + let (kind, severity) = test_case.policy; + let policy = ExportPolicy { + kind: kind.to_owned(), + severity, + priority: 0, + }; + let filters: Vec = test_case + .filters + .iter() + .map(|(kinds, min_severity)| ExporterFilter { + event_kinds: kinds.iter().map(|kind| (*kind).to_owned()).collect(), + min_severity: *min_severity, + }) + .collect(); + let (counts, eligible) = route_lifecycle_event( + &policy, + &filters, + vec![ExporterCounts::default(); filters.len()], + ); + let filtered: u64 = counts.iter().map(|counts| counts.filtered).sum(); + assert_eq!( + (eligible.as_slice(), filtered), + (test_case.expected_eligible, test_case.expected_filtered), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_exporter_counts_when_snapshotting_then_flush_completes_only_when_settled_and_drained() { + let test_cases = [ + SnapshotTestCase { + description: "settled drained accounting is complete", + counts: &[(3, 2, 1, 0), (2, 1, 0, 1)], + queue_depth: 0, + drained: true, + expected_accepted: 5, + expected_flush_complete: true, + }, + SnapshotTestCase { + description: "a point-in-time snapshot is never complete", + counts: &[(1, 1, 0, 0)], + queue_depth: 0, + drained: false, + expected_accepted: 1, + expected_flush_complete: false, + }, + SnapshotTestCase { + description: "unsettled attempts are incomplete", + counts: &[(2, 1, 0, 0)], + queue_depth: 0, + drained: true, + expected_accepted: 2, + expected_flush_complete: false, + }, + ]; + + for test_case in test_cases { + let exporters: Vec<(String, ExporterCounts)> = test_case + .counts + .iter() + .map(|(accepted, delivered, failed, dropped)| { + ( + "sink".to_owned(), + ExporterCounts { + accepted: *accepted, + filtered: 0, + delivered: *delivered, + failed: *failed, + dropped: *dropped, + }, + ) + }) + .collect(); + let snapshot = export_snapshot(SnapshotInputs { + exporters: &exporters, + queue_depth: test_case.queue_depth, + queue_capacity: 8, + drained: test_case.drained, + }); + assert_eq!( + (snapshot.aggregate.accepted, snapshot.flush_complete), + ( + test_case.expected_accepted, + test_case.expected_flush_complete + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/event_export/tests/test_types.rs b/crates/sqlbuild-core/src/event_export/tests/test_types.rs new file mode 100644 index 0000000000..d3960e9af0 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/tests/test_types.rs @@ -0,0 +1,60 @@ +use crate::event_export::models::ExportSeverity; + +/// One catalogued event type and the export dimensions it maps to. +pub(crate) struct PolicyTestCase { + pub(crate) description: &'static str, + pub(crate) event_type: &'static str, + pub(crate) terminal: bool, + pub(crate) expected_kind: &'static str, + pub(crate) expected_severity: ExportSeverity, + pub(crate) expected_priority: u8, +} + +/// One exception class name and its sanitized diagnostic form. +pub(crate) struct SanitizationTestCase { + pub(crate) description: &'static str, + pub(crate) name: String, + pub(crate) expected_name: String, +} + +/// A required and optional minimum severity and the stricter of the two. +pub(crate) struct StricterSeverityTestCase { + pub(crate) description: &'static str, + pub(crate) severities: (ExportSeverity, Option), + pub(crate) expected_severity: ExportSeverity, +} + +/// One full single-slot queue receiving a second event. +pub(crate) struct DisplacementTestCase { + pub(crate) description: &'static str, + pub(crate) priorities: (u8, u8), + pub(crate) expected_inserted: bool, + pub(crate) expected_displaced: Option, + pub(crate) expected_retained: u64, +} + +/// Queued `(priority, invocation terminal)` events and their delivery order by sequence. +pub(crate) struct DeliveryOrderTestCase { + pub(crate) description: &'static str, + pub(crate) events: &'static [(u8, bool)], + pub(crate) expected_sequences: &'static [u64], +} + +/// One event routed across sink filters, and the resulting accounting. +pub(crate) struct RoutingTestCase { + pub(crate) description: &'static str, + pub(crate) policy: (&'static str, ExportSeverity), + pub(crate) filters: &'static [(&'static [&'static str], ExportSeverity)], + pub(crate) expected_eligible: &'static [usize], + pub(crate) expected_filtered: u64, +} + +/// Per-exporter `(accepted, delivered, failed, dropped)` counts and the snapshot completion. +pub(crate) struct SnapshotTestCase { + pub(crate) description: &'static str, + pub(crate) counts: &'static [(u64, u64, u64, u64)], + pub(crate) queue_depth: usize, + pub(crate) drained: bool, + pub(crate) expected_accepted: u64, + pub(crate) expected_flush_complete: bool, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 0a7ad4ba94..4d3832c03d 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -16,6 +16,7 @@ pub mod contract_command; pub mod cost_allocation; pub mod cost_command; pub mod cost_report; +pub mod dagster_invocation; pub mod dbt_cli; pub mod dbt_command; pub mod dbt_events; @@ -23,6 +24,7 @@ pub mod dbt_graph; pub mod dbt_plan; pub mod dbt_process; pub mod debug_report; +pub mod event_export; pub mod format_command; pub mod freshness_text; pub mod init_command; diff --git a/crates/sqlbuild-discovery/src/yaml_entries/_helpers/integrations.rs b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/integrations.rs index 337f57a5a1..419ee50b64 100644 --- a/crates/sqlbuild-discovery/src/yaml_entries/_helpers/integrations.rs +++ b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/integrations.rs @@ -8,12 +8,13 @@ use crate::yaml_entries::_helpers::fields::{ Site, get, optional_mapping, optional_text, reject_unknown_keys, required_text, stripped_strings, }; +use crate::yaml_entries::constants::DLT_LOADER_KIND; +use crate::yaml_entries::main::integration_loader_name::integration_loader_name; use crate::yaml_entries::models::{ ParsedDltResource, ParsedDltSource, ParsedIngestrConfig, ParsedIntegrationLoader, ParsedSourceEntry, }; -const DLT_LOADER_KIND: &str = "dlt"; const INGESTR_KEYS: [&str; 8] = [ "columns", "extra_args", @@ -56,15 +57,6 @@ const RAW_CONFIG_EXCLUDED: [&str; 8] = [ "description", ]; -/// Python's `integration_loader_name`. -pub(crate) fn integration_loader_name(kind: &str, source_name: &str) -> String { - if kind == DLT_LOADER_KIND { - source_name.to_owned() - } else { - format!("{kind}__{source_name}") - } -} - /// `parse_source_integration_loader`: the source's `ingestr` block, if any. pub(crate) fn source_integration_loader( entries: &[(N, N)], diff --git a/crates/sqlbuild-discovery/src/yaml_entries/_helpers/sources.rs b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/sources.rs index eb356f6e80..af199d8c9d 100644 --- a/crates/sqlbuild-discovery/src/yaml_entries/_helpers/sources.rs +++ b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/sources.rs @@ -10,9 +10,8 @@ use crate::yaml_entries::_helpers::fields::{ stripped_strings, }; use crate::yaml_entries::_helpers::freshness::freshness; -use crate::yaml_entries::_helpers::integrations::{ - integration_loader_name, source_integration_loader, -}; +use crate::yaml_entries::_helpers::integrations::source_integration_loader; +use crate::yaml_entries::main::integration_loader_name::integration_loader_name; use crate::yaml_entries::models::{ParsedIntegrationLoader, ParsedSourceEntry, ParsedYamlColumn}; const ENTRY_KEYS: [&str; 18] = [ diff --git a/crates/sqlbuild-discovery/src/yaml_entries/constants.rs b/crates/sqlbuild-discovery/src/yaml_entries/constants.rs index 185f6b92f3..aecfb9e032 100644 --- a/crates/sqlbuild-discovery/src/yaml_entries/constants.rs +++ b/crates/sqlbuild-discovery/src/yaml_entries/constants.rs @@ -1,6 +1,9 @@ -//! The top-level keys of source and schema YAML documents. +//! The top-level keys of source and schema YAML documents and integration loader kinds. /// The keys a schema YAML file under `seeds/` may declare. pub(crate) const SEED_FILE_KEYS: [&str; 1] = ["seeds"]; /// The keys a sources YAML file may declare. pub(crate) const SOURCE_FILE_KEYS: [&str; 2] = ["sources", "dlt_sources"]; + +/// The integration kind whose loaders run under the bare source name. +pub(crate) const DLT_LOADER_KIND: &str = "dlt"; diff --git a/crates/sqlbuild-discovery/src/yaml_entries/main/integration_loader_name.rs b/crates/sqlbuild-discovery/src/yaml_entries/main/integration_loader_name.rs new file mode 100644 index 0000000000..6a496fb4ac --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_entries/main/integration_loader_name.rs @@ -0,0 +1,12 @@ +//! The loader name a declarative integration source runs under. + +use crate::yaml_entries::constants::DLT_LOADER_KIND; + +/// The source name for dlt sources, `__` for every other integration. +pub fn integration_loader_name(kind: &str, source_name: &str) -> String { + if kind == DLT_LOADER_KIND { + source_name.to_owned() + } else { + format!("{kind}__{source_name}") + } +} diff --git a/crates/sqlbuild-discovery/src/yaml_entries/main/mod.rs b/crates/sqlbuild-discovery/src/yaml_entries/main/mod.rs index cb3f72f7bd..2ffa29d821 100644 --- a/crates/sqlbuild-discovery/src/yaml_entries/main/mod.rs +++ b/crates/sqlbuild-discovery/src/yaml_entries/main/mod.rs @@ -1,2 +1,3 @@ +pub mod integration_loader_name; pub mod parse_schema_document; pub mod parse_sources_document; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs index 0599b0309f..e9b5ecf5ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs @@ -14,6 +14,8 @@ use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::models::{CatalogInput, ProjectCatalog}; use crate::bindings::types::CompilerDetach; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + /// Dialects whose quoted identifiers bind case-sensitively unless the session says otherwise. pub(crate) const CASE_SENSITIVE_BINDING_DIALECTS: [&str; 2] = ["snowflake", "postgresql"]; /// Adapter dialect names and the native dialect each validates as. @@ -27,7 +29,6 @@ const SNOWFLAKE_DIALECT: &str = "snowflake"; const QUOTED_CASE_PARAMETER: &str = "QUOTED_IDENTIFIERS_IGNORE_CASE"; const SESSION_PARAMETERS: &str = "session_parameters"; const GENERIC_DIALECT: &str = "generic"; -const MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; /// One native project catalog and the relation shapes the compile has told it about. #[pyclass(module = "sqlbuild._native")] @@ -321,9 +322,12 @@ pub(crate) fn schema_validations<'py>( None => groups.push((catalog, own, vec![index])), } } - let result_type: Bound<'py, PyAny> = py.import(MODELS_MODULE)?.getattr("SqlBindingResult")?; - let diagnostic_type: Bound<'py, PyAny> = - py.import(MODELS_MODULE)?.getattr("SqlBindingDiagnostic")?; + let result_type: Bound<'py, PyAny> = py + .import(NATIVE_RECORDS_MODULE)? + .getattr("SqlBindingResult")?; + let diagnostic_type: Bound<'py, PyAny> = py + .import(NATIVE_RECORDS_MODULE)? + .getattr("SqlBindingDiagnostic")?; let empty: Bound<'py, PyAny> = result_type.call0()?; let mut results: Vec> = vec![empty; requests.len()]; for (catalog, any_explicit, members) in groups { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index b7353adbfc..ea8a364d13 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -1,5 +1,6 @@ //! Compile's explicit-reference checks over plain project facts, run natively. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -348,7 +349,8 @@ fn python_report<'py>( fields.set_item("help", &row.help)?; diagnostics.push(compiler_diagnostic.call((), Some(&fields))?); } - let literal_relation: Bound<'py, PyAny> = reference_models.getattr("LiteralSqlRelation")?; + let literal_relation: Bound<'py, PyAny> = + shape_class(reference_models.as_any(), "LiteralSqlRelation")?; let relation_name: Bound<'py, PyAny> = reference_models.getattr("RelationName")?; let mut unmatched: Vec> = Vec::with_capacity(report.unmatched.len()); for row in &report.unmatched { @@ -369,9 +371,7 @@ fn python_report<'py>( let fields: Bound<'py, PyDict> = PyDict::new(py); fields.set_item("diagnostics", PyTuple::new(py, diagnostics)?)?; fields.set_item("unmatched", PyTuple::new(py, unmatched)?)?; - compile_models - .getattr("PythonSqlReferenceReport")? - .call((), Some(&fields)) + shape_class(compile_models.as_any(), "PythonSqlReferenceReport")?.call((), Some(&fields)) } /// `(method, sql, line)` for each literal SQL call to the context in a dedented function source. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs index 4d1ddd93fe..cd70197f7c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs @@ -10,7 +10,8 @@ use sqlbuild_analysis::semantic_validation::models::NormalizationInput; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::types::CompilerDetach; -const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const GENERIC_DIALECT: &str = "generic"; /// The query, relation schemas and binding context of one schema-validation request. @@ -60,7 +61,7 @@ pub(crate) fn schema_validation_request<'py>( parts.quoted_identifiers_ignore_case, )?; arguments.set_item("catalog", parts.catalog)?; - py.import(SQL_ANALYSIS_MODELS_MODULE)? + py.import(NATIVE_RECORDS_MODULE)? .getattr("SqlSchemaValidationRequest")? .call((), Some(&arguments)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs index 28f000ad13..2ff4d0fb6d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs @@ -16,7 +16,9 @@ use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::types::CompilerDetach; -const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + +const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const PROJECTION_FACTS_CONTEXT: &str = "native fixture projection facts"; @@ -78,7 +80,7 @@ fn infer_fixture_column_facts<'py>( PyFrozenSet::new(py, facts.null_literal_names)?, )?; arguments.set_item("quoted_names", PyFrozenSet::new(py, facts.quoted_names)?)?; - py.import(COMPILE_MODELS_MODULE)? + py.import(NATIVE_RECORDS_MODULE)? .getattr("FixtureColumnInference")? .call((), Some(&arguments)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index fdb93624a8..9dadfd103b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::cache::native_store::current_code_identity; use crate::bindings::_helpers::compile_driver::analysis_cache::record_analysis_cache_metrics; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const LOG_DEBUG_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs index b3d9bb9962..28a3b907ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs @@ -1,5 +1,6 @@ //! Finished session outcomes and pivot proof rows as Python's analysis objects. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -7,8 +8,9 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::keyword_request::Request; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const LINEAGE_NATIVE: &str = "native"; @@ -57,7 +59,7 @@ fn session_outcome_analysis<'py>( PyTuple::empty(py).into_any() }; let binding_class: Bound<'py, PyAny> = py - .import(SQL_ANALYSIS_MODELS_MODULE)? + .import(NATIVE_RECORDS_MODULE)? .getattr("SqlBindingDiagnostic")?; let mut bindings: Vec> = Vec::new(); for row in raw.try_iter()? { @@ -66,7 +68,7 @@ fn session_outcome_analysis<'py>( bindings.push(binding_class.call1(row)?); } let result: Bound<'py, PyAny> = call( - &models.getattr("PolyglotAnalysisResult")?, + &shape_class(models.as_any(), "PolyglotAnalysisResult")?, &[ ("analysis_succeeded", succeeded), ("columns", inferred_columns(&columns)?), @@ -81,7 +83,7 @@ fn session_outcome_analysis<'py>( ], )?; call( - &models.getattr("ModelSqlAnalysis")?, + &shape_class(models.as_any(), "ModelSqlAnalysis")?, &[ ("polyglot_analysis", result), ("placeholders", request.get("placeholders")?), @@ -106,7 +108,7 @@ fn proof<'py>(row: &Bound<'py, PyAny>) -> PyResult> { let (output_proven, columns, families, inputs, failure_reason, bare): ProofObjects<'py> = row.extract()?; let models = py.import(COMPILE_MODELS_MODULE)?; - let family_class: Bound<'py, PyAny> = models.getattr("DynamicColumnFamilyProof")?; + let family_class: Bound<'py, PyAny> = shape_class(models.as_any(), "DynamicColumnFamilyProof")?; let mut proofs: Vec> = Vec::new(); for family in families.try_iter()? { let (name, inferred_type): (Bound<'py, PyAny>, Bound<'py, PyAny>) = family?.extract()?; @@ -122,7 +124,7 @@ fn proof<'py>(row: &Bound<'py, PyAny>) -> PyResult> { PyTuple::empty(py).into_any() }; call( - &models.getattr("DynamicColumnContractProof")?, + &shape_class(models.as_any(), "DynamicColumnContractProof")?, &[ ("output_proven", output_proven), ("fixed_columns", fixed), @@ -228,7 +230,8 @@ fn compact_lineage<'py>(rows: &Bound<'py, PyAny>) -> PyResult> fn lineage_facts<'py>(rows: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = rows.py(); let models = py.import(COMPILE_MODELS_MODULE)?; - let column_class: Bound<'py, PyAny> = models.getattr("CompiledLineageColumnFact")?; + let column_class: Bound<'py, PyAny> = + shape_class(models.as_any(), "CompiledLineageColumnFact")?; let source_class: Bound<'py, PyAny> = models.getattr("CompiledLineageSourceFact")?; let compact: Bound<'py, PyAny> = models.getattr("CompactLineageFacts")?; let mut facts: Vec> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs index 7843499028..ef50a1c090 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs @@ -32,7 +32,7 @@ const LINEAGE_RESOURCE_TYPES: [(&str, &str); 4] = [ ("seed", "seed"), ("table_fn", "table_fn"), ]; -const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index a78d925e01..b70e9eaabe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -14,10 +14,11 @@ use sqlbuild_core::cli_text::models::CliStyle; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; -const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const AUDIT_PLAN_MODULE: &str = "sqlbuild._native"; const COST_CONTEXT_MODULE: &str = "sqlbuild._native"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; @@ -119,7 +120,7 @@ fn compile_audits<'py>( ], )?; let options: Bound<'py, PyAny> = call_with( - &imported(py, PIPELINE_MODELS, "CompilePipelineOptions")?, + &imported(py, NATIVE_RECORDS_MODULE, "CompilePipelineOptions")?, &[ ("selected_target", request.getattr("selected_target")?), ("no_sql_validation", request.getattr("no_sql_validation")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs index a6c1b2bd5f..30abd51eec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs @@ -10,7 +10,7 @@ use sqlbuild_analysis::auditing::models::{MeasurementOutcome, Thresholds}; use crate::bindings::_helpers::auditing::identity::thresholds as read_thresholds; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; -const AUDITING_ERRORS_MODULE: &str = "sqlbuild.compiler.auditing.exceptions"; +const AUDITING_ERRORS_MODULE: &str = "sqlbuild._native"; fn measurement_error(py: Python<'_>, message: &str) -> PyErr { match py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs index 3d904ab50e..a139607174 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::contract_classes::contract_classes; use crate::bindings::_helpers::model_config::header_metadata::{ContractClasses, audit_tuple}; -const IDENTITY_ERRORS_MODULE: &str = "sqlbuild.compiler.resource_names.exceptions"; +const IDENTITY_ERRORS_MODULE: &str = "sqlbuild._native"; /// A resource identity error as `ResourceIdentityError`, anything else as `error_class(message)`. fn raised(py: Python<'_>, error: ConfigError, error_class: &Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index 1165ddcea3..1f81f7cb57 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -15,9 +15,9 @@ use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::build_command::policies::build_command_error; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; -const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const CLONE_MODELS: &str = "sqlbuild.executor.clone.models"; const CLONE_TYPES: &str = "sqlbuild.executor.clone.types"; const SPEC_TYPES: &str = "sqlbuild.spec.contracts.types"; @@ -443,7 +443,7 @@ fn clone_boundaries<'py>( ( "destination_connection", call_with( - &imported(py, PIPELINE_MODELS, "ClonePipelineConnection")?, + &imported(py, NATIVE_RECORDS_MODULE, "ClonePipelineConnection")?, &[ ("config", context.invocation.getattr("connection_config")?), ("handle", connection.clone()), @@ -453,7 +453,7 @@ fn clone_boundaries<'py>( ( "options", call_with( - &imported(py, PIPELINE_MODELS, "ClonePipelineOptions")?, + &imported(py, NATIVE_RECORDS_MODULE, "ClonePipelineOptions")?, &[ ("no_sql_validation", request.getattr("no_sql_validation")?), ("no_cache", request.getattr("no_cache")?), @@ -598,7 +598,7 @@ pub(crate) fn run_defer_clone_boundary_prephase( ( "analysis_selection", call_with( - &imported(py, COMPILE_MODELS, "CompileAnalysisSelection")?, + &imported(py, NATIVE_RECORDS_MODULE, "CompileAnalysisSelection")?, &[("no_cache", request.getattr("no_cache")?)], )?, ), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index bae571a778..f1e02d7543 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -17,8 +17,9 @@ use crate::bindings::_helpers::build_command::support::{ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; -const PYTHON_NODE_MODELS: &str = "sqlbuild.compiler.python_nodes.models"; const LOADER_DAG_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; @@ -301,7 +302,7 @@ fn lifecycle_plan<'py>( let plan_output: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; let graph: Bound<'py, PyAny> = python_node_graph(discovered)?; let selection: Bound<'py, PyAny> = call_with( - &imported(py, PYTHON_NODE_MODELS, "PythonSqlRunSelection")?, + &imported(py, NATIVE_RECORDS_MODULE, "PythonSqlRunSelection")?, &[ ( "sql_keys", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index ddc33b18dd..6ba0587a4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -16,9 +16,10 @@ use crate::bindings::_helpers::build_command::support::{ use crate::bindings::_helpers::cli::progress_text::with_verbose_inspection; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; -const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; @@ -50,7 +51,7 @@ fn request_options<'py>(context: &BuildContext<'py>) -> PyResult) -> Bound<'_, PyAny> { /// Refuse `+`, `~` and path selectors, splitting selectors as Python's `split` does. fn refuse_graph_selectors(py: Python<'_>, selectors: &[Bound<'_, PyAny>]) -> PyResult<()> { let parse: Bound<'_, PyAny> = imported(py, SELECTOR_PARSE_MODULE, "parse_project_selector")?; - let path_selector: Bound<'_, PyAny> = imported(py, PLANNER_MODELS, "PathSelector")?; + let path_selector: Bound<'_, PyAny> = imported(py, NATIVE_RECORDS_MODULE, "PathSelector")?; for selector in selectors { for token in selector.call_method0("split")?.try_iter()? { for part in token?.call_method1("split", (",",))?.try_iter()? { @@ -230,7 +229,7 @@ fn compile_pipeline<'py>( ], )?; let options: Bound<'py, PyAny> = call_with( - &imported(py, PIPELINE_MODELS, "CompilePipelineOptions")?, + &imported(py, NATIVE_RECORDS_MODULE, "CompilePipelineOptions")?, &[ ("selected_target", request.getattr("selected_target")?), ("no_sql_validation", request.getattr("no_sql_validation")?), @@ -323,7 +322,7 @@ fn prepare<'py>( } } let selection: Bound<'py, PyAny> = call_with( - &imported(py, PYTHON_NODE_MODELS, "PythonSqlRunSelection")?, + &imported(py, NATIVE_RECORDS_MODULE, "PythonSqlRunSelection")?, &[ ("sql_keys", PyFrozenSet::empty(py)?.into_any()), ("python_node_names", PyFrozenSet::empty(py)?.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 2f2ffd4833..aa61b40fff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -36,7 +36,7 @@ const DIAGNOSTICS_MODULE: &str = "sqlbuild.diagnostics.main.diagnostics_context" const TRANSIENT_LINE_MODULE: &str = "sqlbuild._native"; const SKILLS_MODULE: &str = "sqlbuild._native"; const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; -const LINT_ERRORS_MODULE: &str = "sqlbuild.lint.exceptions"; +const LINT_ERRORS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; const NATIVE_MODULE: &str = "sqlbuild._native"; const CLI_LOGGER: &str = "sqlbuild.cli"; @@ -50,12 +50,12 @@ const EXPECTED_COMMAND_FAILURES: [(&str, &str); 4] = [ "sqlbuild.executor.pipeline.exceptions", "AuditExecutionError", ), - ("sqlbuild.compiler.discovery.exceptions", "DiscoveryError"), + ("sqlbuild._native", "DiscoveryError"), ( "sqlbuild.runtime.execution_limits.exceptions", "ExecutionDurationLimitError", ), - ("sqlbuild.spec.contracts.exceptions", "SpecConfigError"), + ("sqlbuild._native", "SpecConfigError"), ]; /// One parsed invocation: its namespace, injected handlers and error colour. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs index 37eceaccbb..d12c2955c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs @@ -9,8 +9,9 @@ use sqlbuild_cli::argv::models::NamespaceDefault; use crate::bindings::_helpers::compile_driver::python_calls::imported; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const COMPILE_LINEAGE_MODULE: &str = "sqlbuild._native"; -const COLUMN_LINEAGE_MODULE: &str = "sqlbuild.compiler.lineage.types"; /// Typed namespace for all CLI arguments across all commands. #[pyclass(module = "sqlbuild._native", dict, frozen)] @@ -28,7 +29,7 @@ fn default_object(py: Python<'_>, default: NamespaceDefault) -> PyResult { - imported(py, COLUMN_LINEAGE_MODULE, "ColumnLineageMode")?.getattr("RICH") + imported(py, NATIVE_RECORDS_MODULE, "ColumnLineageMode")?.getattr("RICH") } } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs index 44a51c59a8..f90097dd99 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -11,6 +11,7 @@ use pyo3::types::{PyBool, PyString, PyTuple, PyTupleMethods}; use pyo3::{IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_cli::dispatch::main::event_export_warning::event_export_warning; use sqlbuild_cli::dispatch::models::{ExportSummary, ExporterLoss}; +use sqlbuild_core::event_export::constants::DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS; use crate::bindings::_helpers::cli::launch_state::withhold_launch_record; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; @@ -21,11 +22,6 @@ const EVENT_DISPATCHER_MODULE: &str = "sqlbuild.runtime.observability.classes.ev const DISPATCHER_SCOPE_MODULE: &str = "sqlbuild.runtime.observability.main.dispatcher_scope"; const IDENTITY_MODULE: &str = "sqlbuild.runtime.observability.main.current_execution_identity"; const EXTENSIONS_MODULE: &str = "sqlbuild.compiler.discovery.main.runtime_extensions"; -const EXPORTER_DISPATCHER_MODULE: &str = "sqlbuild.runtime.event_exporting.classes.dispatcher"; -const EXPORTER_SCOPE_CLASS_MODULE: &str = "sqlbuild.runtime.event_exporting.classes.command_scope"; -const EXPORTER_SCOPE_MODULE: &str = - "sqlbuild.runtime.event_exporting.main._event_exporter_command_scope"; -const EXPORTING_CONSTANTS_MODULE: &str = "sqlbuild.runtime.event_exporting.constants"; const OUTPUT_CAPTURE_MODULE: &str = "sqlbuild.runtime.output_capture.main.configured_output_capture_scope"; const DEBUG_EVENT_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; @@ -343,17 +339,14 @@ impl Observed { withhold_launch_record(py)?; let timeout: Bound<'_, PyAny> = extensions.getattr("lifecycle_shutdown_timeout_seconds")?; - let timeout: Bound<'_, PyAny> = if timeout.is_none() { - imported( - py, - EXPORTING_CONSTANTS_MODULE, - "DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS", - )? + let timeout: f64 = if timeout.is_none() { + DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS } else { - timeout.extract::()?.into_bound_py_any(py)? + timeout.extract::()? }; + let timeout: Bound<'_, PyAny> = timeout.into_bound_py_any(py)?; let delivery: Bound<'_, PyAny> = call_with( - &imported(py, EXPORTER_DISPATCHER_MODULE, "EventExporterDispatcher")?, + &imported(py, NATIVE_MODULE, "EventExporterDispatcher")?, &[ ("shutdown_timeout_seconds", timeout), ( @@ -367,7 +360,7 @@ impl Observed { ], )?; let scope: Bound<'_, PyAny> = call_with( - &imported(py, EXPORTER_SCOPE_CLASS_MODULE, "EventExporterCommandScope")?, + &imported(py, NATIVE_MODULE, "EventExporterCommandScope")?, &[("dispatcher", delivery.clone())], )?; self.exporter_scope = Some(scope.clone().unbind()); @@ -400,10 +393,7 @@ impl Observed { else { return Ok(()); }; - self.enter( - imported(py, EXPORTER_SCOPE_MODULE, "event_exporter_command_scope")? - .call1((&scope,))?, - )?; + self.enter(imported(py, NATIVE_MODULE, "event_exporter_command_scope")?.call1((&scope,))?)?; let identity: Bound<'_, PyAny> = imported(py, IDENTITY_MODULE, "current_execution_identity")?.call0()?; if !identity.is_none() && command_output_sinks { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index c09f1aea65..3875d8810a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -9,9 +9,10 @@ use crate::bindings::_helpers::cli::{ use crate::bindings::_helpers::{ audit_command, build_command, build_progress, check, clone, command_runtime, compute_logs, contract_command, cost_command, cost_report, cost_telemetry, dbt_command, dbt_events, - dbt_interop, debug_command, diff_report, execution_history, format_command, freshness, - init_command, janitor, load, output_capture, plan_command, presentation_text, query_command, - result_output, scenario_command, seed_command, sql_test_command, workspace_commands, + dbt_interop, debug_command, diff_report, event_exporting, execution_history, format_command, + freshness, init_command, janitor, load, output_capture, plan_command, presentation_text, + query_command, result_output, scenario_command, seed_command, sql_test_command, + workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -30,6 +31,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { compute_logs::registration::register(module)?; execution_history::registration::register(module)?; output_capture::registration::register(module)?; + event_exporting::registration::register(module)?; load::registration::register(module)?; result_output::registration::register(module)?; cost_report::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index faae3454c8..dc63e2b38a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -27,6 +27,8 @@ use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; @@ -34,7 +36,6 @@ const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; -const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const NO_DESTINATION: (&str, &str) = ( "C406", @@ -254,11 +255,11 @@ fn prepare<'py>( ], )?; let destination_connection: Bound<'py, PyAny> = call_with( - &imported(py, PIPELINE_MODELS, "ClonePipelineConnection")?, + &imported(py, NATIVE_RECORDS_MODULE, "ClonePipelineConnection")?, &[("config", config.clone()), ("handle", connection.clone())], )?; let options: Bound<'py, PyAny> = call_with( - &imported(py, PIPELINE_MODELS, "ClonePipelineOptions")?, + &imported(py, NATIVE_RECORDS_MODULE, "ClonePipelineOptions")?, &[ ("no_sql_validation", request.getattr("no_sql_validation")?), ("select", request.getattr("select")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs index 98bad823cb..ffe3319388 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs @@ -8,9 +8,9 @@ use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const CLONE_PLANNING_MODULE: &str = "sqlbuild._native"; /// The `ClonePipelineResult` fields after the two projects, in `run_clone_planning` order. const PLANNED_FIELDS: [&str; 8] = [ @@ -54,7 +54,7 @@ fn compiled_environment<'py>( ( "analysis_selection", call_with( - &imported(py, COMPILE_MODELS_MODULE, "CompileAnalysisSelection")?, + &imported(py, NATIVE_RECORDS_MODULE, "CompileAnalysisSelection")?, &[("no_cache", options.getattr("no_cache")?)], )?, ), @@ -131,7 +131,7 @@ pub(crate) fn prepared_clone_pipeline<'py>( fields.push((*name, planned.get_item(index)?)); } call_with( - &imported(py, PIPELINE_MODELS_MODULE, "ClonePipelineResult")?, + &imported(py, NATIVE_RECORDS_MODULE, "ClonePipelineResult")?, &fields, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs index 441cc01201..1d8619cffc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs @@ -7,7 +7,7 @@ use sqlbuild_config::command_connection::errors::CommandConnectionError; use crate::bindings::_helpers::compile_driver::python_calls::imported; const CLI_ERRORS_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; /// The `CliUserError` or `ProjectConfigError` for `error`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs index 65c0ff2b63..ebd1627a41 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs @@ -6,7 +6,8 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const CURSOR_INPUTS_KEY: &str = "cursor_inputs"; const FILTER_ROLE: &str = "filter"; const WATERMARK_ROLE: &str = "watermark"; @@ -93,7 +94,7 @@ pub(crate) fn resolve_cursor_input_roles<'py>( } } call_with( - &imported(py, COMPILE_MODELS_MODULE, "CursorInputRoles")?, + &imported(py, NATIVE_RECORDS_MODULE, "CursorInputRoles")?, &[ ("filter_inputs", filter.into_any()), ("watermark_inputs", watermark.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs index d3af02c52d..00e47e1ad6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -11,11 +11,12 @@ use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; use crate::bindings::_helpers::scopes::scope_validation::validate_scope_index; use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const SCAN_CACHE_MODULE: &str = "sqlbuild._native"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; fn compile_error(py: Python<'_>, message: Bound<'_, PyAny>) -> PyResult { Ok(PyErr::from_value( @@ -33,7 +34,7 @@ fn scope_build<'py>( let resolver: Bound<'py, PyAny> = declaration_scope_resolver(discovered, &index, (loaded_macros, &lookup))?; call_with( - &imported(py, COMPILE_MODELS_MODULE, "DeclarationScopeBuild")?, + &imported(py, NATIVE_RECORDS_MODULE, "DeclarationScopeBuild")?, &[ ("loaded_macros", loaded_macros.clone()), ("index", index), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs index 2bca63eb99..49da3e115f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs @@ -1,5 +1,6 @@ //! The native config builder, validator and inherited storage policies of one compile. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class_at; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -9,8 +10,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::compiled_project::loop_classes::table_backed; use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; @@ -34,7 +36,7 @@ pub(crate) fn native_model_config_session<'py>( py, ["SqlHookEntry", "NamedSqlHookEntry", "PythonHookEntry"] .iter() - .map(|name| imported(py, DISCOVERY_MODELS_MODULE, name)) + .map(|name| shape_class_at(py, DISCOVERY_MODELS_MODULE, name)) .collect::>>()?, )?; let target_namespace: Bound<'py, PyAny> = if target.is_none() { @@ -67,7 +69,7 @@ pub(crate) fn native_model_config_session<'py>( inherited.set_item(&materialized, storage)?; } call_with( - &imported(py, COMPILE_MODELS_MODULE, "NativeModelConfigSession")?, + &imported(py, NATIVE_RECORDS_MODULE, "NativeModelConfigSession")?, &[ ("path_defaults", project_config.getattr("path_defaults")?), ("builder", builder), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index b312e9695b..4afff18d09 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -16,6 +16,8 @@ use crate::bindings::_helpers::model_config::unicode_environment::unicode_enviro use crate::bindings::_helpers::render::declaration_values::constant_declaration_sql; use crate::bindings::_helpers::render::macro_expansion_result::macro_expansion_classes; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; @@ -132,7 +134,7 @@ fn sql_hook_inputs<'py>( vec![ ( "named_sql_hook_entry", - imported(py, DISCOVERY_MODELS_MODULE, "NamedSqlHookEntry")?, + imported(py, NATIVE_RECORDS_MODULE, "NamedSqlHookEntry")?, ), ( "python_hook_entry", @@ -215,7 +217,7 @@ pub(crate) fn build_model_inputs<'py>( "PROJECT_CONFIG_FILENAME", )?)?; let config_inputs: Bound<'py, PyAny> = call_with( - &imported(py, COMPILE_MODELS_MODULE, "NativeModelConfigInputs")?, + &imported(py, NATIVE_RECORDS_MODULE, "NativeModelConfigInputs")?, &[ ( "project_config", @@ -253,7 +255,7 @@ pub(crate) fn build_model_inputs<'py>( ), ( "visible_declarations", - imported(py, COMPILE_MODELS_MODULE, "VisibleModelDeclarations")?, + imported(py, NATIVE_RECORDS_MODULE, "VisibleModelDeclarations")?, ), ("config_session", session), ("syntax_help", opt_out_help(py, true)?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs index 5bed96ebad..7f8b50d169 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs @@ -8,6 +8,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const USER_MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; @@ -176,7 +178,7 @@ pub(crate) fn build_sql_expansion_context<'py>( ], )?; call_with( - &imported(py, COMPILE_MODELS_MODULE, "SqlExpansionContext")?, + &imported(py, NATIVE_RECORDS_MODULE, "SqlExpansionContext")?, &[ ("effective_vars", effective_vars), ("loaded_macros", scope.getattr("loaded_macros")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index 2a39b14a90..fcbb4a85bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -22,6 +22,8 @@ use crate::bindings::_helpers::discovery::constants::SQL_HOOK_OUTPUT_FIELDS; use crate::bindings::_helpers::discovery::hook_entries::hook_entries; use crate::bindings::_helpers::semantic_shapes::coverage::semantic_coverage; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const METADATA_MODULE: &str = "importlib.metadata"; @@ -761,7 +763,7 @@ fn test_resource<'py>(test: &Bound<'py, PyAny>) -> PyResult> let payload: Bound<'py, PyAny> = test.getattr("payload")?; let expected_models: Bound<'py, PyList> = if payload.is_instance(&imported( py, - COMPILE_MODELS_MODULE, + NATIVE_RECORDS_MODULE, "CompiledModelSqlTestPayload", )?)? { PyList::new( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs index da85adc00a..a9c0a96441 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs @@ -13,12 +13,13 @@ use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::compile_outputs::json_report::{compact_sorted_json, orjson_sorted}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const CACHE_VERSION: i64 = 4; const ALGORITHM_FINGERPRINT: &str = "sql-test-artifact-v4-orjson-identity"; const CACHE_FILE_NAME: &str = "sql-test-artifacts.json"; const MAX_CACHE_BYTES: u64 = 10_000_000; const MAX_CACHE_RECORDS: usize = 100_000; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const METADATA_MODULE: &str = "importlib.metadata"; /// The digest shared by every test artifact identity, and each model's own identity. @@ -217,7 +218,7 @@ fn test_payload<'py>(test: &Bound<'py, PyAny>) -> PyResult> { let test_payload: Bound<'py, PyAny> = test.getattr("payload")?; if test_payload.is_instance(&imported( py, - COMPILE_MODELS_MODULE, + NATIVE_RECORDS_MODULE, "CompiledModelSqlTestPayload", )?)? { let block: Bound<'py, PyAny> = test.getattr("test_block")?; @@ -256,7 +257,7 @@ fn test_payload<'py>(test: &Bound<'py, PyAny>) -> PyResult> { )?; } else if test_payload.is_instance(&imported( py, - COMPILE_MODELS_MODULE, + NATIVE_RECORDS_MODULE, "CompiledDirectLogicSqlTestPayload", )?)? { payload.set_item( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index f972159849..2997c586de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -15,31 +15,30 @@ use crate::bindings::_helpers::rules::early_sql_lint::EarlySqlLint; use crate::bindings::_helpers::rules::results::rules_progress; use crate::bindings::_helpers::rules::rules_config::load_rules_config; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const DISCOVER_MODULE: &str = "sqlbuild._native"; const REJECT_SELECTORS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const SELECTED_GRAPH_MODULE: &str = "sqlbuild._native"; -const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const RICH_LINEAGE: &str = "rich"; const FAST_LINEAGE: &str = "fast"; const RICH_COLUMN_LINEAGE: &str = "RICH"; const FAST_COLUMN_LINEAGE: &str = "FAST"; const CONTRACTS_MODULE: &str = "sqlbuild._native"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.compiler.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const SELECTION_MODULE: &str = "sqlbuild._native"; const RULE_THREADS_MODULE: &str = "sqlbuild._native"; -const RULE_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const CLI_COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const GENERIC_DIALECT: &str = "generic"; const TARGET_DIRECTORY: &str = "target"; /// `ColumnLineageMode.`. fn column_lineage_mode<'py>(py: Python<'py>, member: &str) -> PyResult> { - imported(py, LINEAGE_TYPES_MODULE, "ColumnLineageMode")?.getattr(member) + imported(py, NATIVE_RECORDS_MODULE, "ColumnLineageMode")?.getattr(member) } /// Column lineage in the compile's mode; `none` skips the analyzer. @@ -264,7 +263,7 @@ fn analyzed<'py>( fn empty_rules_result(py: Python<'_>) -> PyResult> { let zero: Bound<'_, PyAny> = 0_i64.into_pyobject(py)?.into_any(); call_with( - &imported(py, RULE_MODELS_MODULE, "RulesRunResult")?, + &imported(py, NATIVE_RECORDS_MODULE, "RulesRunResult")?, &[ ("findings", PyTuple::empty(py).into_any()), ("evaluated_models", zero.clone()), @@ -525,9 +524,9 @@ fn with_rule_diagnostics<'py>( let diagnostic_class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS_MODULE, "CompilerDiagnostic")?; let phase: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "DiagnosticPhase")?.getattr("RULE")?; + imported(py, NATIVE_RECORDS_MODULE, "DiagnosticPhase")?.getattr("RULE")?; let severity: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "DiagnosticSeverity")?.getattr("ERROR")?; + imported(py, NATIVE_RECORDS_MODULE, "DiagnosticSeverity")?.getattr("ERROR")?; let fix_note: Bound<'py, PyAny> = PyString::new(py, RULE_FIX_AVAILABLE_NOTE).into_any(); let mut diagnostics: Vec> = core.iter().collect(); for fault in rules_result.getattr("findings")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs index 4028306fef..051a46aec1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs @@ -14,12 +14,13 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::planner_identity::json_values::{COMPACT, unicode_sorted_dumps}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const ANALYSIS_CACHE_VERSION: i64 = 13; const ANALYSIS_ALGORITHM_FINGERPRINT: &str = "model-sql-analysis-v21-completed-star-shapes"; const LOCAL_QUALNAME_MARKER: &str = ""; const PACKAGE_NAME: &str = "sqlbuild"; const UNKNOWN_VERSION: &str = "unknown"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; /// Sorted; the dialects whose expression types compile checks. const TYPE_CHECKED_DIALECTS: [&str; 4] = ["bigquery", "duckdb", "postgresql", "snowflake"]; const METADATA_MODULE: &str = "importlib.metadata"; @@ -73,7 +74,7 @@ pub(crate) fn analysis_cache_context<'py>( Err(error) => return Err(error), }; let context = call_with( - &imported(py, COMPILE_MODELS_MODULE, "AnalysisCacheContext")?, + &imported(py, NATIVE_RECORDS_MODULE, "AnalysisCacheContext")?, &[ ("root", root.clone()), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index 6f79a80240..e58b1509b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -13,9 +13,11 @@ use crate::bindings::_helpers::compile_driver::compile_timings::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; -pub(crate) const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +pub(crate) const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; const GENERIC_DIALECT: &str = "generic"; @@ -334,7 +336,7 @@ fn model_analyses<'py>( let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); let names: &Bound<'py, PyAny> = request.analysis_model_names.bind(py); let mode: &Bound<'py, PyAny> = request.column_lineage_mode.bind(py); - let lineage_mode: Bound<'py, PyAny> = imported(py, LINEAGE_TYPES_MODULE, "ColumnLineageMode")?; + let lineage_mode: Bound<'py, PyAny> = imported(py, NATIVE_RECORDS_MODULE, "ColumnLineageMode")?; let rich: bool = mode.eq(lineage_mode.getattr("RICH")?)?; let allow_compact: bool = mode.eq(lineage_mode.getattr("FAST")?)? || rich; let flags: (Bound<'py, PyAny>, Bound<'py, PyAny>) = ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs index 65482d6dc3..fb915fda00 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs @@ -8,6 +8,8 @@ use crate::bindings::_helpers::compile_driver::assembly_project::ADAPTER_NAME_MO use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::render::declaration_values::declaration_scope_resolver; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -37,7 +39,7 @@ pub(crate) fn test_model_query_overrides<'py>( let payload: Bound<'py, PyAny> = test_input.getattr("payload")?; if !payload.is_instance(&imported( py, - COMPILE_MODELS_MODULE, + NATIVE_RECORDS_MODULE, "CompileModelSqlTestInputPayload", )?)? { return Ok(overrides); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs index aa9e48ede4..9a89d802c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs @@ -9,7 +9,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::project_assembly::models::{NativeModelFacts, NativeProjectFacts}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; /// `NativeProjectFacts` of `inputs`: resources, syntax validity, deps and standalone pivot proofs. pub(crate) fn native_project_facts<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index 321999347e..6069d8cbeb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -12,8 +12,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::hand_built::compiled_model::compiled_model as hand_built_model; use crate::bindings::_helpers::render::macro_calls::top_level_macro_call_names; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; @@ -172,7 +173,7 @@ fn compiled_model<'py>( ); } let analysis: Bound<'py, PyAny> = call_with( - &imported(py, COMPILE_MODELS_MODULE, "HandBuiltAnalysis")?, + &imported(py, NATIVE_RECORDS_MODULE, "HandBuiltAnalysis")?, &[ ("session", session_analysis), ("fallback", fallback), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs index 48343b56db..543432df89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs @@ -7,8 +7,9 @@ use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::compile_driver::assembly_checks::test_model_query_overrides; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COLLECTOR_MODULE: &str = "sqlbuild._native"; const DECIMAL_MODULE: &str = "decimal"; @@ -38,7 +39,7 @@ pub(crate) fn assembled_sql_tests<'py>( } let report: Bound<'py, PyAny> = imported(py, COLLECTOR_MODULE, "report_compile_diagnostic")?; let mocked_input: Bound<'py, PyAny> = - imported(py, COMPILE_MODELS_MODULE, "CompileModelSqlTestInputPayload")?; + imported(py, NATIVE_RECORDS_MODULE, "CompileModelSqlTestInputPayload")?; let mut tests: Vec> = Vec::with_capacity(assemblies.len()); for (test_input, assembly) in test_inputs.iter().zip(assemblies) { let failure: Bound<'py, PyAny> = assembly.getattr("failure")?; @@ -100,7 +101,7 @@ fn with_model_queries<'py>( let payload: Bound<'py, PyAny> = test.getattr("payload")?; if !payload.is_instance(&imported( py, - COMPILE_MODELS_MODULE, + NATIVE_RECORDS_MODULE, "CompiledModelSqlTestPayload", )?)? { return Ok(test.clone()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 445f559674..d7ba67d013 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -27,10 +27,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::rules::results::format_skipped_type_proof_rules; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const BYPASS_OUTCOME: &str = "bypass"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; /// What one full compile reports for reuse recording. struct CommandOutcome<'py> { @@ -331,7 +331,7 @@ fn analysis_options<'py>( ) -> PyResult> { let py: Python<'py> = request.py(); let selection: Bound<'py, PyAny> = call_with( - &imported(py, COMPILE_MODELS_MODULE, "CompileAnalysisSelection")?, + &imported(py, NATIVE_RECORDS_MODULE, "CompileAnalysisSelection")?, &[ ("select", request.getattr("select")?), ("exclude", request.getattr("exclude")?), @@ -358,7 +358,7 @@ fn analysis_options<'py>( fn rules_failed(analysis: &Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = analysis.py(); let rule_phase: Bound<'_, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "DiagnosticPhase")?.getattr("RULE")?; + imported(py, NATIVE_RECORDS_MODULE, "DiagnosticPhase")?.getattr("RULE")?; for diagnostic in analysis.getattr("diagnostics")?.try_iter()? { let diagnostic: Bound<'_, PyAny> = diagnostic?; if diagnostic.getattr("is_error")?.is_truthy()? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs index 68fcd2cedf..42557c8924 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs @@ -7,8 +7,8 @@ use sqlbuild_config::project::main::unsupported_target_warehouse::unsupported_ta use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; -const SPEC_ERRORS_MODULE: &str = "sqlbuild.spec.contracts.exceptions"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; +const SPEC_ERRORS_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main._session_warehouse_support"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs index b6d143c779..f3b8062bda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs @@ -13,8 +13,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; -const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PROJECT_VARS_MODULE: &str = "sqlbuild._native"; const PRESERVE_TARGET_VALUE: &str = "preserve"; @@ -321,7 +322,7 @@ fn resolve_model_deferral_inputs<'py>( &[], ), )?; - let inputs: Bound<'py, PyAny> = imported(py, PLANNER_MODELS_MODULE, "DeferralInputs")?; + let inputs: Bound<'py, PyAny> = imported(py, NATIVE_RECORDS_MODULE, "DeferralInputs")?; let defer_to: Bound<'py, PyAny> = arguments.value("defer_to")?; if defer_to.is_none() { return inputs.call0(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index 6048f80a10..273d43ecf8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -7,6 +7,7 @@ use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::config_validation::{ ProjectConfiguration, reject_invalid_project_configuration, }; +use crate::bindings::_helpers::compile_driver::integration_loaders::integration_loader_functions; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, within, }; @@ -27,19 +28,16 @@ use crate::bindings::_helpers::discovery::sql_test_records::{ use crate::bindings::_helpers::discovery::yaml_records::{ discover_schema_files, discover_source_files, }; +use crate::bindings::_helpers::event_exporting::scope::configure_discovered_event_exporters; const FRONTIER_MODULE: &str = "sqlbuild._native"; -const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; +const FRONTIER_TYPES_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const VALIDATION_MODULE: &str = "sqlbuild.compiler.discovery._helpers.validation.discovery"; const NATIVE_MODULE: &str = "sqlbuild._native"; const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; -const EVENT_EXPORTERS_MODULE: &str = - "sqlbuild.runtime.event_exporting.main._configure_discovered_event_exporters"; -const FILESYSTEM_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.core"; +const FILESYSTEM_MODULE: &str = "sqlbuild._native"; const SINKS_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks"; -const INGESTR_LOADERS_MODULE: &str = "sqlbuild.integrations.ingestr.main.loaders"; -const DLT_LOADERS_MODULE: &str = "sqlbuild.integrations.dlt.main.loaders"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; /// Declaration file collections counted into the declaration-parse lifecycle, in field order. @@ -243,12 +241,7 @@ fn assembled_inputs<'py>( ), )?; imported(py, VALIDATION_MODULE, "validate_discovered_inputs")?.call1((&discovered,))?; - imported( - py, - EVENT_EXPORTERS_MODULE, - "configure_discovered_event_exporters", - )? - .call1((&discovered,))?; + configure_discovered_event_exporters(&discovered)?; Ok(discovered) } @@ -442,13 +435,7 @@ fn project_inputs<'py>( .into_any()) }; let source_files: Bound<'py, PyAny> = declarations.get_item("source_files")?; - let integration_loaders: Bound<'py, PyAny> = - imported(py, INGESTR_LOADERS_MODULE, "build_ingestr_loader_functions")? - .call1((&source_files,))? - .add( - imported(py, DLT_LOADERS_MODULE, "build_dlt_loader_functions")? - .call1((&source_files,))?, - )?; + let integration_loaders: Bound<'py, PyAny> = integration_loader_functions(py, &source_files)?; let loader_functions: Bound<'py, PyAny> = tuple_of("loaders")?.add(integration_loaders)?; let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ ("project_config", project_config.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs new file mode 100644 index 0000000000..22dcb11e42 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs @@ -0,0 +1,85 @@ +//! Synthetic loader records for sources declaring an ingestr or dlt integration loader. + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_discovery::yaml_entries::main::integration_loader_name::integration_loader_name; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_TYPES_MODULE: &str = "sqlbuild.compiler.discovery.types"; + +/// One integration: its kind constant and the runtime module building its loader functions. +struct Integration { + constants_module: &'static str, + kind_constant: &'static str, + loaders_module: &'static str, + loader_factory: &'static str, +} + +/// Integrations in loader order: every ingestr loader precedes every dlt loader. +const INTEGRATIONS: [Integration; 2] = [ + Integration { + constants_module: "sqlbuild.integrations.ingestr.constants", + kind_constant: "INGESTR_INTEGRATION_KIND", + loaders_module: "sqlbuild.integrations.ingestr.main._loaders", + loader_factory: "ingestr_loader_function", + }, + Integration { + constants_module: "sqlbuild.integrations.dlt.constants", + kind_constant: "DLT_INTEGRATION_KIND", + loaders_module: "sqlbuild.integrations.dlt.main._loaders", + loader_factory: "dlt_loader_function", + }, +]; + +/// The external-connection `DiscoveredLoaderFunction` of every integration source, in file order. +pub(crate) fn integration_loader_functions<'py>( + py: Python<'py>, + source_files: &Bound<'py, PyAny>, +) -> PyResult> { + let record_class: Bound<'py, PyAny> = + imported(py, DISCOVERY_MODELS_MODULE, "DiscoveredLoaderFunction")?; + let external: Bound<'py, PyAny> = + imported(py, DISCOVERY_TYPES_MODULE, "LoaderConnectionMode")?.getattr("EXTERNAL")?; + let mut loaders: Vec> = Vec::new(); + for integration in &INTEGRATIONS { + let kind: String = + imported(py, integration.constants_module, integration.kind_constant)?.extract()?; + let factory: Bound<'py, PyAny> = + imported(py, integration.loaders_module, integration.loader_factory)?; + for source_file in source_files.try_iter()? { + let source_file: Bound<'py, PyAny> = source_file?; + for source_entry in source_file.getattr("source_entries")?.try_iter()? { + let source_entry: Bound<'py, PyAny> = source_entry?; + let declared: Bound<'py, PyAny> = source_entry.getattr("integration_loader")?; + if declared.is_none() || declared.getattr("kind")?.extract::()? != kind { + continue; + } + let name: String = integration_loader_name( + &kind, + &source_entry.getattr("name")?.extract::()?, + ); + let function: Bound<'py, PyAny> = call_with( + &factory, + &[ + ("source_entry", source_entry), + ("name", name.clone().into_pyobject(py)?.into_any()), + ], + )?; + loaders.push(call_with( + &record_class, + &[ + ("file_path", source_file.getattr("file_path")?), + ("relative_path", source_file.getattr("relative_path")?), + ("name", name.into_pyobject(py)?.into_any()), + ("function", function), + ("connection_mode", external.clone()), + ], + )?); + } + } + } + Ok(PyTuple::new(py, loaders)?.into_any()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index cd7dafa09c..ef8fdc85aa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -16,6 +16,7 @@ pub(crate) mod diff; pub(crate) mod discovery; pub(crate) mod input_reads; pub(crate) mod inputs; +pub(crate) mod integration_loaders; pub(crate) mod phases; pub(crate) mod pipeline; pub(crate) mod pipeline_entries; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index 2a3cf543f1..075ad1d7af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -13,13 +13,14 @@ use crate::bindings::_helpers::compile_driver::python_calls::{Status, call_with, use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const EMPTY_DAG_PATH: &str = ""; const TARGET_DIRECTORY: &str = "target"; const DEFAULT_DAG_FILE: &str = "sqlbuild_dag.json"; @@ -185,11 +186,11 @@ pub(crate) fn sql_test_planning_diagnostics<'py>( &[ ( "phase", - imported(py, COMPILE_TYPES_MODULE, "DiagnosticPhase")?.getattr("TEST")?, + imported(py, NATIVE_RECORDS_MODULE, "DiagnosticPhase")?.getattr("TEST")?, ), ( "severity", - imported(py, COMPILE_TYPES_MODULE, "DiagnosticSeverity")?.getattr("ERROR")?, + imported(py, NATIVE_RECORDS_MODULE, "DiagnosticSeverity")?.getattr("ERROR")?, ), ("code", value.getattr("code")?), ("message", value.str()?.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index a53986c818..063eda4ee2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -18,12 +18,13 @@ use crate::bindings::_helpers::frontier::stage_capture::write_plan_capture; use crate::bindings::_helpers::rules::results::{format_rules_result, rules_progress}; use crate::bindings::_helpers::rules::rules_config::load_rules_config; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild._native"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; -const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const TIMING_TRACKER_MODULE: &str = "sqlbuild._native"; const RULE_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; const RULE_THREADS_MODULE: &str = "sqlbuild._native"; @@ -80,7 +81,7 @@ fn compiled_phase<'py>( let options: Bound<'py, PyAny> = if options.is_truthy()? { options.clone() } else { - imported(py, PIPELINE_MODELS_MODULE, "CompilePipelineOptions")?.call0()? + imported(py, NATIVE_RECORDS_MODULE, "CompilePipelineOptions")?.call0()? }; let configured: Bound<'py, PyAny> = options.getattr("connection_config")?; let connection_config: Bound<'py, PyAny> = if configured.is_none() { @@ -132,7 +133,7 @@ fn compiled_phase<'py>( &format!("Compiled project. ({compile_seconds:.2}s)"), )?; call_with( - &imported(py, PIPELINE_MODELS_MODULE, "CompiledProjectPhaseResult")?, + &imported(py, NATIVE_RECORDS_MODULE, "CompiledProjectPhaseResult")?, &[ ("project", project), ("connection_config", connection_config), @@ -263,7 +264,7 @@ fn run_rules_gate<'py>( load_rules_config(py, project_dir.extract()?)?.into_bound(py); if options.getattr("no_cache")?.is_truthy()? { let cache: Bound<'py, PyAny> = call_with( - &imported(py, RULE_MODELS_MODULE, "RulesCacheConfig")?, + &imported(py, NATIVE_RECORDS_MODULE, "RulesCacheConfig")?, &[("enabled", false.into_pyobject(py)?.to_owned().into_any())], )?; config = imported(py, "dataclasses", "replace")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs index af827ae3a3..2a9f54957d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs @@ -8,7 +8,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -62,7 +63,7 @@ pub(crate) fn compile_analysis_selection<'py>( } } let selection: Bound<'py, PyAny> = - imported(py, COMPILE_MODELS_MODULE, "CompileAnalysisSelection")?; + imported(py, NATIVE_RECORDS_MODULE, "CompileAnalysisSelection")?; let no_cache: Bound<'py, PyAny> = options.getattr("no_cache")?; if (options .getattr("resolve_python_run_selectors")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index f023e6ef3d..7821a065e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -1,5 +1,6 @@ //! Plan a compiled project over an open warehouse connection, and open that connection. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; @@ -11,9 +12,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::graph::python_node_graph::run_selection_from_inputs; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const EXECUTION_PLAN_MODULE: &str = "sqlbuild._native"; -const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const MATERIALIZATIONS_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.materializations"; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const SINGLE_CONNECTION: i64 = 1; @@ -70,7 +72,7 @@ pub(crate) fn pipeline_result<'py>( ], )?; } - let planner_models = py.import(PLANNER_MODELS_MODULE)?; + let planner_models = py.import(NATIVE_RECORDS_MODULE)?; let selection: Bound<'py, PyAny> = call_with( &planner_models.getattr("PlannerSelection")?, &[ @@ -82,7 +84,7 @@ pub(crate) fn pipeline_result<'py>( ], )?; let overrides: Bound<'py, PyAny> = call_with( - &planner_models.getattr("PlannerOverrides")?, + &shape_class(planner_models.as_any(), "PlannerOverrides")?, &option_fields( options, &[ @@ -101,10 +103,12 @@ pub(crate) fn pipeline_result<'py>( options, &["defer_sources_to", "source_deferral_enabled"], )?); - let deferral: Bound<'py, PyAny> = - call_with(&planner_models.getattr("DeferralInputs")?, &deferral_fields)?; + let deferral: Bound<'py, PyAny> = call_with( + &shape_class(planner_models.as_any(), "DeferralInputs")?, + &deferral_fields, + )?; let policies: Bound<'py, PyAny> = call_with( - &planner_models.getattr("PlannerPolicies")?, + &shape_class(planner_models.as_any(), "PlannerPolicies")?, &option_fields( options, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index e4f8b44114..81b9bcee82 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -18,16 +18,16 @@ use crate::bindings::_helpers::compile_driver::targets::{ project_targets, }; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const FRONTIER_MODULE: &str = "sqlbuild._native"; -const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; +const FRONTIER_TYPES_MODULE: &str = "sqlbuild._native"; const BUILD_INPUTS_MODULE: &str = "sqlbuild._native"; const ASSEMBLE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; const PROMOTION_MODULE: &str = "sqlbuild._native"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; -const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const COLLECTION_RENDERING_MODULE: &str = "sqlbuild._native"; @@ -62,7 +62,7 @@ pub(crate) fn lineage_mode<'py>(arguments: &KeywordArguments<'py>) -> PyResult( ], )?; let adapter_context: Bound<'py, PyAny> = call_with( - &imported(py, COMPILE_MODELS_MODULE, "CompileAdapterContext")?, + &imported(py, NATIVE_RECORDS_MODULE, "CompileAdapterContext")?, &[ ("value_renderer", adapter.clone()), ("collection_rendering", collection_rendering), @@ -374,7 +374,7 @@ fn build_project_graph<'py>( (&["discovered_inputs", "adapter"], &optional), )?; let selection: Bound<'py, PyAny> = call_with( - &imported(py, COMPILE_MODELS_MODULE, "CompileAnalysisSelection")?, + &imported(py, NATIVE_RECORDS_MODULE, "CompileAnalysisSelection")?, &[("no_cache", py_bool(py, arguments.flag("no_cache")?))], )?; selected_graph(&arguments, selection) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs index 8491a94905..0fa45a689e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs @@ -14,9 +14,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; -const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; -const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.compiler.python_nodes.models"; const PYTHON_NODE_TYPES_MODULE: &str = "sqlbuild.compiler.python_nodes.types"; const REF_TYPES_MODULE: &str = "sqlbuild.python_nodes.types"; const SKIPPED_UPSTREAM_CODE: &str = "P501"; @@ -123,7 +123,7 @@ fn build_python_run_plan_outputs<'py>( &[], ), )?; - let outputs: Bound<'py, PyAny> = imported(py, PIPELINE_MODELS_MODULE, "PythonRunPlanOutputs")?; + let outputs: Bound<'py, PyAny> = imported(py, NATIVE_RECORDS_MODULE, "PythonRunPlanOutputs")?; let plan_output: Bound<'py, PyAny> = arguments.value("plan_output")?; let run_selection: Bound<'py, PyAny> = arguments.value("run_selection")?; let selected: Bound<'py, PyAny> = arguments.value("selected_python_node_names")?; @@ -156,7 +156,7 @@ fn build_python_run_plan_outputs<'py>( )?), )?; let selection: Bound<'py, PyAny> = call_with( - &imported(py, PYTHON_NODE_MODELS_MODULE, "PythonSqlRunSelection")?, + &imported(py, NATIVE_RECORDS_MODULE, "PythonSqlRunSelection")?, &[ ("sql_keys", plan_output.getattr("selected_keys")?), ("python_node_names", selected.clone()), @@ -217,8 +217,8 @@ fn plan_entries<'py>( previous_identities: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = lifecycle.py(); - let phases: Bound<'py, PyAny> = imported(py, PYTHON_NODE_TYPES_MODULE, "PythonRunPhase")?; - let entry_class: Bound<'py, PyAny> = imported(py, PIPELINE_MODELS_MODULE, "PythonPlanEntry")?; + let phases: Bound<'py, PyAny> = imported(py, NATIVE_RECORDS_MODULE, "PythonRunPhase")?; + let entry_class: Bound<'py, PyAny> = imported(py, NATIVE_RECORDS_MODULE, "PythonPlanEntry")?; let upstream: HashMap> = upstream_names(graph)?; let nodes: Bound<'py, PyAny> = graph.getattr("nodes_by_name")?; let mut entries: Vec> = Vec::new(); @@ -303,11 +303,8 @@ fn identity_status<'py>( identity: &Bound<'py, PyAny>, previous_identities: &Bound<'py, PyAny>, ) -> PyResult> { - let statuses: Bound<'py, PyAny> = imported( - identity.py(), - PYTHON_NODE_TYPES_MODULE, - "PythonIdentityStatus", - )?; + let statuses: Bound<'py, PyAny> = + imported(identity.py(), NATIVE_RECORDS_MODULE, "PythonIdentityStatus")?; if identity.is_none() { return statuses.getattr("UNKNOWN"); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index 11ae284645..9ab6883540 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -7,9 +7,9 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; -const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; -const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const SPLIT_SELECTORS_MODULE: &str = "sqlbuild._native"; const PLANNER_SCOPES_MODULE: &str = "sqlbuild._native"; const STATIC_SCOPE_MODULE: &str = "sqlbuild._native"; @@ -28,7 +28,7 @@ fn unit_test_selectors_applied<'py>( ) -> PyResult> { let py: Python<'py> = discovered.py(); let resolved: Bound<'py, PyAny> = if options.is_none() { - imported(py, PIPELINE_MODELS_MODULE, "CompilePipelineOptions")?.call0()? + imported(py, NATIVE_RECORDS_MODULE, "CompilePipelineOptions")?.call0()? } else { options.clone() }; @@ -77,7 +77,7 @@ fn planner_scopes<'py>( ( "selection", call_with( - &imported(py, PLANNER_MODELS_MODULE, "PlannerSelection")?, + &imported(py, NATIVE_RECORDS_MODULE, "PlannerSelection")?, &[ ("select", selection.getattr("select")?), ("exclude", selection.getattr("exclude")?), @@ -87,7 +87,7 @@ fn planner_scopes<'py>( ( "policies", call_with( - &imported(py, PLANNER_MODELS_MODULE, "PlannerPolicies")?, + &imported(py, NATIVE_RECORDS_MODULE, "PlannerPolicies")?, &[("auto_load_sources", selection.getattr("auto_load_sources")?)], )?, ), @@ -262,7 +262,7 @@ fn compile_static_command_context<'py>( ( "selection", call_with( - &imported(py, PLANNER_MODELS_MODULE, "PlannerSelection")?, + &imported(py, NATIVE_RECORDS_MODULE, "PlannerSelection")?, &[ ("select", options.getattr("select")?), ("exclude", options.getattr("exclude")?), @@ -287,7 +287,7 @@ fn compile_static_command_context<'py>( ( "deferral", call_with( - &imported(py, PLANNER_MODELS_MODULE, "DeferralInputs")?, + &imported(py, NATIVE_RECORDS_MODULE, "DeferralInputs")?, &[ ("deferred_locations", deferred), ("defer_sources_to", options.getattr("defer_sources_to")?), @@ -307,7 +307,7 @@ fn compile_static_command_context<'py>( ], )?; call_with( - &imported(py, PIPELINE_MODELS_MODULE, "StaticCommandContext")?, + &imported(py, NATIVE_RECORDS_MODULE, "StaticCommandContext")?, &[ ("project", project), ("scope", scope), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs new file mode 100644 index 0000000000..f18c81c805 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs @@ -0,0 +1,863 @@ +//! Layouts of the compile and discovery value records hosted natively. + +use crate::bindings::_helpers::records::models::{FieldDefault, FieldSpec, RecordSpec}; + +const R_RELATED_LOCATION: RecordSpec = RecordSpec { + name: "RelatedLocation", + doc: "A secondary authored location that adds context to a diagnostic.", + fields: &[ + FieldSpec::required("label"), + FieldSpec::required("location"), + FieldSpec::new("message", FieldDefault::None), + ], +}; +const R_FIXTURE_COLUMN_INFERENCE: RecordSpec = RecordSpec { + name: "FixtureColumnInference", + doc: "Inferred fixture columns plus outputs that are explicitly bare NULL literals.", + fields: &[ + FieldSpec::required("columns"), + FieldSpec::required("null_literal_names"), + FieldSpec::required("quoted_names"), + ], +}; +const R_DYNAMIC_COLUMN_FAMILY_PROOF: RecordSpec = RecordSpec { + name: "DynamicColumnFamilyProof", + doc: "Compiler evidence for one declared runtime-generated column family.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::new("inferred_type", FieldDefault::None), + ], +}; +const R_DYNAMIC_COLUMN_CONTRACT_PROOF: RecordSpec = RecordSpec { + name: "DynamicColumnContractProof", + doc: "Closed output-shape evidence for a dynamic column contract.", + fields: &[ + FieldSpec::required("output_proven"), + FieldSpec::new("fixed_columns", FieldDefault::EmptyTuple), + FieldSpec::new("families", FieldDefault::EmptyTuple), + FieldSpec::new("input_relations", FieldDefault::EmptyTuple), + FieldSpec::new("failure_reason", FieldDefault::None), + FieldSpec::new("bare_dynamic_pivot", FieldDefault::False), + ], +}; +const R_COMPILE_MODEL_CONFIG: RecordSpec = RecordSpec { + name: "CompileModelConfig", + doc: "Pre-semantic effective config layers attached to a model input.", + fields: &[ + FieldSpec::new("values", FieldDefault::EmptyDict), + FieldSpec::new("model_header_keys", FieldDefault::EmptyTuple), + FieldSpec::new("matched_path_default", FieldDefault::None), + FieldSpec::new("logical_schema", FieldDefault::None), + FieldSpec::new("layer_schema", FieldDefault::None), + FieldSpec::new("logical_database", FieldDefault::None), + FieldSpec::new( + "time_travel_retention", + FieldDefault::Call( + "sqlbuild.spec.contracts.models", + "ResolvedTimeTravelRetention", + ), + ), + FieldSpec::new( + "table_type", + FieldDefault::Call("sqlbuild.spec.contracts.models", "ResolvedTableType"), + ), + ], +}; +const R_CURSOR_INPUT_ROLES: RecordSpec = RecordSpec { + name: "CursorInputRoles", + doc: "Effective filter and watermark inputs for one compiled model.", + fields: &[ + FieldSpec::required("filter_inputs"), + FieldSpec::required("watermark_inputs"), + ], +}; +const R_DECLARATION_RUNTIME_PROJECTION: RecordSpec = RecordSpec { + name: "DeclarationRuntimeProjection", + doc: "Process-only declaration values keyed by canonical static identity.", + fields: &[FieldSpec::new("declarations", FieldDefault::EmptyDict)], +}; +const R_DECLARATION_SCOPE_RESOLVER: RecordSpec = RecordSpec { + name: "DeclarationScopeResolver", + doc: "Process-only lookup state used to project visible runtime declarations.", + fields: &[ + FieldSpec::required("project_dir"), + FieldSpec::required("lookup"), + FieldSpec::required("projection"), + FieldSpec { + name: "native_contexts", + default: FieldDefault::Required, + compared: false, + }, + FieldSpec::new("resource_specific", FieldDefault::None), + FieldSpec { + name: "contexts_by_directory", + default: FieldDefault::EmptyDict, + compared: false, + }, + ], +}; +const R_DECLARATION_SCOPE_BUILD: RecordSpec = RecordSpec { + name: "DeclarationScopeBuild", + doc: "Static artifact and process-local values shared by compile attachment phases.", + fields: &[ + FieldSpec::required("loaded_macros"), + FieldSpec::required("index"), + FieldSpec::required("resolver"), + FieldSpec::required("sql_test_scans"), + ], +}; +const R_MACRO_EXPANSION_RESULT: RecordSpec = RecordSpec { + name: "MacroExpansionResult", + doc: "Expanded SQL and resolved macro usage facts from one authored string.", + fields: &[ + FieldSpec::required("sql"), + FieldSpec::new("spans", FieldDefault::EmptyTuple), + FieldSpec::new("dependencies", FieldDefault::EmptyTuple), + FieldSpec::new("usages", FieldDefault::EmptyTuple), + FieldSpec::new("argument_references", FieldDefault::EmptyTuple), + ], +}; +const R_VISIBLE_MODEL_DECLARATIONS: RecordSpec = RecordSpec { + name: "VisibleModelDeclarations", + doc: "One model's visible declarations and macros, as the native model loop resolves them.", + fields: &[ + FieldSpec::required("local_enums"), + FieldSpec::required("local_constants"), + FieldSpec::required("enums"), + FieldSpec::required("constants"), + FieldSpec::required("inaccessible_enums"), + FieldSpec::required("inaccessible_constants"), + FieldSpec::required("enum_visibility"), + FieldSpec::required("constant_visibility"), + FieldSpec::required("macros"), + FieldSpec::required("macro_records"), + FieldSpec::required("inaccessible_macros"), + ], +}; +const R_DECLARATION_EXPANSION_CONTEXT: RecordSpec = RecordSpec { + name: "DeclarationExpansionContext", + doc: "Declarations and adapter rendering used by one authored SQL owner.", + fields: &[ + FieldSpec::required("declarations"), + FieldSpec::required("value_renderer"), + FieldSpec::required("collection_rendering"), + FieldSpec::new("resolver", FieldDefault::None), + ], +}; +const R_COMPILE_ADAPTER_CONTEXT: RecordSpec = RecordSpec { + name: "CompileAdapterContext", + doc: "Cycle-free adapter behavior required while building compile inputs.", + fields: &[ + FieldSpec::required("value_renderer"), + FieldSpec::required("collection_rendering"), + FieldSpec::required("python_functions_inherit_default_namespace"), + FieldSpec::required("sql_lexical_syntax"), + ], +}; +const R_INVALID_SQL_REFERENCE_CALL: RecordSpec = RecordSpec { + name: "InvalidSqlReferenceCall", + doc: "One reference call whose arguments compile could not replace with a relation.", + fields: &[ + FieldSpec::required("ref_kind"), + FieldSpec::required("call"), + FieldSpec::required("start"), + FieldSpec::required("message"), + FieldSpec::required("help"), + FieldSpec::required("corrected_call"), + ], +}; +const R_SQL_REFERENCE_SCAN: RecordSpec = RecordSpec { + name: "SqlReferenceScan", + doc: "Valid references and rejected reference calls found in one SQL text.", + fields: &[ + FieldSpec::required("references"), + FieldSpec::new("invalid_calls", FieldDefault::EmptyTuple), + ], +}; +const R_COMPILED_LINEAGE_COLUMN_FACT: RecordSpec = RecordSpec { + name: "CompiledLineageColumnFact", + doc: "Compact output column lineage fact extracted during SQL analysis.", + fields: &[ + FieldSpec::required("output_column"), + FieldSpec::new("upstream_columns", FieldDefault::EmptyTuple), + FieldSpec::new( + "transform_kind", + FieldDefault::Member("sqlbuild._native", "ColumnTransformKind", "UNKNOWN"), + ), + FieldSpec::new( + "confidence", + FieldDefault::Member("sqlbuild._native", "ColumnLineageConfidence", "UNKNOWN"), + ), + ], +}; +const R_POLYGLOT_ANALYSIS_RESULT: RecordSpec = RecordSpec { + name: "PolyglotAnalysisResult", + doc: "Outcome of one Polyglot column and lineage analysis pass.", + fields: &[ + FieldSpec::required("analysis_succeeded"), + FieldSpec::new("columns", FieldDefault::None), + FieldSpec::new("lineage_columns", FieldDefault::EmptyTuple), + FieldSpec::new("has_star", FieldDefault::False), + FieldSpec::new("star_resolved", FieldDefault::False), + FieldSpec::new("binding_diagnostics", FieldDefault::EmptyTuple), + FieldSpec::new("binding_validated", FieldDefault::False), + ], +}; +const R_ANALYSIS_CACHE_CONTEXT: RecordSpec = RecordSpec { + name: "AnalysisCacheContext", + doc: "Shared project-local analysis cache identity for one compile invocation.", + fields: &[ + FieldSpec::required("root"), + FieldSpec::required("shared_fingerprint"), + FieldSpec::new("signature_namespace", FieldDefault::Text("default")), + ], +}; +const R_COMPILE_ANALYSIS_SELECTION: RecordSpec = RecordSpec { + name: "CompileAnalysisSelection", + doc: "Deep-analysis selection and invocation-local preparation observer.", + fields: &[ + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("auto_load_sources", FieldDefault::False), + FieldSpec::new("no_cache", FieldDefault::False), + FieldSpec { + name: "on_inputs_ready", + default: FieldDefault::None, + compared: false, + }, + ], +}; +const R_COMPILE_SQL_SCENARIO_CTE: RecordSpec = RecordSpec { + name: "CompileSqlScenarioCte", + doc: "One top-level SQL-native scenario CTE extracted after macro expansion.", + fields: &[FieldSpec::required("name"), FieldSpec::required("sql_body")], +}; +const R_FUNCTION_ARGUMENT: RecordSpec = RecordSpec { + name: "FunctionArgument", + doc: "One SQL function argument with an adapter-native type string.", + fields: &[FieldSpec::required("name"), FieldSpec::required("type")], +}; +const R_FUNCTION_RETURN_COLUMN: RecordSpec = RecordSpec { + name: "FunctionReturnColumn", + doc: "One named table-function return column with an adapter-native type string.", + fields: &[FieldSpec::required("name"), FieldSpec::required("type")], +}; +const R_HAND_BUILT_ANALYSIS: RecordSpec = RecordSpec { + name: "HandBuiltAnalysis", + doc: "A hand-built model input's session position, fallback columns and standalone proof.", + fields: &[ + FieldSpec::required("session"), + FieldSpec::required("fallback"), + FieldSpec::required("standalone_proof"), + ], +}; +const R_MODEL_SQL_ANALYSIS: RecordSpec = RecordSpec { + name: "ModelSqlAnalysis", + doc: "Completed model analysis and its in-memory binding proof.", + fields: &[ + FieldSpec::required("polyglot_analysis"), + FieldSpec::required("placeholders"), + FieldSpec::new("cached", FieldDefault::False), + FieldSpec::new("fused_binding_validated", FieldDefault::False), + FieldSpec::new("cleaned_sql", FieldDefault::None), + FieldSpec::new("validated_schema", FieldDefault::None), + FieldSpec::new("dynamic_column_contract", FieldDefault::None), + ], +}; +const R_COMPILE_SQL_SCENARIO_INPUT: RecordSpec = RecordSpec { + name: "CompileSqlScenarioInput", + doc: "One discovered SQL-native scenario with compile-time SQL expansion applied.", + fields: &[ + FieldSpec::required("scenario_file"), + FieldSpec::required("sql_body"), + FieldSpec::new("authored_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("expected_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("source_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("ref_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("seed_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("dbt_ref_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("expected_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_names", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_target_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("target_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("declaration_usages", FieldDefault::EmptyTuple), + ], +}; +const R_COMPILE_PROJECT_INPUTS: RecordSpec = RecordSpec { + name: "CompileProjectInputs", + doc: "Attached project view used as the first pre-semantic compile input snapshot.", + fields: &[ + FieldSpec::required("project_config"), + FieldSpec::required("local_config"), + FieldSpec::required("discovered_inputs"), + FieldSpec::required("sql_lexical_syntax"), + FieldSpec::new("run_id", FieldDefault::Text("")), + FieldSpec::new("effective_target_name", FieldDefault::None), + FieldSpec::new("effective_target", FieldDefault::None), + FieldSpec::new("compile_cache_dir", FieldDefault::None), + FieldSpec::new("effective_connection", FieldDefault::EmptyDict), + FieldSpec::new( + "effective_settings", + FieldDefault::Call("sqlbuild.spec.contracts.models", "SettingsConfig"), + ), + FieldSpec::new("no_sql_validation", FieldDefault::False), + FieldSpec::new("effective_vars", FieldDefault::EmptyDict), + FieldSpec { + name: "macro_context", + default: FieldDefault::None, + compared: false, + }, + FieldSpec::new("loaded_macros", FieldDefault::EmptyDict), + FieldSpec::new("public_enums", FieldDefault::EmptyDict), + FieldSpec::new("public_constants", FieldDefault::EmptyDict), + FieldSpec::new("model_inputs", FieldDefault::EmptyTuple), + FieldSpec::new("seed_inputs", FieldDefault::EmptyTuple), + FieldSpec::new("source_inputs", FieldDefault::EmptyTuple), + FieldSpec::new("sql_function_inputs", FieldDefault::EmptyTuple), + FieldSpec::new("test_inputs", FieldDefault::EmptyTuple), + FieldSpec::new("scenario_inputs", FieldDefault::EmptyTuple), + FieldSpec::new("audit_inputs", FieldDefault::EmptyTuple), + FieldSpec::new("diagnostics", FieldDefault::EmptyTuple), + FieldSpec::new("external_sql_reference_resolver", FieldDefault::None), + FieldSpec::new( + "scope_index", + FieldDefault::Call("sqlbuild.compiler.scopes.models", "ScopeIndex"), + ), + FieldSpec { + name: "declaration_scope", + default: FieldDefault::None, + compared: false, + }, + FieldSpec { + name: "native_project", + default: FieldDefault::None, + compared: false, + }, + ], +}; +const R_COMPILED_SOURCE: RecordSpec = RecordSpec { + name: "CompiledSource", + doc: "Planner-ready compiled source metadata.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("deps"), + FieldSpec::required("name"), + FieldSpec::required("source_entry"), + FieldSpec::required("source_file"), + ], +}; +const R_COMPILED_SEED: RecordSpec = RecordSpec { + name: "CompiledSeed", + doc: "Planner-ready compiled seed metadata.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("deps"), + FieldSpec::required("name"), + FieldSpec::required("seed_file"), + FieldSpec::required("schema_entry"), + FieldSpec::required("schema_file"), + FieldSpec::required("destination"), + FieldSpec::new("external", FieldDefault::False), + ], +}; +const R_COMPILED_FUNCTION: RecordSpec = RecordSpec { + name: "CompiledFunction", + doc: "Planner-ready compiled SQL function metadata.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("deps"), + FieldSpec::required("name"), + FieldSpec::required("relative_path"), + FieldSpec::required("arguments"), + FieldSpec::required("returns"), + FieldSpec::required("body_sql"), + FieldSpec::required("destination"), + FieldSpec::required("fingerprint_destination"), + FieldSpec::new("return_columns", FieldDefault::EmptyTuple), + FieldSpec::new("references", FieldDefault::EmptyTuple), + FieldSpec::new( + "language", + FieldDefault::Member("sqlbuild.compiler.compile.types", "FunctionLanguage", "SQL"), + ), + FieldSpec::new("source_file_path", FieldDefault::None), + FieldSpec::new("runtime_version", FieldDefault::None), + FieldSpec::new("entry_point", FieldDefault::None), + FieldSpec::new("packages", FieldDefault::EmptyTuple), + FieldSpec::new("tags", FieldDefault::EmptyTuple), + FieldSpec::new("description", FieldDefault::None), + ], +}; +const R_PYTHON_SQL_REFERENCE_REPORT: RecordSpec = RecordSpec { + name: "PythonSqlReferenceReport", + doc: "P008 diagnostics plus literal SQL relations that named no project relation.", + fields: &[ + FieldSpec::new("diagnostics", FieldDefault::EmptyTuple), + FieldSpec::new("unmatched", FieldDefault::EmptyTuple), + ], +}; +const R_COMPILE_SQL_TEST_CTE: RecordSpec = RecordSpec { + name: "CompileSqlTestCte", + doc: "One top-level SQL-native test CTE extracted after macro expansion.", + fields: &[FieldSpec::required("name"), FieldSpec::required("sql_body")], +}; +const R_COMPILE_MODEL_SQL_TEST_INPUT_PAYLOAD: RecordSpec = RecordSpec { + name: "CompileModelSqlTestInputPayload", + doc: "Model-mode SQL test compile payload.", + fields: &[ + FieldSpec::new("authored_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("macro_mocks", FieldDefault::EmptyDict), + FieldSpec::new("mock_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("mock_source_names", FieldDefault::EmptyTuple), + FieldSpec::new("mock_seed_names", FieldDefault::EmptyTuple), + FieldSpec::new("mock_dbt_ref_names", FieldDefault::EmptyTuple), + FieldSpec::new("mock_table_function_names", FieldDefault::EmptyTuple), + FieldSpec::new("expected_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("expected_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_names", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_target_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("read_helper_names", FieldDefault::EmptyTuple), + FieldSpec::new("reference_target_model_names", FieldDefault::EmptyTuple), + ], +}; +const R_COMPILE_DIRECT_LOGIC_SQL_TEST_INPUT_PAYLOAD: RecordSpec = RecordSpec { + name: "CompileDirectLogicSqlTestInputPayload", + doc: "Direct-logic SQL test compile payload.", + fields: &[ + FieldSpec::required("actual_cte"), + FieldSpec::required("expected_cte"), + FieldSpec::required("mode"), + FieldSpec::new("helper_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("tested_resource_names", FieldDefault::EmptyTuple), + ], +}; +const R_COMPILE_SQL_TEST_INPUT: RecordSpec = RecordSpec { + name: "CompileSqlTestInput", + doc: "One discovered SQL-native test block with compile-time SQL expansion applied.", + fields: &[ + FieldSpec::required("test_file"), + FieldSpec::required("test_block"), + FieldSpec::required("sql_body"), + FieldSpec::new( + "mode", + FieldDefault::Member("sqlbuild.compiler.compile.types", "SqlTestMode", "MODEL"), + ), + FieldSpec::new( + "payload", + FieldDefault::Call("sqlbuild._native", "CompileModelSqlTestInputPayload"), + ), + FieldSpec::new("declaration_usages", FieldDefault::EmptyTuple), + FieldSpec::new("parent_name", FieldDefault::None), + FieldSpec::new("case_name", FieldDefault::None), + FieldSpec::new("case_index", FieldDefault::None), + FieldSpec::new("parameter_schema", FieldDefault::EmptyTuple), + FieldSpec::new("parameter_values", FieldDefault::EmptyTuple), + ], +}; +const R_COMPILED_MODEL_SQL_TEST_PAYLOAD: RecordSpec = RecordSpec { + name: "CompiledModelSqlTestPayload", + doc: "Compiled model-mode SQL test payload.", + fields: &[ + FieldSpec::new("authored_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("macro_mocks", FieldDefault::EmptyDict), + FieldSpec::new("model_query_overrides", FieldDefault::EmptyDict), + FieldSpec::new("mock_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("mock_source_names", FieldDefault::EmptyTuple), + FieldSpec::new("mock_seed_names", FieldDefault::EmptyTuple), + FieldSpec::new("mock_dbt_ref_names", FieldDefault::EmptyTuple), + FieldSpec::new("mock_table_function_names", FieldDefault::EmptyTuple), + FieldSpec::new("expected_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("expected_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_names", FieldDefault::EmptyTuple), + ], +}; +const R_SQL_EXPANSION_CONTEXT: RecordSpec = RecordSpec { + name: "SqlExpansionContext", + doc: "Everything needed to expand one project's authored SQL bodies.", + fields: &[ + FieldSpec::required("effective_vars"), + FieldSpec::required("loaded_macros"), + FieldSpec::required("macro_context"), + FieldSpec::required("enums"), + FieldSpec::required("constants"), + FieldSpec::required("value_renderer"), + FieldSpec::required("collection_rendering"), + FieldSpec::new("local_declarations", FieldDefault::EmptyDict), + FieldSpec::new("declaration_resolver", FieldDefault::None), + ], +}; +const R_EXPANSION_SPAN: RecordSpec = RecordSpec { + name: "ExpansionSpan", + doc: "One substituted region, pairing its source range with its rendered range.", + fields: &[ + FieldSpec::required("source_start"), + FieldSpec::required("source_end"), + FieldSpec::required("output_start"), + FieldSpec::required("output_end"), + ], +}; +const R_COMPILED_SQL_EXPANSION: RecordSpec = RecordSpec { + name: "CompiledSqlExpansion", + doc: "Process-local authored-to-expanded SQL evidence shared with compiler checks.", + fields: &[ + FieldSpec::required("authored_sql"), + FieldSpec::required("expanded_sql"), + FieldSpec::required("passes"), + ], +}; +const R_MAPPED_OFFSET: RecordSpec = RecordSpec { + name: "MappedOffset", + doc: "An offset in rendered SQL resolved back onto the text that produced it.", + fields: &[ + FieldSpec::required("offset"), + FieldSpec::required("generated"), + ], +}; +const R_COMPILED_DIRECT_LOGIC_SQL_TEST_PAYLOAD: RecordSpec = RecordSpec { + name: "CompiledDirectLogicSqlTestPayload", + doc: "Compiled direct-logic SQL test payload.", + fields: &[ + FieldSpec::required("actual_cte"), + FieldSpec::required("expected_cte"), + FieldSpec::new( + "mode", + FieldDefault::Member("sqlbuild.compiler.compile.types", "SqlTestMode", "MODEL"), + ), + FieldSpec::new("helper_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("tested_resource_names", FieldDefault::EmptyTuple), + ], +}; +const R_COMPILED_SQL_TEST_RESOURCE: RecordSpec = RecordSpec { + name: "CompiledSqlTestResource", + doc: "One direct SQL test resource with its authored resource kind.", + fields: &[FieldSpec::required("kind"), FieldSpec::required("name")], +}; +const R_NATIVE_MODEL_CONFIG_SESSION: RecordSpec = RecordSpec { + name: "NativeModelConfigSession", + doc: "The native config builder, validator, and inherited storage policies for one compile.", + fields: &[ + FieldSpec::required("path_defaults"), + FieldSpec::required("builder"), + FieldSpec::required("validator"), + FieldSpec::required("inherited_storage"), + ], +}; +const R_NATIVE_MODEL_CONFIG_INPUTS: RecordSpec = RecordSpec { + name: "NativeModelConfigInputs", + doc: "The project and run inputs every model's native config build reads.", + fields: &[ + FieldSpec::required("project_config"), + FieldSpec::required("target_config"), + FieldSpec::required("effective_vars"), + FieldSpec::required("effective_target_name"), + FieldSpec::required("run_id"), + FieldSpec::required("microbatch_concurrency"), + ], +}; +const R_MODEL_VALIDATION_REQUEST: RecordSpec = RecordSpec { + name: "ModelValidationRequest", + doc: "One model's effective config and the facts its validators read.", + fields: &[ + FieldSpec::required("model_file"), + FieldSpec::required("config"), + FieldSpec::required("references"), + FieldSpec::required("declared_columns"), + FieldSpec::required("query_sql"), + ], +}; +const R_NAMED_SQL_HOOK_ENTRY: RecordSpec = RecordSpec { + name: "NamedSqlHookEntry", + doc: "An unresolved invocation of a discovered SQL hook resource.", + fields: &[FieldSpec::required("name"), FieldSpec::required("kwargs")], +}; +const R_DISCOVERED_SQL_FUNCTION_FILE: RecordSpec = RecordSpec { + name: "DiscoveredSqlFunctionFile", + doc: "A discovered SQL function file and its raw contents.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("contents"), + FieldSpec::required("header_values"), + FieldSpec::required("body_sql"), + ], +}; +const R_DISCOVERED_SQL_HOOK_FILE: RecordSpec = RecordSpec { + name: "DiscoveredSqlHookFile", + doc: "A discovered named SQL lifecycle hook resource.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("contents"), + FieldSpec::required("header_values"), + FieldSpec::required("sql_body"), + FieldSpec::required("name"), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new( + "scope_kind", + FieldDefault::Member("sqlbuild.compiler.scopes.types", "ScopeKind", "GLOBAL"), + ), + FieldSpec::new("ownership_root", FieldDefault::None), + FieldSpec::new("owning_path", FieldDefault::None), + FieldSpec::new("declaration_root", FieldDefault::None), + ], +}; +const R_ENUM_MEMBER: RecordSpec = RecordSpec { + name: "EnumMember", + doc: "One named member in a SQLBuild enum declaration.", + fields: &[FieldSpec::required("name"), FieldSpec::required("value")], +}; +const R_DISCOVERED_ENUM_FILE: RecordSpec = RecordSpec { + name: "DiscoveredEnumFile", + doc: "A public SQL file containing one or more enum declarations.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("contents"), + FieldSpec::required("declarations"), + FieldSpec::new( + "scope_kind", + FieldDefault::Member("sqlbuild.compiler.scopes.types", "ScopeKind", "GLOBAL"), + ), + FieldSpec::new("ownership_root", FieldDefault::None), + FieldSpec::new("owning_path", FieldDefault::None), + FieldSpec::new("declaration_root", FieldDefault::None), + ], +}; +const R_DISCOVERED_CONSTANT_FILE: RecordSpec = RecordSpec { + name: "DiscoveredConstantFile", + doc: "A public SQL file containing one or more constant declarations.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("contents"), + FieldSpec::required("declarations"), + FieldSpec::new( + "scope_kind", + FieldDefault::Member("sqlbuild.compiler.scopes.types", "ScopeKind", "GLOBAL"), + ), + FieldSpec::new("ownership_root", FieldDefault::None), + FieldSpec::new("owning_path", FieldDefault::None), + FieldSpec::new("declaration_root", FieldDefault::None), + ], +}; +const R_MODEL_SCHEMA_DECLARATION: RecordSpec = RecordSpec { + name: "ModelSchemaDeclaration", + doc: "One public reusable model column schema declaration.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("columns"), + FieldSpec::required("relative_path"), + FieldSpec::new("extends", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + ], +}; +const R_DISCOVERED_MODEL_SCHEMA_FILE: RecordSpec = RecordSpec { + name: "DiscoveredModelSchemaFile", + doc: "A public SQL file containing reusable model schemas.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("contents"), + FieldSpec::required("declarations"), + FieldSpec::new( + "scope_kind", + FieldDefault::Member("sqlbuild.compiler.scopes.types", "ScopeKind", "GLOBAL"), + ), + FieldSpec::new("ownership_root", FieldDefault::None), + FieldSpec::new("owning_path", FieldDefault::None), + FieldSpec::new("declaration_root", FieldDefault::None), + ], +}; +const R_DISCOVERED_PYTHON_FUNCTION_FILE: RecordSpec = RecordSpec { + name: "DiscoveredPythonFunctionFile", + doc: "A discovered Python function file and its parsed UDF metadata.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("contents"), + FieldSpec::required("header_values"), + FieldSpec::required("entry_point"), + FieldSpec::required("body_python"), + ], +}; +const R_DISCOVERED_SCHEMA_FILE: RecordSpec = RecordSpec { + name: "DiscoveredSchemaFile", + doc: "A discovered schema.yml file and its raw contents.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("contents"), + FieldSpec::required("model_entries"), + FieldSpec::required("seed_entries"), + ], +}; +const R_DISCOVERED_SEED_FILE: RecordSpec = RecordSpec { + name: "DiscoveredSeedFile", + doc: "A discovered seed file.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + ], +}; +const R_DISCOVERED_SQL_TEST_CASE: RecordSpec = RecordSpec { + name: "DiscoveredSqlTestCase", + doc: "One ordered named case belonging to a SQL-native test template.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("values"), + FieldSpec::required("case_index"), + ], +}; +const R_DISCOVERED_SQL_TEST_BLOCK: RecordSpec = RecordSpec { + name: "DiscoveredSqlTestBlock", + doc: "One raw TEST(...) block discovered from a SQL-native test file.", + fields: &[ + FieldSpec::required("test_index"), + FieldSpec::required("header_values"), + FieldSpec::required("sql_body"), + FieldSpec::new("name", FieldDefault::None), + FieldSpec::new( + "mode", + FieldDefault::Member("sqlbuild.compiler.compile.types", "SqlTestMode", "MODEL"), + ), + FieldSpec::new("parameters", FieldDefault::EmptyTuple), + FieldSpec::new("cases", FieldDefault::EmptyTuple), + FieldSpec::new("cursor_start", FieldDefault::None), + FieldSpec::new("cursor_end", FieldDefault::None), + FieldSpec::new("sql_body_span", FieldDefault::None), + ], +}; +const R_DISCOVERED_AUDIT_FILE: RecordSpec = RecordSpec { + name: "DiscoveredAuditFile", + doc: "A discovered audit SQL file and its raw contents.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("contents"), + FieldSpec::required("blocks"), + FieldSpec::new( + "declaration_kind", + FieldDefault::Member("sqlbuild.compiler.scopes.types", "DeclarationKind", "AUDIT"), + ), + FieldSpec::new( + "scope_kind", + FieldDefault::Member("sqlbuild.compiler.scopes.types", "ScopeKind", "GLOBAL"), + ), + FieldSpec::new("ownership_root", FieldDefault::None), + FieldSpec::new("owning_path", FieldDefault::None), + FieldSpec::new("declaration_root", FieldDefault::None), + ], +}; +const R_DISCOVERED_AUDIT_BLOCK: RecordSpec = RecordSpec { + name: "DiscoveredAuditBlock", + doc: "One raw AUDIT(...) block discovered from a SQL audit file.", + fields: &[ + FieldSpec::required("audit_index"), + FieldSpec::required("header_values"), + FieldSpec::required("sql_body"), + FieldSpec::new("name", FieldDefault::None), + FieldSpec::new( + "evaluation_mode", + FieldDefault::Member( + "sqlbuild.compiler.auditing.types", + "AuditEvaluationMode", + "VIOLATIONS", + ), + ), + FieldSpec::new("measurement_contract", FieldDefault::None), + FieldSpec::new("measure_sql", FieldDefault::None), + FieldSpec::new("evidence_sql", FieldDefault::None), + ], +}; +const R_DISCOVERED_ADAPTER_FILE: RecordSpec = RecordSpec { + name: "DiscoveredAdapterFile", + doc: "A detected project adapter Python file.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + ], +}; +const R_DISCOVERY_FILE_FAULT: RecordSpec = RecordSpec { + name: "DiscoveryFileFault", + doc: "One project-relative authored file fault captured during tolerant discovery.", + fields: &[FieldSpec::required("path"), FieldSpec::required("message")], +}; +const R_TOLERANT_SCOPE_DISCOVERY: RecordSpec = RecordSpec { + name: "TolerantScopeDiscovery", + doc: "Parsed scope inputs and independently categorized tolerant discovery faults.", + fields: &[ + FieldSpec::required("discovered_inputs"), + FieldSpec::new("resource_faults", FieldDefault::EmptyTuple), + FieldSpec::new("declaration_faults", FieldDefault::EmptyTuple), + FieldSpec::new("relationship_faults", FieldDefault::EmptyTuple), + FieldSpec::new("config_faults", FieldDefault::EmptyTuple), + ], +}; + +/// Compile and discovery value records. +pub(crate) const COMPILE_VALUE_RECORDS: &[RecordSpec] = &[ + R_RELATED_LOCATION, + R_FIXTURE_COLUMN_INFERENCE, + R_DYNAMIC_COLUMN_FAMILY_PROOF, + R_DYNAMIC_COLUMN_CONTRACT_PROOF, + R_COMPILE_MODEL_CONFIG, + R_CURSOR_INPUT_ROLES, + R_DECLARATION_RUNTIME_PROJECTION, + R_DECLARATION_SCOPE_RESOLVER, + R_DECLARATION_SCOPE_BUILD, + R_MACRO_EXPANSION_RESULT, + R_VISIBLE_MODEL_DECLARATIONS, + R_DECLARATION_EXPANSION_CONTEXT, + R_COMPILE_ADAPTER_CONTEXT, + R_INVALID_SQL_REFERENCE_CALL, + R_SQL_REFERENCE_SCAN, + R_COMPILED_LINEAGE_COLUMN_FACT, + R_POLYGLOT_ANALYSIS_RESULT, + R_ANALYSIS_CACHE_CONTEXT, + R_COMPILE_ANALYSIS_SELECTION, + R_COMPILE_SQL_SCENARIO_CTE, + R_FUNCTION_ARGUMENT, + R_FUNCTION_RETURN_COLUMN, + R_HAND_BUILT_ANALYSIS, + R_MODEL_SQL_ANALYSIS, + R_COMPILE_SQL_SCENARIO_INPUT, + R_COMPILE_PROJECT_INPUTS, + R_COMPILED_SOURCE, + R_COMPILED_SEED, + R_COMPILED_FUNCTION, + R_PYTHON_SQL_REFERENCE_REPORT, + R_COMPILE_SQL_TEST_CTE, + R_COMPILE_MODEL_SQL_TEST_INPUT_PAYLOAD, + R_COMPILE_DIRECT_LOGIC_SQL_TEST_INPUT_PAYLOAD, + R_COMPILE_SQL_TEST_INPUT, + R_COMPILED_MODEL_SQL_TEST_PAYLOAD, + R_SQL_EXPANSION_CONTEXT, + R_EXPANSION_SPAN, + R_COMPILED_SQL_EXPANSION, + R_MAPPED_OFFSET, + R_COMPILED_DIRECT_LOGIC_SQL_TEST_PAYLOAD, + R_COMPILED_SQL_TEST_RESOURCE, + R_NATIVE_MODEL_CONFIG_SESSION, + R_NATIVE_MODEL_CONFIG_INPUTS, + R_MODEL_VALIDATION_REQUEST, + R_NAMED_SQL_HOOK_ENTRY, + R_DISCOVERED_SQL_FUNCTION_FILE, + R_DISCOVERED_SQL_HOOK_FILE, + R_ENUM_MEMBER, + R_DISCOVERED_ENUM_FILE, + R_DISCOVERED_CONSTANT_FILE, + R_MODEL_SCHEMA_DECLARATION, + R_DISCOVERED_MODEL_SCHEMA_FILE, + R_DISCOVERED_PYTHON_FUNCTION_FILE, + R_DISCOVERED_SCHEMA_FILE, + R_DISCOVERED_SEED_FILE, + R_DISCOVERED_SQL_TEST_CASE, + R_DISCOVERED_SQL_TEST_BLOCK, + R_DISCOVERED_AUDIT_FILE, + R_DISCOVERED_AUDIT_BLOCK, + R_DISCOVERED_ADAPTER_FILE, + R_DISCOVERY_FILE_FAULT, + R_TOLERANT_SCOPE_DISCOVERY, +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/mod.rs new file mode 100644 index 0000000000..61d449b063 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/mod.rs @@ -0,0 +1 @@ +pub(crate) mod constants; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs index c2d41b0a42..5f9f2b902b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs @@ -1,5 +1,6 @@ //! Seeds, functions, sources, SQL tests, scenarios and audits attached from discovered inputs. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class_at; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; @@ -24,6 +25,8 @@ use crate::bindings::_helpers::compiled_project::sql_test_loop::{ }; use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_CTE_SCAN_ALGORITHM; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_RECORDS: &str = "sqlbuild._native"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; @@ -31,7 +34,6 @@ const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; const SQL_VALUE_ERRORS_MODULE: &str = "sqlbuild.sql_values.exceptions"; const YAML_FILE_SUFFIXES: [&str; 2] = [".yml", ".yaml"]; @@ -341,7 +343,7 @@ fn sql_test_settings<'py>( ("scenario_input", "CompileSqlScenarioInput"), ("scenario_cte", "CompileSqlScenarioCte"), ] { - settings.set_item(key, imported(py, COMPILE_MODELS_MODULE, class)?)?; + settings.set_item(key, shape_class_at(py, COMPILE_MODELS_MODULE, class)?)?; } settings.set_item("modes", by_value(py, COMPILE_TYPES_MODULE, "SqlTestMode")?)?; settings.set_item( @@ -417,7 +419,7 @@ fn builtin_audit_file<'py>( let relative_path: Bound<'py, PyAny> = path_class.call1((format!("audits/generic/{name}.sql"),))?; let block: Bound<'py, PyAny> = call_with( - &imported(py, DISCOVERY_MODELS_MODULE, "DiscoveredAuditBlock")?, + &imported(py, NATIVE_RECORDS_MODULE, "DiscoveredAuditBlock")?, &[ ("audit_index", 0.into_bound_py_any(py)?), ("header_values", PyDict::new(py).into_any()), @@ -425,7 +427,7 @@ fn builtin_audit_file<'py>( ], )?; call_with( - &imported(py, DISCOVERY_MODELS_MODULE, "DiscoveredAuditFile")?, + &imported(py, NATIVE_RECORDS_MODULE, "DiscoveredAuditFile")?, &[ ( "file_path", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/expanded_offsets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/expanded_offsets.rs index b7a8abb148..e626aedc67 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/expanded_offsets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/expanded_offsets.rs @@ -6,7 +6,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::main::map_expanded_offset::map_expanded_offset as mapped; use sqlbuild_analysis::compiled_project::models::SplicedSpan; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; /// `map_expanded_offset(*, offset, passes)`: a `MappedOffset` in the first pass's source. #[pyfunction] @@ -34,7 +34,7 @@ fn map_expanded_offset<'py>( let arguments: Bound<'py, PyDict> = PyDict::new(py); arguments.set_item("offset", offset)?; arguments.set_item("generated", generated)?; - py.import(COMPILE_MODELS_MODULE)? + py.import(NATIVE_RECORDS_MODULE)? .getattr("MappedOffset")? .call((), Some(&arguments)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs index d60a1188ab..0c5ed72558 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs @@ -6,6 +6,8 @@ use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::imported; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; @@ -14,7 +16,7 @@ const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; -const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; @@ -74,7 +76,7 @@ pub(crate) fn native_config_classes<'py>( ("table_backed", PyTuple::new(py, table_backed)?.into_any()), ( "compile_model_config", - class(py, COMPILE_MODELS_MODULE, "CompileModelConfig")?, + class(py, NATIVE_RECORDS_MODULE, "CompileModelConfig")?, ), ( "resolved_retention", @@ -282,7 +284,7 @@ fn input_view_classes<'py>(py: Python<'py>) -> PyResult> { ), ( "lineage_column_fact", - class(py, COMPILE_MODELS_MODULE, "CompiledLineageColumnFact")?, + class(py, NATIVE_RECORDS_MODULE, "CompiledLineageColumnFact")?, ), ( "lineage_source_fact", @@ -290,11 +292,11 @@ fn input_view_classes<'py>(py: Python<'py>) -> PyResult> { ), ( "contract_proof", - class(py, COMPILE_MODELS_MODULE, "DynamicColumnContractProof")?, + class(py, NATIVE_RECORDS_MODULE, "DynamicColumnContractProof")?, ), ( "family_proof", - class(py, COMPILE_MODELS_MODULE, "DynamicColumnFamilyProof")?, + class(py, NATIVE_RECORDS_MODULE, "DynamicColumnFamilyProof")?, ), ]; dict( @@ -314,11 +316,11 @@ fn input_view_classes<'py>(py: Python<'py>) -> PyResult> { ), ( "sql_expansion", - class(py, COMPILE_MODELS_MODULE, "CompiledSqlExpansion")?, + class(py, NATIVE_RECORDS_MODULE, "CompiledSqlExpansion")?, ), ( "expansion_span", - class(py, COMPILE_MODELS_MODULE, "ExpansionSpan")?, + class(py, NATIVE_RECORDS_MODULE, "ExpansionSpan")?, ), ( "audits", @@ -344,11 +346,11 @@ fn input_view_classes<'py>(py: Python<'py>) -> PyResult> { vec![ ( "function_argument", - class(py, COMPILE_MODELS_MODULE, "FunctionArgument")?, + class(py, NATIVE_RECORDS_MODULE, "FunctionArgument")?, ), ( "function_return_column", - class(py, COMPILE_MODELS_MODULE, "FunctionReturnColumn")?, + class(py, NATIVE_RECORDS_MODULE, "FunctionReturnColumn")?, ), ( "function_language", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs index 272d1aecb9..e286a8f9a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs @@ -1,5 +1,6 @@ //! Compiled records and views: projecting native views as records, and recording later stages. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class_at; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, }; @@ -13,6 +14,8 @@ use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelIn use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -36,7 +39,7 @@ pub(crate) fn replace<'py>( /// `value` when it already is a `class` record, else `class(**{field: getattr(value, field)})`. fn record<'py>(value: &Bound<'py, PyAny>, class: &str) -> PyResult> { let py: Python<'py> = value.py(); - let class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS_MODULE, class)?; + let class: Bound<'py, PyAny> = shape_class_at(py, COMPILE_MODELS_MODULE, class)?; if value.is_instance(&class)? { return Ok(value.clone()); } @@ -220,7 +223,7 @@ fn with_changes<'py>( ) -> PyResult> { let py: Python<'py> = value.py(); let changes: Bound<'py, PyDict> = changes.unwrap_or_else(|| PyDict::new(py)); - if value.is_instance(&imported(py, COMPILE_MODELS_MODULE, class)?)? { + if value.is_instance(&shape_class_at(py, COMPILE_MODELS_MODULE, class)?)? { return imported(py, DATACLASSES_MODULE, "replace")?.call((value,), Some(&changes)); } value.call_method("with_changes", (), Some(&changes)) @@ -250,7 +253,7 @@ pub(crate) fn compile_inputs_view<'py>( } let keywords: Bound<'py, PyDict> = fields.copy()?; keywords.set_item("native_project", native_project)?; - imported(py, COMPILE_MODELS_MODULE, "CompileProjectInputs")?.call((), Some(&keywords)) + imported(py, NATIVE_RECORDS_MODULE, "CompileProjectInputs")?.call((), Some(&keywords)) } /// A typed SQL value as the plain value native consumers read. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs new file mode 100644 index 0000000000..048b70cc3d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs @@ -0,0 +1,1711 @@ +//! Layouts of the compiler, rule, lint and config value records hosted natively. + +use crate::bindings::_helpers::records::constants::{EXCEPTION_PARENT, VALUE_ERROR_PARENT}; +use crate::bindings::_helpers::records::models::{ + ClassValue, EnumSpec, ErrorSpec, FieldDefault, FieldSpec, RecordSpec, +}; + +const R_NATIVE_MODEL_ANALYSIS_REQUEST: RecordSpec = RecordSpec { + name: "NativeModelAnalysisRequest", + doc: "The Python model analysis's arguments plus the facts dynamic pivot proofs need.", + fields: &[ + FieldSpec::required("known_functions"), + FieldSpec::required("known_types"), + FieldSpec::required("model_inputs"), + FieldSpec::required("column_nullability_by_table"), + FieldSpec::required("column_types_by_table"), + FieldSpec::required("inference_profile"), + FieldSpec::required("allow_compact_analysis"), + FieldSpec::required("rich_type_inference"), + FieldSpec::required("analysis_cache"), + FieldSpec::required("complete_binding_schemas"), + FieldSpec::required("dynamic_families_by_table"), + ], +}; +const R_NATIVE_MODEL_ANALYSES: RecordSpec = RecordSpec { + name: "NativeModelAnalyses", + doc: "Each analysed model's position in the finished session that keeps its outcome.", + fields: &[ + FieldSpec::required("positions"), + FieldSpec::required("succeeded"), + FieldSpec::required("session"), + ], +}; +const R_NATIVE_PIVOT_TABLES: RecordSpec = RecordSpec { + name: "NativePivotTables", + doc: "The relation facts Python's dynamic pivot proofs read, for proofs outside a session.", + fields: &[ + FieldSpec::required("dialect"), + FieldSpec::required("column_types_by_table"), + FieldSpec::required("authoritative_column_types_by_table"), + FieldSpec::required("column_nullability_by_table"), + FieldSpec::required("dynamic_families_by_table"), + ], +}; +const R_NATIVE_COLUMN_QUERY: RecordSpec = RecordSpec { + name: "NativeColumnQuery", + doc: "One query outside the session whose output columns Python's analysis inferred.", + fields: &[ + FieldSpec::required("sql"), + FieldSpec::required("mode"), + FieldSpec::new("placeholders", FieldDefault::None), + FieldSpec::new("references", FieldDefault::EmptyTuple), + FieldSpec::new("recover_cte_facts", FieldDefault::True), + ], +}; +const R_NATIVE_QUERY_COLUMNS: RecordSpec = RecordSpec { + name: "NativeQueryColumns", + doc: "One query's analysis: whether it succeeded, its columns and whether it projects a star.", + fields: &[ + FieldSpec::required("succeeded"), + FieldSpec::required("columns"), + FieldSpec::required("has_star"), + ], +}; +const R_CONTRACT_FINDING: RecordSpec = RecordSpec { + name: "ContractFinding", + doc: "", + fields: &[ + FieldSpec::required("resource_type"), + FieldSpec::required("resource_name"), + FieldSpec::required("kind"), + FieldSpec::required("message"), + FieldSpec::new("column_name", FieldDefault::None), + FieldSpec::new("declared_type", FieldDefault::None), + FieldSpec::new("physical_type", FieldDefault::None), + ], +}; +const R_CONTRACT_EVIDENCE: RecordSpec = RecordSpec { + name: "ContractEvidence", + doc: "", + fields: &[ + FieldSpec::required("resource_type"), + FieldSpec::required("resource_name"), + FieldSpec::required("database"), + FieldSpec::required("schema"), + FieldSpec::required("relation"), + FieldSpec::required("declared_columns"), + FieldSpec::required("physical_columns"), + FieldSpec::required("source_path"), + FieldSpec::new("dynamic_columns", FieldDefault::EmptyTuple), + FieldSpec::new("findings", FieldDefault::EmptyTuple), + ], +}; +const R_CONTRACT_ADOPTION_RESULT: RecordSpec = RecordSpec { + name: "ContractAdoptionResult", + doc: "", + fields: &[ + FieldSpec::required("from_target"), + FieldSpec::required("evidence"), + FieldSpec::new("written_paths", FieldDefault::EmptyTuple), + ], +}; +const R_CONTRACT_VALIDATION_RESULT: RecordSpec = RecordSpec { + name: "ContractValidationResult", + doc: "Diagnostics emitted by model contract validation.", + fields: &[FieldSpec::new("diagnostics", FieldDefault::EmptyTuple)], +}; +const R_LINEAGE_GRAPH_VIEWS: RecordSpec = RecordSpec { + name: "LineageGraphViews", + doc: "A native project graph's lineage edges and selector indexes as Python dicts.", + fields: &[ + FieldSpec::required("upstream_deps"), + FieldSpec::required("downstream_deps"), + FieldSpec::required("tag_index"), + FieldSpec::required("path_index"), + FieldSpec::required("all_keys"), + ], +}; +const R_ATTACHED_AUDIT_GATE_EDGE: RecordSpec = RecordSpec { + name: "AttachedAuditGateEdge", + doc: "One ordering edge: the gated node waits for a resource its attached audit reads.", + fields: &[ + FieldSpec::required("audit_name"), + FieldSpec::required("target"), + FieldSpec::required("gated"), + FieldSpec::required("read"), + ], +}; +const R_HOOK_READ_EDGE: RecordSpec = RecordSpec { + name: "HookReadEdge", + doc: "One ordering edge: a model waits for a resource one of its hooks reads.", + fields: &[ + FieldSpec::required("hook_name"), + FieldSpec::required("gated"), + FieldSpec::required("read"), + FieldSpec::new( + "hook_type", + FieldDefault::Member("sqlbuild._native", "HookReadType", "PYTHON"), + ), + FieldSpec::new("hook_path", FieldDefault::None), + ], +}; +const R_QUALIFIED_LINEAGE_COLUMN: RecordSpec = RecordSpec { + name: "QualifiedLineageColumn", + doc: "A resource-qualified column in the collapsed SQLBuild lineage graph.", + fields: &[ + FieldSpec::required("resource_type"), + FieldSpec::required("resource_name"), + FieldSpec::required("column_name"), + ], +}; +const R_COLUMN_LINEAGE_SOURCE: RecordSpec = RecordSpec { + name: "ColumnLineageSource", + doc: "One upstream column dependency for an output column.", + fields: &[ + FieldSpec::required("resource_type"), + FieldSpec::required("resource_name"), + FieldSpec::required("column_name"), + ], +}; +const R_COLUMN_LINEAGE_EDGE: RecordSpec = RecordSpec { + name: "ColumnLineageEdge", + doc: "One collapsed SQLBuild lineage graph edge.", + fields: &[ + FieldSpec::required("source"), + FieldSpec::required("target"), + FieldSpec::new( + "transform_kind", + FieldDefault::Member("sqlbuild._native", "ColumnTransformKind", "UNKNOWN"), + ), + FieldSpec::new( + "confidence", + FieldDefault::Member("sqlbuild._native", "ColumnLineageConfidence", "UNKNOWN"), + ), + ], +}; +const R_DIRECT_SEMANTIC_COLUMN_USE: RecordSpec = RecordSpec { + name: "DirectSemanticColumnUse", + doc: "One direct non-projection use of a resolved resource column.", + fields: &[ + FieldSpec::required("consumer_model"), + FieldSpec::required("context"), + FieldSpec::required("expression_sql"), + FieldSpec::required("source"), + FieldSpec::new( + "confidence", + FieldDefault::Member("sqlbuild._native", "ColumnLineageConfidence", "UNKNOWN"), + ), + FieldSpec::new("line", FieldDefault::None), + FieldSpec::new("column", FieldDefault::None), + ], +}; +const R_COLUMN_LINEAGE: RecordSpec = RecordSpec { + name: "ColumnLineage", + doc: "Lineage for one output column in one model.", + fields: &[ + FieldSpec::required("output_column"), + FieldSpec::required("transform_kind"), + FieldSpec::required("expression_sql"), + FieldSpec::required("upstream_columns"), + FieldSpec::new( + "nullability", + FieldDefault::Member( + "sqlbuild.compiler.lineage.types", + "InferredNullability", + "UNKNOWN", + ), + ), + FieldSpec::new( + "confidence", + FieldDefault::Member("sqlbuild._native", "ColumnLineageConfidence", "UNKNOWN"), + ), + ], +}; +const R_MODEL_COLUMN_LINEAGE: RecordSpec = RecordSpec { + name: "ModelColumnLineage", + doc: "Column lineage extracted for one compiled model.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("columns"), + FieldSpec::new("has_star", FieldDefault::False), + ], +}; +const R_REFACTOR_PROJECT: RecordSpec = RecordSpec { + name: "RefactorProject", + doc: "Compiler facts of one project, as the refactoring planner reads them.", + fields: &[ + FieldSpec::required("project_dir"), + FieldSpec::required("graph"), + FieldSpec::required("discovered"), + ], +}; +const R_MANUAL_LOCATION: RecordSpec = RecordSpec { + name: "ManualLocation", + doc: "A location a refactoring cannot rewrite safely.", + fields: &[ + FieldSpec::required("path"), + FieldSpec::required("line"), + FieldSpec::required("column"), + FieldSpec::required("reason"), + ], +}; +const R_NATIVE_SQL_TEST_PLANNING_REQUEST: RecordSpec = RecordSpec { + name: "NativeSqlTestPlanningRequest", + doc: "Compiled models and tests to plan, with the adapter facts native planning cannot derive.", + fields: &[ + FieldSpec::required("models"), + FieldSpec::required("rendered_model_sql"), + FieldSpec::required("functions"), + FieldSpec::required("tests"), + FieldSpec::required("rendered_test_overrides"), + FieldSpec::required("sql_analysis_enabled"), + FieldSpec::required("sql_analysis_dialect"), + FieldSpec::required("set_difference_operator"), + FieldSpec::required("requires_derived_table_aliases"), + FieldSpec::required("lexical_syntax"), + FieldSpec::required("render_sql"), + FieldSpec::required("include_plan"), + ], +}; +const R_NATIVE_SQL_TEST_CHAIN_REQUEST: RecordSpec = RecordSpec { + name: "NativeSqlTestChainRequest", + doc: "Compiled models and the tests whose unmocked model chains to resolve.", + fields: &[ + FieldSpec::required("models"), + FieldSpec::required("tests"), + FieldSpec::required("lexical_syntax"), + ], +}; +const R_NATIVE_SQL_TEST_FAILURE: RecordSpec = RecordSpec { + name: "NativeSqlTestFailure", + doc: "The error one test's assembly raises: its kind and message.", + fields: &[FieldSpec::required("kind"), FieldSpec::required("message")], +}; +const R_NATIVE_SQL_TEST_ASSEMBLY: RecordSpec = RecordSpec { + name: "NativeSqlTestAssembly", + doc: "One natively assembled test, its keyed diagnostics, and the error its assembly raises.", + fields: &[ + FieldSpec::required("test"), + FieldSpec::required("diagnostics"), + FieldSpec::new("failure", FieldDefault::None), + ], +}; +const R_HEADER_SPAN: RecordSpec = RecordSpec { + name: "HeaderSpan", + doc: "A located DSL header region inside one file.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("start"), + FieldSpec::required("end"), + ], +}; +const R_LINT_FILE_ROLE: RecordSpec = RecordSpec { + name: "LintFileRole", + doc: "Path-derived classification of one lint input, derived once per file.", + fields: &[ + FieldSpec::new("declaration_kind", FieldDefault::None), + FieldSpec::new("in_project", FieldDefault::False), + FieldSpec::new("in_hook_directory", FieldDefault::False), + ], +}; +const R_INTERPOLATION_SITE: RecordSpec = RecordSpec { + name: "InterpolationSite", + doc: "One sqlbuild interpolation occurrence replaced by a unique sentinel.", + fields: &[ + FieldSpec::required("sentinel"), + FieldSpec::required("neutralized_start"), + FieldSpec::required("neutralized_end"), + FieldSpec::required("original_start"), + FieldSpec::required("original_end"), + FieldSpec::required("original_text"), + ], +}; +const R_LINT_BODY: RecordSpec = RecordSpec { + name: "LintBody", + doc: "One authored SQL body prepared for linting, with its expansion spans.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("body_start"), + FieldSpec::required("body_end"), + FieldSpec::required("lint_text"), + FieldSpec::required("passes"), + FieldSpec::new("external_identifiers", FieldDefault::EmptyTuple), + FieldSpec::new("dependency_identifiers", FieldDefault::EmptyTuple), + FieldSpec::new("externally_referenced_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("allows_ceremonial_select", FieldDefault::False), + FieldSpec::new("allows_dynamic_output_star", FieldDefault::False), + FieldSpec::new("allows_empty_fixture_star", FieldDefault::False), + FieldSpec::new("dependency_relations", FieldDefault::EmptyTuple), + ], +}; +const R_PENDING_LINT_BODY: RecordSpec = RecordSpec { + name: "PendingLintBody", + doc: "One expanded authored body awaiting its native lexical preparation.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("body_start"), + FieldSpec::required("body_end"), + FieldSpec::required("expansion_input"), + FieldSpec::required("expanded"), + FieldSpec::required("expansion_passes"), + FieldSpec::required("pre_expansion_sites"), + FieldSpec::required("external_identifiers"), + FieldSpec::required("allows_ceremonial_select"), + FieldSpec::required("allows_dynamic_output_star"), + FieldSpec::required("allows_empty_fixture_star"), + FieldSpec::required("request"), + ], +}; +const R_EXPANDED_LINT_FILE: RecordSpec = RecordSpec { + name: "ExpandedLintFile", + doc: "The expanded bodies of one file, up to the first body that could not be expanded.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("bodies"), + FieldSpec::required("failure"), + ], +}; +const R_PREPARED_LINT_FILES: RecordSpec = RecordSpec { + name: "PreparedLintFiles", + doc: "Lint bodies of every expandable file, and why the skipped files could not expand.", + fields: &[ + FieldSpec::required("bodies"), + FieldSpec::required("unexpandable"), + ], +}; +const R_LINT_EDIT: RecordSpec = RecordSpec { + name: "LintEdit", + doc: "One deterministic authored-source edit proposed by a lint rule.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("code"), + FieldSpec::required("start"), + FieldSpec::required("end"), + FieldSpec::required("replacement"), + ], +}; +const R_LINT_VIOLATION: RecordSpec = RecordSpec { + name: "LintViolation", + doc: "One lint diagnostic reported against an authored file.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("line"), + FieldSpec::required("column"), + FieldSpec::required("code"), + FieldSpec::required("message"), + FieldSpec::required("severity"), + FieldSpec::required("engine"), + FieldSpec::new("end_line", FieldDefault::None), + FieldSpec::new("end_column", FieldDefault::None), + FieldSpec::new("remediation", FieldDefault::None), + FieldSpec::new("fix", FieldDefault::None), + FieldSpec::new("fix_unavailable_reason", FieldDefault::None), + ], +}; +const R_RULE_FIX_RESULT: RecordSpec = RecordSpec { + name: "RuleFixResult", + doc: "One applied, refused, or unavailable semantic Rule fix.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("code"), + FieldSpec::required("line"), + FieldSpec::required("status"), + FieldSpec::required("reason"), + ], +}; +const R_LINT_CONFIG: RecordSpec = RecordSpec { + name: "LintConfig", + doc: "Resolved lint and format configuration for one run.", + fields: &[ + FieldSpec::new("native_enabled", FieldDefault::True), + FieldSpec::new("max_description_lines", FieldDefault::Int(10)), + FieldSpec::new("line_width", FieldDefault::Int(100)), + FieldSpec::new("dialect", FieldDefault::Text("generic")), + FieldSpec::new("enabled_native_rules", FieldDefault::None), + FieldSpec::new("ignored_native_rules", FieldDefault::EmptyTuple), + FieldSpec::new("header_rules_enabled", FieldDefault::True), + FieldSpec::new("max_literal_length", FieldDefault::Int(100)), + FieldSpec::new("max_ranking_order_by", FieldDefault::Int(5)), + FieldSpec::new("allow_suppressions", FieldDefault::True), + FieldSpec { + name: "relation_keys", + default: FieldDefault::EmptyDict, + compared: false, + }, + ], +}; +const R_LINT_RUN_RESULT: RecordSpec = RecordSpec { + name: "LintRunResult", + doc: "Aggregated outcome of one lint or format run.", + fields: &[ + FieldSpec::required("files_checked"), + FieldSpec::required("violations"), + FieldSpec::required("formatted_files"), + FieldSpec { + name: "source_texts", + default: FieldDefault::EmptyDict, + compared: false, + }, + FieldSpec::new("unexpandable", FieldDefault::EmptyDict), + ], +}; +const R_COMPILE_FACTS: RecordSpec = RecordSpec { + name: "CompileFacts", + doc: "Per-model outputs and per-owner diagnostics of one compilation.", + fields: &[ + FieldSpec::required("models"), + FieldSpec::required("model_paths"), + FieldSpec::required("diagnostics"), + ], +}; +const R_FIX_VERDICT: RecordSpec = RecordSpec { + name: "FixVerdict", + doc: "Edited files that fail verification, and whether any change is unattributable.", + fields: &[ + FieldSpec::required("failing"), + FieldSpec::required("unattributed"), + ], +}; +const R_DISPLAY_OPTIONS: RecordSpec = RecordSpec { + name: "DisplayOptions", + doc: "Options controlling bounded human output.", + fields: &[ + FieldSpec::new("max_entries_per_section", FieldDefault::Int(20)), + FieldSpec::new("overflow_flag", FieldDefault::Text("--verbose")), + ], +}; +const R_COMPILE_PIPELINE_OPTIONS: RecordSpec = RecordSpec { + name: "CompilePipelineOptions", + doc: "Selection, deferral, and planning options for one compile pipeline run.", + fields: &[ + FieldSpec::new("selected_target", FieldDefault::None), + FieldSpec::new("no_sql_validation", FieldDefault::False), + FieldSpec::new("defer_to", FieldDefault::None), + FieldSpec::new("defer_sources_to", FieldDefault::None), + FieldSpec::new("source_deferral_enabled", FieldDefault::True), + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("cursor_overrides", FieldDefault::None), + FieldSpec::new("full_refresh", FieldDefault::False), + FieldSpec::new("auto_load_sources", FieldDefault::False), + FieldSpec::new("reload_sources", FieldDefault::False), + FieldSpec::new("connection_config", FieldDefault::None), + FieldSpec::new("cli_vars", FieldDefault::None), + FieldSpec::new("external_sql_reference_resolver", FieldDefault::None), + FieldSpec::new("resolve_python_run_selectors", FieldDefault::False), + FieldSpec::new("no_cache", FieldDefault::False), + FieldSpec::new("max_microbatches", FieldDefault::None), + FieldSpec::new("selection_diagnostics", FieldDefault::False), + FieldSpec::new("plan_sql_tests", FieldDefault::True), + FieldSpec::new("record_migration_fingerprints", FieldDefault::True), + FieldSpec::new("accepts_unit_test_selectors", FieldDefault::False), + FieldSpec::new( + "sql_test_selection", + FieldDefault::Call("sqlbuild._native", "SqlTestSelection"), + ), + ], +}; +const R_PYTHON_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "PythonPlanEntry", + doc: "Display-ready Python node entry for plan output.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("kind"), + FieldSpec::required("phase"), + FieldSpec::new( + "identity_status", + FieldDefault::Member("sqlbuild._native", "PythonIdentityStatus", "UNKNOWN"), + ), + FieldSpec::new("current_definition_json", FieldDefault::None), + FieldSpec::new("previous_definition_json", FieldDefault::None), + FieldSpec::new("current_metadata_json", FieldDefault::None), + FieldSpec::new("previous_metadata_json", FieldDefault::None), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + ], +}; +const R_PYTHON_RUN_PLAN_OUTPUTS: RecordSpec = RecordSpec { + name: "PythonRunPlanOutputs", + doc: "Plan output and Python entries after python-aware run selection.", + fields: &[ + FieldSpec::required("plan_output"), + FieldSpec::required("python_plan_entries"), + FieldSpec::required("selected_python_node_names"), + ], +}; +const R_COMPILED_PROJECT_PHASE_RESULT: RecordSpec = RecordSpec { + name: "CompiledProjectPhaseResult", + doc: "Canonical project compilation output before command-specific planning.", + fields: &[ + FieldSpec::required("project"), + FieldSpec::required("connection_config"), + FieldSpec::required("compile_seconds"), + ], +}; +const R_STATIC_COMMAND_CONTEXT: RecordSpec = RecordSpec { + name: "StaticCommandContext", + doc: "Shared compile, selection, and relation phases for focused commands.", + fields: &[ + FieldSpec::required("project"), + FieldSpec::required("scope"), + FieldSpec::required("relations"), + FieldSpec::required("connection_config"), + FieldSpec::required("compile_seconds"), + ], +}; +const R_CLONE_PIPELINE_OPTIONS: RecordSpec = RecordSpec { + name: "ClonePipelineOptions", + doc: "Compilation and selection options for one clone pipeline.", + fields: &[ + FieldSpec::new("no_sql_validation", FieldDefault::False), + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("cli_vars", FieldDefault::None), + FieldSpec::new("no_cache", FieldDefault::False), + ], +}; +const R_CLONE_PIPELINE_CONNECTION: RecordSpec = RecordSpec { + name: "ClonePipelineConnection", + doc: "Resolved destination connection used throughout clone compilation and planning.", + fields: &[FieldSpec::required("config"), FieldSpec::required("handle")], +}; +const R_CLONE_PIPELINE_RESULT: RecordSpec = RecordSpec { + name: "ClonePipelineResult", + doc: "Prepared clone inputs for origin and destination target environments.", + fields: &[ + FieldSpec::required("origin_project"), + FieldSpec::required("destination_project"), + FieldSpec::required("clone_plan"), + FieldSpec::new("destination_source_entries", FieldDefault::EmptyTuple), + FieldSpec::new("destination_model_entries", FieldDefault::EmptyTuple), + FieldSpec::new("destination_seed_entries", FieldDefault::EmptyTuple), + FieldSpec::new("destination_function_entries", FieldDefault::EmptyTuple), + FieldSpec::new("origin_model_entries", FieldDefault::EmptyTuple), + FieldSpec::new("origin_seed_entries", FieldDefault::EmptyTuple), + FieldSpec::new("origin_source_entries", FieldDefault::EmptyTuple), + ], +}; +const R_PARSED_SELECTOR: RecordSpec = RecordSpec { + name: "ParsedSelector", + doc: "One parsed selector token before graph resolution.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("value"), + FieldSpec::new("upstream", FieldDefault::False), + FieldSpec::new("downstream", FieldDefault::False), + ], +}; +const R_PATH_SELECTOR: RecordSpec = RecordSpec { + name: "PathSelector", + doc: "A directed path selector between two model names with optional endpoint expansion.", + fields: &[ + FieldSpec::required("start_name"), + FieldSpec::required("end_name"), + FieldSpec::new("upstream", FieldDefault::False), + FieldSpec::new("downstream", FieldDefault::False), + ], +}; +const R_SQL_TEST_SELECTION: RecordSpec = RecordSpec { + name: "SqlTestSelection", + doc: "Unit tests selected or excluded by name, beside the tests of the selected resources.", + fields: &[ + FieldSpec::new("names", FieldDefault::EmptyFrozenset), + FieldSpec::new("excluded_names", FieldDefault::EmptyFrozenset), + FieldSpec::new("models_selected", FieldDefault::True), + ], +}; +const R_PLANNER_SCOPE: RecordSpec = RecordSpec { + name: "PlannerScope", + doc: "Resolved graph scope for one planner invocation.", + fields: &[ + FieldSpec::required("upstream_deps"), + FieldSpec::required("downstream_deps"), + FieldSpec::required("all_keys"), + FieldSpec::required("models_by_name"), + FieldSpec::required("selected_keys"), + FieldSpec::required("execution_order"), + FieldSpec::new("user_selected_keys", FieldDefault::EmptyFrozenset), + FieldSpec::new("python_read_source_names", FieldDefault::EmptyFrozenset), + FieldSpec::new( + "sql_test_selection", + FieldDefault::Call("sqlbuild._native", "SqlTestSelection"), + ), + ], +}; +const R_PLANNER_RELATIONS_CONTEXT: RecordSpec = RecordSpec { + name: "PlannerRelationsContext", + doc: "Resolved relation and source inputs for plan entry construction.", + fields: &[ + FieldSpec::required("model_locations"), + FieldSpec::required("seed_locations"), + FieldSpec::required("function_locations"), + FieldSpec::required("source_map"), + FieldSpec::required("source_read_map"), + FieldSpec::required("python_source_read_map"), + FieldSpec::required("source_warehouse_columns"), + FieldSpec::required("star_exclude_keyword"), + FieldSpec::new("listed_source_names", FieldDefault::EmptyFrozenset), + ], +}; +const R_SQL_TEST_ASSERTION_STEP: RecordSpec = RecordSpec { + name: "SqlTestAssertionStep", + doc: "One zero-row assertion step in a SQL-native unit test.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("resolved_sql"), + FieldSpec::new("lifted_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("comparison_body_sql", FieldDefault::None), + ], +}; +const R_PLANNER_SELECTION: RecordSpec = RecordSpec { + name: "PlannerSelection", + doc: "User-facing selection inputs for one planner invocation.", + fields: &[ + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("selected_keys", FieldDefault::None), + FieldSpec::new("python_read_source_names", FieldDefault::EmptyFrozenset), + FieldSpec::new( + "sql_test_selection", + FieldDefault::Call("sqlbuild._native", "SqlTestSelection"), + ), + ], +}; +const R_PLANNER_OVERRIDES: RecordSpec = RecordSpec { + name: "PlannerOverrides", + doc: "Explicit user overrides that force or bound planner decisions.", + fields: &[ + FieldSpec::new("cursor_overrides", FieldDefault::None), + FieldSpec::new("full_refresh", FieldDefault::False), + FieldSpec::new("reload_sources", FieldDefault::False), + FieldSpec::new("forced_stale_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("external_blocked_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("max_microbatches", FieldDefault::None), + ], +}; +const R_DEFERRAL_INPUTS: RecordSpec = RecordSpec { + name: "DeferralInputs", + doc: "Deferred relation and source-deferral inputs for planning.", + fields: &[ + FieldSpec::new("deferred_locations", FieldDefault::None), + FieldSpec::new("deferred_relations", FieldDefault::None), + FieldSpec::new("defer_sources_to", FieldDefault::None), + FieldSpec::new("source_deferral_enabled", FieldDefault::True), + ], +}; +const R_PLANNER_POLICIES: RecordSpec = RecordSpec { + name: "PlannerPolicies", + doc: "Behavior policies selected by the caller for one planner invocation.", + fields: &[ + FieldSpec::new("auto_load_sources", FieldDefault::False), + FieldSpec::new("selection_diagnostics", FieldDefault::False), + FieldSpec::new("plan_sql_tests", FieldDefault::True), + FieldSpec::new("record_migration_fingerprints", FieldDefault::True), + ], +}; +const R_DISCOVERED_PYTHON_LOADER_METADATA: RecordSpec = RecordSpec { + name: "DiscoveredPythonLoaderMetadata", + doc: "Loader-specific metadata carried by an internal Python loader node.", + fields: &[ + FieldSpec::new("destination", FieldDefault::None), + FieldSpec::new("write_strategy", FieldDefault::None), + FieldSpec::new("cursor_column", FieldDefault::None), + FieldSpec::new("unique_key", FieldDefault::EmptyTuple), + FieldSpec::new("columns", FieldDefault::EmptyTuple), + FieldSpec::new("contract", FieldDefault::None), + FieldSpec::new( + "connection_mode", + FieldDefault::Member( + "sqlbuild.compiler.discovery.types", + "LoaderConnectionMode", + "SQLBUILD", + ), + ), + ], +}; +const R_DISCOVERED_PYTHON_TASK_METADATA: RecordSpec = RecordSpec { + name: "DiscoveredPythonTaskMetadata", + doc: "Task-specific metadata carried by an internal Python task node.", + fields: &[FieldSpec::new("retry", FieldDefault::None)], +}; +const R_DISCOVERED_PYTHON_ASSET_METADATA: RecordSpec = RecordSpec { + name: "DiscoveredPythonAssetMetadata", + doc: "Asset-specific metadata carried by an internal Python asset node.", + fields: &[ + FieldSpec::new("columns", FieldDefault::EmptyTuple), + FieldSpec::new("column_lineage", FieldDefault::None), + FieldSpec::new("retry", FieldDefault::None), + ], +}; +const R_DISCOVERED_PYTHON_CHECK_METADATA: RecordSpec = RecordSpec { + name: "DiscoveredPythonCheckMetadata", + doc: "Check-specific metadata carried by an internal Python check node.", + fields: &[FieldSpec::new( + "severity", + FieldDefault::Member( + "sqlbuild.python_nodes.types", + "PythonCheckSeverity", + "ERROR", + ), + )], +}; +const R_PYTHON_NODE_DEPENDENCY_EDGE: RecordSpec = RecordSpec { + name: "PythonNodeDependencyEdge", + doc: "Internal dependency edge between two discovered Python nodes.", + fields: &[ + FieldSpec::required("upstream_name"), + FieldSpec::required("downstream_name"), + ], +}; +const R_PYTHON_SQL_RUN_SELECTION: RecordSpec = RecordSpec { + name: "PythonSqlRunSelection", + doc: "Unified run-command selection excluding Python checks.", + fields: &[ + FieldSpec::required("sql_keys"), + FieldSpec::required("python_node_names"), + ], +}; +const R_LITERAL_SQL_RELATION: RecordSpec = RecordSpec { + name: "LiteralSqlRelation", + doc: "A relation named in literal SQL in Python code that matched no project relation.", + fields: &[ + FieldSpec::required("owner_label"), + FieldSpec::required("owner_kind"), + FieldSpec::required("relative_path"), + FieldSpec::required("line"), + FieldSpec::required("method"), + FieldSpec::required("relation"), + ], +}; +const R_DIRECT_SOURCE_FRESHNESS_PLANNING_RESULT: RecordSpec = RecordSpec { + name: "DirectSourceFreshnessPlanningResult", + doc: "Direct planning-time source freshness observations and comparisons.", + fields: &[ + FieldSpec::new("observed_records", FieldDefault::EmptyTuple), + FieldSpec::new("previous_records", FieldDefault::EmptyTuple), + FieldSpec::new("changed_identities", FieldDefault::EmptyFrozenset), + FieldSpec::new("unchanged_identities", FieldDefault::EmptyFrozenset), + FieldSpec::new("unknown_source_names", FieldDefault::EmptyTuple), + FieldSpec::new("unknown_sources", FieldDefault::EmptyDict), + FieldSpec::new("age_statuses", FieldDefault::EmptyDict), + FieldSpec::new("propagation", FieldDefault::None), + ], +}; +const R_SQL_BINDING_DIAGNOSTIC: RecordSpec = RecordSpec { + name: "SqlBindingDiagnostic", + doc: "One native binding error or non-blocking runtime-conversion warning.", + fields: &[ + FieldSpec::required("code"), + FieldSpec::required("message"), + FieldSpec::new("line", FieldDefault::None), + FieldSpec::new("column", FieldDefault::None), + FieldSpec::new("start", FieldDefault::None), + FieldSpec::new("end", FieldDefault::None), + FieldSpec::new("severity", FieldDefault::Text("error")), + ], +}; +const R_SQL_BINDING_RESULT: RecordSpec = RecordSpec { + name: "SqlBindingResult", + doc: "Stable result of schema-aware semantic SQL validation.", + fields: &[FieldSpec::new("diagnostics", FieldDefault::EmptyTuple)], +}; +const R_SQL_SCHEMA_VALIDATION_REQUEST: RecordSpec = RecordSpec { + name: "SqlSchemaValidationRequest", + doc: "One expanded SQL query and its complete relation schemas.", + fields: &[ + FieldSpec::required("sql"), + FieldSpec::required("dialect"), + FieldSpec::required("schema"), + FieldSpec::new("known_functions", FieldDefault::EmptyTuple), + FieldSpec::new("known_types", FieldDefault::EmptyTuple), + FieldSpec::new("quoted_identifiers_ignore_case", FieldDefault::False), + FieldSpec { + name: "catalog", + default: FieldDefault::None, + compared: false, + }, + ], +}; +const R_DBT_CLI_OPTIONS: RecordSpec = RecordSpec { + name: "DbtCliOptions", + doc: "Resolved dbt CLI options shared by dbt commands.", + fields: &[ + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("profiles_dir", FieldDefault::None), + FieldSpec::new("target", FieldDefault::None), + FieldSpec::new("target_path", FieldDefault::None), + FieldSpec::new("vars", FieldDefault::None), + FieldSpec::new("state", FieldDefault::None), + FieldSpec::new("defer", FieldDefault::False), + ], +}; +const R_NORMALIZED_DBT_PROFILE_CONNECTION: RecordSpec = RecordSpec { + name: "NormalizedDbtProfileConnection", + doc: "Rendered dbt profile output normalized for SQLBuild.", + fields: &[ + FieldSpec::required("adapter"), + FieldSpec::required("connection"), + FieldSpec::new("target_schema", FieldDefault::None), + FieldSpec::new("target_database", FieldDefault::None), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + ], +}; +const R_DBT_INIT_PROGRESS_CALLBACKS: RecordSpec = RecordSpec { + name: "DbtInitProgressCallbacks", + doc: "Optional progress hooks for `sqb dbt init` phases.", + fields: &[ + FieldSpec::new("start", FieldDefault::None), + FieldSpec::new("complete", FieldDefault::None), + ], +}; +const R_DBT_INIT_REQUEST: RecordSpec = RecordSpec { + name: "DbtInitRequest", + doc: "Request to initialize a SQLBuild project from a dbt project.", + fields: &[ + FieldSpec::required("cwd"), + FieldSpec::required("dbt_project_dir"), + FieldSpec::required("profiles_dir"), + FieldSpec::required("profile_name"), + FieldSpec::required("target_name"), + FieldSpec::required("sqb_output_dir"), + FieldSpec::new("dry_run", FieldDefault::False), + FieldSpec::new("overwrite", FieldDefault::False), + FieldSpec::new("skip_dbt_debug", FieldDefault::False), + FieldSpec::new( + "progress_callbacks", + FieldDefault::Call("sqlbuild._native", "DbtInitProgressCallbacks"), + ), + ], +}; +const R_DBT_INIT_RESULT: RecordSpec = RecordSpec { + name: "DbtInitResult", + doc: "Result from `sqb dbt init`.", + fields: &[ + FieldSpec::required("output_dir"), + FieldSpec::required("project_file"), + FieldSpec::required("project_name"), + FieldSpec::required("adapter"), + FieldSpec::required("target_name"), + FieldSpec::required("profile_name"), + FieldSpec::required("toml"), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + FieldSpec::new("dry_run", FieldDefault::False), + ], +}; +const R_DBT_COMMAND_RESULT: RecordSpec = RecordSpec { + name: "DbtCommandResult", + doc: "Completed dbt command output.", + fields: &[ + FieldSpec::required("argv"), + FieldSpec::required("returncode"), + FieldSpec::new("stdout", FieldDefault::Text("")), + FieldSpec::new("stderr", FieldDefault::Text("")), + ], +}; +const R_DBT_NODE_EXECUTION_RESULT: RecordSpec = RecordSpec { + name: "DbtNodeExecutionResult", + doc: "One dbt node execution result parsed from JSON logs.", + fields: &[ + FieldSpec::required("unique_id"), + FieldSpec::required("resource_type"), + FieldSpec::required("node_name"), + FieldSpec::required("status"), + FieldSpec::required("index"), + FieldSpec::required("total"), + FieldSpec::required("execution_time"), + FieldSpec::new("materialized", FieldDefault::None), + FieldSpec::new("relation_name", FieldDefault::None), + FieldSpec::new("database", FieldDefault::None), + FieldSpec::new("schema", FieldDefault::None), + FieldSpec::new("node_checksum", FieldDefault::None), + FieldSpec::new("messages", FieldDefault::EmptyTuple), + ], +}; +const R_DBT_LS_RESULT: RecordSpec = RecordSpec { + name: "DbtLsResult", + doc: "Parsed result from one dbt ls invocation.", + fields: &[FieldSpec::required("nodes"), FieldSpec::required("command")], +}; +const R_RULE_GUIDANCE: RecordSpec = RecordSpec { + name: "RuleGuidance", + doc: "Structured authoring guidance shared by diagnostics and generated documentation.", + fields: &[ + FieldSpec::required("good_example"), + FieldSpec::required("anti_tautology"), + FieldSpec::required("mutation_check"), + ], +}; +const R_RULE_EXEMPTION: RecordSpec = RecordSpec { + name: "RuleExemption", + doc: "", + fields: &[ + FieldSpec::required("rule"), + FieldSpec::required("path"), + FieldSpec::required("reason"), + ], +}; +const R_GRAPH_EDGE_EXCLUSION: RecordSpec = RecordSpec { + name: "GraphEdgeExclusion", + doc: "One exact, temporary internal-layer dependency exception.", + fields: &[ + FieldSpec::required("consumer"), + FieldSpec::required("dependency"), + FieldSpec::required("reason"), + ], +}; +const R_RULE_IGNORE: RecordSpec = RecordSpec { + name: "RuleIgnore", + doc: "", + fields: &[ + FieldSpec::required("rules"), + FieldSpec::required("paths"), + FieldSpec::required("reason"), + FieldSpec::new("selectors", FieldDefault::EmptyTuple), + ], +}; +const R_SELECT_STAR_ALLOW: RecordSpec = RecordSpec { + name: "SelectStarAllow", + doc: "", + fields: &[FieldSpec::required("paths"), FieldSpec::required("reason")], +}; +const R_THRESHOLD_OVERRIDE: RecordSpec = RecordSpec { + name: "ThresholdOverride", + doc: "", + fields: &[ + FieldSpec::required("paths"), + FieldSpec::required("thresholds"), + FieldSpec::required("reason"), + ], +}; +const R_RULES_CACHE_CONFIG: RecordSpec = RecordSpec { + name: "RulesCacheConfig", + doc: "", + fields: &[FieldSpec::new("enabled", FieldDefault::True)], +}; +const R_SQL_TEST_RULES_CONFIG: RecordSpec = RecordSpec { + name: "SqlTestRulesConfig", + doc: "", + fields: &[FieldSpec::new( + "pipeline_directory", + FieldDefault::Text("pipelines"), + )], +}; +const R_PREPARED_SQL_LINT_RESULT: RecordSpec = RecordSpec { + name: "PreparedSqlLintResult", + doc: "Reusable expansion context and completed preliminary SQL checks.", + fields: &[ + FieldSpec::required("result"), + FieldSpec::required("context"), + ], +}; +const R_SQL_EXPANSION_REUSE: RecordSpec = RecordSpec { + name: "SqlExpansionReuse", + doc: "Compiler declaration scope for post-assembly SQL checks; assembly still runs its macros.", + fields: &[ + FieldSpec::required("discovered_inputs"), + FieldSpec::required("declaration_scope"), + ], +}; +const R_PREPARED_SQL_LINT: RecordSpec = RecordSpec { + name: "PreparedSqlLint", + doc: "Process-local SQL checks started after compiler expansion has completed.", + fields: &[ + FieldSpec::required("codes"), + FieldSpec::required("future"), + FieldSpec::new("project_dir", FieldDefault::None), + ], +}; +const R_SQL_RULES_EVALUATION: RecordSpec = RecordSpec { + name: "SqlRulesEvaluation", + doc: "SQL rule findings of one run, their cache accounting and elapsed time.", + fields: &[ + FieldSpec::required("findings"), + FieldSpec::required("cache_hits"), + FieldSpec::required("cache_misses"), + FieldSpec::new("sql_ms", FieldDefault::Int(0)), + ], +}; +const R_RULE_RUN_PLAN: RecordSpec = RecordSpec { + name: "RuleRunPlan", + doc: "The catalogue, selected rules, project slice and settings one rules run evaluates.", + fields: &[ + FieldSpec::required("effective_config"), + FieldSpec::required("project_dir"), + FieldSpec::required("catalogue"), + FieldSpec::required("custom_payloads"), + FieldSpec::required("selected"), + FieldSpec::required("skipped_type_proof_rules"), + FieldSpec::required("model_paths"), + FieldSpec::required("project"), + FieldSpec::required("config"), + ], +}; +const R_RULES_RUN_RESULT: RecordSpec = RecordSpec { + name: "RulesRunResult", + doc: "Unified findings and phase measurements for configured rules.", + fields: &[ + FieldSpec::required("findings"), + FieldSpec::required("evaluated_models"), + FieldSpec::required("built_in_ms"), + FieldSpec::required("custom_ms"), + FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), + FieldSpec::new("unevaluated_resources", FieldDefault::Int(0)), + FieldSpec::new("cache_hits", FieldDefault::Int(0)), + FieldSpec::new("cache_misses", FieldDefault::Int(0)), + FieldSpec::new("skipped_type_proof_rules", FieldDefault::EmptyTuple), + ], +}; +const R_NATIVE_RULES_EVALUATION: RecordSpec = RecordSpec { + name: "NativeRulesEvaluation", + doc: "The built-in rules response, decoded natively.", + fields: &[ + FieldSpec::required("findings"), + FieldSpec::required("selected_codes"), + FieldSpec::required("evaluated_models"), + FieldSpec::required("cache_hits"), + FieldSpec::required("cache_misses"), + FieldSpec::required("built_in_ms"), + FieldSpec::required("reused"), + ], +}; +const R_CUSTOM_RULES_OUTCOME: RecordSpec = RecordSpec { + name: "CustomRulesOutcome", + doc: "Custom-rule findings in canonical order and their cache accounting.", + fields: &[ + FieldSpec::required("findings"), + FieldSpec::required("cache_hits"), + FieldSpec::required("cache_misses"), + FieldSpec::required("custom_ms"), + FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), + ], +}; +const E_CONTRACT_FINDING_KIND: EnumSpec = EnumSpec { + name: "ContractFindingKind", + doc: "Stable physical-to-authored contract difference categories.", + members: &[ + ("MISSING_DECLARATION", "missing_declaration"), + ("MISSING_PHYSICAL_COLUMN", "missing_physical_column"), + ("TYPE_MISMATCH", "type_mismatch"), + ("RELATION_MISSING", "relation_missing"), + ("INSPECTION_UNAVAILABLE", "inspection_unavailable"), + ("OWNERSHIP_CONFLICT", "ownership_conflict"), + ], +}; +const E_COMPILER_STAGE: EnumSpec = EnumSpec { + name: "CompilerStage", + doc: "Frontier objects handed from one compiler stage to the next, in pipeline order.", + members: &[ + ("DISCOVERED_PROJECT_INPUTS", "discovered_project_inputs"), + ("COMPILE_PROJECT_INPUTS", "compile_project_inputs"), + ("COMPILED_PROJECT", "compiled_project"), + ], +}; +const E_HOOK_READ_TYPE: EnumSpec = EnumSpec { + name: "HookReadType", + doc: "Kind of model hook that declares or writes a read.", + members: &[ + ("PYTHON", "Python"), + ("SQL", "SQL"), + ("INLINE_SQL", "inline SQL"), + ], +}; +const E_LINT_SEVERITY: EnumSpec = EnumSpec { + name: "LintSeverity", + doc: "Severity assigned to one lint violation.", + members: &[("FAULT", "fault"), ("WARNING", "warning")], +}; +const E_DIAGNOSTIC_PHASE: EnumSpec = EnumSpec { + name: "DiagnosticPhase", + doc: "Phase that produced a compiler diagnostic.", + members: &[ + ("COMPILE", "compile"), + ("CONTRACT", "contract"), + ("RULE", "rule"), + ("PLAN", "plan"), + ("BUILD", "build"), + ("AUDIT", "audit"), + ("TEST", "test"), + ("CONNECTION", "connection"), + ], +}; +const E_DIAGNOSTIC_SEVERITY: EnumSpec = EnumSpec { + name: "DiagnosticSeverity", + doc: "Severity for a compiler diagnostic.", + members: &[("ERROR", "error"), ("WARNING", "warning"), ("INFO", "info")], +}; +const E_COLUMN_TRANSFORM_KIND: EnumSpec = EnumSpec { + name: "ColumnTransformKind", + doc: "High-level transform classification for one output column.", + members: &[ + ("DIRECT", "direct"), + ("CAST", "cast"), + ("EXPRESSION", "expression"), + ("AGGREGATION", "aggregation"), + ("STAR", "star"), + ("CONSTANT", "constant"), + ("UNKNOWN", "unknown"), + ], +}; +const E_COLUMN_LINEAGE_CONFIDENCE: EnumSpec = EnumSpec { + name: "ColumnLineageConfidence", + doc: "Coarse confidence that a lineage edge is fully understood.", + members: &[ + ("HIGH", "high"), + ("MEDIUM", "medium"), + ("LOW", "low"), + ("UNKNOWN", "unknown"), + ], +}; +const E_COLUMN_LINEAGE_MODE: EnumSpec = EnumSpec { + name: "ColumnLineageMode", + doc: "Column lineage analyzer mode.", + members: &[("RICH", "rich"), ("FAST", "fast")], +}; +const E_POLYGLOT_ANALYSIS_DIALECT: EnumSpec = EnumSpec { + name: "PolyglotAnalysisDialect", + doc: "Dialect names accepted by Polyglot analyze_query.", + members: &[ + ("GENERIC", "generic"), + ("POSTGRESQL", "postgresql"), + ("MYSQL", "mysql"), + ("BIGQUERY", "bigquery"), + ("SNOWFLAKE", "snowflake"), + ("DUCKDB", "duckdb"), + ("SQLITE", "sqlite"), + ("HIVE", "hive"), + ("SPARK", "spark"), + ("TRINO", "trino"), + ("PRESTO", "presto"), + ("REDSHIFT", "redshift"), + ("TSQL", "tsql"), + ("ORACLE", "oracle"), + ("CLICKHOUSE", "clickhouse"), + ("DATABRICKS", "databricks"), + ("ATHENA", "athena"), + ("TERADATA", "teradata"), + ("DORIS", "doris"), + ("STARROCKS", "starrocks"), + ("MATERIALIZE", "materialize"), + ("RISINGWAVE", "risingwave"), + ("SINGLESTORE", "singlestore"), + ("COCKROACHDB", "cockroachdb"), + ("TIDB", "tidb"), + ("DRUID", "druid"), + ("SOLR", "solr"), + ("TABLEAU", "tableau"), + ("DUNE", "dune"), + ("FABRIC", "fabric"), + ("DRILL", "drill"), + ("DREMIO", "dremio"), + ("EXASOL", "exasol"), + ("DATAFUSION", "datafusion"), + ], +}; +const E_MIGRATION_DISCOVERY: EnumSpec = EnumSpec { + name: "MigrationDiscovery", + doc: "How a migration was requested.", + members: &[("MANUAL", "manual"), ("AUTOMATIC", "automatic")], +}; +const E_MIGRATION_COMPATIBILITY: EnumSpec = EnumSpec { + name: "MigrationCompatibility", + doc: "Outcome of checking the origin relation against the destination model.", + members: &[ + ("COMPATIBLE", "compatible"), + ("INCOMPATIBLE", "incompatible"), + ("NOT_CHECKED", "not_checked"), + ], +}; +const E_OLD_NAME_VIEW_ACTION: EnumSpec = EnumSpec { + name: "OldNameViewAction", + doc: "What a build does at a migrated model's old name.", + members: &[ + ("ARCHIVE_AND_VIEW", "archive_and_view"), + ("VIEW_ONLY", "view_only"), + ("LIVE", "live"), + ("NONE", "none"), + ], +}; +const E_SELECTOR_KIND: EnumSpec = EnumSpec { + name: "SelectorKind", + doc: "", + members: &[ + ("NAME", "name"), + ("SEED", "seed"), + ("SOURCE", "source"), + ("TASK", "task"), + ("ASSET", "asset"), + ("LOADER", "loader"), + ("CHECK", "check"), + ("TAG", "tag"), + ("PATH", "path"), + ("TEST", "test"), + ], +}; +const E_PYTHON_RUN_PHASE: EnumSpec = EnumSpec { + name: "PythonRunPhase", + doc: "Lifecycle-aware Python execution phase.", + members: &[ + ("PRE_SQL_INGRESS", "pre_sql_ingress"), + ("READ_SIDE", "read_side"), + ], +}; +const E_PYTHON_IDENTITY_STATUS: EnumSpec = EnumSpec { + name: "PythonIdentityStatus", + doc: "Read-only identity diff status for Python plan display.", + members: &[ + ("NEW", "new"), + ("CHANGED", "changed"), + ("UNCHANGED", "unchanged"), + ("UNKNOWN", "unknown"), + ], +}; +const E_COLLECTION_RENDERING: EnumSpec = EnumSpec { + name: "CollectionRendering", + doc: "Physical rendering choices for homogeneous SQL collections.", + members: &[("VALUE_LIST", "value_list"), ("ARRAY", "array")], +}; + +/// Compiler, rule, lint and config value records. +pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ + R_NATIVE_MODEL_ANALYSIS_REQUEST, + R_NATIVE_MODEL_ANALYSES, + R_NATIVE_PIVOT_TABLES, + R_NATIVE_COLUMN_QUERY, + R_NATIVE_QUERY_COLUMNS, + R_CONTRACT_FINDING, + R_CONTRACT_EVIDENCE, + R_CONTRACT_ADOPTION_RESULT, + R_CONTRACT_VALIDATION_RESULT, + R_LINEAGE_GRAPH_VIEWS, + R_ATTACHED_AUDIT_GATE_EDGE, + R_HOOK_READ_EDGE, + R_QUALIFIED_LINEAGE_COLUMN, + R_COLUMN_LINEAGE_SOURCE, + R_COLUMN_LINEAGE_EDGE, + R_DIRECT_SEMANTIC_COLUMN_USE, + R_COLUMN_LINEAGE, + R_MODEL_COLUMN_LINEAGE, + R_REFACTOR_PROJECT, + R_MANUAL_LOCATION, + R_NATIVE_SQL_TEST_PLANNING_REQUEST, + R_NATIVE_SQL_TEST_CHAIN_REQUEST, + R_NATIVE_SQL_TEST_FAILURE, + R_NATIVE_SQL_TEST_ASSEMBLY, + R_HEADER_SPAN, + R_LINT_FILE_ROLE, + R_INTERPOLATION_SITE, + R_LINT_BODY, + R_PENDING_LINT_BODY, + R_EXPANDED_LINT_FILE, + R_PREPARED_LINT_FILES, + R_LINT_EDIT, + R_LINT_VIOLATION, + R_RULE_FIX_RESULT, + R_LINT_CONFIG, + R_LINT_RUN_RESULT, + R_COMPILE_FACTS, + R_FIX_VERDICT, + R_DISPLAY_OPTIONS, + R_COMPILE_PIPELINE_OPTIONS, + R_PYTHON_PLAN_ENTRY, + R_PYTHON_RUN_PLAN_OUTPUTS, + R_COMPILED_PROJECT_PHASE_RESULT, + R_STATIC_COMMAND_CONTEXT, + R_CLONE_PIPELINE_OPTIONS, + R_CLONE_PIPELINE_CONNECTION, + R_CLONE_PIPELINE_RESULT, + R_PARSED_SELECTOR, + R_PATH_SELECTOR, + R_SQL_TEST_SELECTION, + R_PLANNER_SCOPE, + R_PLANNER_RELATIONS_CONTEXT, + R_SQL_TEST_ASSERTION_STEP, + R_PLANNER_SELECTION, + R_PLANNER_OVERRIDES, + R_DEFERRAL_INPUTS, + R_PLANNER_POLICIES, + R_DISCOVERED_PYTHON_LOADER_METADATA, + R_DISCOVERED_PYTHON_TASK_METADATA, + R_DISCOVERED_PYTHON_ASSET_METADATA, + R_DISCOVERED_PYTHON_CHECK_METADATA, + R_PYTHON_NODE_DEPENDENCY_EDGE, + R_PYTHON_SQL_RUN_SELECTION, + R_LITERAL_SQL_RELATION, + R_DIRECT_SOURCE_FRESHNESS_PLANNING_RESULT, + R_SQL_BINDING_DIAGNOSTIC, + R_SQL_BINDING_RESULT, + R_SQL_SCHEMA_VALIDATION_REQUEST, + R_DBT_CLI_OPTIONS, + R_NORMALIZED_DBT_PROFILE_CONNECTION, + R_DBT_INIT_PROGRESS_CALLBACKS, + R_DBT_INIT_REQUEST, + R_DBT_INIT_RESULT, + R_DBT_COMMAND_RESULT, + R_DBT_NODE_EXECUTION_RESULT, + R_DBT_LS_RESULT, + R_RULE_GUIDANCE, + R_RULE_EXEMPTION, + R_GRAPH_EDGE_EXCLUSION, + R_RULE_IGNORE, + R_SELECT_STAR_ALLOW, + R_THRESHOLD_OVERRIDE, + R_RULES_CACHE_CONFIG, + R_SQL_TEST_RULES_CONFIG, + R_PREPARED_SQL_LINT_RESULT, + R_SQL_EXPANSION_REUSE, + R_PREPARED_SQL_LINT, + R_SQL_RULES_EVALUATION, + R_RULE_RUN_PLAN, + R_RULES_RUN_RESULT, + R_NATIVE_RULES_EVALUATION, + R_CUSTOM_RULES_OUTCOME, +]; +/// Compiler, rule, lint and config string enums. +pub(crate) const COMPILER_ENUMS: &[EnumSpec] = &[ + E_CONTRACT_FINDING_KIND, + E_COMPILER_STAGE, + E_HOOK_READ_TYPE, + E_LINT_SEVERITY, + E_DIAGNOSTIC_PHASE, + E_DIAGNOSTIC_SEVERITY, + E_COLUMN_TRANSFORM_KIND, + E_COLUMN_LINEAGE_CONFIDENCE, + E_COLUMN_LINEAGE_MODE, + E_POLYGLOT_ANALYSIS_DIALECT, + E_MIGRATION_DISCOVERY, + E_MIGRATION_COMPATIBILITY, + E_OLD_NAME_VIEW_ACTION, + E_SELECTOR_KIND, + E_PYTHON_RUN_PHASE, + E_PYTHON_IDENTITY_STATUS, + E_COLLECTION_RENDERING, +]; +/// Compiler, rule, lint and config expected-error classes, each after its parent. +pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ + ErrorSpec { + name: "DiscoveryError", + parent: VALUE_ERROR_PARENT, + doc: "Base error for expected discovery failures.", + class_attributes: &[("code", ClassValue::Text("D000"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ProjectConfigError", + parent: "DiscoveryError", + doc: "Raised when shared or local project config files are invalid.", + class_attributes: &[("code", ClassValue::Text("D001"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ModelSqlParseError", + parent: "DiscoveryError", + doc: "Raised when an authored SQL model file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D002"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "SqlTestParseError", + parent: "DiscoveryError", + doc: "Raised when an authored SQL-native test file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D003"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "SqlAuditParseError", + parent: "DiscoveryError", + doc: "Raised when an authored SQL audit file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D004"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "SchemaParseError", + parent: "DiscoveryError", + doc: "Raised when a schema.yml file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D005"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "SourceParseError", + parent: "DiscoveryError", + doc: "Raised when a sources/*.yml file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D006"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "DiscoveryConflictError", + parent: "DiscoveryError", + doc: "Raised when discovered project inputs conflict across files.", + class_attributes: &[("code", ClassValue::Text("D007"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "SeedDiscoveryError", + parent: "DiscoveryError", + doc: "Raised when declared seed metadata does not match local seed files.", + class_attributes: &[("code", ClassValue::Text("D008"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "SqlScenarioParseError", + parent: "DiscoveryError", + doc: "Raised when an authored SQL scenario file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D009"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "PythonNodeDiscoveryError", + parent: "DiscoveryError", + doc: "Raised when project Python nodes cannot be discovered.", + class_attributes: &[("code", ClassValue::Text("D011"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ProviderDiscoveryError", + parent: "DiscoveryError", + doc: "Raised when project providers cannot be discovered.", + class_attributes: &[("code", ClassValue::Text("D012"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "DeclarationParseError", + parent: "DiscoveryError", + doc: "Raised when an authored public SQL declaration is invalid.", + class_attributes: &[("code", ClassValue::Text("D013"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "SqlHookParseError", + parent: "DiscoveryError", + doc: "Raised when an authored SQL hook resource has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D014"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "EventExporterDiscoveryError", + parent: "DiscoveryError", + doc: "Raised when project lifecycle event exporters are invalid.", + class_attributes: &[("code", ClassValue::Text("D015"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ProjectPythonPathError", + parent: "DiscoveryError", + doc: "Raised when project-owned Python lives outside a supported extension root.", + class_attributes: &[("code", ClassValue::Text("D016"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ProjectPathError", + parent: "DiscoveryError", + doc: "Raised when an authored project path cannot be read as valid UTF-8.", + class_attributes: &[("code", ClassValue::Text("D016"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ResourceIdentityError", + parent: "DiscoveryError", + doc: "Raised when an authored resource identity is not canonical snake_case.", + class_attributes: &[("code", ClassValue::Text("D016"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "MeasurementAuditError", + parent: "builtins.ValueError", + doc: "Raised when a measurement audit contract or value is invalid.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ContractAdoptionError", + parent: "builtins.RuntimeError", + doc: "Raised when safe contract source editing cannot continue.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "FingerprintInputError", + parent: VALUE_ERROR_PARENT, + doc: "Raised when fingerprint storage inputs or rows are invalid.", + class_attributes: &[("code", ClassValue::Text("F000"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "QueryFingerprintError", + parent: "builtins.ValueError", + doc: "Raised when SQL cannot be tokenized into a change fingerprint.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "NativeStageMismatchError", + parent: "builtins.RuntimeError", + doc: "Raised when native state the compiler replays is inconsistent with its inputs.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "UnknownLineageDialectError", + parent: "builtins.ValueError", + doc: "Raised, as Python's parser raised `ValueError`, for a dialect Polyglot does not know.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "RefactorValueError", + parent: "builtins.ValueError", + doc: "An unhandled value failure the Python planner also leaves to the caller.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "RefactorIOError", + parent: "builtins.OSError", + doc: "A file read or write failure the Python planner also leaves to the caller.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ScopeError", + parent: VALUE_ERROR_PARENT, + doc: "Base error for invalid scope-domain input.", + class_attributes: &[("code", ClassValue::Text("S000"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "InvalidScopePathError", + parent: "ScopeError", + doc: "Raised when a project-relative scope path is invalid.", + class_attributes: &[("code", ClassValue::Text("S011"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "InvalidQualifiedIdentityError", + parent: "ScopeError", + doc: "Raised when a qualified scope identity cannot be parsed.", + class_attributes: &[("code", ClassValue::Text("S002"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "UnindexableScopeError", + parent: "ScopeError", + doc: "Raised when the native scope index cannot place a discovered path or resource kind.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ScopeValidationError", + parent: "ScopeError", + doc: "Aggregate error raised only when strict scope validation is requested.", + class_attributes: &[], + required: &["diagnostics"], + fixed: &[], + }, + ErrorSpec { + name: "UnsupportedLexicalSyntaxError", + parent: "builtins.ValueError", + doc: "Raised when an adapter declares lexical rules the native SQL scanners do not support.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "LintError", + parent: EXCEPTION_PARENT, + doc: "A lint or format run could not proceed.", + class_attributes: &[("code", ClassValue::Text("L001"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "NativeLintError", + parent: "LintError", + doc: "The SQLBuild-owned native lint engine rejected a request or response.", + class_attributes: &[("code", ClassValue::Text("L003"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ProjectCompileError", + parent: "LintError", + doc: "The project could not be compiled, so its SQL cannot be linted.", + class_attributes: &[("code", ClassValue::Text("L005"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "LintStoppedError", + parent: "builtins.Exception", + doc: "A speculative lint run was stopped because its result is no longer needed.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "SpecConfigError", + parent: "builtins.RuntimeError", + doc: "Raised when project specification configuration is invalid.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ConfigValueTypeError", + parent: VALUE_ERROR_PARENT, + doc: "Raised when an authored config value has the wrong runtime type.", + class_attributes: &[], + required: &["key", "expected", "actual_type"], + fixed: &[], + }, + ErrorSpec { + name: "DbtInteropError", + parent: VALUE_ERROR_PARENT, + doc: "Base class for expected dbt interop errors.", + class_attributes: &[("code", ClassValue::Text("C240"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "DbtInteropConfigError", + parent: "DbtInteropError", + doc: "Raised when dbt project configuration is missing or invalid.", + class_attributes: &[("code", ClassValue::Text("C240"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "DbtInteropRuntimeError", + parent: "DbtInteropError", + doc: "Raised when dbt command execution fails.", + class_attributes: &[("code", ClassValue::Text("C241"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "DbtInteropArgumentError", + parent: "DbtInteropError", + doc: "Raised when `sqb dbt` arguments cannot be routed safely.", + class_attributes: &[("code", ClassValue::Text("C230"))], + required: &[], + fixed: &[], + }, +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/mod.rs new file mode 100644 index 0000000000..16fd0baad8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod constants; +pub(crate) mod shape_classes; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/shape_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/shape_classes.rs new file mode 100644 index 0000000000..1c9fbfd410 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/shape_classes.rs @@ -0,0 +1,41 @@ +//! Resolve a shape class by name: the native class once it is hosted natively, else the Python one. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::compile_records::constants::COMPILE_VALUE_RECORDS; +use crate::bindings::_helpers::compiler_records::constants::{ + COMPILER_ENUMS, COMPILER_ERRORS, COMPILER_RECORDS, +}; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +/// Whether `class` is one of the compiler shapes hosted natively. +fn hosted(class: &str) -> bool { + COMPILER_RECORDS.iter().any(|spec| spec.name == class) + || COMPILE_VALUE_RECORDS.iter().any(|spec| spec.name == class) + || COMPILER_ENUMS.iter().any(|spec| spec.name == class) + || COMPILER_ERRORS.iter().any(|spec| spec.name == class) +} + +/// `class` from the native module when hosted natively, else from the imported `module`. +pub(crate) fn shape_class<'py, T>( + module: &Bound<'py, T>, + class: &str, +) -> PyResult> { + if hosted(class) { + return imported(module.py(), NATIVE_MODULE, class); + } + module.as_any().getattr(class) +} + +/// `class` from the native module when hosted natively, else from the module at `module_path`. +pub(crate) fn shape_class_at<'py>( + py: Python<'py>, + module_path: &str, + class: &str, +) -> PyResult> { + if hosted(class) { + return imported(py, NATIVE_MODULE, class); + } + imported(py, module_path, class) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index ef2324bcad..606b246d5d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -24,12 +24,13 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::contract_command::write::{ContractWrite, write_contracts}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; -const ADOPTION_MODELS: &str = "sqlbuild.compiler.contract_adoption.models"; +const ADOPTION_MODELS: &str = "sqlbuild._native"; const GENERATE_ACTION: &str = "generate"; /// A `CliUserError` with its code. @@ -80,7 +81,7 @@ fn select_resources<'py>( ) -> PyResult> { let py: Python<'py> = request.py(); let analysis: Bound<'py, PyAny> = call_with( - &imported(py, COMPILE_MODELS, "CompileAnalysisSelection")?, + &imported(py, NATIVE_RECORDS_MODULE, "CompileAnalysisSelection")?, &[("no_cache", PyBool::new(py, true).to_owned().into_any())], )?; let project: Bound<'py, PyAny> = call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs index b5f59917dd..bc88c90805 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs @@ -6,8 +6,9 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const UTF8: &str = "utf-8"; @@ -139,7 +140,7 @@ fn applied<'py>( let keywords: Bound<'py, PyDict> = PyDict::new(py); keywords.set_item("no_cache", true)?; let selection: Bound<'py, PyAny> = - imported(py, COMPILE_MODELS_MODULE, "CompileAnalysisSelection")? + imported(py, NATIVE_RECORDS_MODULE, "CompileAnalysisSelection")? .call((), Some(&keywords))?; call_with( &native.getattr("build_compiled_project")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs index 78ed48d8ea..291232f396 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs @@ -13,10 +13,12 @@ use crate::bindings::_helpers::contracts::model_contracts::{ }; use crate::bindings::_helpers::sqltext::model_headers::{ColumnLocation, location_rows}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.compiler.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const SPEC_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; const PATHLIB_MODULE: &str = "pathlib"; const IMPLICIT_COLUMN_CONTRACTS: &str = "implicit"; @@ -118,7 +120,7 @@ fn related_locations<'py>( return Ok(PyTuple::empty(py)); }; let related: Bound<'py, PyAny> = keyword_call( - &py.import(COMPILE_MODELS_MODULE)? + &py.import(NATIVE_RECORDS_MODULE)? .getattr("RelatedLocation")?, vec![ ("label", OUTPUT_LABEL.into_pyobject(py)?.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs index 7169a9df1e..8f4c9e76a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs @@ -18,7 +18,7 @@ use sqlbuild_config::project::models::{Built, ConfigFile, ConfigPaths}; use crate::bindings::_helpers::boundary::config_values::config_value_to_python; const MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; -const ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const PROJECT_CONFIG_ERROR: &str = "ProjectConfigError"; const FILE_NAMES: [&str; 4] = [ "sqlbuild_project.toml", @@ -40,7 +40,7 @@ const ENUM_MODULES: [(&str, &str); 10] = [ ), ("EventExportSeverity", "sqlbuild.spec.contracts.types"), ("AuditSeverity", "sqlbuild.compiler.auditing.types"), - ("CollectionRendering", "sqlbuild.sql_values.types"), + ("CollectionRendering", "sqlbuild._native"), ]; /// The four configuration file paths of a project directory, as Python and native paths. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs index 7eb4fc6175..0df7568b94 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs @@ -7,7 +7,7 @@ use pyo3::types::{PyDict, PyString, PyStringMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::resource_identities::main::resource_identity_failure::resource_identity_failure; -const RESOURCE_NAME_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.resource_names.exceptions"; +const RESOURCE_NAME_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const TABLE_FUNCTION_NODE_TYPE: &str = "table_fn"; const UDF_NODE_TYPE: &str = "udf"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs index 097936c985..09eadce62f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs @@ -18,7 +18,7 @@ use crate::bindings::_helpers::contracts::effective_config::{ }; const MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; -const ERRORS_MODULE: &str = "sqlbuild.spec.contracts.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const TARGET_GROUPS: [(&str, &str); 4] = [ ("clone", "ClonePolicy"), ("execution_limits", "ExecutionLimitsConfig"), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs index a85b7a51d6..bc1c66c99a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs @@ -12,7 +12,7 @@ use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::dag_artifact::rows::{project_input, python_graph_input}; use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -const SPEC_EXCEPTIONS_MODULE: &str = "sqlbuild.spec.contracts.exceptions"; +const SPEC_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; fn python_error(error: DagArtifactError, py: pyo3::Python<'_>) -> PyErr { match error { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs index 70ba39f2c8..55b6276cfe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs @@ -3,8 +3,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::dag_artifact::{artifact, selection}; +use crate::bindings::_helpers::dagster_invocation; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { artifact::register(module)?; - selection::register(module) + selection::register(module)?; + dagster_invocation::registration::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/access.rs b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/access.rs new file mode 100644 index 0000000000..a9adb97882 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/access.rs @@ -0,0 +1,156 @@ +//! Reads of DAG artifact and execution document mappings, as the Dagster integration reads them. + +use std::collections::HashSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +pub(crate) const CONSTANTS_MODULE: &str = "sqlbuild.integrations.dagster.constants"; + +/// An asset path as Dagster spells it. +pub(crate) type AssetPath = Vec; +/// An asset path and one of its check names. +pub(crate) type CheckKey = (AssetPath, String); + +/// `mapping.get(key)`. +pub(crate) fn get<'py>(mapping: &Bound<'py, PyAny>, key: &str) -> PyResult> { + mapping.call_method1("get", (key,)) +} + +/// `mapping.get(key, default)`. +pub(crate) fn value_or<'py>( + mapping: &Bound<'py, PyAny>, + key: &str, + default: Bound<'py, PyAny>, +) -> PyResult> { + mapping.call_method1("get", (key, default)) +} + +/// `str(mapping.get(key))`. +pub(crate) fn text_of(mapping: &Bound<'_, PyAny>, key: &str) -> PyResult { + get(mapping, key)?.str()?.extract() +} + +/// The items of `mapping.get(key, ())`. +pub(crate) fn items<'py>( + mapping: &Bound<'py, PyAny>, + key: &str, +) -> PyResult>> { + let py: Python<'py> = mapping.py(); + value_or(mapping, key, PyTuple::empty(py).into_any())? + .try_iter()? + .collect() +} + +/// `[str(part) for part in node["asset_key"]]`. +pub(crate) fn asset_path(node: &Bound<'_, PyAny>) -> PyResult { + node.get_item("asset_key")? + .try_iter()? + .map(|part| part?.str()?.extract()) + .collect() +} + +/// `[str(part) for part in node.get("asset_key", ())]`. +pub(crate) fn asset_path_or_empty(node: &Bound<'_, PyAny>) -> PyResult { + items(node, "asset_key")? + .iter() + .map(|part| part.str()?.extract()) + .collect() +} + +/// A Dagster integration constant. +pub(crate) fn constant<'py>(py: Python<'py>, name: &str) -> PyResult> { + imported(py, CONSTANTS_MODULE, name) +} + +/// Whether a constant set holds `value`. +pub(crate) fn constant_contains(py: Python<'_>, (name, value): (&str, &str)) -> PyResult { + constant(py, name)?.contains(value) +} + +/// `bool(record.get(, True))`. +pub(crate) fn enabled(record: &Bound<'_, PyAny>, field_constant: &str) -> PyResult { + let py: Python<'_> = record.py(); + let field: Bound<'_, PyAny> = constant(py, field_constant)?; + record.call_method1("get", (field, true))?.is_truthy() +} + +/// Whether a translated DAG node is owned by the SQLBuild asset definition. +pub(crate) fn asset_enabled(node: &Bound<'_, PyAny>) -> PyResult { + enabled(node, "DAGSTER_ASSET_ENABLED_FIELD") +} + +/// Whether a translated DAG check is owned by the SQLBuild asset definition. +pub(crate) fn check_enabled(check: &Bound<'_, PyAny>) -> PyResult { + enabled(check, "DAGSTER_CHECK_ENABLED_FIELD") +} + +/// `{str(node.get("id")): node for node in dag.get("nodes", ())}`, keeping the last node per id. +pub(crate) fn nodes_by_id<'py>( + dag: &Bound<'py, PyAny>, +) -> PyResult)>> { + let mut nodes: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); + for node in items(dag, "nodes")? { + let id: String = text_of(&node, "id")?; + match nodes.iter_mut().find(|(listed, _)| *listed == id) { + Some(entry) => entry.1 = node, + None => nodes.push((id, node)), + } + } + Ok(nodes) +} + +/// The node with `id`, if any. +pub(crate) fn node_with_id<'a, 'py>( + nodes: &'a [(String, Bound<'py, PyAny>)], + id: &str, +) -> Option<&'a Bound<'py, PyAny>> { + nodes + .iter() + .find(|(listed, _)| listed == id) + .map(|(_, node)| node) +} + +/// Selected asset paths passed from the Dagster context. +pub(crate) fn path_set(values: &Bound<'_, PyAny>) -> PyResult> { + if values.is_none() { + return Ok(HashSet::new()); + } + values.try_iter()?.map(|path| path?.extract()).collect() +} + +/// Selected asset check keys passed from the Dagster context. +pub(crate) fn check_key_set(values: &Bound<'_, PyAny>) -> PyResult> { + if values.is_none() { + return Ok(HashSet::new()); + } + values.try_iter()?.map(|key| key?.extract()).collect() +} + +/// A Python list of strings. +pub(crate) fn string_list<'py>(py: Python<'py>, values: &[String]) -> PyResult> { + Ok(PyList::new(py, values)?.into_any()) +} + +/// A Python tuple of strings. +pub(crate) fn string_tuple<'py>(py: Python<'py>, values: &[String]) -> PyResult> { + Ok(PyTuple::new(py, values)?.into_any()) +} + +/// `{**base, **extra}`, keeping each key's first position. +pub(crate) fn merged<'py>( + base: &[(&str, Bound<'py, PyAny>)], + extra: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = extra.py(); + let merged: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in base { + merged.set_item(*key, value)?; + } + for (key, value) in extra.iter() { + merged.set_item(key, value)?; + } + Ok(merged) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/arguments.rs b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/arguments.rs new file mode 100644 index 0000000000..a01d52fa43 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/arguments.rs @@ -0,0 +1,355 @@ +//! SQLBuild CLI arguments for one Dagster invocation: selections, JSON output and event output. + +use std::collections::HashSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_core::dagster_invocation::constants::{ + JSON_OUTPUT_FLAG, SCENARIO_TEST_ARGUMENT_COUNT, +}; +use sqlbuild_core::dagster_invocation::main::caller_json_output_path::caller_json_output_path; +use sqlbuild_core::dagster_invocation::main::has_explicit_scenario_selector::has_explicit_scenario_selector; +use sqlbuild_core::dagster_invocation::main::json_output_requested::json_output_requested; +use sqlbuild_core::dagster_invocation::main::wrap_selectors::wrap_selectors; + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::dagster_invocation::access::{ + AssetPath, CheckKey, asset_path, asset_path_or_empty, check_enabled, check_key_set, constant, + constant_contains, get, items, node_with_id, nodes_by_id, path_set, string_tuple, text_of, +}; +use crate::bindings::_helpers::dagster_invocation::results::{dagster_check_name, sorted_nodes}; + +const SELECT_FILE_FLAG: &str = "--select-file"; +const EVENT_OUTPUT_FLAG: &str = "--event-output"; +const SCENARIO_TEST_COMMAND: [&str; 2] = ["scenario", "test"]; +const SELECTOR_FILE: (&str, &str) = ("sqlbuild-dagster-select-", ".txt"); +const EXECUTION_JSON_FILE: (&str, &str) = ("sqlbuild-dagster-execution-", ".json"); +const EVENT_FILE: (&str, &str) = ("sqlbuild-dagster-events-", ".jsonl"); +const LOAD_COMMAND: &str = "load"; +const CHECK_COMMAND: &str = "check"; +const SELECTOR_LOG_WIDTH: usize = 100; + +/// A new, closed temporary file holding `lines`, as a `pathlib.Path`. +fn temporary_file<'py>( + py: Python<'py>, + (prefix, suffix): (&str, &str), + lines: &[String], +) -> PyResult> { + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("mode", "w")?; + options.set_item("encoding", "utf-8")?; + options.set_item("prefix", prefix)?; + options.set_item("suffix", suffix)?; + options.set_item("delete", false)?; + let handle: Bound<'py, PyAny> = + imported(py, "tempfile", "NamedTemporaryFile")?.call((), Some(&options))?; + for line in lines { + handle.call_method1("write", (format!("{line}\n"),))?; + } + handle.call_method0("close")?; + imported(py, "pathlib", "Path")?.call1((handle.getattr("name")?,)) +} + +fn is_scenario_test(args: &[String]) -> bool { + args.len() >= SCENARIO_TEST_ARGUMENT_COUNT + && args[..SCENARIO_TEST_ARGUMENT_COUNT] == SCENARIO_TEST_COMMAND +} + +fn check_is_selected( + check: &Bound<'_, PyAny>, + (check_name, nodes): (&str, &[(String, Bound<'_, PyAny>)]), + selected_check_keys: &HashSet, +) -> PyResult { + for asset_id in items(check, "checked_asset_ids")? { + let Some(node) = node_with_id(nodes, &asset_id.str()?.extract::()?) else { + continue; + }; + if selected_check_keys.contains(&(asset_path_or_empty(node)?, check_name.to_owned())) { + return Ok(true); + } + } + Ok(false) +} + +/// Selectors of every enabled DAG check matching a selected check key, without duplicates. +fn selected_check_selectors( + dag: &Bound<'_, PyAny>, + selected_check_keys: &HashSet, +) -> PyResult> { + let nodes = nodes_by_id(dag)?; + let mut selectors: Vec = Vec::new(); + for check in items(dag, "checks")? { + if !check_enabled(&check)? { + continue; + } + let name: String = dagster_check_name(&check)?; + if check_is_selected(&check, (&name, &nodes), selected_check_keys)? { + let selector: String = text_of(&check, "name")?; + if !selectors.contains(&selector) { + selectors.push(selector); + } + } + } + Ok(selectors) +} + +/// Asset paths checked by every selected check. +fn selected_check_asset_paths( + dag: &Bound<'_, PyAny>, + selected_check_keys: &HashSet, +) -> PyResult> { + let nodes = nodes_by_id(dag)?; + let mut paths: HashSet = HashSet::new(); + for check in items(dag, "checks")? { + if !check_enabled(&check)? { + continue; + } + let name: String = dagster_check_name(&check)?; + if !check_is_selected(&check, (&name, &nodes), selected_check_keys)? { + continue; + } + for asset_id in items(&check, "checked_asset_ids")? { + if let Some(node) = node_with_id(&nodes, &asset_id.str()?.extract::()?) { + paths.insert(asset_path_or_empty(node)?); + } + } + } + Ok(paths) +} + +/// The selection appended to the arguments and the selector file holding it, if any. +struct Selection<'py> { + args: Vec, + selectors: Vec, + selector_file: Option>, +} + +fn unselected<'py>(args: &[String]) -> Selection<'py> { + Selection { + args: args.to_vec(), + selectors: Vec::new(), + selector_file: None, + } +} + +fn with_selector_file<'py>( + py: Python<'py>, + args: &[String], + selectors: Vec, +) -> PyResult> { + let file: Bound<'py, PyAny> = temporary_file(py, SELECTOR_FILE, &selectors)?; + let mut selected: Vec = args.to_vec(); + selected.push(SELECT_FILE_FLAG.to_owned()); + selected.push(file.str()?.extract()?); + Ok(Selection { + args: selected, + selectors, + selector_file: Some(file), + }) +} + +fn scenario_selection<'py>( + (args, dag): (&[String], &Bound<'py, PyAny>), + read_selection: &Bound<'py, PyAny>, +) -> PyResult> { + if has_explicit_scenario_selector(args) { + return Ok(unselected(args)); + } + let (selected_paths, selected_check_keys) = selection_facts(read_selection)?; + let selected_check_keys: &HashSet = &selected_check_keys; + let selected_paths: HashSet = if selected_paths.is_empty() { + selected_check_keys + .iter() + .map(|(path, _)| path.clone()) + .collect() + } else { + selected_paths + }; + if selected_paths.is_empty() && selected_check_keys.is_empty() { + return Ok(unselected(args)); + } + let nodes = nodes_by_id(dag)?; + let mut selected_ids: HashSet = HashSet::new(); + for node in items(dag, "nodes")? { + if selected_paths.contains(&asset_path_or_empty(&node)?) { + selected_ids.insert(text_of(&node, "id")?); + } + } + let scenario_kind: String = constant(dag.py(), "SCENARIO_CHECK_KIND")?.extract()?; + let mut selectors: Vec = Vec::new(); + for check in items(dag, "checks")? { + if !check_enabled(&check)? || text_of(&check, "kind")? != scenario_kind { + continue; + } + let mut checked: bool = false; + for asset_id in items(&check, "checked_asset_ids")? { + checked |= selected_ids.contains(&asset_id.str()?.extract::()?); + } + if !checked { + continue; + } + let name: String = dagster_check_name(&check)?; + if !selected_check_keys.is_empty() + && !check_is_selected(&check, (&name, &nodes), selected_check_keys)? + { + continue; + } + let selector: Bound<'py, PyAny> = get(&check, "name")?; + if !selector.is_none() { + selectors.push(selector.str()?.extract()?); + } + } + if selectors.is_empty() { + return Ok(unselected(args)); + } + let mut selected: Vec = args.to_vec(); + selected.extend(selectors.iter().cloned()); + Ok(Selection { + args: selected, + selectors, + selector_file: None, + }) +} + +fn asset_selection<'py>( + (args, dag): (&[String], &Bound<'py, PyAny>), + read_selection: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = dag.py(); + if is_scenario_test(args) { + return scenario_selection((args, dag), read_selection); + } + if !constant_contains(py, ("ASSET_SELECTION_COMMANDS", &args[0]))? { + return Ok(unselected(args)); + } + let explicit: Bound<'py, PyAny> = constant(py, "EXPLICIT_SELECTION_FLAGS")?; + for argument in args { + if explicit.contains(argument)? { + return Ok(unselected(args)); + } + } + let (mut selected_paths, selected_check_keys) = selection_facts(read_selection)?; + let selected_check_keys: &HashSet = &selected_check_keys; + if args[0] == CHECK_COMMAND && !selected_check_keys.is_empty() { + let selectors: Vec = selected_check_selectors(dag, selected_check_keys)?; + if !selectors.is_empty() { + return with_selector_file(py, args, selectors); + } + } + if !selected_check_keys.is_empty() { + selected_paths.extend(selected_check_asset_paths(dag, selected_check_keys)?); + } + if selected_paths.is_empty() { + return Ok(unselected(args)); + } + let kinds: Bound<'py, PyAny> = constant( + py, + if args[0] == LOAD_COMMAND { + "LOAD_SELECTABLE_NODE_KINDS" + } else { + "DEFAULT_SELECTABLE_NODE_KINDS" + }, + )?; + let mut selectors: Vec = Vec::new(); + for node in sorted_nodes(dag)? { + if !selected_paths.contains(&asset_path(&node)?) { + continue; + } + if !kinds.contains(text_of(&node, "kind")?)? { + continue; + } + let selector: Bound<'py, PyAny> = get(&node, "name")?; + if !selector.is_none() { + selectors.push(selector.str()?.extract()?); + } + } + if selectors.is_empty() { + return Ok(unselected(args)); + } + with_selector_file(py, args, selectors) +} + +/// The selected asset paths and check keys the Dagster context reports. +fn selection_facts( + read_selection: &Bound<'_, PyAny>, +) -> PyResult<(HashSet, HashSet)> { + let facts: Bound<'_, PyAny> = read_selection.call0()?; + Ok(( + path_set(&facts.get_item(0)?)?, + check_key_set(&facts.get_item(1)?)?, + )) +} + +fn optional<'py>(py: Python<'py>, value: Option>) -> Bound<'py, PyAny> { + value.unwrap_or_else(|| py.None().into_bound(py)) +} + +/// The CLI arguments, Dagster selection, selector file and output files of one invocation. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn dagster_cli_arguments<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "dagster_cli_arguments", + options, + (&["args", "dag", "has_context", "read_selection"], &[]), + )?; + let args: Vec = arguments.value("args")?.extract()?; + let dag: Option> = Some(arguments.value("dag")?); + let has_context: bool = arguments.flag("has_context")?; + let read_selection: Bound<'py, PyAny> = arguments.value("read_selection")?; + let managed_dag: Option> = + dag.filter(|dag| !dag.is_none() && has_context && !args.is_empty()); + let selection: Selection<'py> = match &managed_dag { + Some(dag) => asset_selection((&args, dag), &read_selection)?, + None => unselected(&args), + }; + let managed: bool = managed_dag.is_some(); + let mut resolved: Vec = selection.args.clone(); + let mut json_path: Option> = None; + if managed + && !json_output_requested(&resolved) + && (constant_contains(py, ("ASSET_SELECTION_COMMANDS", &resolved[0]))? + || is_scenario_test(&resolved)) + { + let path: Bound<'py, PyAny> = temporary_file(py, EXECUTION_JSON_FILE, &[])?; + resolved.push(JSON_OUTPUT_FLAG.to_owned()); + resolved.push(path.str()?.extract()?); + json_path = Some(path); + } + let json_owned: bool = json_path.is_some(); + if json_path.is_none() + && let Some(path) = caller_json_output_path(&resolved) + { + json_path = Some(imported(py, "pathlib", "Path")?.call1((path,))?); + } + let event_path: Option> = if managed + && constant_contains(py, ("LIVE_EVENT_COMMANDS", &resolved[0]))? + && !resolved + .iter() + .any(|argument| argument == EVENT_OUTPUT_FLAG) + { + Some(temporary_file(py, EVENT_FILE, &[])?) + } else { + None + }; + let result: Bound<'py, PyDict> = PyDict::new(py); + result.set_item("args", string_tuple(py, &resolved)?)?; + result.set_item("selection", string_tuple(py, &selection.selectors)?)?; + result.set_item("selector_file", optional(py, selection.selector_file))?; + result.set_item("execution_json_path", optional(py, json_path))?; + result.set_item("execution_json_owned", json_owned)?; + result.set_item("event_jsonl_path", optional(py, event_path))?; + Ok(result) +} + +/// Selectors packed into log lines of at most 100 characters. +#[pyfunction] +#[pyo3(signature = (*, selectors))] +pub(crate) fn dagster_selector_log_lines(selectors: Vec) -> Vec { + wrap_selectors(&selectors, SELECTOR_LOG_WIDTH) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/failures.rs b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/failures.rs new file mode 100644 index 0000000000..3cf7b1e638 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/failures.rs @@ -0,0 +1,184 @@ +//! Failure descriptions and metadata of failed SQLBuild CLI invocations. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use sqlbuild_core::dagster_invocation::main::asset_failure_text::asset_failure_text; +use sqlbuild_core::dagster_invocation::main::diagnostic_tail::diagnostic_tail; +use sqlbuild_core::dagster_invocation::models::AssetFailure; + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::dagster_invocation::access::{ + constant_contains, get, items, text_of, +}; + +const LIVE_FAILURE_MESSAGE: &str = + "SQLBuild resource attempt failed; see final execution diagnostics"; +const UNKNOWN: &str = "unknown"; + +fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.extract()?)) +} + +fn sql_text(value: &Bound<'_, PyAny>) -> PyResult> { + if !value.is_instance_of::() { + return Ok(None); + } + Ok(Some(value.extract()?)) +} + +fn asset_failure(asset: &Bound<'_, PyAny>) -> PyResult { + Ok(AssetFailure { + kind: text_of(asset, "kind")?, + name: text_of(asset, "name")?, + status: text_of(asset, "status")?, + failed_phase: optional_text(&get(asset, "failed_phase")?)?, + error_message: optional_text(&get(asset, "error_message")?)?, + error_code: optional_text(&get(asset, "error_code")?)?, + staging_relation: optional_text(&get(asset, "staging_relation")?)?, + failed_sql: sql_text(&get(asset, "failed_sql")?)?, + last_recorded_sql: sql_text(&get(asset, "last_recorded_sql")?)?, + }) +} + +/// The Dagster failure description and metadata of a failed invocation. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn dagster_failure<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult<(String, Bound<'py, PyDict>)> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "dagster_failure", + options, + ( + &[ + "command", + "project_dir", + "selection", + "selector_file", + "returncode", + "execution_payload", + "stdout", + "stderr", + ], + &[], + ), + )?; + let command: Vec = arguments.value("command")?.extract()?; + let project_dir: Bound<'py, PyAny> = arguments.value("project_dir")?; + let selection: Vec = arguments.value("selection")?.extract()?; + let selector_file: String = arguments.value("selector_file")?.extract()?; + let returncode: i64 = arguments.value("returncode")?.extract()?; + let payload_value: Bound<'py, PyAny> = arguments.value("execution_payload")?; + let execution_payload: Option> = + (!payload_value.is_none()).then_some(payload_value); + let stdout: String = arguments.value("stdout")?.extract()?; + let stderr: String = arguments.value("stderr")?.extract()?; + let command_text: String = command.join(" "); + let metadata: Bound<'py, PyDict> = PyDict::new(py); + metadata.set_item("command", &command_text)?; + metadata.set_item("project_dir", project_dir.str()?)?; + metadata.set_item("selection", selection.join(" "))?; + metadata.set_item("selector_file", selector_file)?; + let mut description: String = + format!("SQLBuild CLI command failed with exit code {returncode}: {command_text}"); + match execution_payload { + Some(payload) => { + let mut incomplete: Vec = Vec::new(); + let mut details: Vec = Vec::new(); + let limits: Bound<'py, PyDict> = PyDict::new(py); + for asset in items(&payload, "assets")? { + let status: String = text_of(&asset, "status")?; + if constant_contains(py, ("COMPLETED_EXECUTION_STATUSES", &status))? { + continue; + } + let failure: AssetFailure = asset_failure(&asset)?; + incomplete.push(format!("{}:{} ({status})", failure.kind, failure.name)); + details.push(asset_failure_text(&failure)); + let microbatch: Bound<'py, PyAny> = get(&asset, "microbatch")?; + if microbatch.is_instance_of::() && !get(µbatch, "limit")?.is_none() { + limits.set_item(text_of(&asset, "name")?, microbatch)?; + } + } + metadata.set_item("execution_status", text_of(&payload, "status")?)?; + metadata.set_item("incomplete_assets", incomplete.join(", "))?; + if !limits.is_empty() { + metadata.set_item("microbatch_limits", limits)?; + } + if !details.is_empty() { + description.push_str("\n\nFailures:\n\n"); + description.push_str(&details.join("\n\n")); + } + } + None => { + let stdout_tail: String = diagnostic_tail(&stdout); + let stderr_tail: String = diagnostic_tail(&stderr); + if !stdout_tail.is_empty() { + metadata.set_item("stdout_tail", &stdout_tail)?; + } + if !stderr_tail.is_empty() { + metadata.set_item("stderr_tail", &stderr_tail)?; + } + let diagnostic: &str = match stderr_tail.trim() { + "" => stdout_tail.trim(), + trimmed => trimmed, + }; + if !diagnostic.is_empty() { + description.push_str(&format!("\n\n{diagnostic}")); + } + } + } + Ok((description, metadata)) +} + +/// The log arguments and structured fields of one failed live resource attempt. +#[pyfunction] +#[pyo3(signature = (*, asset, envelope, command))] +pub(crate) fn dagster_live_failure<'py>( + asset: Bound<'py, PyAny>, + envelope: Bound<'py, PyAny>, + command: Vec, +) -> PyResult<(Vec, Bound<'py, PyDict>)> { + let py: Python<'py> = asset.py(); + let kind: String = asset.getattr("kind")?.extract()?; + let name: String = asset.getattr("name")?.extract()?; + let label: String = format!("{kind}:{name}"); + let or_unknown = |value: Bound<'py, PyAny>| -> PyResult { + if value.is_truthy()? { + value.str()?.extract() + } else { + Ok(UNKNOWN.to_owned()) + } + }; + let phase: String = or_unknown(asset.getattr("failed_phase")?)?; + let code: String = or_unknown(envelope.getattr("error_code")?)?; + let metadata: Bound<'py, PyDict> = PyDict::new(py); + metadata.set_item("sqlbuild_asset", &label)?; + metadata.set_item("sqlbuild_asset_kind", &kind)?; + metadata.set_item("sqlbuild_asset_name", &name)?; + metadata.set_item("sqlbuild_phase", &phase)?; + metadata.set_item("sqlbuild_error_code", &code)?; + metadata.set_item("sqlbuild_error_message", LIVE_FAILURE_MESSAGE)?; + metadata.set_item("sqlbuild_command", command.join(" "))?; + for (key, value) in [ + ("sqlbuild_target", asset.getattr("target")?), + ( + "sqlbuild_staging_relation", + asset.getattr("staging_relation")?, + ), + ("sqlbuild_duration_ms", envelope.getattr("duration_ms")?), + ] { + if !value.is_none() { + metadata.set_item(key, value)?; + } + } + Ok(( + vec![label, phase, code, LIVE_FAILURE_MESSAGE.to_owned()], + metadata, + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/mod.rs new file mode 100644 index 0000000000..9d1391a8de --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod access; +pub(crate) mod arguments; +pub(crate) mod failures; +pub(crate) mod registration; +pub(crate) mod results; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/registration.rs new file mode 100644 index 0000000000..58ff98354a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/registration.rs @@ -0,0 +1,29 @@ +//! Register Dagster invocation assembly and event planning. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +use crate::bindings::_helpers::dagster_invocation::{arguments, failures, results}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(arguments::dagster_cli_arguments, module)?)?; + module.add_function(wrap_pyfunction!( + arguments::dagster_selector_log_lines, + module + )?)?; + module.add_function(wrap_pyfunction!(results::dagster_payload_results, module)?)?; + module.add_function(wrap_pyfunction!( + results::dagster_integration_results, + module + )?)?; + module.add_function(wrap_pyfunction!( + results::dagster_selected_asset_results, + module + )?)?; + module.add_function(wrap_pyfunction!( + results::dagster_execution_payload, + module + )?)?; + module.add_function(wrap_pyfunction!(failures::dagster_failure, module)?)?; + module.add_function(wrap_pyfunction!(failures::dagster_live_failure, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/results.rs new file mode 100644 index 0000000000..7dbad46a46 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/results.rs @@ -0,0 +1,668 @@ +//! Dagster event plans from execution documents, integration results and DAG artifacts. + +use std::collections::HashSet; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction}; +use sqlbuild_core::dagster_invocation::main::normalize_check_name::normalize_check_name; +use sqlbuild_core::dagster_invocation::main::topological_order::topological_order; + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::dagster_invocation::access::{ + AssetPath, CheckKey, asset_enabled, asset_path, check_enabled, check_key_set, constant, + constant_contains, get, items, merged, node_with_id, nodes_by_id, path_set, string_list, + text_of, value_or, +}; + +const CHECK_KIND_FALLBACK: &str = "check"; +const CHECK_NAME_JOIN: &str = "__"; +const EXECUTION_DOCUMENT_VERSION: i64 = 1; +const CHECK_ASSET_KINDS: [&str; 5] = ["model", "source", "seed", "udf", "table_fn"]; +const MATERIALIZE: &str = "materialize"; +const MATERIALIZATION: &str = "materialization"; +const CHECK: &str = "check"; + +/// DAG nodes upstream first, or in artifact order when edges form a cycle. +pub(crate) fn sorted_nodes<'py>(dag: &Bound<'py, PyAny>) -> PyResult>> { + let nodes: Vec> = items(dag, "nodes")?; + let ids: Vec = nodes + .iter() + .map(|node| text_of(node, "id")) + .collect::>()?; + let mut edges: Vec<(String, String)> = Vec::new(); + for edge in items(dag, "edges")? { + edges.push((text_of(&edge, "from_id")?, text_of(&edge, "to_id")?)); + } + Ok(match topological_order(&ids, &edges) { + Some(order) => order + .into_iter() + .map(|position| nodes[position].clone()) + .collect(), + None => nodes, + }) +} + +/// The translated Dagster check name, or one built from the check's kind, name and target. +pub(crate) fn dagster_check_name(check: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = check.py(); + let field: Bound<'_, PyAny> = constant(py, "DAGSTER_CHECK_NAME_FIELD")?; + let translated: Bound<'_, PyAny> = check.call_method1("get", (field,))?; + if !translated.is_none() { + return translated.str()?.extract(); + } + let mut parts: Vec = vec![ + value_or( + check, + "kind", + PyString::new(py, CHECK_KIND_FALLBACK).into_any(), + )? + .str()? + .extract()?, + value_or( + check, + "name", + PyString::new(py, CHECK_KIND_FALLBACK).into_any(), + )? + .str()? + .extract()?, + ]; + for attached in ["attached_column_name", "attached_target_name"] { + let value: Bound<'_, PyAny> = get(check, attached)?; + if !value.is_none() { + parts.push(check.get_item(attached)?.str()?.extract()?); + break; + } + } + Ok(parts + .iter() + .filter(|part| !part.is_empty()) + .map(|part| normalize_check_name(part)) + .collect::>() + .join(CHECK_NAME_JOIN)) +} + +/// A metadata value with enums replaced by their values and containers made plain. +fn metadata_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + if value.is_instance(&imported(py, "enum", "Enum")?)? { + return value.getattr("value"); + } + if value.is_instance(&imported(py, "collections.abc", "Mapping")?)? { + let plain: Bound<'py, PyDict> = PyDict::new(py); + for pair in value.call_method0("items")?.try_iter()? { + let (key, item): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; + plain.set_item(key.str()?, metadata_value(&item)?)?; + } + return Ok(plain.into_any()); + } + if value.is_instance_of::() || value.is_instance_of::() { + let plain: Vec> = value + .try_iter()? + .map(|item| metadata_value(&item?)) + .collect::>()?; + return Ok(PyList::new(py, plain)?.into_any()); + } + Ok(value.clone()) +} + +/// A record's non-null metadata, without check internals. +fn metadata_from_mapping<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + let excluded: Bound<'py, PyAny> = constant(py, "CHECK_METADATA_EXCLUDED_KEYS")?; + let metadata: Bound<'py, PyDict> = PyDict::new(py); + for pair in value.call_method0("items")?.try_iter()? { + let (key, item): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; + if item.is_none() || excluded.contains(&key)? { + continue; + } + metadata.set_item(key.str()?, metadata_value(&item)?)?; + } + Ok(metadata) +} + +/// One Dagster event: `(kind, asset_path, metadata, check_name, passed, warn)`. +fn plan<'py>( + py: Python<'py>, + (kind, path): (&str, &AssetPath), + metadata: Bound<'py, PyDict>, + check: Option<(&str, bool, bool)>, +) -> PyResult> { + let none = || py.None().into_bound(py); + let (name, passed, warn) = match check { + Some((name, passed, warn)) => ( + PyString::new(py, name).into_any(), + PyBool::new(py, passed).to_owned().into_any(), + PyBool::new(py, warn).to_owned().into_any(), + ), + None => (none(), none(), none()), + }; + Ok(PyTuple::new( + py, + [ + PyString::new(py, kind).into_any(), + string_list(py, path)?, + metadata.into_any(), + name, + passed, + warn, + ], + )? + .into_any()) +} + +fn command_text(command: &[String]) -> String { + command.join(" ") +} + +/// The loader feeding a source, with the loader's own execution facts. +fn loader_result<'py>( + dag: &Bound<'py, PyAny>, + (source_node, payload_asset): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + loader_name: &str, +) -> PyResult)>> { + let py: Python<'py> = dag.py(); + let nodes = nodes_by_id(dag)?; + let loader_kind: String = constant(py, "LOADER_NODE_KIND")?.extract()?; + let source_id: String = text_of(source_node, "id")?; + for edge in items(dag, "edges")? { + if text_of(&edge, "to_id")? != source_id { + continue; + } + let Some(upstream) = node_with_id(&nodes, &text_of(&edge, "from_id")?) else { + continue; + }; + if text_of(upstream, "kind")? != loader_kind || text_of(upstream, "name")? != loader_name { + continue; + } + let facts: Bound<'py, PyDict> = PyDict::new(py); + facts.set_item("kind", "loader")?; + facts.set_item("name", get(upstream, "name")?)?; + facts.set_item("resource_id", get(upstream, "id")?)?; + facts.set_item("source", get(source_node, "name")?)?; + facts.set_item("source_relation", get(payload_asset, "target")?)?; + facts.set_item("status", get(payload_asset, "status")?)?; + return Ok(Some((text_of(upstream, "id")?, facts))); + } + Ok(None) +} + +/// Dagster selection facts of the running asset or check. +struct SelectionFacts { + paths: HashSet, + check_keys: HashSet, + explicit_checks: bool, +} + +impl SelectionFacts { + fn from_arguments(arguments: &KeywordArguments<'_>) -> PyResult { + Ok(Self { + paths: path_set(&arguments.value("selected_asset_paths")?)?, + check_keys: check_key_set(&arguments.value("selected_check_keys")?)?, + explicit_checks: arguments.flag("check_selection_is_explicit")?, + }) + } + + fn check_only(&self) -> bool { + !self.check_keys.is_empty() && self.paths.is_empty() + } +} + +/// How check results are filtered. +struct CheckFilter<'a> { + selection: &'a SelectionFacts, + selected_paths: &'a HashSet, + emitted_asset_paths: Option<&'a HashSet>, +} + +fn warn_severity(check: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = check.py(); + let warning: String = constant(py, "WARNING_CHECK_SEVERITY")?.extract()?; + let insufficient: String = constant(py, "INSUFFICIENT_CHECK_STATUS")?.extract()?; + let status: String = text_of(check, "status")?; + Ok(status == insufficient || status == warning || text_of(check, "severity")? == warning) +} + +fn dag_check_for<'py>( + dag: &Bound<'py, PyAny>, + check: &Bound<'py, PyAny>, +) -> PyResult>> { + let mut wanted: Bound<'py, PyAny> = get(check, "dag_check_id")?; + if !wanted.is_truthy()? { + wanted = get(check, "check_id")?; + } + for dag_check in items(dag, "checks")? { + if get(&dag_check, "id")?.eq(&wanted)? { + return Ok(Some(dag_check)); + } + } + Ok(None) +} + +fn asset_ids_for_check(dag: &Bound<'_, PyAny>, check: &Bound<'_, PyAny>) -> PyResult> { + let asset_name: Bound<'_, PyAny> = get(check, "asset_name")?; + if asset_name.is_none() { + return Ok(Vec::new()); + } + let name: String = asset_name.str()?.extract()?; + for kind in CHECK_ASSET_KINDS { + if let Some(node) = node_by_kind_and_name(dag, (kind, &name))? { + return Ok(vec![text_of(&node, "id")?]); + } + } + Ok(Vec::new()) +} + +fn check_plans<'py>( + (dag, nodes): (&Bound<'py, PyAny>, &[(String, Bound<'py, PyAny>)]), + check: &Bound<'py, PyAny>, + (filter, seen): (&CheckFilter<'_>, HashSet), +) -> PyResult<(Vec>, HashSet)> { + let mut seen: HashSet = seen; + let py: Python<'py> = dag.py(); + let dag_check: Option> = dag_check_for(dag, check)?; + let (asset_ids, check_name): (Vec, String) = match &dag_check { + Some(dag_check) if !check_enabled(dag_check)? => return Ok((Vec::new(), seen)), + Some(dag_check) => ( + items(dag_check, "checked_asset_ids")? + .iter() + .map(|id| id.str()?.extract()) + .collect::>()?, + dagster_check_name(dag_check)?, + ), + None => (asset_ids_for_check(dag, check)?, dagster_check_name(check)?), + }; + let mut plans: Vec> = Vec::new(); + for asset_id in asset_ids { + let Some(node) = node_with_id(nodes, &asset_id) else { + continue; + }; + if !asset_enabled(node)? { + continue; + } + let path: AssetPath = asset_path(node)?; + let key: CheckKey = (path.clone(), check_name.clone()); + if filter + .emitted_asset_paths + .is_some_and(|emitted| !emitted.contains(&path)) + { + continue; + } + if filter.selection.explicit_checks && !filter.selection.check_keys.contains(&key) { + continue; + } + if !filter.selection.explicit_checks + && !filter.selected_paths.is_empty() + && !filter.selected_paths.contains(&path) + { + continue; + } + if !seen.insert(key) { + continue; + } + plans.push(plan( + py, + (CHECK, &path), + metadata_from_mapping(check)?, + Some(( + &check_name, + get(check, "passed")?.is_truthy()?, + warn_severity(check)?, + )), + )?); + } + Ok((plans, seen)) +} + +fn node_by_kind_and_name<'py>( + dag: &Bound<'py, PyAny>, + (kind, name): (&str, &str), +) -> PyResult>> { + let mut found: Option> = None; + for node in items(dag, "nodes")? { + if text_of(&node, "kind")? == kind && text_of(&node, "name")? == name { + found = Some(node); + } + } + Ok(found) +} + +fn keep_checks_only<'py>( + plans: Vec>, + selection: &SelectionFacts, +) -> PyResult>> { + if !selection.check_only() { + return Ok(plans); + } + let mut kept: Vec> = Vec::new(); + for planned in plans { + if planned.get_item(0)?.extract::()? == CHECK { + kept.push(planned); + } + } + Ok(kept) +} + +fn completed(py: Python<'_>, status: &str) -> PyResult { + constant_contains(py, ("COMPLETED_EXECUTION_STATUSES", status)) +} + +/// Dagster events for an execution JSON document. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn dagster_payload_results<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult>> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "dagster_payload_results", + options, + ( + &[ + "dag", + "payload", + "command", + "selected_asset_paths", + "selected_check_keys", + "check_selection_is_explicit", + ], + &[], + ), + )?; + let dag: Bound<'py, PyAny> = arguments.value("dag")?; + let payload: Bound<'py, PyAny> = arguments.value("payload")?; + let command: Vec = arguments.value("command")?.extract()?; + let selection = SelectionFacts::from_arguments(&arguments)?; + let clone_command: String = constant(py, "CLONE_COMMAND")?.extract()?; + let success: String = constant(py, "SUCCESS_EXECUTION_STATUS")?.extract()?; + let source_kind: String = constant(py, "SOURCE_NODE_KIND")?.extract()?; + let is_clone: bool = text_of(&payload, "command")? == clone_command; + let selected_paths: HashSet = if is_clone { + HashSet::new() + } else { + selection.paths.clone() + }; + let nodes = nodes_by_id(&dag)?; + let mut asset_results: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); + let mut record = |id: String, facts: Bound<'py, PyAny>| match asset_results + .iter_mut() + .find(|(listed, _)| *listed == id) + { + Some(entry) => entry.1 = facts, + None => asset_results.push((id, facts)), + }; + for payload_asset in items(&payload, "assets")? { + let status: String = text_of(&payload_asset, "status")?; + if !completed(py, &status)? || (is_clone && status != success) { + continue; + } + let has_loader: bool = !get(&payload_asset, "loader")?.is_none(); + let kind: String = if has_loader { + source_kind.clone() + } else { + text_of(&payload_asset, "kind")? + }; + let Some(node) = node_by_kind_and_name(&dag, (&kind, &text_of(&payload_asset, "name")?))? + else { + continue; + }; + record(text_of(&node, "id")?, payload_asset.clone()); + if text_of(&node, "kind")? == source_kind && has_loader { + let loader_name: String = payload_asset.get_item("loader")?.str()?.extract()?; + if let Some((loader_id, facts)) = + loader_result(&dag, (&node, &payload_asset), &loader_name)? + { + record(loader_id, facts.into_any()); + } + } + } + let mut plans: Vec> = Vec::new(); + let kind: &str = if is_clone { + MATERIALIZATION + } else { + MATERIALIZE + }; + for node in sorted_nodes(&dag)? { + let id: String = text_of(&node, "id")?; + let Some((_, facts)) = asset_results.iter().find(|(listed, _)| *listed == id) else { + continue; + }; + if !asset_enabled(&node)? { + continue; + } + let path: AssetPath = asset_path(&node)?; + if !selected_paths.is_empty() && !selected_paths.contains(&path) { + continue; + } + let metadata: Bound<'py, PyDict> = merged( + &[ + ( + "command", + PyString::new(py, &command_text(&command)).into_any(), + ), + ("sqlbuild_id", get(&node, "id")?), + ], + &metadata_from_mapping(facts)?, + )?; + plans.push(plan(py, (kind, &path), metadata, None)?); + } + let filter = CheckFilter { + selection: &selection, + selected_paths: &selected_paths, + emitted_asset_paths: None, + }; + let mut seen: HashSet = HashSet::new(); + for check in items(&payload, "checks")? { + let (check_results, updated) = check_plans((&dag, &nodes), &check, (&filter, seen))?; + plans.extend(check_results); + seen = updated; + } + keep_checks_only(plans, &selection) +} + +fn envelope_fields<'py>( + envelope: &Bound<'py, PyAny>, + names: &[&str], +) -> PyResult)>> { + names + .iter() + .map(|name| Ok(((*name).to_owned(), envelope.getattr(*name)?))) + .collect() +} + +/// Dagster events for one live integration result record. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn dagster_integration_results<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult>> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "dagster_integration_results", + options, + ( + &[ + "dag", + "envelope", + "command", + "selected_asset_paths", + "selected_check_keys", + "check_selection_is_explicit", + "emitted_asset_paths", + ], + &[], + ), + )?; + let dag: Bound<'py, PyAny> = arguments.value("dag")?; + let envelope: Bound<'py, PyAny> = arguments.value("envelope")?; + let command: Vec = arguments.value("command")?.extract()?; + let emitted_asset_paths: Bound<'py, PyAny> = arguments.value("emitted_asset_paths")?; + let selection = SelectionFacts::from_arguments(&arguments)?; + let nodes = nodes_by_id(&dag)?; + let source_kind: String = constant(py, "SOURCE_NODE_KIND")?.extract()?; + let clone_command: String = constant(py, "CLONE_COMMAND")?.extract()?; + let envelope_command: String = envelope.getattr("command")?.str()?.extract()?; + let mut plans: Vec> = Vec::new(); + let asset: Bound<'py, PyAny> = envelope.getattr("asset")?; + if !asset.is_none() && completed(py, &asset.getattr("status")?.str()?.extract::()?)? { + let resource_id: String = envelope.getattr("resource_id")?.str()?.extract()?; + let node: Option<&Bound<'py, PyAny>> = node_with_id(&nodes, &resource_id); + let asset_facts: Bound<'py, PyAny> = asset.call_method0("as_dict")?; + let mut projected: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); + if let Some(node) = node { + projected.push((text_of(node, "id")?, asset_facts.clone())); + let loader: Bound<'py, PyAny> = asset.getattr("loader")?; + let loader_facts: Option<(String, Bound<'py, PyDict>)> = + if text_of(node, "kind")? == source_kind && !loader.is_none() { + loader_result( + &dag, + (node, &asset_facts), + &loader.str()?.extract::()?, + )? + } else { + None + }; + if let Some((loader_id, facts)) = loader_facts + && node_with_id(&nodes, &loader_id).is_some() + { + projected.push((loader_id, facts.into_any())); + } + } + let kind: &str = if envelope_command == clone_command { + MATERIALIZATION + } else { + MATERIALIZE + }; + let fields = envelope_fields( + &envelope, + &[ + "event_id", + "resource_attempt_id", + "resource_id", + "event_sequence", + "duration_ms", + ], + )?; + for candidate in sorted_nodes(&dag)? { + let id: String = text_of(&candidate, "id")?; + let Some((_, facts)) = projected.iter().rev().find(|(listed, _)| *listed == id) else { + continue; + }; + if !asset_enabled(&candidate)? { + continue; + } + let path: AssetPath = asset_path(&candidate)?; + if !selection.paths.is_empty() && !selection.paths.contains(&path) { + continue; + } + let mut base: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ( + "command", + PyString::new(py, &command_text(&command)).into_any(), + ), + ("sqlbuild_id", get(&candidate, "id")?), + ]; + for (name, value) in &fields { + base.push((name.as_str(), value.clone())); + } + plans.push(plan( + py, + (kind, &path), + merged(&base, &metadata_from_mapping(facts)?)?, + None, + )?); + } + } + let standalone: bool = constant_contains(py, ("STANDALONE_CHECK_COMMANDS", &envelope_command))?; + let emitted: HashSet = path_set(&emitted_asset_paths)?; + let filter = CheckFilter { + selection: &selection, + selected_paths: &selection.paths, + emitted_asset_paths: (!selection.check_only() && !standalone).then_some(&emitted), + }; + let check_fields = envelope_fields( + &envelope, + &[ + "event_id", + "resource_attempt_id", + "resource_id", + "event_sequence", + ], + )?; + let mut seen: HashSet = HashSet::new(); + for check in envelope.getattr("checks")?.try_iter()? { + let facts: Bound<'py, PyDict> = check?.call_method0("as_dict")?.cast_into()?; + let record: Bound<'py, PyDict> = facts.copy()?; + for (name, value) in &check_fields { + record.set_item(name, value)?; + } + let (check_results, updated) = + check_plans((&dag, &nodes), record.as_any(), (&filter, seen))?; + plans.extend(check_results); + seen = updated; + } + keep_checks_only(plans, &selection) +} + +/// Dagster events for selected materializable assets when no execution document exists. +#[pyfunction] +#[pyo3(signature = (*, dag, command, selected_asset_paths))] +pub(crate) fn dagster_selected_asset_results<'py>( + dag: Bound<'py, PyAny>, + command: Vec, + selected_asset_paths: Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = dag.py(); + let selected: HashSet = path_set(&selected_asset_paths)?; + let materializable: Bound<'py, PyAny> = constant(py, "MATERIALIZABLE_NODE_KINDS")?; + let mut plans: Vec> = Vec::new(); + for node in sorted_nodes(&dag)? { + if !selected.is_empty() && !selected.contains(&asset_path(&node)?) { + continue; + } + if !asset_enabled(&node)? || !materializable.contains(text_of(&node, "kind")?)? { + continue; + } + let path: AssetPath = asset_path(&node)?; + let metadata: Bound<'py, PyDict> = PyDict::new(py); + metadata.set_item("command", command_text(&command))?; + metadata.set_item("sqlbuild_id", get(&node, "id")?)?; + plans.push(plan(py, (MATERIALIZE, &path), metadata, None)?); + } + Ok(plans) +} + +/// The version-1 execution document in `text`, or `None` when it is absent or not one. +#[pyfunction] +#[pyo3(signature = (*, text))] +pub(crate) fn dagster_execution_payload<'py>( + text: Bound<'py, PyString>, +) -> PyResult>> { + let py: Python<'py> = text.py(); + let stripped: Bound<'py, PyAny> = text.call_method0("strip")?; + if !stripped.is_truthy()? { + return Ok(None); + } + let payload: Bound<'py, PyAny> = match imported(py, "json", "loads")?.call1((stripped,)) { + Ok(payload) => payload, + Err(error) if error.is_instance_of::(py) => { + let decode: Bound<'py, PyAny> = imported(py, "json", "JSONDecodeError")?; + if error.value(py).is_instance(&decode)? { + return Ok(None); + } + return Err(error); + } + Err(error) => return Err(error), + }; + if !payload.is_instance(&imported(py, "collections.abc", "Mapping")?)? { + return Ok(None); + } + let version: Bound<'py, PyAny> = get(&payload, "version")?; + if !version.eq(EXECUTION_DOCUMENT_VERSION.into_pyobject(py)?)? { + return Ok(None); + } + Ok(Some(payload)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs index 1631392a1a..b5b97e059d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs @@ -14,10 +14,11 @@ use sqlbuild_core::dbt_command::models::ForwardedArg; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::dbt_command::support::{color_enabled, flag, path, stream}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const DISCOVERY_CONSTANTS: &str = "sqlbuild.compiler.discovery.constants"; const PROGRESS_REPORTER_MODULE: &str = "sqlbuild._native"; -const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; fn project_config_found(directory: &Bound<'_, PyAny>) -> PyResult { let name: Bound<'_, PyAny> = imported( @@ -91,7 +92,7 @@ pub(crate) fn ensure_twin_project<'py>( ], )?; let callbacks: Bound<'py, PyAny> = call_with( - &imported(py, DBT_MODELS, "DbtInitProgressCallbacks")?, + &imported(py, NATIVE_RECORDS_MODULE, "DbtInitProgressCallbacks")?, &[ ("start", progress.getattr("start")?), ("complete", progress.getattr("complete")?), @@ -107,7 +108,7 @@ pub(crate) fn ensure_twin_project<'py>( None => none.clone(), }; let request: Bound<'py, PyAny> = call_with( - &imported(py, DBT_MODELS, "DbtInitRequest")?, + &imported(py, NATIVE_RECORDS_MODULE, "DbtInitRequest")?, &[ ("cwd", effective_dir), ("dbt_project_dir", dbt_project_dir), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs index 847766f82d..7f1200a679 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::dbt_interop::runtime_config::validate_dbt_executi const NATIVE_MODULE: &str = "sqlbuild._native"; const PLAN_MODULE: &str = "sqlbuild._native"; const EXECUTE_MODULE: &str = "sqlbuild._native"; -const PRESENTATION_MODELS: &str = "sqlbuild.presentation.models"; +const PRESENTATION_MODELS: &str = "sqlbuild._native"; const PLAN_COMMAND: &str = "plan"; const DEBUG_COMMAND: &str = "debug"; const BOUNDED_SECTION_ENTRIES: i64 = 10; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs index 8533136121..5ead8833fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs @@ -14,10 +14,11 @@ use sqlbuild_core::dbt_command::models::DbtInitSummary; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::dbt_command::support::{color_enabled, flag, path, stream}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild._native"; const PROGRESS_REPORTER_MODULE: &str = "sqlbuild._native"; -const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; fn missing_project(py: Python<'_>) -> PyErr { let raised = (|| { @@ -90,14 +91,14 @@ fn run_dbt_init_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult< ], )?; let callbacks: Bound<'_, PyAny> = call_with( - &imported(py, DBT_MODELS, "DbtInitProgressCallbacks")?, + &imported(py, NATIVE_RECORDS_MODULE, "DbtInitProgressCallbacks")?, &[ ("start", progress.getattr("start")?), ("complete", progress.getattr("complete")?), ], )?; let init_request: Bound<'_, PyAny> = call_with( - &imported(py, DBT_MODELS, "DbtInitRequest")?, + &imported(py, NATIVE_RECORDS_MODULE, "DbtInitRequest")?, &[ ("cwd", request.getattr("cwd")?), ("dbt_project_dir", project_dir), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs index 55a258af72..ae69b05319 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs @@ -20,8 +20,9 @@ use crate::bindings::_helpers::dbt_interop::runner::launch_os_error; use crate::bindings::types::CompilerDetach; use sqlbuild_config::dbt_profile::models::DbtProjectMetadata; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; -const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; const DBT_EXCEPTIONS: &str = "sqlbuild.integrations.dbt.exceptions"; const PROFILE_RENDER_MODULE: &str = "sqlbuild.integrations.dbt._helpers.profile.render"; const PROJECT_FILE: &str = "sqlbuild_project.toml"; @@ -147,7 +148,7 @@ fn build_dbt_init_project<'py>(request: &Bound<'py, PyAny>) -> PyResult>( py: Python<'py>, @@ -48,7 +49,7 @@ pub(crate) fn python_result<'py>( }) .collect::>()?; call_with( - &imported(py, DBT_MODELS, "DbtNodeExecutionResult")?, + &imported(py, NATIVE_RECORDS_MODULE, "DbtNodeExecutionResult")?, &[ ("unique_id", text(py, &result.unique_id)), ("resource_type", text(py, &result.resource_type)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs index 371202fafb..e8af0fab78 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs @@ -1,5 +1,6 @@ //! dbt argv building, executable choice, `dbt ls` output, `sqb dbt` argument routing and config. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class_at; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -250,8 +251,8 @@ pub(crate) fn raised(py: Python<'_>, class_name: &str, error: DbtCliError) -> Py if let Some(help) = error.help { keywords.set_item("help", help)?; } - let instance: Bound<'_, PyAny> = - imported(py, DBT_EXCEPTIONS, class_name)?.call((error.message,), Some(&keywords))?; + let instance: Bound<'_, PyAny> = shape_class_at(py, DBT_EXCEPTIONS, class_name)? + .call((error.message,), Some(&keywords))?; Ok(PyErr::from_value(instance)) }; created().unwrap_or_else(|failure| failure) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs index ae9fcfc08d..c29ad70a61 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs @@ -24,8 +24,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::dbt_interop::manifest::resolve_sqlbuild_model_dbt_refs; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_RECORDS: &str = "sqlbuild._native"; -const DBT_EXCEPTIONS: &str = "sqlbuild.integrations.dbt.exceptions"; const SELECTOR_EXPANSION_MODULE: &str = "sqlbuild._native"; const MODEL_SELECTORS_MODULE: &str = "sqlbuild._native"; const MODEL_RESOURCE_TYPE: &str = "model"; @@ -232,7 +233,7 @@ fn selection_error(py: Python<'_>, error: DbtSelectionError) -> PyErr { let keywords: Bound<'_, PyDict> = PyDict::new(py); keywords.set_item("code", PATH_SELECTOR_ERROR_CODE)?; let error: Bound<'_, PyAny> = - imported(py, DBT_EXCEPTIONS, "DbtInteropArgumentError")? + imported(py, NATIVE_RECORDS_MODULE, "DbtInteropArgumentError")? .call((message,), Some(&keywords))?; Ok(PyErr::from_value(error)) }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs index 43226da479..ab89bc5a93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs @@ -64,12 +64,8 @@ fn listed<'py>( || py.None().into_bound(py), |detail| PyString::new(py, &detail).into_any(), ); - let error: Bound<'py, PyAny> = imported( - py, - "sqlbuild.integrations.dbt.exceptions", - "DbtInteropRuntimeError", - )? - .call(("dbt ls failed",), Some(&keywords(py, &[("help", help)])?))?; + let error: Bound<'py, PyAny> = imported(py, "sqlbuild._native", "DbtInteropRuntimeError")? + .call(("dbt ls failed",), Some(&keywords(py, &[("help", help)])?))?; Err(PyErr::from_value(error)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs index 83de2c34b8..d22fbee4a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs @@ -15,8 +15,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::dbt_interop::pipeline_prologue::progress; use crate::bindings::_helpers::dbt_interop::runtime_config::config_overrides; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_RECORDS: &str = "sqlbuild._native"; -const MODELS: &str = "sqlbuild.integrations.dbt.models"; const PLAN_OPTIONS: (&[&str], &[&str]) = ( &["project_dir", "args"], &[ @@ -107,12 +108,7 @@ fn execute_dbt_interop_from_project(request: &Bound<'_, PyAny>) -> PyResult let kind = imported(py, "sqlbuild._native", "DbtInteropCommand")?; if !command.eq(kind.getattr("RUN")?)? && !command.eq(kind.getattr("BUILD")?)? { return Err(PyErr::from_value( - imported( - py, - "sqlbuild.integrations.dbt.exceptions", - "DbtInteropArgumentError", - )? - .call1((format!( + imported(py, "sqlbuild._native", "DbtInteropArgumentError")?.call1((format!( "unsupported dbt interop execution command: {}", command.str()? ),))?, @@ -348,7 +344,7 @@ fn debug_dbt_from_project(py: Python<'_>, options: Option>) -> ], )?; let dbt_options = call_with( - &imported(py, MODELS, "DbtCliOptions")?, + &imported(py, NATIVE_RECORDS_MODULE, "DbtCliOptions")?, &[ ("project_dir", resolved.getattr("project_dir")?), ("profiles_dir", resolved.getattr("profiles_dir")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs index 485fddb95b..e2c81bcc2a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs @@ -98,7 +98,7 @@ fn write_dbt_execution_plan_text( ( "display_options", call_with( - &imported(py, "sqlbuild.presentation.models", "DisplayOptions")?, + &imported(py, "sqlbuild._native", "DisplayOptions")?, &[("max_entries_per_section", limit)], )?, ), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs index 939f0d94b2..f77d959915 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs @@ -145,12 +145,7 @@ fn load_compiled_dbt_manifest<'py>( )?; if result.getattr("returncode")?.extract::()? != 0 { let detail = imported(py, "sqlbuild._native", "dbt_failure_detail")?.call1((result,))?; - let error = imported( - py, - "sqlbuild.integrations.dbt.exceptions", - "DbtInteropRuntimeError", - )? - .call( + let error = imported(py, "sqlbuild._native", "DbtInteropRuntimeError")?.call( ("dbt compile failed",), Some( &crate::bindings::_helpers::compile_driver::python_calls::keywords( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs index 764aa3e0ce..c47a3cf46f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const DBT_TYPES: &str = "sqlbuild._native"; -const PRESENTATION_MODELS: &str = "sqlbuild.presentation.models"; +const PRESENTATION_MODELS: &str = "sqlbuild._native"; const BUILD_OPTIONS: (&[&str], &[&str]) = ( &[ "command", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index 084519dff4..61ba953612 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -13,6 +13,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_RECORDS: &str = "sqlbuild._native"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const HOOKS: &str = "sqlbuild.runtime.contracts.models"; @@ -167,11 +169,11 @@ pub(crate) fn build_sqlbuild_plan_output<'py>( )?; let full_refresh = boolean(py, args.contains("--full-refresh")?); let selection = call_with( - &imported(py, PLANNER_MODELS, "PlannerSelection")?, + &imported(py, NATIVE_RECORDS_MODULE, "PlannerSelection")?, &[("select", selected.clone())], )?; let overrides = call_with( - &imported(py, PLANNER_MODELS, "PlannerOverrides")?, + &imported(py, NATIVE_RECORDS_MODULE, "PlannerOverrides")?, &[ ("cursor_overrides", cursor), ("full_refresh", full_refresh.clone()), @@ -193,7 +195,7 @@ pub(crate) fn build_sqlbuild_plan_output<'py>( ("deferral", deferral), ( "policies", - imported(py, PLANNER_MODELS, "PlannerPolicies")?.call0()?, + imported(py, NATIVE_RECORDS_MODULE, "PlannerPolicies")?.call0()?, ), ("on_progress", hooks.getattr("on_progress")?), ("project_config", discovered.getattr("project_config")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs index 289cde0880..2e1672a8f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs @@ -16,7 +16,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const DBT_ERRORS_MODULE: &str = "sqlbuild.integrations.dbt.exceptions"; const INIT_TOML_OPTIONS: (&[&str], &[&str]) = ( &[ @@ -82,7 +83,7 @@ fn normalized_connection<'py>( connection.set_item(key, value)?; } call_with( - &imported(py, DBT_MODELS, "NormalizedDbtProfileConnection")?, + &imported(py, NATIVE_RECORDS_MODULE, "NormalizedDbtProfileConnection")?, &[ ("adapter", PyString::new(py, profile.adapter).into_any()), ("connection", connection.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs index 5f66035ae2..2a17ec7c68 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs @@ -22,8 +22,8 @@ use crate::bindings::_helpers::dbt_interop::cli::{ }; use crate::bindings::types::CompilerDetach; -const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; -const DBT_EXCEPTIONS: &str = "sqlbuild.integrations.dbt.exceptions"; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const LAUNCH_FAILURE: &str = "failed to execute dbt"; const POISONED_CACHE: &str = "dbt ls cache is poisoned"; @@ -123,7 +123,7 @@ impl DbtRunner { let result: Bound<'py, PyAny> = self.run(command, &options.getattr("project_dir")?)?; let stdout: String = result.getattr("stdout")?.extract()?; let listing: Bound<'py, PyAny> = call_with( - &imported(py, DBT_MODELS, "DbtLsResult")?, + &imported(py, NATIVE_RECORDS_MODULE, "DbtLsResult")?, &[ ("nodes", ls_nodes(py, &stdout)?.into_any()), ("command", result), @@ -191,7 +191,7 @@ impl DbtRunner { }); let output: DbtProcessOutput = output.map_err(|error| launch_failure(py, error))?; call_with( - &imported(py, DBT_MODELS, "DbtCommandResult")?, + &imported(py, NATIVE_RECORDS_MODULE, "DbtCommandResult")?, &[ ("argv", command.into_any()), ("returncode", output.returncode.into_pyobject_any(py)?), @@ -282,8 +282,9 @@ pub(crate) fn launch_failure(py: Python<'_>, error: PyErr) -> PyErr { let created = || -> PyResult { let options: Bound<'_, PyDict> = PyDict::new(py); options.set_item("help", error.value(py).str()?)?; - let instance: Bound<'_, PyAny> = imported(py, DBT_EXCEPTIONS, "DbtInteropRuntimeError")? - .call((LAUNCH_FAILURE,), Some(&options))?; + let instance: Bound<'_, PyAny> = + imported(py, NATIVE_RECORDS_MODULE, "DbtInteropRuntimeError")? + .call((LAUNCH_FAILURE,), Some(&options))?; let raised: PyErr = PyErr::from_value(instance); raised.set_cause(py, Some(error.clone_ref(py))); Ok(raised) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs index b2316c094d..6236e085bd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs @@ -15,7 +15,8 @@ use sqlbuild_core::dbt_cli::models::DbtParsedFlags; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::dbt_interop::cli::{raised, resolved_path}; -const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const BUILTIN_ADAPTERS_MODULE: &str = "sqlbuild.adapter.discovery.main.builtins"; const PROJECT_ADAPTERS_MODULE: &str = "sqlbuild.adapter.discovery.main.project_adapters"; @@ -284,7 +285,7 @@ pub(crate) fn plan_options<'py>( )? }; call_with( - &imported(py, DBT_MODELS, "DbtCliOptions")?, + &imported(py, NATIVE_RECORDS_MODULE, "DbtCliOptions")?, &[ ("project_dir", resolved.getattr("project_dir")?), ("profiles_dir", resolved.getattr("profiles_dir")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs index bd0752a308..b49aeecd87 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs @@ -8,8 +8,9 @@ use pyo3::types::{ }; use std::collections::HashSet; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_RECORDS: &str = "sqlbuild._native"; -const MODELS: &str = "sqlbuild.integrations.dbt.models"; pub(crate) fn result_ids(results: &Bound<'_, PyAny>) -> PyResult> { results @@ -129,7 +130,7 @@ pub(crate) fn append_missing<'py>( py.None().into_bound(py) }; appended.push(call_with( - &imported(py, MODELS, "DbtNodeExecutionResult")?, + &imported(py, NATIVE_RECORDS_MODULE, "DbtNodeExecutionResult")?, &[ ("unique_id", unique_id), ("resource_type", PyString::new(py, resource_type).into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs index 5c0e6178eb..667afa6b83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs @@ -1,5 +1,6 @@ //! Declaration file collections discovered natively and built as Python discovery records. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class_at; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; @@ -30,6 +31,8 @@ use crate::bindings::_helpers::model_config::header_columns::schema_columns; use crate::bindings::_helpers::sqltext::authored_values::map_to_python; use crate::bindings::models::{NativeDiscoverySession, NativeProjectTree}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const AUDITING_MODELS_MODULE: &str = "sqlbuild.compiler.auditing.models"; @@ -73,7 +76,7 @@ impl<'py> FileContext<'_, 'py> { } fn record(&self, class: &str, fields: &Bound<'py, PyDict>) -> PyResult> { - imported(self.py(), MODELS_MODULE, class)?.call((), Some(fields)) + shape_class_at(self.py(), MODELS_MODULE, class)?.call((), Some(fields)) } } @@ -129,7 +132,7 @@ fn single_value<'py>( fn enum_file<'py>(context: &FileContext<'_, 'py>, file: &EnumFile) -> PyResult> { let py: Python<'py> = context.py(); - let member_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "EnumMember")?; + let member_class: Bound<'py, PyAny> = imported(py, NATIVE_RECORDS_MODULE, "EnumMember")?; let declaration_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "EnumDeclaration")?; let mut declarations: Vec> = Vec::with_capacity(file.declarations.len()); for declaration in &file.declarations { @@ -217,7 +220,7 @@ fn schema_file<'py>( let py: Python<'py> = context.py(); let file_path: String = context.file_path.str()?.to_string(); let declaration_class: Bound<'py, PyAny> = - imported(py, MODELS_MODULE, "ModelSchemaDeclaration")?; + imported(py, NATIVE_RECORDS_MODULE, "ModelSchemaDeclaration")?; let mut declarations: Vec> = Vec::with_capacity(file.declarations.len()); for declaration in &file.declarations { let raw_columns: Bound<'py, PyAny> = @@ -313,7 +316,7 @@ fn audit_block<'py>(py: Python<'py>, block: &AuditBlock) -> PyResult( files: &[ScopedFile], ) -> PyResult> { let py: Python<'py> = faults.project_dir.py(); - let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "DiscoveredSeedFile")?; + let class: Bound<'py, PyAny> = imported(py, NATIVE_RECORDS_MODULE, "DiscoveredSeedFile")?; let seeds: Vec> = files .iter() .map(|scoped| { @@ -669,7 +672,7 @@ pub(crate) fn discover_adapter_file<'py>( return Ok(py.None().into_bound(py)); } call_with( - &imported(py, MODELS_MODULE, "DiscoveredAdapterFile")?, + &imported(py, NATIVE_RECORDS_MODULE, "DiscoveredAdapterFile")?, &[ ("file_path", file_path), ("relative_path", path_class(py)?.call1(("adapter.py",))?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs index 8105fa7314..56b7b29f06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs @@ -6,7 +6,7 @@ use pyo3::types::PyTuple; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const FAULT_CLASS: &str = "DiscoveryFileFault"; const PROJECT_DIR_MARKER: &str = "."; @@ -46,7 +46,7 @@ impl<'py> FileFaults<'py> { .str()? .call_method1("replace", (self.project_dir.str()?, PROJECT_DIR_MARKER))?; on_fault.call1((call_with( - &imported(py, DISCOVERY_MODELS_MODULE, FAULT_CLASS)?, + &imported(py, NATIVE_RECORDS_MODULE, FAULT_CLASS)?, &[("path", relative_path.clone()), ("message", message)], )?,))?; Ok(None) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs index 93c2ccccc8..bceaa896b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs @@ -15,14 +15,16 @@ use sqlbuild_model_config::declarations::models::{ use sqlbuild_model_config::sql_values::errors::SqlValueFailure; use sqlbuild_model_config::sql_values::models::RawSqlValue; +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class; use crate::bindings::_helpers::contracts::sql_values::{ SqlValueClasses, raw_value, sql_value, sql_value_error, }; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; -const IDENTITY_ERRORS_MODULE: &str = "sqlbuild.compiler.resource_names.exceptions"; -const RENDERING_MODULE: &str = "sqlbuild.sql_values.types"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; +const IDENTITY_ERRORS_MODULE: &str = "sqlbuild._native"; fn declaration_error(py: Python<'_>, failure: DeclarationFailure) -> PyResult { Ok(match failure { @@ -58,7 +60,7 @@ fn enum_record<'py>( owner: &Owner<'py>, ) -> PyResult> { let models = py.import(MODELS_MODULE)?; - let member_class = models.getattr("EnumMember")?; + let member_class = shape_class(models.as_any(), "EnumMember")?; let int_class = py.import("builtins")?.getattr("int")?; let members = facts .members @@ -88,7 +90,7 @@ fn constant_record<'py>( ) -> PyResult> { let render_as = match facts.render_as { Some(rendering) => py - .import(RENDERING_MODULE)? + .import(NATIVE_RECORDS_MODULE)? .getattr("CollectionRendering")? .call1((rendering,))?, None => py.None().into_bound(py), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs index 8ada475801..a5d2082a40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs @@ -8,7 +8,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::keyword_request::Request; const NATIVE_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; const HEADER_CACHE_SIZE: usize = 4096; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs new file mode 100644 index 0000000000..6f8dde6114 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs @@ -0,0 +1,189 @@ +//! Shared reads for imported project Python discovery: paths, shim calls and records. + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::event_exporting::scope::current_scope; + +pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; +pub(crate) const SHIM_MODULE: &str = + "sqlbuild.compiler.discovery._helpers.filesystem.project_modules"; +pub(crate) const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +pub(crate) const ERRORS_MODULE: &str = "sqlbuild._native"; + +/// One project directory under discovery. +pub(crate) struct ProjectDir<'py> { + pub(crate) path: Bound<'py, PyAny>, +} + +impl<'py> ProjectDir<'py> { + pub(crate) fn py(&self) -> Python<'py> { + self.path.py() + } + + /// `project_dir / name`. + pub(crate) fn child(&self, name: &str) -> PyResult> { + self.path.div(name) + } + + /// `project_relative_path(path=file_path, project_dir=project_dir)`. + pub(crate) fn relative(&self, file_path: &Bound<'py, PyAny>) -> PyResult> { + call_with( + &imported(self.py(), NATIVE_MODULE, "project_relative_path")?, + &[ + ("path", file_path.clone()), + ("project_dir", self.path.clone()), + ], + ) + } + + /// `str(file_path.relative_to(project_dir))`. + pub(crate) fn relative_to(&self, file_path: &Bound<'py, PyAny>) -> PyResult { + file_path + .call_method1("relative_to", (&self.path,))? + .str()? + .extract() + } + + /// Python files under `root`, in discovery order. + pub(crate) fn module_files( + &self, + root: &str, + public_only: bool, + ) -> PyResult>> { + let py: Python<'py> = self.py(); + call_with( + &imported(py, NATIVE_MODULE, "discover_python_module_files")?, + &[ + ("project_dir", self.path.clone()), + ("root", PyString::new(py, root).into_any()), + ( + "public_only", + pyo3::types::PyBool::new(py, public_only) + .to_owned() + .into_any(), + ), + ], + )? + .try_iter()? + .collect() + } + + /// `shim.(file_path=file_path, project_dir=project_dir)`. + pub(crate) fn load( + &self, + loader: &str, + file_path: &Bound<'py, PyAny>, + ) -> PyResult> { + call_with( + &shim(self.py(), loader)?, + &[ + ("file_path", file_path.clone()), + ("project_dir", self.path.clone()), + ], + ) + } + + /// The cached providers, lifecycle sinks and output sinks of a warm command, if any. + pub(crate) fn cached_extensions(&self) -> PyResult>> { + let scope: Bound<'py, PyAny> = current_scope(self.py())?; + if scope.is_none() { + return Ok(None); + } + let cached: Bound<'py, PyAny> = call_with( + &scope.getattr("cached_extensions")?, + &[("project_dir", self.path.clone())], + )?; + Ok((!cached.is_none()).then_some(cached)) + } +} + +/// `project_modules.`. +pub(crate) fn shim<'py>(py: Python<'py>, name: &str) -> PyResult> { + imported(py, SHIM_MODULE, name) +} + +/// Functions defined in one loaded module. +pub(crate) fn module_functions<'py>( + module: &Bound<'py, PyAny>, +) -> PyResult>> { + shim(module.py(), "module_functions")? + .call1((module,))? + .try_iter()? + .collect() +} + +/// `sqlbuild.compiler.discovery.models.(**pairs)`. +pub(crate) fn discovered<'py>( + py: Python<'py>, + class_name: &str, + pairs: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with(&imported(py, MODELS_MODULE, class_name)?, pairs) +} + +/// `sqlbuild._native.(message)`. +pub(crate) fn discovery_error(py: Python<'_>, class_name: &str, message: String) -> PyErr { + imported(py, ERRORS_MODULE, class_name) + .and_then(|class| class.call1((message,))) + .map_or_else(|error| error, PyErr::from_value) +} + +/// `discovery_error` raised from `cause`. +pub(crate) fn discovery_error_from( + py: Python<'_>, + (class_name, message): (&str, String), + cause: PyErr, +) -> PyErr { + let error: PyErr = discovery_error(py, class_name, message); + error.set_cause(py, Some(cause)); + error +} + +/// Whether `error` is an `Exception`, which discovery wraps; other exceptions propagate. +pub(crate) fn is_exception(py: Python<'_>, error: &PyErr) -> bool { + error.is_instance_of::(py) +} + +/// `{provider.name: provider for provider in providers}`. +pub(crate) fn provider_by_name<'py>(providers: &Bound<'py, PyAny>) -> PyResult> { + let by_name: Bound<'py, PyDict> = PyDict::new(providers.py()); + for provider in providers.try_iter()? { + let provider: Bound<'py, PyAny> = provider?; + by_name.set_item(provider.getattr("name")?, provider)?; + } + Ok(by_name) +} + +/// The provider parameters of one loaded user function. +pub(crate) fn provider_usages<'py>( + function: &Bound<'py, PyAny>, + provider_by_name: &Bound<'py, PyDict>, +) -> PyResult> { + call_with( + &shim(function.py(), "provider_usages")?, + &[ + ("function", function.clone()), + ("provider_by_name", provider_by_name.clone().into_any()), + ], + ) +} + +/// A tuple of the given items. +pub(crate) fn tuple_of<'py>( + py: Python<'py>, + items: Vec>, +) -> PyResult> { + Ok(PyTuple::new(py, items)?.into_any()) +} + +/// `name` of the read definition, or `None` when the function carries no such definition. +pub(crate) fn definition<'py>( + function: &Bound<'py, PyAny>, + (module, reader): (&str, &str), +) -> PyResult>> { + let read: Bound<'py, PyAny> = imported(function.py(), module, reader)?.call1((function,))?; + Ok((!read.is_none()).then_some(read)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs new file mode 100644 index 0000000000..1ea77ab871 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs @@ -0,0 +1,373 @@ +//! Provider classes and lifecycle-event sinks from imported providers/ and sinks/ modules. + +use std::collections::HashSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyString}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::discovery::imported_modules::{ + NATIVE_MODULE, ProjectDir, discovered, discovery_error, discovery_error_from, module_functions, + provider_by_name, provider_usages, shim, tuple_of, +}; + +const PROVIDER_ERROR: &str = "ProviderDiscoveryError"; +const SINK_ERROR: &str = "EventExporterDiscoveryError"; +const PROVIDER_INPUT_ERROR: (&str, &str) = ("sqlbuild.provider.exceptions", "ProviderInputError"); +const VALIDATION_ERROR: (&str, &str) = ("pydantic", "ValidationError"); +const SINK_DEFINITION: (&str, &str) = ( + "sqlbuild.runtime.event_exporting.main.get_lifecycle_event_sink_definition", + "get_lifecycle_event_sink_definition", +); +const ROOT_LOCATION: &str = ""; +const DEFAULT_MESSAGE: &str = "invalid value"; +const DEFAULT_ERROR_TYPE: &str = "validation_error"; +const DEFAULT_SETTINGS_ERROR: &str = "invalid provider settings"; + +fn reject_invalid_identity( + name: &Bound<'_, PyAny>, + relative_path: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = name.py(); + call_with( + &imported(py, NATIVE_MODULE, "validate_resource_identity")?, + &[ + ("name", name.clone()), + ("kind", PyString::new(py, "provider").into_any()), + ("path", relative_path.clone()), + ], + )?; + Ok(()) +} + +/// The provider's validated name, from its explicit `provider_name` or `name()`. +fn provider_name<'py>( + provider_class: &Bound<'py, PyAny>, + relative_path: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = provider_class.py(); + let explicit: Bound<'py, PyAny> = provider_class.getattr("provider_name")?; + if !explicit.is_none() { + reject_invalid_identity(&explicit, relative_path)?; + } + let name: Bound<'py, PyAny> = match provider_class.call_method0("name") { + Ok(name) => name, + Err(error) => { + let input_error: Bound<'py, PyAny> = + imported(py, PROVIDER_INPUT_ERROR.0, PROVIDER_INPUT_ERROR.1)?; + if !error.value(py).is_instance(&input_error)? { + return Err(error); + } + let message: String = format!( + "Provider class {} in {} has an invalid provider name: {}", + provider_class.getattr("__name__")?.str()?, + relative_path.str()?, + error.value(py) + ); + return Err(discovery_error_from(py, (PROVIDER_ERROR, message), error)); + } + }; + reject_invalid_identity(&name, relative_path)?; + Ok(name) +} + +/// `": []"` per pydantic error, one per line. +fn validation_details(error: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = error.py(); + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("include_input", false)?; + let mut details: Vec = Vec::new(); + for item in error + .call_method("errors", (), Some(&options))? + .try_iter()? + { + let item: Bound<'_, PyAny> = item?; + let location: Bound<'_, PyAny> = item.call_method1("get", ("loc", py.None()))?; + let location_text: String = if location.is_none() || !location.is_truthy()? { + ROOT_LOCATION.to_owned() + } else { + location + .try_iter()? + .map(|part| part?.str()?.extract::()) + .collect::>>()? + .join(".") + }; + let message: Bound<'_, PyAny> = item.call_method1("get", ("msg", DEFAULT_MESSAGE))?; + let error_type: Bound<'_, PyAny> = + item.call_method1("get", ("type", DEFAULT_ERROR_TYPE))?; + details.push(format!( + "{location_text}: {} [{}]", + message.str()?, + error_type.str()? + )); + } + if details.is_empty() { + return Ok(DEFAULT_SETTINGS_ERROR.to_owned()); + } + Ok(details.join("\n")) +} + +/// An instance of the provider class, reporting invalid settings with each field's error. +fn provider_instance<'py>( + provider_class: &Bound<'py, PyAny>, + (provider_name, relative_path): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = provider_class.py(); + provider_class.call0().or_else(|error| { + let validation_error: Bound<'py, PyAny> = + imported(py, VALIDATION_ERROR.0, VALIDATION_ERROR.1)?; + if !error.value(py).is_instance(&validation_error)? { + return Err(error); + } + let message: String = format!( + "Provider '{}' in {} has invalid settings:\n{}", + provider_name.str()?, + relative_path.str()?, + validation_details(error.value(py))? + ); + Err(discovery_error_from(py, (PROVIDER_ERROR, message), error)) + }) +} + +/// Provider classes under providers/. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +pub(crate) fn discover_provider_classes<'py>( + project_dir: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let project = ProjectDir { path: project_dir }; + let record_read: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "record_settings_class_read")?; + if let Some(cached) = project.cached_extensions()? { + let providers: Bound<'py, PyAny> = cached.get_item(0)?; + for provider in providers.try_iter()? { + record_read.call1((provider?.getattr("provider_class")?,))?; + } + return Ok(providers); + } + if !project + .child("providers")? + .call_method0("is_dir")? + .is_truthy()? + { + return tuple_of(py, Vec::new()); + } + let mut providers: Vec> = Vec::new(); + let mut seen: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); + for file_path in project.module_files("providers", true)? { + let module: Bound<'py, PyAny> = project.load("load_provider_module", &file_path)?; + let classes: Bound<'py, PyAny> = shim(py, "module_provider_classes")?.call1((&module,))?; + for provider_class in classes.try_iter()? { + let provider_class: Bound<'py, PyAny> = provider_class?; + let relative_path: Bound<'py, PyAny> = project.relative(&file_path)?; + let name: Bound<'py, PyAny> = provider_name(&provider_class, &relative_path)?; + let name_text: String = name.extract()?; + if let Some((_, existing)) = seen.iter().find(|(listed, _)| *listed == name_text) { + return Err(discovery_error( + py, + PROVIDER_ERROR, + format!( + "Duplicate provider name '{name_text}' found in {} and {}", + project.relative_to(existing)?, + project.relative_to(&file_path)? + ), + )); + } + seen.push((name_text, file_path.clone())); + record_read.call1((&provider_class,))?; + let settings: Bound<'py, PyAny> = + provider_instance(&provider_class, (&name, &relative_path))?; + providers.push(discovered( + py, + "DiscoveredProvider", + &[ + ("file_path", file_path.clone()), + ("relative_path", relative_path), + ("name", name), + ("provider_class", provider_class), + ("settings", settings), + ], + )?); + } + } + tuple_of(py, providers) +} + +/// Sink modules imported and their lifecycle declarations collected, without providers. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +pub(crate) fn discover_event_exporter_declarations<'py>( + project_dir: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let project = ProjectDir { path: project_dir }; + if project + .child("event_exporters")? + .call_method0("exists")? + .is_truthy()? + { + return Err(discovery_error( + py, + SINK_ERROR, + "event_exporters/ was replaced by sinks/ and @lifecycle_event_sink".to_owned(), + )); + } + if !project + .child("sinks")? + .call_method0("is_dir")? + .is_truthy()? + { + return tuple_of(py, Vec::new()); + } + let read_definition: Bound<'py, PyAny> = imported(py, SINK_DEFINITION.0, SINK_DEFINITION.1)?; + let mut declarations: Vec> = Vec::new(); + let mut seen: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); + for file_path in project.module_files("sinks", true)? { + let module: Bound<'py, PyAny> = project.load("load_sink_module", &file_path)?; + let mut seen_functions: HashSet = HashSet::new(); + for function in module_functions(&module)? { + if !seen_functions.insert(function.as_ptr() as usize) { + continue; + } + let read: Bound<'py, PyAny> = read_definition.call1((&function,))?; + if read.is_none() { + continue; + } + let name: String = read.getattr("name")?.extract()?; + if let Some((_, existing)) = seen.iter().find(|(listed, _)| *listed == name) { + return Err(discovery_error( + py, + SINK_ERROR, + format!( + "Duplicate event exporter name '{name}' found in {} and {}", + project.relative_to(existing)?, + project.relative_to(&file_path)? + ), + )); + } + let relative_path: Bound<'py, PyAny> = project.relative(&file_path)?; + call_with( + &shim(py, "validate_event_exporter_signature")?, + &[ + ("function", function.clone()), + ("exporter_name", read.getattr("name")?), + ("relative_path", relative_path.clone()), + ], + )?; + seen.push((name, file_path.clone())); + declarations.push(discovered( + py, + "DiscoveredEventExporterDeclaration", + &[ + ("file_path", file_path.clone()), + ("relative_path", relative_path), + ("name", read.getattr("name")?), + ("function", function.clone()), + ("event_kinds", read.getattr("event_kinds")?), + ("min_severity", read.getattr("min_severity")?), + ], + )?); + } + } + tuple_of(py, declarations) +} + +/// Provider parameters after the event, each naming a known provider of the right class. +fn exporter_provider_usages<'py>( + declaration: &Bound<'py, PyAny>, + (project, providers): (&ProjectDir<'py>, &Bound<'py, PyDict>), +) -> PyResult> { + let py: Python<'py> = project.py(); + let function: Bound<'py, PyAny> = declaration.getattr("function")?; + let name: String = declaration.getattr("name")?.extract()?; + let relative_path: String = project + .relative(&declaration.getattr("file_path")?)? + .str()? + .extract()?; + let empty: Bound<'py, PyAny> = imported(py, "inspect", "Parameter")?.getattr("empty")?; + let annotations: Bound<'py, PyAny> = shim(py, "parameter_annotations")?.call1((&function,))?; + for (index, pair) in annotations.try_iter()?.enumerate() { + if index == 0 { + continue; + } + let (parameter, annotation): (String, Bound<'py, PyAny>) = pair?.extract()?; + let Some(provider) = providers.get_item(¶meter)? else { + return Err(discovery_error( + py, + SINK_ERROR, + format!( + "Event exporter '{name}' in {relative_path} requires unknown provider '{parameter}'" + ), + )); + }; + let provider_class: Bound<'py, PyAny> = provider.getattr("provider_class")?; + if !annotation.is(&empty) && !annotation.is(&provider_class) { + return Err(discovery_error( + py, + SINK_ERROR, + format!( + "Event exporter '{name}' in {relative_path} provider parameter '{parameter}' must be unannotated or exactly {}", + provider_class.getattr("__name__")?.str()? + ), + )); + } + } + provider_usages(&function, providers) +} + +/// Declarations bound to the discovered project providers they name. +#[pyfunction] +#[pyo3(signature = (*, declarations, providers, project_dir))] +pub(crate) fn bind_event_exporter_declarations<'py>( + declarations: Bound<'py, PyAny>, + providers: Bound<'py, PyAny>, + project_dir: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let project = ProjectDir { path: project_dir }; + let providers: Bound<'py, PyDict> = provider_by_name(&providers)?; + let mut bound: Vec> = Vec::new(); + for declaration in declarations.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + let usages: Bound<'py, PyAny> = + exporter_provider_usages(&declaration, (&project, &providers))?; + let mut pairs: Vec<(&str, Bound<'py, PyAny>)> = Vec::new(); + for field in [ + "file_path", + "relative_path", + "name", + "function", + "event_kinds", + "min_severity", + ] { + pairs.push((field, declaration.getattr(field)?)); + } + pairs.push(("provider_usages", usages)); + bound.push(discovered(py, "DiscoveredEventExporter", &pairs)?); + } + tuple_of(py, bound) +} + +/// Validated lifecycle-event sink declarations under sinks/, bound to providers. +#[pyfunction] +#[pyo3(signature = (*, project_dir, providers = None))] +pub(crate) fn discover_event_exporter_functions<'py>( + project_dir: Bound<'py, PyAny>, + providers: Option>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let project = ProjectDir { + path: project_dir.clone(), + }; + if let Some(cached) = project.cached_extensions()? { + return cached.get_item(1); + } + let declarations: Bound<'py, PyAny> = + discover_event_exporter_declarations(project_dir.clone())?; + bind_event_exporter_declarations( + declarations, + providers.map_or_else(|| tuple_of(py, Vec::new()), Ok)?, + project_dir, + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs new file mode 100644 index 0000000000..b6f1b12efc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs @@ -0,0 +1,578 @@ +//! Decorated Python nodes, audit factories, hooks and materializations from imported modules. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::discovery::imported_modules::{ + MODELS_MODULE, NATIVE_MODULE, ProjectDir, definition, discovered, discovery_error, + discovery_error_from, is_exception, module_functions, provider_by_name, provider_usages, shim, + tuple_of, +}; +use crate::bindings::_helpers::discovery::imported_providers; + +const PYTHON_NODE_ROOT: &str = "python"; +const NODE_ERROR: &str = "PythonNodeDiscoveryError"; +const AUDIT_CASE: (&str, &str) = ("sqlbuild.python_nodes.models", "AuditCase"); +const READ_FACTORY: (&str, &str) = ( + "sqlbuild.python_nodes.main.read_factory_definition", + "read_factory_definition", +); +const READ_AUDIT_FACTORY: (&str, &str) = ( + "sqlbuild.python_nodes.main.read_audit_factory_definition", + "read_audit_factory_definition", +); +const READ_HOOK: (&str, &str) = ( + "sqlbuild.python_nodes.main.read_hook_definition", + "read_hook_definition", +); +const NOT_A_NODE: &str = "that is not a SQLBuild task, asset, loader, or check"; + +/// One Python-node kind: its reader, record class, bucket and copied definition fields. +struct NodeKind { + reader: (&'static str, &'static str), + record: &'static str, + bucket: usize, + fields: &'static [&'static str], +} + +/// Readers in precedence order; the first matching definition decides the kind. +const NODE_KINDS: [NodeKind; 4] = [ + NodeKind { + reader: ( + "sqlbuild.python_nodes.main.read_loader_definition", + "read_loader_definition", + ), + record: "DiscoveredLoaderFunction", + bucket: 0, + fields: &[ + "depends_on", + "destination", + "write_strategy", + "cursor_column", + "unique_key", + "columns", + "contract", + "description", + ], + }, + NodeKind { + reader: ( + "sqlbuild.python_nodes.main.read_task_definition", + "read_task_definition", + ), + record: "DiscoveredTaskFunction", + bucket: 1, + fields: &[ + "depends_on", + "tags", + "group", + "description", + "meta", + "retry", + ], + }, + NodeKind { + reader: ( + "sqlbuild.python_nodes.main.read_asset_definition", + "read_asset_definition", + ), + record: "DiscoveredAssetFunction", + bucket: 2, + fields: &[ + "depends_on", + "tags", + "group", + "description", + "meta", + "columns", + "column_lineage", + "retry", + ], + }, + NodeKind { + reader: ( + "sqlbuild.python_nodes.main.read_check_definition", + "read_check_definition", + ), + record: "DiscoveredCheckFunction", + bucket: 3, + fields: &[ + "depends_on", + "severity", + "tags", + "group", + "description", + "meta", + ], + }, +]; + +/// Discovered loaders, tasks, assets, checks and audit factories, in discovery order. +struct Buckets<'py> { + nodes: [Vec>; 4], + audit_factories: Vec<(String, String, Bound<'py, PyAny>)>, +} + +/// The module and the file it was imported from. +struct ImportedModule<'py> { + file_path: Bound<'py, PyAny>, + module: Bound<'py, PyAny>, +} + +impl<'py> Buckets<'py> { + /// Append `function` to its kind's bucket; `false` when it carries no node definition. + fn append_node( + &mut self, + (project, file_path): (&ProjectDir<'py>, &Bound<'py, PyAny>), + function: &Bound<'py, PyAny>, + providers: &Bound<'py, PyDict>, + ) -> PyResult { + let py: Python<'py> = project.py(); + for kind in &NODE_KINDS { + let Some(read) = definition(function, kind.reader)? else { + continue; + }; + let mut pairs: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("file_path", file_path.clone()), + ("relative_path", project.relative(file_path)?), + ("name", read.getattr("name")?), + ("function", function.clone()), + ]; + for field in kind.fields { + pairs.push((field, read.getattr(*field)?)); + } + pairs.push(("provider_usages", provider_usages(function, providers)?)); + self.nodes[kind.bucket].push(discovered(py, kind.record, &pairs)?); + return Ok(true); + } + Ok(false) + } + + /// Collect audit factories without routing them through Python-node discovery. + fn append_audit_factories( + &mut self, + project: &ProjectDir<'py>, + imported_module: &ImportedModule<'py>, + ) -> PyResult<()> { + let py: Python<'py> = project.py(); + for function in module_functions(&imported_module.module)? { + let Some(read) = definition(&function, READ_AUDIT_FACTORY)? else { + continue; + }; + let name: String = read.getattr("name")?.extract()?; + let relative: Bound<'py, PyAny> = project.relative(&imported_module.file_path)?; + let relative_text: String = relative.str()?.extract()?; + let kinds: Vec<&str> = node_definition_names(&function)?; + if !kinds.is_empty() { + return Err(discovery_error( + py, + NODE_ERROR, + format!( + "Audit factory '{name}' in {relative_text} cannot also be decorated as a Python-node kind ({})", + kinds.join(", ") + ), + )); + } + if let Some((_, existing, _)) = self + .audit_factories + .iter() + .find(|(listed, _, _)| *listed == name) + { + return Err(discovery_error( + py, + NODE_ERROR, + format!( + "Duplicate audit factory name '{name}' found in {existing} and {relative_text}" + ), + )); + } + let cases: Vec> = audit_cases(&function, (&name, &relative_text))?; + let line: Bound<'py, PyAny> = shim(py, "source_line")?.call1((&function,))?; + let record: Bound<'py, PyAny> = discovered( + py, + "DiscoveredAuditFactory", + &[ + ("name", read.getattr("name")?), + ("function", function.clone()), + ("file_path", imported_module.file_path.clone()), + ("relative_path", relative), + ("line", line), + ("cases", tuple_of(py, cases)?), + ], + )?; + self.audit_factories.push((name, relative_text, record)); + } + Ok(()) + } + + fn append_factory_nodes( + &mut self, + (project, imported_module): (&ProjectDir<'py>, &ImportedModule<'py>), + providers: &Bound<'py, PyDict>, + direct: &HashMap>, + ) -> PyResult<()> { + let py: Python<'py> = project.py(); + let location: String = project.relative_to(&imported_module.file_path)?; + for function in module_functions(&imported_module.module)? { + let Some(read) = definition(&function, READ_FACTORY)? else { + continue; + }; + let name: String = read.getattr("name")?.extract()?; + let result: Bound<'py, PyAny> = call_factory(&function, ("Factory", &name, &location))?; + for (index, generated) in factory_functions(&result, (&name, &location))? + .into_iter() + .enumerate() + { + if direct + .get(&(generated.as_ptr() as usize)) + .is_some_and(|known| known.is(&generated)) + { + continue; + } + if !self.append_node( + (project, &imported_module.file_path), + &generated, + providers, + )? { + return Err(discovery_error( + py, + NODE_ERROR, + format!( + "Factory '{name}' in {location} returned item {index} {NOT_A_NODE}" + ), + )); + } + } + } + Ok(()) + } + + fn discover_nodes( + &mut self, + project: &ProjectDir<'py>, + providers: &Bound<'py, PyDict>, + ) -> PyResult<()> { + let mut modules: Vec> = Vec::new(); + for file_path in project.module_files(PYTHON_NODE_ROOT, false)? { + let module: Bound<'py, PyAny> = + project.load("import_python_root_module", &file_path)?; + modules.push(ImportedModule { file_path, module }); + } + let mut direct: HashMap> = HashMap::new(); + for imported_module in &modules { + self.append_audit_factories(project, imported_module)?; + for function in module_functions(&imported_module.module)? { + if self.append_node((project, &imported_module.file_path), &function, providers)? { + direct.insert(function.as_ptr() as usize, function); + } + } + } + for imported_module in &modules { + self.append_factory_nodes((project, imported_module), providers, &direct)?; + } + Ok(()) + } +} + +/// The Python-node kinds an audit factory is also decorated as. +fn node_definition_names(function: &Bound<'_, PyAny>) -> PyResult> { + let mut names: Vec<&'static str> = Vec::new(); + if definition(function, READ_FACTORY)?.is_some() { + names.push("factory"); + } + for (kind, name) in NODE_KINDS.iter().zip(["loader", "task", "asset", "check"]) { + if definition(function, kind.reader)?.is_some() { + names.push(name); + } + } + Ok(names) +} + +fn requires_arguments(function: &Bound<'_, PyAny>) -> PyResult { + shim(function.py(), "requires_arguments")? + .call1((function,))? + .is_truthy() +} + +/// Call a zero-argument discovery factory, wrapping its failure. +fn call_factory<'py>( + factory: &Bound<'py, PyAny>, + (label, name, location): (&str, &str, &str), +) -> PyResult> { + let py: Python<'py> = factory.py(); + if requires_arguments(factory)? { + return Err(discovery_error( + py, + NODE_ERROR, + format!("{label} '{name}' in {location} must not require arguments"), + )); + } + factory.call0().map_err(|error| { + if !is_exception(py, &error) { + return error; + } + let message: String = format!( + "{label} '{name}' in {location} failed during discovery: {}", + error.value(py) + ); + discovery_error_from(py, (NODE_ERROR, message), error) + }) +} + +fn audit_cases<'py>( + factory: &Bound<'py, PyAny>, + (name, relative_path): (&str, &str), +) -> PyResult>> { + let py: Python<'py> = factory.py(); + let result: Bound<'py, PyAny> = call_factory(factory, ("Audit factory", name, relative_path))?; + if !result.is_instance_of::() && !result.is_instance_of::() { + return Err(discovery_error( + py, + NODE_ERROR, + format!( + "Audit factory '{name}' in {relative_path} must return a list or tuple of AuditCase instances" + ), + )); + } + let audit_case: Bound<'py, PyAny> = imported(py, AUDIT_CASE.0, AUDIT_CASE.1)?; + let mut cases: Vec> = Vec::new(); + for (index, item) in result.try_iter()?.enumerate() { + let item: Bound<'py, PyAny> = item?; + if !item.is_instance(&audit_case)? { + return Err(discovery_error( + py, + NODE_ERROR, + format!( + "Audit factory '{name}' in {relative_path} returned item {index} that is not an AuditCase" + ), + )); + } + cases.push(item); + } + Ok(cases) +} + +/// The functions a factory returned, rejecting anything but node functions. +fn factory_functions<'py>( + result: &Bound<'py, PyAny>, + (name, location): (&str, &str), +) -> PyResult>> { + let py: Python<'py> = result.py(); + if result.is_callable() { + return Ok(vec![result.clone()]); + } + let builtins: Bound<'py, PyAny> = py.import("builtins")?.into_any(); + let excluded: bool = result.is_instance_of::() + || result.is_instance_of::() + || result.is_instance_of::(); + let collection: bool = result.is_instance_of::() + || result.is_instance_of::() + || result.is_instance(&builtins.getattr("set")?)?; + if excluded || !collection { + return Err(discovery_error( + py, + NODE_ERROR, + format!( + "Factory '{name}' in {location} must return a SQLBuild node function or a list, tuple, or set of node functions" + ), + )); + } + let mut functions: Vec> = Vec::new(); + for (index, item) in result.try_iter()?.enumerate() { + let item: Bound<'py, PyAny> = item?; + if !item.is_callable() { + return Err(discovery_error( + py, + NODE_ERROR, + format!("Factory '{name}' in {location} returned item {index} {NOT_A_NODE}"), + )); + } + functions.push(item); + } + Ok(functions) +} + +/// Decorated Python DAG node functions under python/. +#[pyfunction] +#[pyo3(signature = (*, project_dir, providers = None))] +pub(crate) fn discover_python_node_functions<'py>( + project_dir: Bound<'py, PyAny>, + providers: Option>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let project = ProjectDir { path: project_dir }; + let providers: Bound<'py, PyDict> = match providers { + Some(providers) => provider_by_name(&providers)?, + None => PyDict::new(py), + }; + let mut buckets = Buckets { + nodes: [Vec::new(), Vec::new(), Vec::new(), Vec::new()], + audit_factories: Vec::new(), + }; + let node_root: Bound<'py, PyAny> = project.child(PYTHON_NODE_ROOT)?; + if node_root.call_method0("is_dir")?.is_truthy()? { + let saved: Bound<'py, PyAny> = call_with( + &shim(py, "install_python_root_package")?, + &[ + ("node_root", node_root), + ("project_dir", project.path.clone()), + ], + )?; + let discovered_nodes: PyResult<()> = buckets.discover_nodes(&project, &providers); + call_with( + &shim(py, "restore_python_root_modules")?, + &[("saved_modules", saved)], + )?; + discovered_nodes?; + } + let [loaders, tasks, assets, checks] = buckets.nodes; + discovered( + py, + "DiscoveredPythonNodeFunctions", + &[ + ("loaders", tuple_of(py, loaders)?), + ("tasks", tuple_of(py, tasks)?), + ("assets", tuple_of(py, assets)?), + ("checks", tuple_of(py, checks)?), + ( + "audit_factories", + tuple_of( + py, + buckets + .audit_factories + .into_iter() + .map(|(_, _, record)| record) + .collect(), + )?, + ), + ], + ) +} + +/// Decorated model lifecycle hook functions under hooks/python/. +#[pyfunction] +#[pyo3(signature = (*, project_dir, providers = None))] +pub(crate) fn discover_hook_functions<'py>( + project_dir: Bound<'py, PyAny>, + providers: Option>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let project = ProjectDir { path: project_dir }; + let providers: Bound<'py, PyDict> = match providers { + Some(providers) => provider_by_name(&providers)?, + None => PyDict::new(py), + }; + let hook_files: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_python_hook_files")?, + &[("project_dir", project.path.clone())], + )?; + let mut hooks: Vec> = Vec::new(); + let mut seen: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); + for hook_file in hook_files.try_iter()? { + let hook_file: Bound<'py, PyDict> = hook_file?.cast_into()?; + let file_path: Bound<'py, PyAny> = hook_file.as_any().get_item("file_path")?; + let module: Bound<'py, PyAny> = project.load("load_python_node_module", &file_path)?; + for function in module_functions(&module)? { + let Some(read) = definition(&function, READ_HOOK)? else { + continue; + }; + let name: String = read.getattr("name")?.extract()?; + if let Some((_, existing)) = seen.iter().find(|(listed, _)| *listed == name) { + return Err(discovery_error( + py, + NODE_ERROR, + format!( + "Duplicate hook name '{name}' found in {} and {}", + project.relative_to(existing)?, + project.relative_to(&file_path)? + ), + )); + } + seen.push((name, file_path.clone())); + let arguments: Bound<'py, PyDict> = hook_file.copy()?; + for (field, value) in [ + ("name", read.getattr("name")?), + ("function", function.clone()), + ("description", read.getattr("description")?), + ("reads", read.getattr("reads")?), + ("provider_usages", provider_usages(&function, &providers)?), + ] { + arguments.set_item(field, value)?; + } + hooks.push( + imported(py, MODELS_MODULE, "DiscoveredHookFunction")? + .call((), Some(&arguments))?, + ); + } + } + tuple_of(py, hooks) +} + +/// Custom materialization Python files under materializations/. +#[pyfunction] +#[pyo3(signature = (*, project_dir, providers = None))] +pub(crate) fn discover_materialization_files<'py>( + project_dir: Bound<'py, PyAny>, + providers: Option>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let project = ProjectDir { path: project_dir }; + if !project + .child("materializations")? + .call_method0("is_dir")? + .is_truthy()? + { + return tuple_of(py, Vec::new()); + } + let providers: Bound<'py, PyDict> = match providers { + Some(providers) => provider_by_name(&providers)?, + None => PyDict::new(py), + }; + let mut files: Vec> = Vec::new(); + for file_path in project.module_files("materializations", false)? { + let materialize: Bound<'py, PyAny> = + project.load("load_materialize_function", &file_path)?; + let usages: Bound<'py, PyAny> = if materialize.is_none() { + tuple_of(py, Vec::new())? + } else { + provider_usages(&materialize, &providers)? + }; + files.push(discovered( + py, + "DiscoveredMaterializationFile", + &[ + ("file_path", file_path.clone()), + ("relative_path", project.relative(&file_path)?), + ("name", file_path.getattr("stem")?), + ("provider_usages", usages), + ], + )?); + } + tuple_of(py, files) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(discover_python_node_functions, module)?)?; + module.add_function(wrap_pyfunction!(discover_hook_functions, module)?)?; + module.add_function(wrap_pyfunction!(discover_materialization_files, module)?)?; + module.add_function(wrap_pyfunction!( + imported_providers::discover_provider_classes, + module + )?)?; + module.add_function(wrap_pyfunction!( + imported_providers::discover_event_exporter_declarations, + module + )?)?; + module.add_function(wrap_pyfunction!( + imported_providers::bind_event_exporter_declarations, + module + )?)?; + module.add_function(wrap_pyfunction!( + imported_providers::discover_event_exporter_functions, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index 5b7d67aa84..285db345a7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -8,6 +8,9 @@ pub(crate) mod guidance; pub(crate) mod header_declarations; pub(crate) mod header_values; pub(crate) mod hook_entries; +pub(crate) mod imported_modules; +pub(crate) mod imported_providers; +pub(crate) mod imported_python_nodes; pub(crate) mod key_checks; pub(crate) mod locations; pub(crate) mod model_file_record; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_function_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_function_files.rs index 6e98da3393..52a298c6c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_function_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_function_files.rs @@ -24,8 +24,9 @@ use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, st use crate::bindings::_helpers::discovery::snapshot::{in_snapshot, project_root, project_tree}; use crate::bindings::models::NativeProjectTree; -const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; const PARSE_ERROR_CLASS: &str = "ModelSqlParseError"; const AST_MODULE: &str = "ast"; const DESCRIPTION_KEY: &str = "description"; @@ -249,7 +250,7 @@ fn function_record<'py>( }; let (header_values, entry_point, body_python) = parsed_function(&source)?; call_with( - &imported(py, MODELS_MODULE, "DiscoveredPythonFunctionFile")?, + &imported(py, NATIVE_RECORDS_MODULE, "DiscoveredPythonFunctionFile")?, &[ ("file_path", source.file_path.clone()), ("relative_path", source.relative_path.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs index 459dd535d1..4dda7e1e1d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs @@ -11,8 +11,8 @@ use sqlbuild_discovery::tree::models::ProjectTree; use crate::bindings::_helpers::discovery::project_files::path_object; -const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; -const IDENTITY_ERRORS_MODULE: &str = "sqlbuild.compiler.resource_names.exceptions"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; +const IDENTITY_ERRORS_MODULE: &str = "sqlbuild._native"; const BUILTINS_MODULE: &str = "builtins"; const OS_MODULE: &str = "os"; const DECODE_ENCODING: &str = "utf-8"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs index 93148c6a47..37d4db2725 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs @@ -4,9 +4,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::discovery::{ declaration_files, declaration_records, description_inputs, guidance, header_declarations, - header_values, hook_entries, key_checks, locations, model_records, project_files, - project_validation, python_function_files, python_hook_files, python_module_files, raised, - scope_snapshot, snapshot, sql_test_records, yaml_entries, yaml_records, + header_values, hook_entries, imported_python_nodes, key_checks, locations, model_records, + project_files, project_validation, python_function_files, python_hook_files, + python_module_files, raised, scope_snapshot, snapshot, sql_test_records, yaml_entries, + yaml_records, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -31,5 +32,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { guidance::register(module)?; description_inputs::register(module)?; hook_entries::register(module)?; + imported_python_nodes::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs index 021b709b93..4be4207f44 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs @@ -24,6 +24,8 @@ use crate::bindings::_helpers::discovery::sql_test_records::{ }; use crate::bindings::_helpers::discovery::yaml_records::discover_source_files; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; @@ -66,7 +68,7 @@ impl<'py> Faults<'py> { None => py.None().into_bound(py), }; call_with( - &imported(py, MODELS_MODULE, "DiscoveryFileFault")?, + &imported(py, NATIVE_RECORDS_MODULE, "DiscoveryFileFault")?, &[ ("path", path), ("message", PyString::new(py, &message).into_any()), @@ -229,7 +231,7 @@ fn tolerant_discovery<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult = imported(py, MODELS_MODULE, "DiscoveredProjectInputs")?.call((), Some(&inputs))?; call_with( - &imported(py, MODELS_MODULE, "TolerantScopeDiscovery")?, + &imported(py, NATIVE_RECORDS_MODULE, "TolerantScopeDiscovery")?, &[ ("discovered_inputs", discovered_inputs), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs index 5ef774ed56..2d4011a816 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs @@ -1,5 +1,6 @@ //! SQL `TEST(...)` blocks validated natively and built as discovery records. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -10,8 +11,10 @@ use sqlbuild_model_config::sql_values::models::{RawKey, RawSqlValue}; use crate::bindings::_helpers::contracts::sql_values::{SqlValueClasses, raw_value, sql_value}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const SQL_VALUE_TYPES_MODULE: &str = "sqlbuild.sql_values.types"; @@ -47,7 +50,7 @@ fn case_record<'py>( [name.into_bound_py_any(py)?, sql_value(value, classes)?], )?); } - py.import(MODELS_MODULE)? + py.import(NATIVE_RECORDS_MODULE)? .getattr("DiscoveredSqlTestCase")? .call1((case.name.as_str(), PyTuple::new(py, values)?, case_index)) } @@ -93,9 +96,7 @@ fn block_record<'py>( keywords.set_item("cursor_start", block.cursor_start)?; keywords.set_item("cursor_end", block.cursor_end)?; keywords.set_item("sql_body_span", &native.2)?; - models - .getattr("DiscoveredSqlTestBlock")? - .call((), Some(&keywords)) + shape_class(models.as_any(), "DiscoveredSqlTestBlock")?.call((), Some(&keywords)) } /// Each `(header, sql_body, span)` block of a SQL test file as a `DiscoveredSqlTestBlock`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs index 5173582032..1f360f8c84 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs @@ -17,8 +17,8 @@ use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::contract_classes::yaml_entry_classes; use crate::bindings::_helpers::model_config::header_metadata::{ContractClasses, audit_tuple}; -const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; -const IDENTITY_ERRORS_MODULE: &str = "sqlbuild.compiler.resource_names.exceptions"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; +const IDENTITY_ERRORS_MODULE: &str = "sqlbuild._native"; const IDENTITY_ERROR: &str = "ResourceIdentityError"; /// The directory whose schema YAML files may declare seeds. const SEEDS_DIRECTORY: &str = "seeds"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs index 15a8adf641..0c0cfa71f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs @@ -31,6 +31,8 @@ use crate::bindings::_helpers::discovery::yaml_entries::{ }; use crate::bindings::models::{NativeProjectTree, YamlDiscoveryRequest}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; /// The most YAML bytes one native load reads before its files are built and released. const YAML_BATCH_BYTES: u64 = 4 * 1024 * 1024; @@ -282,7 +284,7 @@ pub(crate) fn discover_schema_files<'py>( if paths.is_empty() { return record_tuple(py, Vec::new()); } - let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "DiscoveredSchemaFile")?; + let class: Bound<'py, PyAny> = imported(py, NATIVE_RECORDS_MODULE, "DiscoveredSchemaFile")?; let mut records: Vec> = Vec::new(); load_files( project_dir, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs new file mode 100644 index 0000000000..b03687364e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs @@ -0,0 +1,27 @@ +//! Python modules, thread names and wait limits of native lifecycle-event export. + +pub(crate) const EXCEPTIONS_MODULE: &str = "sqlbuild.runtime.event_exporting.exceptions"; +pub(crate) const SEVERITY_MODULE: &str = "sqlbuild.spec.contracts.types"; +pub(crate) const CATALOG_MODULE: &str = "sqlbuild.runtime.observability.constants"; +pub(crate) const INVOKE_LIFECYCLE_MODULE: &str = + "sqlbuild.runtime.event_exporting.main._invoke_lifecycle_event_sink"; +pub(crate) const RUN_OUTPUT_SINKS_MODULE: &str = + "sqlbuild.runtime.event_exporting.main._run_command_output_sinks"; +pub(crate) const SESSION_MODULE: &str = "sqlbuild.provider.main.session"; +pub(crate) const OUTPUT_MODELS_MODULE: &str = "sqlbuild.runtime.output_capture.models"; +pub(crate) const SCOPE_CONTEXT_VARIABLE: &str = "sqlbuild_event_exporter_command_scope"; +pub(crate) const DISPATCHER_THREAD_NAME: &str = "sqlbuild-event-exporter-dispatcher"; +pub(crate) const NOTIFIER_THREAD_NAME: &str = "sqlbuild-event-exporter-notifier"; +pub(crate) const HEALTH_THREAD_NAME: &str = "sqlbuild-event-exporter-health"; +pub(crate) const OUTPUT_SINK_THREAD_NAME: &str = "sqlbuild-command-output-sink"; +pub(crate) const INVOCATION_THREAD_PREFIX: &str = "sqlbuild-event-exporter-"; +/// Attribute naming the worker of a long-lived dispatcher thread, for the exit hook. +pub(crate) const WORKER_ATTRIBUTE: &str = "_sqlbuild_event_exporter_worker"; +/// The longest single wait of a polling delivery loop, as the Python dispatcher polled. +pub(crate) const POLL_SECONDS: f64 = 0.01; +/// Extra time `shutdown` waits for the dispatcher thread beyond the drain deadline. +pub(crate) const JOIN_GRACE_SECONDS: f64 = 0.05; +/// How long shutdown and interpreter exit wait for each stopped dispatcher thread. +pub(crate) const EXIT_JOIN_SECONDS: f64 = 1.0; +/// No time left before a deadline. +pub(crate) const NO_SECONDS: f64 = 0.0; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs new file mode 100644 index 0000000000..676c3bccf0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs @@ -0,0 +1,945 @@ +//! `EventExporterDispatcher`: bounded priority delivery of lifecycle events on Python daemon threads. + +use std::collections::{HashSet, VecDeque}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyResult, Python, pyclass, pyfunction, pymethods, +}; +use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyString}; +use sqlbuild_core::command_output::constants::COMMAND_OUTPUT_RECORD_TYPE; +use sqlbuild_core::event_export::constants::{ + DEFAULT_EVENT_EXPORT_HEALTH_INTERVAL_SECONDS, DEFAULT_EVENT_EXPORT_INVOCATION_TIMEOUT_SECONDS, + DEFAULT_EVENT_EXPORT_NOTIFICATION_QUEUE_CAPACITY, DEFAULT_EVENT_EXPORT_QUEUE_CAPACITY, + DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS, INVOCATION_TERMINAL_EVENT_TYPES, +}; +use sqlbuild_core::event_export::main::drop_lifecycle_attempts::drop_lifecycle_attempts; +use sqlbuild_core::event_export::main::export_snapshot::export_snapshot; +use sqlbuild_core::event_export::main::queue_lifecycle_event::queue_lifecycle_event; +use sqlbuild_core::event_export::main::route_lifecycle_event::route_lifecycle_event; +use sqlbuild_core::event_export::models::{ + EventQueue, ExportPolicy, ExportSeverity, ExportSnapshot, ExporterCounts, ExporterFilter, + QueuedEvent, SnapshotInputs, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::event_exporting::constants::{ + DISPATCHER_THREAD_NAME, EXIT_JOIN_SECONDS, HEALTH_THREAD_NAME, JOIN_GRACE_SECONDS, NO_SECONDS, + NOTIFIER_THREAD_NAME, OUTPUT_SINK_THREAD_NAME, POLL_SECONDS, RUN_OUTPUT_SINKS_MODULE, + WORKER_ATTRIBUTE, +}; +use crate::bindings::_helpers::event_exporting::models::{ + BoundEventExporter, EventExportSummary, EventExporterFailure, +}; +use crate::bindings::_helpers::event_exporting::values::{ + export_policy, input_error, severity_from, state_error, +}; +use crate::bindings::_helpers::event_exporting::workers::{DispatcherWorker, WorkerRole}; +use crate::bindings::_helpers::native_threads::constants::OUTPUT_SUPPRESSED_ATTRIBUTE; +use crate::bindings::_helpers::native_threads::exit_signal::{ + interpreter_exiting, signal_interpreter_exit, +}; + +/// One bound lifecycle-event sink. +pub(crate) struct Exporter { + pub(crate) name: String, + pub(crate) function: Py, + pub(crate) provider_arguments: Py, +} + +/// One bound command-output sink and the streams it accepts. +pub(crate) struct OutputSink { + function: Py, + provider_arguments: Py, + streams: Vec, +} + +/// A queued event with the policy dimensions its failures report. +pub(crate) struct Queued { + pub(crate) event: Py, + pub(crate) kind: String, + pub(crate) severity: ExportSeverity, +} + +/// Delivery state, read and written only while holding the dispatcher's Python condition. +pub(crate) struct State { + pub(crate) bound: bool, + pub(crate) exporters: Vec>, + pub(crate) filters: Vec, + pub(crate) counts: Vec, + output_sinks: Arc>, + /// Calls queued to the single command-output sink thread, once it has started. + output_lane: Option>, + pub(crate) queue: EventQueue, + sequence: u64, + accepting: bool, + pub(crate) deadline: Option, + pub(crate) stopping: bool, + pub(crate) force_stop: bool, + pub(crate) blocked: HashSet, + pub(crate) live_invocations: HashSet, + pub(crate) next_invocation: u64, + idle_finalizers: Vec>, + dispatcher_running: bool, + shutdown_started: bool, + shutdown_complete: bool, + final_summary: Option>, + pub(crate) failures: VecDeque, + pub(crate) notification_stopping: bool, + pub(crate) health_stopping: bool, + pub(crate) pending_periodic: Option, + pub(crate) periodic_inflight: Option<[u64; 7]>, + pub(crate) last_published_periodic: [u64; 7], + pub(crate) final_notification: Option>, + pub(crate) final_delivered: bool, +} + +/// What the dispatcher and its threads share. +pub(crate) struct Shared { + condition: Py, + state: Mutex, + pub(crate) invocation_timeout: Duration, + shutdown_timeout: Duration, + pub(crate) health_interval: Duration, + notification_capacity: usize, + pub(crate) failure_callback: Option>, + pub(crate) summary_callback: Option>, + notifications: bool, +} + +/// Deliver filtered lifecycle events off execution threads with bounded memory. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct EventExporterDispatcher { + shared: Arc, + thread: Py, + notification_thread: Option>, + health_thread: Option>, +} + +/// The dispatcher's Python condition, entered while alive and exited on drop. +pub(crate) struct Held<'py>(Bound<'py, PyAny>); + +/// How one command-output sink call ended. +pub(crate) enum Delivery { + /// The sink returned; `true` when it raised. + Delivered(bool), + /// Interpreter exit began while the sink was still running. + Abandoned, +} + +/// Validated dispatcher limits, in seconds and items. +struct Limits { + queue_capacity: usize, + shutdown_timeout: Duration, + invocation_timeout: Duration, + notification_capacity: usize, + health_interval: Duration, +} + +impl Drop for Held<'_> { + fn drop(&mut self) { + let py = self.0.py(); + drop( + self.0 + .call_method1("__exit__", (py.None(), py.None(), py.None())), + ); + } +} + +fn seconds(value: f64) -> Duration { + Duration::try_from_secs_f64(value.max(NO_SECONDS)).unwrap_or(Duration::MAX) +} + +fn remaining_seconds(deadline: Instant) -> f64 { + deadline + .saturating_duration_since(Instant::now()) + .as_secs_f64() +} + +/// Run finalizers, ignoring their failures. +pub(crate) fn run_finalizers(py: Python<'_>, finalizers: Vec>) { + for finalizer in finalizers { + drop(finalizer.bind(py).call0()); + } +} + +/// End a retired command-output sink thread once it has run every call queued before. +fn retire_output_lane(py: Python<'_>, retired: Option>) -> PyResult<()> { + if let Some(calls) = retired { + calls.bind(py).call_method1("put", (py.None(),))?; + } + Ok(()) +} + +/// `thread.join(timeout=seconds)` when the thread is alive. +fn join(thread: &Bound<'_, PyAny>, seconds: f64) -> PyResult<()> { + if thread.call_method0("is_alive")?.is_truthy()? { + call_with( + &thread.getattr("join")?, + &[("timeout", PyFloat::new(thread.py(), seconds).into_any())], + )?; + } + Ok(()) +} + +/// Bound exporters read from `BoundEventExporter` values or objects with the same attributes. +fn read_exporters( + exporters: &Bound<'_, PyAny>, +) -> PyResult<(Vec>, Vec)> { + let mut bound: Vec> = Vec::new(); + let mut filters: Vec = Vec::new(); + for exporter in exporters.try_iter()? { + let exporter: Bound<'_, PyAny> = exporter?; + if let Ok(native) = exporter.downcast::() { + let native = native.get(); + filters.push(native.filter.clone()); + bound.push(Arc::new(Exporter { + name: native.name.clone(), + function: native.function.clone_ref(exporter.py()), + provider_arguments: native.provider_arguments.clone_ref(exporter.py()), + })); + continue; + } + filters.push(ExporterFilter { + event_kinds: exporter + .getattr("event_kinds")? + .try_iter()? + .map(|kind| kind.and_then(|kind| kind.extract::())) + .collect::>()?, + min_severity: severity_from(&exporter.getattr("min_severity")?)?, + }); + bound.push(Arc::new(Exporter { + name: exporter.getattr("name")?.extract()?, + function: exporter.getattr("function")?.unbind(), + provider_arguments: exporter.getattr("provider_arguments")?.unbind(), + })); + } + Ok((bound, filters)) +} + +fn read_output_sinks(sinks: &Bound<'_, PyAny>) -> PyResult> { + let mut bound: Vec = Vec::new(); + for sink in sinks.try_iter()? { + let sink: Bound<'_, PyAny> = sink?; + bound.push(OutputSink { + function: sink.getattr("function")?.unbind(), + provider_arguments: sink.getattr("provider_arguments")?.unbind(), + streams: sink + .getattr("streams")? + .try_iter()? + .map(|stream| stream.and_then(|stream| stream.extract::())) + .collect::>()?, + }); + } + Ok(bound) +} + +/// `health_interval_seconds` when it is a positive, finite non-boolean number. +fn health_interval(value: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = value.py(); + if value.is_none() { + return Ok(DEFAULT_EVENT_EXPORT_HEALTH_INTERVAL_SECONDS); + } + let invalid = || { + input_error( + py, + "event exporter health_interval_seconds must be positive and finite", + ) + }; + if value.is_instance_of::() + || !(value.is_instance_of::() || value.is_instance_of::()) + { + return Err(invalid()); + } + let number: f64 = value.extract()?; + if !number.is_finite() || number <= NO_SECONDS { + return Err(invalid()); + } + Ok(number) +} + +/// The constructor's limits, validated in the Python dispatcher's order. +fn limits(options: &KeywordArguments<'_>) -> PyResult { + let integer = |name: &str, default: i64| -> PyResult { + let value = options.value(name)?; + if value.is_none() { + Ok(default) + } else { + value.extract() + } + }; + let float = |name: &str, default: f64| -> PyResult { + let value = options.value(name)?; + if value.is_none() { + Ok(default) + } else { + value.extract() + } + }; + let py: Python<'_> = options.value("exporters")?.py(); + let queue_capacity: i64 = integer("queue_capacity", DEFAULT_EVENT_EXPORT_QUEUE_CAPACITY)?; + let shutdown_timeout: f64 = float( + "shutdown_timeout_seconds", + DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS, + )?; + let invocation_timeout: f64 = float( + "invocation_timeout_seconds", + DEFAULT_EVENT_EXPORT_INVOCATION_TIMEOUT_SECONDS, + )?; + let notification_capacity: i64 = integer( + "notification_queue_capacity", + DEFAULT_EVENT_EXPORT_NOTIFICATION_QUEUE_CAPACITY, + )?; + let capacity_error = || input_error(py, "event exporter queue capacities must be at least 1"); + if notification_capacity < 1 { + return Err(capacity_error()); + } + if shutdown_timeout < NO_SECONDS || invocation_timeout <= NO_SECONDS { + return Err(input_error(py, "event exporter timeouts must be positive")); + } + let health: f64 = health_interval(&options.value("health_interval_seconds")?)?; + if queue_capacity < 1 { + return Err(capacity_error()); + } + Ok(Limits { + queue_capacity: usize::try_from(queue_capacity).unwrap_or(usize::MAX), + shutdown_timeout: seconds(shutdown_timeout), + invocation_timeout: seconds(invocation_timeout), + notification_capacity: usize::try_from(notification_capacity).unwrap_or(usize::MAX), + health_interval: seconds(health), + }) +} + +/// Stop every live dispatcher's threads and join them before finalization ends them inside native code. +#[pyfunction] +pub(crate) fn stop_event_exporter_threads(py: Python<'_>) -> PyResult<()> { + signal_interpreter_exit(py)?; + let mut threads: Vec> = Vec::new(); + for thread in imported(py, "threading", "enumerate")? + .call0()? + .try_iter()? + { + let thread: Bound<'_, PyAny> = thread?; + let Ok(worker) = thread.getattr(WORKER_ATTRIBUTE) else { + continue; + }; + if let Ok(worker) = worker.downcast::() { + worker.get().shared().stop_for_exit(py)?; + threads.push(thread); + } + } + for thread in threads { + join(&thread, EXIT_JOIN_SECONDS)?; + } + Ok(()) +} + +impl Shared { + pub(crate) fn hold<'py>(&self, py: Python<'py>) -> PyResult> { + let condition = self.condition.bind(py).clone(); + condition.call_method0("__enter__")?; + Ok(Held(condition)) + } + + pub(crate) fn locked(&self) -> PyResult> { + self.state + .lock() + .map_err(|_| PyRuntimeError::new_err("event exporter state lock is poisoned")) + } + + /// Wake every waiting thread; the caller holds the condition. + pub(crate) fn notify_all(&self, py: Python<'_>) -> PyResult<()> { + self.condition.bind(py).call_method0("notify_all")?; + Ok(()) + } + + /// Wait on the held condition for at most `timeout` seconds. + pub(crate) fn wait(&self, py: Python<'_>, timeout: f64) -> PyResult<()> { + self.condition + .bind(py) + .call_method1("wait", (timeout.max(NO_SECONDS),))?; + Ok(()) + } + + /// Whether the shutdown drain deadline has passed. + pub(crate) fn past_deadline(state: &State) -> bool { + state + .deadline + .is_some_and(|deadline| Instant::now() >= deadline) + } + + fn snapshot(state: &State, final_snapshot: bool) -> ExportSnapshot { + let exporters: Vec<(String, ExporterCounts)> = state + .exporters + .iter() + .zip(&state.counts) + .map(|(exporter, counts)| (exporter.name.clone(), *counts)) + .collect(); + export_snapshot(SnapshotInputs { + exporters: &exporters, + queue_depth: state.queue.items.len(), + queue_capacity: state.queue.capacity, + drained: final_snapshot + && !state.dispatcher_running + && state.live_invocations.is_empty(), + }) + } + + /// A point-in-time snapshot taken under the condition. + pub(crate) fn current_snapshot(&self, py: Python<'_>) -> PyResult { + let _held = self.hold(py)?; + Ok(Self::snapshot(&*self.locked()?, false)) + } + + /// Count one failed attempt and queue its sanitized diagnostic. + pub(crate) fn record_failure( + &self, + py: Python<'_>, + (index, kind, severity): (usize, &str, ExportSeverity), + error_type: String, + ) -> PyResult<()> { + let _held = self.hold(py)?; + let mut state = self.locked()?; + let mut exporter_name: String = String::new(); + if let Some(counts) = state.counts.get_mut(index) { + counts.failed += 1; + } + if let Some(exporter) = state.exporters.get(index) { + exporter_name.clone_from(&exporter.name); + } + if !self.notifications + || self.failure_callback.is_none() + || state.failures.len() >= self.notification_capacity + { + return Ok(()); + } + state.failures.push_back(EventExporterFailure { + exporter_name, + error_type, + event_kind: kind.to_owned(), + event_severity: severity, + }); + drop(state); + self.notify_all(py) + } + + /// Forget one finished invocation; once idle, run the finalizers waiting for it. + pub(crate) fn invocation_finished(&self, py: Python<'_>, invocation: u64) -> PyResult<()> { + let (finalizers, retired): (Vec>, Option>) = { + let _held = self.hold(py)?; + let mut state = self.locked()?; + state.live_invocations.remove(&invocation); + if state.live_invocations.is_empty() && !state.dispatcher_running { + let retired: Option> = if state.shutdown_complete { + state.output_lane.take() + } else { + None + }; + (std::mem::take(&mut state.idle_finalizers), retired) + } else { + (Vec::new(), None) + } + }; + retire_output_lane(py, retired)?; + run_finalizers(py, finalizers); + Ok(()) + } + + /// Mark the dispatcher thread finished and run idle finalizers. + pub(crate) fn dispatcher_finished(&self, py: Python<'_>) -> PyResult<()> { + let finalizers: Vec> = { + let _held = self.hold(py)?; + let mut state = self.locked()?; + state.dispatcher_running = false; + let finalizers = if state.live_invocations.is_empty() { + std::mem::take(&mut state.idle_finalizers) + } else { + Vec::new() + }; + drop(state); + self.notify_all(py)?; + finalizers + }; + run_finalizers(py, finalizers); + Ok(()) + } + + /// Register a running invocation, returning its identity. + pub(crate) fn begin_invocation(&self, py: Python<'_>) -> PyResult { + let _held = self.hold(py)?; + let mut state = self.locked()?; + state.next_invocation += 1; + let invocation: u64 = state.next_invocation; + state.live_invocations.insert(invocation); + Ok(invocation) + } + + /// Stop acceptance, delivery, health and notifications so every long-lived thread returns. + fn stop_for_exit(&self, py: Python<'_>) -> PyResult<()> { + let _held = self.hold(py)?; + let mut state = self.locked()?; + state.accepting = false; + state.stopping = true; + state.force_stop = true; + state.health_stopping = true; + state.notification_stopping = true; + drop(state); + self.notify_all(py) + } + + /// The queue feeding this dispatcher's single command-output sink thread, started on first use. + fn output_lane<'py>(&self, py: Python<'py>) -> PyResult> { + let _held = self.hold(py)?; + let existing: Option> = self + .locked()? + .output_lane + .as_ref() + .map(|calls| calls.clone_ref(py)); + if let Some(calls) = existing { + return Ok(calls.into_bound(py)); + } + let calls: Bound<'py, PyAny> = imported(py, "queue", "SimpleQueue")?.call0()?; + let arguments = PyDict::new(py); + arguments.set_item("calls", &calls)?; + let thread = call_with( + &imported(py, "threading", "Thread")?, + &[ + ( + "target", + imported(py, RUN_OUTPUT_SINKS_MODULE, "run_command_output_sinks")?, + ), + ("kwargs", arguments.into_any()), + ( + "name", + PyString::new(py, OUTPUT_SINK_THREAD_NAME).into_any(), + ), + ("daemon", PyBool::new(py, true).to_owned().into_any()), + ], + )?; + thread.setattr(OUTPUT_SUPPRESSED_ATTRIBUTE, true)?; + thread.call_method0("start")?; + self.locked()?.output_lane = Some(calls.clone().unbind()); + Ok(calls) + } + + /// Call one command-output sink on the sink thread and wait until it returns or exit begins. + fn call_output_sink( + &self, + py: Python<'_>, + sink: &OutputSink, + record: &Bound<'_, PyAny>, + ) -> PyResult { + let Ok(calls) = self.output_lane(py) else { + return Ok(Delivery::Delivered(true)); + }; + let outcome: Bound<'_, PyAny> = imported(py, "queue", "SimpleQueue")?.call0()?; + calls.call_method1( + "put", + (( + sink.function.bind(py), + sink.provider_arguments.bind(py), + record, + &outcome, + ),), + )?; + let empty: Bound<'_, PyAny> = imported(py, "queue", "Empty")?; + let get: Bound<'_, PyAny> = outcome.getattr("get")?; + loop { + match call_with( + &get, + &[("timeout", PyFloat::new(py, POLL_SECONDS).into_any())], + ) { + Ok(failed) => return Ok(Delivery::Delivered(failed.is_truthy()?)), + Err(error) if error.is_instance(py, &empty) => {} + Err(error) => return Err(error), + } + if interpreter_exiting(py)? { + return Ok(Delivery::Abandoned); + } + } + } + + /// The final summary once another caller's shutdown completes. + fn await_shutdown(&self, py: Python<'_>) -> PyResult> { + let _held = self.hold(py)?; + loop { + let state = self.locked()?; + if state.shutdown_complete { + return state + .final_summary + .as_ref() + .map(|summary| summary.clone_ref(py)) + .ok_or_else(|| { + state_error(py, "event exporter shutdown completed without summary") + }); + } + drop(state); + self.wait(py, POLL_SECONDS)?; + } + } +} + +/// A started daemon thread running `worker`, tagged so the exit hook can stop it. +fn start_worker<'py>( + py: Python<'py>, + shared: &Arc, + (role, name): (WorkerRole, &str), +) -> PyResult> { + let worker: Bound<'py, PyAny> = Py::new(py, DispatcherWorker::new(Arc::clone(shared), role))? + .into_bound(py) + .into_any(); + let thread = call_with( + &imported(py, "threading", "Thread")?, + &[ + ("target", worker.clone()), + ("name", PyString::new(py, name).into_any()), + ("daemon", PyBool::new(py, true).to_owned().into_any()), + ], + )?; + thread.setattr(WORKER_ATTRIBUTE, worker)?; + Ok(thread) +} + +#[pymethods] +impl EventExporterDispatcher { + #[new] + #[pyo3(signature = (**options))] + fn new(py: Python<'_>, options: Option>) -> PyResult { + let options: KeywordArguments<'_> = KeywordArguments::read( + py, + "EventExporterDispatcher", + options, + ( + &[], + &[ + "exporters", + "queue_capacity", + "shutdown_timeout_seconds", + "invocation_timeout_seconds", + "notification_queue_capacity", + "health_interval_seconds", + "failure_callback", + "summary_callback", + ], + ), + )?; + let limits: Limits = limits(&options)?; + let exporters: Bound<'_, PyAny> = options.value("exporters")?; + let (bound_exporters, filters) = if exporters.is_none() { + (Vec::new(), Vec::new()) + } else { + read_exporters(&exporters)? + }; + let callback = |name: &str| -> PyResult>> { + let value: Bound<'_, PyAny> = options.value(name)?; + Ok((!value.is_none()).then(|| value.unbind())) + }; + let failure_callback: Option> = callback("failure_callback")?; + let summary_callback: Option> = callback("summary_callback")?; + let notifications: bool = failure_callback.is_some() || summary_callback.is_some(); + let has_summary: bool = summary_callback.is_some(); + let shared = Arc::new(Shared { + condition: imported(py, "threading", "Condition")?.call0()?.unbind(), + state: Mutex::new(State { + bound: !exporters.is_none(), + counts: vec![ExporterCounts::default(); bound_exporters.len()], + exporters: bound_exporters, + filters, + output_sinks: Arc::new(Vec::new()), + output_lane: None, + queue: EventQueue::new(limits.queue_capacity), + sequence: 0, + accepting: true, + deadline: None, + stopping: false, + force_stop: false, + blocked: HashSet::new(), + live_invocations: HashSet::new(), + next_invocation: 0, + idle_finalizers: Vec::new(), + dispatcher_running: true, + shutdown_started: false, + shutdown_complete: false, + final_summary: None, + failures: VecDeque::new(), + notification_stopping: false, + health_stopping: false, + pending_periodic: None, + periodic_inflight: None, + last_published_periodic: [ + 0, + 0, + 0, + 0, + 0, + 0, + u64::try_from(limits.queue_capacity).unwrap_or(u64::MAX), + ], + final_notification: None, + final_delivered: false, + }), + invocation_timeout: limits.invocation_timeout, + shutdown_timeout: limits.shutdown_timeout, + health_interval: limits.health_interval, + notification_capacity: limits.notification_capacity, + failure_callback, + summary_callback, + notifications, + }); + let notification_thread: Option> = notifications + .then(|| start_worker(py, &shared, (WorkerRole::Notify, NOTIFIER_THREAD_NAME))) + .transpose()?; + let health_thread: Option> = has_summary + .then(|| start_worker(py, &shared, (WorkerRole::Health, HEALTH_THREAD_NAME))) + .transpose()?; + for thread in notification_thread.iter().chain(&health_thread) { + thread.call_method0("start")?; + } + let thread = start_worker(py, &shared, (WorkerRole::Dispatch, DISPATCHER_THREAD_NAME))?; + thread.call_method0("start")?; + Ok(Self { + shared, + thread: thread.unbind(), + notification_thread: notification_thread.map(Bound::unbind), + health_thread: health_thread.map(Bound::unbind), + }) + } + + #[getter] + fn thread(&self, py: Python<'_>) -> Py { + self.thread.clone_ref(py) + } + + #[getter] + fn notification_thread(&self, py: Python<'_>) -> Option> { + self.notification_thread + .as_ref() + .map(|thread| thread.clone_ref(py)) + } + + /// Filter and enqueue eligible exporter attempts without waiting. + fn enqueue(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { + let event_type: String = event.getattr("event_type")?.extract()?; + let policy: ExportPolicy = export_policy(py, &event_type)?; + let shared = &self.shared; + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + if !state.bound { + drop(state); + return Err(state_error( + py, + "event exporters must be bound before enqueue", + )); + } + let counts: Vec = std::mem::take(&mut state.counts); + let (counts, eligible) = route_lifecycle_event(&policy, &state.filters, counts); + state.counts = counts; + if eligible.is_empty() { + return Ok(()); + } + if !state.accepting { + state.counts = drop_lifecycle_attempts(std::mem::take(&mut state.counts), &eligible); + return Ok(()); + } + let sequence: u64 = state.sequence; + state.sequence += 1; + let capacity: usize = state.queue.capacity; + let mut outcome = queue_lifecycle_event( + std::mem::replace(&mut state.queue, EventQueue::new(capacity)), + QueuedEvent { + sequence, + item: Queued { + event: event.clone().unbind(), + kind: policy.kind, + severity: policy.severity, + }, + priority: policy.priority, + invocation_terminal: INVOCATION_TERMINAL_EVENT_TYPES.contains(&event_type.as_str()), + eligible_exporters: eligible, + }, + ); + state.queue = std::mem::replace(&mut outcome.queue, EventQueue::new(capacity)); + for lost in outcome.displaced.iter().chain(&outcome.rejected) { + state.counts = drop_lifecycle_attempts( + std::mem::take(&mut state.counts), + &lost.eligible_exporters, + ); + } + drop(state); + shared.notify_all(py)?; + drop(outcome); + Ok(()) + } + + /// Bind validated exporters once before lifecycle publication starts. + fn bind(&self, py: Python<'_>, exporters: &Bound<'_, PyAny>) -> PyResult<()> { + let (exporters, filters) = read_exporters(exporters)?; + let shared = &self.shared; + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + if state.bound { + drop(state); + return Err(state_error(py, "event exporters are already bound")); + } + state.counts = vec![ExporterCounts::default(); exporters.len()]; + state.exporters = exporters; + state.filters = filters; + state.bound = true; + drop(state); + shared.notify_all(py) + } + + /// Bind explicit command-output sinks before capture starts. + fn bind_command_output_sinks(&self, py: Python<'_>, sinks: &Bound<'_, PyAny>) -> PyResult<()> { + let sinks: Arc> = Arc::new(read_output_sinks(sinks)?); + let previous = { + let _held = self.shared.hold(py)?; + std::mem::replace(&mut self.shared.locked()?.output_sinks, sinks) + }; + drop(previous); + Ok(()) + } + + /// Deliver output records through the already-bound destination providers. + fn export_output(&self, py: Python<'_>, records: &Bound<'_, PyAny>) -> PyResult<()> { + let shared = &self.shared; + let invocation: u64 = shared.begin_invocation(py)?; + let sinks: Arc> = { + let _held = shared.hold(py)?; + Arc::clone(&shared.locked()?.output_sinks) + }; + let mut failed: bool = false; + let delivered = (|| -> PyResult { + for record in records.try_iter()? { + let record: Bound<'_, PyAny> = record?; + let is_output: bool = record + .getattr("record_type")? + .eq(COMMAND_OUTPUT_RECORD_TYPE)?; + let stream: Option = if is_output { + Some(record.getattr("stream")?.extract()?) + } else { + None + }; + for sink in sinks.iter() { + if stream + .as_ref() + .is_some_and(|stream| !sink.streams.contains(stream)) + { + continue; + } + match shared.call_output_sink(py, sink, &record)? { + Delivery::Delivered(sink_failed) => failed |= sink_failed, + Delivery::Abandoned => return Ok(Delivery::Abandoned), + } + } + } + Ok(Delivery::Delivered(failed)) + })(); + if let Ok(Delivery::Abandoned) = delivered { + return Err(state_error( + py, + "output export stopped because the interpreter is exiting", + )); + } + shared.invocation_finished(py, invocation)?; + delivered?; + if failed { + return Err(state_error(py, "one or more output export attempts failed")); + } + Ok(()) + } + + /// Stop acceptance and drain until the fixed shutdown deadline. + fn shutdown(&self, py: Python<'_>) -> PyResult> { + let shared = &self.shared; + let (owns, deadline) = { + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + let owns: bool = !state.shutdown_started; + if owns { + state.shutdown_started = true; + state.accepting = false; + state.deadline = Some(Instant::now() + shared.shutdown_timeout); + state.stopping = true; + state.health_stopping = true; + } + let deadline: Instant = state.deadline.unwrap_or_else(Instant::now); + drop(state); + shared.notify_all(py)?; + (owns, deadline) + }; + if !owns { + return shared.await_shutdown(py); + } + let thread: &Bound<'_, PyAny> = self.thread.bind(py); + call_with( + &thread.getattr("join")?, + &[( + "timeout", + PyFloat::new(py, remaining_seconds(deadline) + JOIN_GRACE_SECONDS).into_any(), + )], + )?; + if thread.call_method0("is_alive")?.is_truthy()? { + { + let _held = shared.hold(py)?; + shared.locked()?.force_stop = true; + shared.notify_all(py)?; + } + thread.call_method0("join")?; + } + let snapshot: ExportSnapshot = { + let _held = shared.hold(py)?; + Shared::snapshot(&*shared.locked()?, true) + }; + let summary: Py = EventExportSummary::object(py, snapshot)?; + { + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + state.final_summary = Some(summary.clone_ref(py)); + state.shutdown_complete = true; + if shared.summary_callback.is_some() + && state.final_notification.is_none() + && !state.final_delivered + { + state.final_notification = Some(summary.clone_ref(py)); + } + state.notification_stopping = true; + let retired: Option> = if state.live_invocations.is_empty() { + state.output_lane.take() + } else { + None + }; + drop(state); + shared.notify_all(py)?; + retire_output_lane(py, retired)?; + } + if let Some(health) = &self.health_thread { + join(health.bind(py), EXIT_JOIN_SECONDS)?; + } + Ok(summary) + } + + /// Run `finalizer` once the dispatcher thread and every live invocation have finished. + fn finalize_when_idle(&self, py: Python<'_>, finalizer: Py) -> PyResult<()> { + { + let _held = self.shared.hold(py)?; + let mut state = self.shared.locked()?; + if !state.live_invocations.is_empty() || state.dispatcher_running { + state.idle_finalizers.push(finalizer); + return Ok(()); + } + } + run_finalizers(py, vec![finalizer]); + Ok(()) + } + + /// Return a thread-safe immutable point-in-time snapshot. + fn summary(&self, py: Python<'_>) -> PyResult> { + let snapshot: ExportSnapshot = self.shared.current_snapshot(py)?; + EventExportSummary::object(py, snapshot) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/mod.rs new file mode 100644 index 0000000000..1c089a997c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod constants; +pub(crate) mod dispatcher; +pub(crate) mod models; +pub(crate) mod policy; +pub(crate) mod registration; +pub(crate) mod scope; +pub(crate) mod values; +pub(crate) mod workers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs new file mode 100644 index 0000000000..bc12c285e0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs @@ -0,0 +1,320 @@ +//! Immutable event exporter accounting, failure and binding values Python reads. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; +use pyo3::types::{PyDict, PyFrozenSet, PyTuple}; +use sqlbuild_core::event_export::constants::EVENT_EXPORT_KINDS; +use sqlbuild_core::event_export::models::{ + ExportSeverity, ExportSnapshot, ExporterCounts, ExporterFilter, +}; + +use crate::bindings::_helpers::event_exporting::values::{ + accounting_repr, bound_exporter_arguments, counts_repr, python_repr, severity_from, + severity_object, text_repr, +}; + +/// Frozen attempt counters for one exporter or their aggregate. +#[pyclass(module = "sqlbuild._native", frozen, eq, hash)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(crate) struct EventExporterCounts { + pub(crate) counts: ExporterCounts, +} + +/// Frozen accounting snapshot for one named exporter. +#[pyclass(module = "sqlbuild._native", frozen, eq, hash)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(crate) struct EventExporterAccounting { + exporter_name: String, + counts: ExporterCounts, +} + +/// Final or point-in-time best-effort delivery accounting. +#[pyclass(module = "sqlbuild._native", frozen, eq, hash)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(crate) struct EventExportSummary { + pub(crate) snapshot: ExportSnapshot, +} + +/// Sanitized exporter failure information safe for diagnostics. +#[pyclass(module = "sqlbuild._native", frozen, eq, hash)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(crate) struct EventExporterFailure { + pub(crate) exporter_name: String, + pub(crate) error_type: String, + pub(crate) event_kind: String, + pub(crate) event_severity: ExportSeverity, +} + +/// One lifecycle-event sink bound to command-scoped provider instances. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub(crate) struct BoundEventExporter { + pub(crate) name: String, + pub(crate) function: Py, + pub(crate) provider_arguments: Py, + pub(crate) filter: ExporterFilter, +} + +#[pymethods] +impl EventExporterCounts { + #[getter] + fn accepted(&self) -> u64 { + self.counts.accepted + } + + #[getter] + fn filtered(&self) -> u64 { + self.counts.filtered + } + + #[getter] + fn delivered(&self) -> u64 { + self.counts.delivered + } + + #[getter] + fn failed(&self) -> u64 { + self.counts.failed + } + + #[getter] + fn dropped(&self) -> u64 { + self.counts.dropped + } + + fn __repr__(&self) -> String { + counts_repr(&self.counts) + } +} + +#[pymethods] +impl EventExporterAccounting { + #[getter] + fn exporter_name(&self) -> &str { + &self.exporter_name + } + + #[getter] + fn counts(&self) -> EventExporterCounts { + EventExporterCounts { + counts: self.counts, + } + } + + fn __repr__(&self, py: Python<'_>) -> PyResult { + accounting_repr(py, (&self.exporter_name, &self.counts)) + } +} + +#[pymethods] +impl EventExportSummary { + #[getter] + fn aggregate(&self) -> EventExporterCounts { + EventExporterCounts { + counts: self.snapshot.aggregate, + } + } + + #[getter] + fn per_exporter<'py>(&self, py: Python<'py>) -> PyResult> { + let items: Vec = self + .snapshot + .per_exporter + .iter() + .map(|(name, counts)| EventExporterAccounting { + exporter_name: name.clone(), + counts: *counts, + }) + .collect(); + PyTuple::new(py, items) + } + + #[getter] + fn queue_depth(&self) -> usize { + self.snapshot.queue_depth + } + + #[getter] + fn queue_capacity(&self) -> usize { + self.snapshot.queue_capacity + } + + #[getter] + fn flush_complete(&self) -> bool { + self.snapshot.flush_complete + } + + #[getter] + fn accepted(&self) -> u64 { + self.snapshot.aggregate.accepted + } + + #[getter] + fn filtered(&self) -> u64 { + self.snapshot.aggregate.filtered + } + + #[getter] + fn delivered(&self) -> u64 { + self.snapshot.aggregate.delivered + } + + #[getter] + fn failed(&self) -> u64 { + self.snapshot.aggregate.failed + } + + #[getter] + fn dropped(&self) -> u64 { + self.snapshot.aggregate.dropped + } + + fn __repr__(&self, py: Python<'_>) -> PyResult { + let per_exporter: Vec = self + .snapshot + .per_exporter + .iter() + .map(|(name, counts)| accounting_repr(py, (name, counts))) + .collect::>>()?; + let per_exporter: String = match per_exporter.as_slice() { + [single] => format!("({single},)"), + items => format!("({})", items.join(", ")), + }; + Ok(format!( + "EventExportSummary(aggregate={}, per_exporter={per_exporter}, queue_depth={}, \ + queue_capacity={}, flush_complete={})", + counts_repr(&self.snapshot.aggregate), + self.snapshot.queue_depth, + self.snapshot.queue_capacity, + if self.snapshot.flush_complete { + "True" + } else { + "False" + } + )) + } +} + +#[pymethods] +impl EventExporterFailure { + #[getter] + fn exporter_name(&self) -> &str { + &self.exporter_name + } + + #[getter] + fn error_type(&self) -> &str { + &self.error_type + } + + #[getter] + fn event_kind(&self) -> &str { + &self.event_kind + } + + #[getter] + fn event_severity<'py>(&self, py: Python<'py>) -> PyResult> { + severity_object(py, self.event_severity) + } + + fn __repr__(&self, py: Python<'_>) -> PyResult { + Ok(format!( + "EventExporterFailure(exporter_name={}, error_type={}, event_kind={}, event_severity={})", + text_repr(py, &self.exporter_name)?, + text_repr(py, &self.error_type)?, + text_repr(py, &self.event_kind)?, + python_repr(&severity_object(py, self.event_severity)?)? + )) + } +} + +#[pymethods] +impl BoundEventExporter { + #[new] + #[pyo3(signature = (*arguments, **options))] + fn new(arguments: &Bound<'_, PyTuple>, options: Option<&Bound<'_, PyDict>>) -> PyResult { + let [ + name, + function, + provider_arguments, + event_kinds, + min_severity, + ] = bound_exporter_arguments(arguments, options)?; + let (Some(name), Some(function), Some(provider_arguments)) = + (name, function, provider_arguments) + else { + return Err(pyo3::exceptions::PyTypeError::new_err( + "BoundEventExporter() missing required arguments", + )); + }; + Self::create( + name.extract()?, + (function.unbind(), provider_arguments.unbind()), + (event_kinds.as_ref(), min_severity.as_ref()), + ) + } + + #[getter] + fn name(&self) -> &str { + &self.name + } + + #[getter] + fn function(&self, py: Python<'_>) -> Py { + self.function.clone_ref(py) + } + + #[getter] + fn provider_arguments(&self, py: Python<'_>) -> Py { + self.provider_arguments.clone_ref(py) + } + + #[getter] + fn event_kinds<'py>(&self, py: Python<'py>) -> PyResult> { + PyFrozenSet::new(py, self.filter.event_kinds.iter()) + } + + #[getter] + fn min_severity<'py>(&self, py: Python<'py>) -> PyResult> { + severity_object(py, self.filter.min_severity) + } +} + +impl BoundEventExporter { + /// A sink bound to `provider_arguments`, filtered by `event_kinds` and `min_severity` when given. + pub(crate) fn create( + name: String, + (function, provider_arguments): (Py, Py), + (event_kinds, min_severity): (Option<&Bound<'_, PyAny>>, Option<&Bound<'_, PyAny>>), + ) -> PyResult { + let event_kinds = match event_kinds { + Some(kinds) => kinds + .try_iter()? + .map(|kind| kind.and_then(|kind| kind.extract::())) + .collect::>()?, + None => EVENT_EXPORT_KINDS + .iter() + .map(|kind| (*kind).to_owned()) + .collect(), + }; + let min_severity: ExportSeverity = match min_severity { + Some(severity) => severity_from(severity)?, + None => ExportSeverity::Debug, + }; + Ok(Self { + name, + function, + provider_arguments, + filter: ExporterFilter { + event_kinds, + min_severity, + }, + }) + } +} + +impl EventExportSummary { + /// A Python summary of `snapshot`. + pub(crate) fn object(py: Python<'_>, snapshot: ExportSnapshot) -> PyResult> { + Py::new(py, Self { snapshot }) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs new file mode 100644 index 0000000000..c5cf0b9ffd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs @@ -0,0 +1,68 @@ +//! Lifecycle export policy, severity comparison and narrowing for Python callers. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyfunction, + wrap_pyfunction, +}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_core::event_export::main::severity_at_least::severity_at_least; +use sqlbuild_core::event_export::main::stricter_severity::stricter_severity; +use sqlbuild_core::event_export::models::{ExportPolicy, ExportSeverity}; + +use crate::bindings::_helpers::event_exporting::values::{ + export_policy, severity_from, severity_object, +}; + +const CATALOG_MODULE: &str = "sqlbuild.runtime.observability.constants"; + +/// `{event_type: (kind, severity, priority)}` for every catalogued lifecycle event, in catalogue order. +#[pyfunction] +fn lifecycle_export_policy_catalog(py: Python<'_>) -> PyResult> { + let catalog: Bound<'_, PyAny> = py + .import(CATALOG_MODULE)? + .getattr("LIFECYCLE_EVENT_CATALOG")?; + let policies: Bound<'_, PyDict> = PyDict::new(py); + for event_type in catalog.try_iter()? { + let event_type: String = event_type?.extract()?; + let policy: ExportPolicy = export_policy(py, &event_type)?; + policies.set_item( + &event_type, + ( + policy.kind, + severity_object(py, policy.severity)?, + policy.priority, + ), + )?; + } + Ok(policies) +} + +/// Whether a catalogued severity meets a configured minimum. +#[pyfunction] +#[pyo3(signature = (*, severity, minimum))] +fn event_export_severity_at_least( + severity: &Bound<'_, PyAny>, + minimum: &Bound<'_, PyAny>, +) -> PyResult { + Ok(severity_at_least( + severity_from(severity)?, + severity_from(minimum)?, + )) +} + +/// The stricter of a required and an optional minimum severity, as `EventExportSeverity`. +#[pyfunction] +#[pyo3(signature = (*, first, second))] +fn stricter_event_export_severity<'py>( + first: &Bound<'py, PyAny>, + second: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let second: Option = second.map(severity_from).transpose()?; + severity_object(first.py(), stricter_severity(severity_from(first)?, second)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(lifecycle_export_policy_catalog, module)?)?; + module.add_function(wrap_pyfunction!(event_export_severity_at_least, module)?)?; + module.add_function(wrap_pyfunction!(stricter_event_export_severity, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs new file mode 100644 index 0000000000..ebe2562456 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs @@ -0,0 +1,30 @@ +//! Register native lifecycle-event export and stop its threads at interpreter exit. + +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::event_exporting::dispatcher::{ + EventExporterDispatcher, stop_event_exporter_threads, +}; +use crate::bindings::_helpers::event_exporting::models::{ + BoundEventExporter, EventExportSummary, EventExporterAccounting, EventExporterCounts, + EventExporterFailure, +}; +use crate::bindings::_helpers::event_exporting::workers::{DispatcherWorker, InvocationCompletion}; +use crate::bindings::_helpers::event_exporting::{policy, scope}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + let hook = wrap_pyfunction!(stop_event_exporter_threads, module)?; + module.add_function(hook.clone())?; + imported(module.py(), "atexit", "register")?.call1((hook,))?; + policy::register(module)?; + scope::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs new file mode 100644 index 0000000000..fbfb35564c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs @@ -0,0 +1,467 @@ +//! `EventExporterCommandScope`: typed sink binding, its provider session and the active command scope. + +use std::sync::{Mutex, MutexGuard}; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyclass, + pyfunction, pymethods, wrap_pyfunction, +}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, context_variable, imported, +}; +use crate::bindings::_helpers::event_exporting::constants::{ + OUTPUT_MODELS_MODULE, SCOPE_CONTEXT_VARIABLE, SESSION_MODULE, +}; +use crate::bindings::_helpers::event_exporting::models::BoundEventExporter; + +static CURRENT_SCOPE: PyOnceLock> = PyOnceLock::new(); + +/// Discovered extensions and the provider session one command bound. +struct ScopeState { + configured: bool, + provider_close_registered: bool, + provider_session: Option>, + project_dir: Option>, + providers: Option>, + event_exporters: Option>, + command_output_sinks: Option>, +} + +/// Bind discovered typed sinks and own their command-scoped provider session. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct EventExporterCommandScope { + dispatcher: Py, + state: Mutex, +} + +/// Releases and force-closes a provider session once sink delivery is idle. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct ProviderSessionCloser { + session: Py, +} + +/// Installs one command-owned exporter scope while entered. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct EventExporterScopeContext { + scope: Py, + token: Mutex>>, +} + +fn empty_tuple(py: Python<'_>) -> Py { + PyTuple::empty(py).into_any().unbind() +} + +fn optional(py: Python<'_>, value: Option<&Py>) -> Py { + value.map_or_else(|| py.None(), |value| value.clone_ref(py)) +} + +/// `{usage.parameter_name: session.get(usage.provider_name)}` for one declaration. +fn provider_arguments<'py>( + session: &Bound<'py, PyAny>, + declaration: &Bound<'py, PyAny>, +) -> PyResult> { + let arguments: Bound<'py, PyDict> = PyDict::new(session.py()); + for usage in declaration.getattr("provider_usages")?.try_iter()? { + let usage: Bound<'py, PyAny> = usage?; + arguments.set_item( + usage.getattr("parameter_name")?, + session.call_method1("get", (usage.getattr("provider_name")?,))?, + )?; + } + Ok(arguments) +} + +/// Bound lifecycle exporters and command-output sinks over `session`'s provider instances. +fn bindings<'py>( + session: &Bound<'py, PyAny>, + (event_exporters, command_output_sinks): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult<(Bound<'py, PyTuple>, Bound<'py, PyTuple>)> { + let py: Python<'py> = session.py(); + let mut exporters: Vec> = Vec::new(); + for exporter in event_exporters.try_iter()? { + let exporter: Bound<'py, PyAny> = exporter?; + let arguments: Bound<'py, PyDict> = provider_arguments(session, &exporter)?; + exporters.push( + Bound::new( + py, + BoundEventExporter::create( + exporter.getattr("name")?.extract()?, + ( + exporter.getattr("function")?.unbind(), + arguments.into_any().unbind(), + ), + ( + Some(&exporter.getattr("event_kinds")?), + Some(&exporter.getattr("min_severity")?), + ), + )?, + )? + .into_any(), + ); + } + let sink_class: Bound<'py, PyAny> = + imported(py, OUTPUT_MODELS_MODULE, "BoundCommandOutputSink")?; + let mut sinks: Vec> = Vec::new(); + for sink in command_output_sinks.try_iter()? { + let sink: Bound<'py, PyAny> = sink?; + let arguments: Bound<'py, PyDict> = provider_arguments(session, &sink)?; + sinks.push(call_with( + &sink_class, + &[ + ("name", sink.getattr("name")?), + ("function", sink.getattr("function")?), + ("provider_arguments", arguments.into_any()), + ("streams", sink.getattr("streams")?), + ], + )?); + } + Ok((PyTuple::new(py, exporters)?, PyTuple::new(py, sinks)?)) +} + +impl EventExporterCommandScope { + fn locked(&self) -> PyResult> { + self.state + .lock() + .map_err(|_| PyRuntimeError::new_err("event exporter scope lock is poisoned")) + } + + /// Bind discovered sinks over one command-scoped provider session, once. + fn bind_extensions<'py>( + &self, + py: Python<'py>, + project_dir: Option>, + (providers, event_exporters, command_output_sinks): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ), + ) -> PyResult<()> { + { + let mut state = self.locked()?; + if state.configured { + return Ok(()); + } + state.configured = true; + } + let project_dir: Option> = project_dir + .map(|project_dir| project_dir.call_method0("resolve")) + .transpose()?; + { + let mut state = self.locked()?; + state.project_dir = project_dir.map(Bound::unbind); + state.providers = Some(providers.clone().unbind()); + state.event_exporters = Some(event_exporters.clone().unbind()); + state.command_output_sinks = Some(command_output_sinks.clone().unbind()); + } + if !event_exporters.is_truthy()? && !command_output_sinks.is_truthy()? { + let empty = PyTuple::empty(py).into_any(); + return self.bind_dispatcher(py, (empty.clone(), empty)); + } + let session: Bound<'_, PyAny> = call_with( + &imported(py, SESSION_MODULE, "build_provider_session")?, + &[ + ("discovered_providers", providers), + ("allow_shared", PyBool::new(py, false).to_owned().into_any()), + ], + )?; + session.call_method0("hold_open")?; + let (exporters, sinks) = match bindings(&session, (&event_exporters, &command_output_sinks)) + { + Ok(bound) => bound, + Err(error) => { + session.call_method0("release")?; + call_with( + &session.getattr("close")?, + &[("force", PyBool::new(py, true).to_owned().into_any())], + )?; + return Err(error); + } + }; + self.locked()?.provider_session = Some(session.unbind()); + self.bind_dispatcher(py, (exporters.into_any(), sinks.into_any())) + } + + fn bind_dispatcher( + &self, + py: Python<'_>, + (exporters, sinks): (Bound<'_, PyAny>, Bound<'_, PyAny>), + ) -> PyResult<()> { + let dispatcher: &Bound<'_, PyAny> = self.dispatcher.bind(py); + dispatcher.call_method1("bind", (exporters,))?; + dispatcher.call_method1("bind_command_output_sinks", (sinks,))?; + Ok(()) + } +} + +#[pymethods] +impl EventExporterCommandScope { + #[new] + #[pyo3(signature = (*, dispatcher))] + fn new(dispatcher: Py) -> Self { + Self { + dispatcher, + state: Mutex::new(ScopeState { + configured: false, + provider_close_registered: false, + provider_session: None, + project_dir: None, + providers: None, + event_exporters: None, + command_output_sinks: None, + }), + } + } + + #[getter] + fn dispatcher(&self, py: Python<'_>) -> Py { + self.dispatcher.clone_ref(py) + } + + #[getter] + fn provider_session(&self, py: Python<'_>) -> PyResult> { + Ok(optional(py, self.locked()?.provider_session.as_ref())) + } + + #[getter] + fn project_dir(&self, py: Python<'_>) -> PyResult> { + Ok(optional(py, self.locked()?.project_dir.as_ref())) + } + + #[getter] + fn providers(&self, py: Python<'_>) -> PyResult> { + let state = self.locked()?; + Ok(state + .providers + .as_ref() + .map_or_else(|| empty_tuple(py), |value| value.clone_ref(py))) + } + + #[getter] + fn event_exporters(&self, py: Python<'_>) -> PyResult> { + let state = self.locked()?; + Ok(state + .event_exporters + .as_ref() + .map_or_else(|| empty_tuple(py), |value| value.clone_ref(py))) + } + + #[getter] + fn command_output_sinks(&self, py: Python<'_>) -> PyResult> { + let state = self.locked()?; + Ok(state + .command_output_sinks + .as_ref() + .map_or_else(|| empty_tuple(py), |value| value.clone_ref(py))) + } + + /// Bind once after normal project discovery has validated declarations. + fn configure(&self, py: Python<'_>, discovered_inputs: &Bound<'_, PyAny>) -> PyResult<()> { + self.bind_extensions( + py, + None, + ( + discovered_inputs.getattr("providers")?, + discovered_inputs.getattr("event_exporters")?, + discovered_inputs.getattr("command_output_sinks")?, + ), + ) + } + + /// Bind declarations discovered at the outer command startup boundary. + #[pyo3(signature = (**options))] + fn configure_extensions( + &self, + py: Python<'_>, + options: Option>, + ) -> PyResult<()> { + let options: KeywordArguments<'_> = KeywordArguments::read( + py, + "configure_extensions", + options, + ( + &["providers", "event_exporters"], + &["project_dir", "command_output_sinks"], + ), + )?; + let project_dir: Bound<'_, PyAny> = options.value("project_dir")?; + let command_output_sinks: Bound<'_, PyAny> = options.value("command_output_sinks")?; + let command_output_sinks: Bound<'_, PyAny> = if command_output_sinks.is_none() { + PyTuple::empty(py).into_any() + } else { + command_output_sinks + }; + self.bind_extensions( + py, + (!project_dir.is_none()).then_some(project_dir), + ( + options.value("providers")?, + options.value("event_exporters")?, + command_output_sinks, + ), + ) + } + + /// Return startup declarations only for the same project root. + #[pyo3(signature = (*, project_dir))] + fn cached_extensions( + &self, + py: Python<'_>, + project_dir: &Bound<'_, PyAny>, + ) -> PyResult> { + let (own, providers, exporters, sinks) = { + let state = self.locked()?; + let Some(own) = state.project_dir.as_ref() else { + return Ok(py.None()); + }; + ( + own.clone_ref(py), + optional(py, state.providers.as_ref()), + optional(py, state.event_exporters.as_ref()), + optional(py, state.command_output_sinks.as_ref()), + ) + }; + if project_dir.call_method0("resolve")?.ne(own.bind(py))? { + return Ok(py.None()); + } + Ok(PyTuple::new(py, [providers, exporters, sinks])? + .into_any() + .unbind()) + } + + /// Flush sink delivery before reverse-order provider teardown. + fn close(&self, py: Python<'_>) -> PyResult> { + let summary: Py = self.dispatcher.bind(py).call_method0("shutdown")?.unbind(); + let session: Py = { + let mut state = self.locked()?; + let Some(session) = state + .provider_session + .as_ref() + .map(|session| session.clone_ref(py)) + else { + return Ok(summary); + }; + if state.provider_close_registered { + return Ok(summary); + } + state.provider_close_registered = true; + session + }; + let closer: Py = Py::new(py, ProviderSessionCloser { session })?; + self.dispatcher + .bind(py) + .call_method1("finalize_when_idle", (closer,))?; + Ok(summary) + } + + /// Reuse bound sink functions and provider instances for output records. + fn export_output(&self, py: Python<'_>, records: &Bound<'_, PyAny>) -> PyResult<()> { + self.dispatcher + .bind(py) + .call_method1("export_output", (records,))?; + Ok(()) + } +} + +#[pymethods] +impl ProviderSessionCloser { + fn __call__(&self, py: Python<'_>) -> PyResult<()> { + let session: &Bound<'_, PyAny> = self.session.bind(py); + let released = session.call_method0("release"); + call_with( + &session.getattr("close")?, + &[("force", PyBool::new(py, true).to_owned().into_any())], + )?; + released?; + Ok(()) + } +} + +#[pymethods] +impl EventExporterScopeContext { + fn __enter__(&self, py: Python<'_>) -> PyResult> { + let token = context_variable(py, &CURRENT_SCOPE, SCOPE_CONTEXT_VARIABLE)? + .call_method1("set", (self.scope.bind(py),))?; + *self + .token + .lock() + .map_err(|_| PyRuntimeError::new_err("event exporter scope lock is poisoned"))? = + Some(token.unbind()); + Ok(self.scope.clone_ref(py)) + } + + #[pyo3(signature = (*_exc_info))] + fn __exit__(&self, py: Python<'_>, _exc_info: &Bound<'_, PyTuple>) -> PyResult { + let token = self + .token + .lock() + .map_err(|_| PyRuntimeError::new_err("event exporter scope lock is poisoned"))? + .take(); + if let Some(token) = token { + context_variable(py, &CURRENT_SCOPE, SCOPE_CONTEXT_VARIABLE)? + .call_method1("reset", (token,))?; + } + Ok(false) + } +} + +/// Install one command-owned exporter scope. +#[pyfunction] +fn event_exporter_command_scope(scope: Py) -> EventExporterScopeContext { + EventExporterScopeContext { + scope, + token: Mutex::new(None), + } +} + +/// The active command-owned exporter scope, or `None`. +pub(crate) fn current_scope(py: Python<'_>) -> PyResult> { + context_variable(py, &CURRENT_SCOPE, SCOPE_CONTEXT_VARIABLE)?.call_method0("get") +} + +/// Return the active command-owned exporter scope, if any. +#[pyfunction] +fn current_event_exporter_command_scope(py: Python<'_>) -> PyResult> { + current_scope(py) +} + +/// Configure the active exporter scope from validated discovery inputs. +pub(crate) fn configure_discovered_event_exporters(discovered: &Bound<'_, PyAny>) -> PyResult<()> { + let scope: Bound<'_, PyAny> = current_scope(discovered.py())?; + if !scope.is_none() { + scope.call_method1("configure", (discovered,))?; + } + Ok(()) +} + +/// Return startup-discovered extensions for the active project root. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +fn cached_event_exporter_extensions<'py>( + py: Python<'py>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let scope: Bound<'py, PyAny> = current_scope(py)?; + if scope.is_none() { + return Ok(scope); + } + call_with( + &scope.getattr("cached_extensions")?, + &[("project_dir", project_dir.clone())], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(event_exporter_command_scope, module)?)?; + module.add_function(wrap_pyfunction!( + current_event_exporter_command_scope, + module + )?)?; + module.add_function(wrap_pyfunction!(cached_event_exporter_extensions, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs new file mode 100644 index 0000000000..cd2ee51981 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs @@ -0,0 +1,170 @@ +//! Event exporter errors, severities, reprs and the lifecycle export policy catalogue across the Python boundary. + +use std::collections::HashMap; + +use pyo3::exceptions::{PyKeyError, PyTypeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; +use sqlbuild_core::event_export::main::lifecycle_export_policy::lifecycle_export_policy; +use sqlbuild_core::event_export::main::sanitized_exception_type::sanitized_exception_type; +use sqlbuild_core::event_export::models::{ExportPolicy, ExportSeverity, ExporterCounts}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::event_exporting::constants::{ + CATALOG_MODULE, EXCEPTIONS_MODULE, SEVERITY_MODULE, +}; + +/// `BoundEventExporter` parameters, in positional order. +const BOUND_EXPORTER_PARAMETERS: [&str; 5] = [ + "name", + "function", + "provider_arguments", + "event_kinds", + "min_severity", +]; +const REQUIRED_BOUND_EXPORTER_PARAMETERS: usize = 3; + +static POLICIES: PyOnceLock> = PyOnceLock::new(); + +/// The exception class `name` of the event exporter exceptions module, raised with `message`. +pub(crate) fn exporter_error(py: Python<'_>, name: &str, message: &str) -> PyErr { + match imported(py, EXCEPTIONS_MODULE, name).and_then(|class| class.call1((message,))) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// `EventExporterStateError(message)`. +pub(crate) fn state_error(py: Python<'_>, message: &str) -> PyErr { + exporter_error(py, "EventExporterStateError", message) +} + +/// `EventExporterInputError(message)`. +pub(crate) fn input_error(py: Python<'_>, message: &str) -> PyErr { + exporter_error(py, "EventExporterInputError", message) +} + +/// The Python `EventExportSeverity` member of `severity`. +pub(crate) fn severity_object( + py: Python<'_>, + severity: ExportSeverity, +) -> PyResult> { + imported(py, SEVERITY_MODULE, "EventExportSeverity")?.call1((severity.value(),)) +} + +/// An `EventExportSeverity` or its string value, as `EventExportSeverity(value)` accepts it. +pub(crate) fn severity_from(value: &Bound<'_, PyAny>) -> PyResult { + let text: String = value.extract()?; + ExportSeverity::from_value(&text).ok_or_else(|| { + PyValueError::new_err(format!("'{text}' is not a valid EventExportSeverity")) + }) +} + +/// The sanitized class name of a raised Python exception. +pub(crate) fn error_type_of(py: Python<'_>, error: &PyErr) -> String { + let name: String = error + .get_type(py) + .getattr("__name__") + .and_then(|name| name.extract()) + .unwrap_or_default(); + sanitized_exception_type(&name) +} + +/// Export dimensions of every catalogued lifecycle event type, derived once per interpreter. +pub(crate) fn export_policies(py: Python<'_>) -> PyResult<&HashMap> { + POLICIES.get_or_try_init(py, || { + let catalog: Bound<'_, PyAny> = imported(py, CATALOG_MODULE, "LIFECYCLE_EVENT_CATALOG")?; + let mut policies: HashMap = HashMap::new(); + for entry in catalog.call_method0("items")?.try_iter()? { + let (event_type, definition): (String, Bound<'_, PyAny>) = entry?.extract()?; + let terminal: bool = definition.getattr("terminal")?.is_truthy()?; + let policy: ExportPolicy = lifecycle_export_policy(&event_type, terminal); + policies.insert(event_type, policy); + } + Ok(policies) + }) +} + +/// The export policy of a validated event's type; `KeyError` for an uncatalogued type. +pub(crate) fn export_policy(py: Python<'_>, event_type: &str) -> PyResult { + export_policies(py)? + .get(event_type) + .cloned() + .ok_or_else(|| PyKeyError::new_err(event_type.to_owned())) +} + +/// `repr(value)`. +pub(crate) fn python_repr(value: &Bound<'_, PyAny>) -> PyResult { + Ok(value.repr()?.to_string()) +} + +/// `repr(text)`. +pub(crate) fn text_repr(py: Python<'_>, text: &str) -> PyResult { + python_repr(&PyString::new(py, text).into_any()) +} + +/// The dataclass-style repr of exporter counts. +pub(crate) fn counts_repr(counts: &ExporterCounts) -> String { + format!( + "EventExporterCounts(accepted={}, filtered={}, delivered={}, failed={}, dropped={})", + counts.accepted, counts.filtered, counts.delivered, counts.failed, counts.dropped + ) +} + +/// The dataclass-style repr of one exporter's accounting. +pub(crate) fn accounting_repr( + py: Python<'_>, + (name, counts): (&str, &ExporterCounts), +) -> PyResult { + Ok(format!( + "EventExporterAccounting(exporter_name={}, counts={})", + text_repr(py, name)?, + counts_repr(counts) + )) +} + +/// `BoundEventExporter(...)` arguments by parameter, as Python binds a dataclass signature. +pub(crate) fn bound_exporter_arguments<'py>( + arguments: &Bound<'py, PyTuple>, + options: Option<&Bound<'py, PyDict>>, +) -> PyResult<[Option>; 5]> { + let mut bound: [Option>; 5] = Default::default(); + if arguments.len() > BOUND_EXPORTER_PARAMETERS.len() { + return Err(PyTypeError::new_err(format!( + "BoundEventExporter() takes at most {} positional arguments", + BOUND_EXPORTER_PARAMETERS.len() + ))); + } + for (slot, argument) in bound.iter_mut().zip(arguments.iter()) { + *slot = Some(argument); + } + for (name, value) in options.into_iter().flat_map(|options| options.iter()) { + let name: String = name.extract()?; + let Some(position) = BOUND_EXPORTER_PARAMETERS + .iter() + .position(|parameter| *parameter == name) + else { + return Err(PyTypeError::new_err(format!( + "BoundEventExporter() got an unexpected keyword argument '{name}'" + ))); + }; + if bound[position].replace(value).is_some() { + return Err(PyTypeError::new_err(format!( + "BoundEventExporter() got multiple values for argument '{name}'" + ))); + } + } + let missing: Option<&&str> = BOUND_EXPORTER_PARAMETERS + .iter() + .take(REQUIRED_BOUND_EXPORTER_PARAMETERS) + .zip(&bound) + .find(|(_, value)| value.is_none()) + .map(|(name, _)| name); + if let Some(name) = missing { + return Err(PyTypeError::new_err(format!( + "BoundEventExporter() missing required argument: '{name}'" + ))); + } + Ok(bound) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs new file mode 100644 index 0000000000..2f4e6f1cce --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs @@ -0,0 +1,407 @@ +//! The dispatcher's delivery, notification, health and per-invocation thread targets. + +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyclass, pymethods}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString}; +use sqlbuild_core::event_export::constants::TIMEOUT_ERROR_TYPE; +use sqlbuild_core::event_export::main::drop_lifecycle_attempts::drop_lifecycle_attempts; +use sqlbuild_core::event_export::main::take_lifecycle_event::take_lifecycle_event; +use sqlbuild_core::event_export::models::{EventQueue, ExportSnapshot, QueuedEvent}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::event_exporting::constants::{ + INVOCATION_THREAD_PREFIX, INVOKE_LIFECYCLE_MODULE, NO_SECONDS, POLL_SECONDS, +}; +use crate::bindings::_helpers::event_exporting::dispatcher::{Exporter, Queued, Shared}; +use crate::bindings::_helpers::event_exporting::models::{ + EventExportSummary, EventExporterFailure, +}; +use crate::bindings::_helpers::event_exporting::values::error_type_of; +use crate::bindings::_helpers::native_threads::exit_signal::interpreter_exiting; + +/// Which long-lived dispatcher thread a worker runs. +#[derive(Clone, Copy)] +pub(crate) enum WorkerRole { + Dispatch, + Notify, + Health, +} + +/// A long-lived dispatcher thread's target. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct DispatcherWorker { + shared: Arc, + role: WorkerRole, +} + +/// Whether an invocation finished, and the sanitized type of what it raised. +#[derive(Default)] +struct Completion { + done: bool, + error_type: Option, +} + +/// Reports how one sink call on its own daemon thread ended. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct InvocationCompletion { + shared: Arc, + invocation: u64, + completion: Arc>, +} + +/// The next thing the notification thread delivers. +enum Notification { + Periodic(ExportSnapshot), + Final(Py), + Failure(EventExporterFailure), + Stop, +} + +/// Whether the call finished, and the sanitized type of what it raised. +fn completion_state(completion: &Mutex) -> PyResult<(bool, Option)> { + let completion = completion + .lock() + .map_err(|_| PyRuntimeError::new_err("event exporter invocation lock is poisoned"))?; + Ok((completion.done, completion.error_type.clone())) +} + +impl DispatcherWorker { + pub(crate) fn new(shared: Arc, role: WorkerRole) -> Self { + Self { shared, role } + } + + /// The state this worker's dispatcher shares. + pub(crate) fn shared(&self) -> &Shared { + &self.shared + } + + /// Wait until bound, returning `false` when stopped first. + fn await_bound(&self, py: Python<'_>) -> PyResult { + let shared = &self.shared; + let _held = shared.hold(py)?; + loop { + let state = shared.locked()?; + if state.bound { + return Ok(true); + } + if state.stopping || state.force_stop { + return Ok(false); + } + drop(state); + shared.wait(py, POLL_SECONDS)?; + } + } + + /// The next queued event, `Ok(None)` to keep polling and `Err(())` once delivery stops. + fn next_event(&self, py: Python<'_>) -> PyResult>, ()>> { + let shared = &self.shared; + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + if Shared::past_deadline(&state) || state.force_stop { + let drained: Vec> = std::mem::take(&mut state.queue.items); + for item in &drained { + state.counts = drop_lifecycle_attempts( + std::mem::take(&mut state.counts), + &item.eligible_exporters, + ); + } + drop(state); + drop(drained); + return Ok(Err(())); + } + if state.queue.items.is_empty() { + drop(state); + shared.wait(py, POLL_SECONDS)?; + state = shared.locked()?; + } + let capacity: usize = state.queue.capacity; + let (queue, next) = take_lifecycle_event(std::mem::replace( + &mut state.queue, + EventQueue::new(capacity), + )); + state.queue = queue; + match next { + Some(item) => Ok(Ok(Some(item))), + None if state.stopping => Ok(Err(())), + None => Ok(Ok(None)), + } + } + + fn run_dispatch(&self, py: Python<'_>) -> PyResult<()> { + let result = (|| -> PyResult<()> { + if !self.await_bound(py)? { + return Ok(()); + } + while let Ok(item) = self.next_event(py)? { + if let Some(item) = item { + self.deliver(py, &item)?; + } + } + Ok(()) + })(); + self.shared.dispatcher_finished(py)?; + result + } + + /// Offer one event to each eligible exporter in order, before the drain deadline. + fn deliver(&self, py: Python<'_>, item: &QueuedEvent) -> PyResult<()> { + let shared = &self.shared; + for (position, index) in item.eligible_exporters.iter().enumerate() { + let exporter: Arc = { + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + if Shared::past_deadline(&state) { + state.counts = drop_lifecycle_attempts( + std::mem::take(&mut state.counts), + &item.eligible_exporters[position..], + ); + return Ok(()); + } + if state.blocked.contains(index) { + state.counts = + drop_lifecycle_attempts(std::mem::take(&mut state.counts), &[*index]); + continue; + } + let Some(exporter) = state.exporters.get(*index) else { + continue; + }; + Arc::clone(exporter) + }; + self.invoke(py, (*index, exporter), item)?; + } + Ok(()) + } + + /// Call one exporter on its own daemon thread and account for its outcome. + fn invoke( + &self, + py: Python<'_>, + (index, exporter): (usize, Arc), + item: &QueuedEvent, + ) -> PyResult<()> { + let shared = &self.shared; + let failure = (index, item.item.kind.as_str(), item.item.severity); + let completion: Arc> = Arc::new(Mutex::new(Completion::default())); + let name: String = format!("{INVOCATION_THREAD_PREFIX}{}", exporter.name); + let invocation: u64 = shared.begin_invocation(py)?; + let reporter = InvocationCompletion::new(Arc::clone(shared), invocation, &completion); + let started = Py::new(py, reporter).and_then(|reporter| { + let arguments = PyDict::new(py); + arguments.set_item("function", exporter.function.bind(py))?; + arguments.set_item("provider_arguments", exporter.provider_arguments.bind(py))?; + arguments.set_item("event", item.item.event.bind(py))?; + arguments.set_item("completion", reporter)?; + let thread = call_with( + &imported(py, "threading", "Thread")?, + &[ + ( + "target", + imported(py, INVOKE_LIFECYCLE_MODULE, "invoke_lifecycle_event_sink")?, + ), + ("kwargs", arguments.into_any()), + ("name", PyString::new(py, &name).into_any()), + ("daemon", PyBool::new(py, true).to_owned().into_any()), + ], + )?; + thread.call_method0("start") + }); + if let Err(error) = started { + shared.invocation_finished(py, invocation)?; + return shared.record_failure(py, failure, error_type_of(py, &error)); + } + let completed: bool = self.await_invocation(py, &completion)?; + let (_, error_type) = completion_state(&completion)?; + if !completed { + { + let _held = shared.hold(py)?; + shared.locked()?.blocked.insert(index); + } + return shared.record_failure(py, failure, TIMEOUT_ERROR_TYPE.to_owned()); + } + if let Some(error_type) = error_type { + return shared.record_failure(py, failure, error_type); + } + let _held = shared.hold(py)?; + if let Some(counts) = shared.locked()?.counts.get_mut(index) { + counts.delivered += 1; + } + Ok(()) + } + + /// Wait for an invocation until its own timeout or the drain deadline, whichever is first. + fn await_invocation(&self, py: Python<'_>, completion: &Mutex) -> PyResult { + let shared = &self.shared; + let invocation_deadline: Instant = Instant::now() + shared.invocation_timeout; + let _held = shared.hold(py)?; + loop { + let state = shared.locked()?; + let effective: Instant = state.deadline.map_or(invocation_deadline, |deadline| { + deadline.min(invocation_deadline) + }); + let force_stop: bool = state.force_stop || interpreter_exiting(py)?; + drop(state); + let remaining: f64 = effective + .saturating_duration_since(Instant::now()) + .as_secs_f64(); + if remaining <= NO_SECONDS || force_stop { + return Ok(completion_state(completion)?.0); + } + shared.wait(py, remaining.min(POLL_SECONDS))?; + if completion_state(completion)?.0 { + return Ok(true); + } + } + } + + fn next_notification(&self, py: Python<'_>) -> PyResult { + let shared = &self.shared; + let _held = shared.hold(py)?; + loop { + let mut state = shared.locked()?; + if let Some(snapshot) = state.pending_periodic.take() { + state.periodic_inflight = Some(snapshot.health_signature()); + return Ok(Notification::Periodic(snapshot)); + } + state.periodic_inflight = None; + if !state.final_delivered + && let Some(summary) = &state.final_notification + { + return Ok(Notification::Final(summary.clone_ref(py))); + } + if let Some(failure) = state.failures.pop_front() { + return Ok(Notification::Failure(failure)); + } + if state.notification_stopping { + return Ok(Notification::Stop); + } + drop(state); + shared.wait(py, POLL_SECONDS)?; + } + } + + fn run_notifications(&self, py: Python<'_>) -> PyResult<()> { + let shared = &self.shared; + loop { + match self.next_notification(py)? { + Notification::Periodic(snapshot) => { + let signature: [u64; 7] = snapshot.health_signature(); + if let Some(callback) = &shared.summary_callback { + let summary = EventExportSummary::object(py, snapshot)?; + drop(callback.bind(py).call1((summary,))); + } + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + state.last_published_periodic = signature; + state.periodic_inflight = None; + } + Notification::Final(summary) => { + if let Some(callback) = &shared.summary_callback { + drop(callback.bind(py).call1((summary,))); + } + let released = { + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + state.final_delivered = true; + state.final_notification.take() + }; + drop(released); + return Ok(()); + } + Notification::Failure(failure) => { + if let Some(callback) = &shared.failure_callback { + let failure = Py::new(py, failure)?; + drop(callback.bind(py).call1((failure,))); + } + } + Notification::Stop => return Ok(()), + } + } + } + + /// Wait one health interval, returning `false` once health publication stops. + fn await_interval(&self, py: Python<'_>) -> PyResult { + let shared = &self.shared; + let deadline: Instant = Instant::now() + shared.health_interval; + let _held = shared.hold(py)?; + loop { + if shared.locked()?.health_stopping { + return Ok(false); + } + let remaining: f64 = deadline + .saturating_duration_since(Instant::now()) + .as_secs_f64(); + if remaining <= NO_SECONDS { + return Ok(true); + } + shared.wait(py, remaining)?; + } + } + + fn run_health(&self, py: Python<'_>) -> PyResult<()> { + let shared = &self.shared; + while self.await_interval(py)? { + let snapshot: ExportSnapshot = shared.current_snapshot(py)?; + let signature: [u64; 7] = snapshot.health_signature(); + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + if state.health_stopping || state.final_notification.is_some() { + return Ok(()); + } + if signature == state.last_published_periodic + || state.periodic_inflight == Some(signature) + { + state.pending_periodic = None; + continue; + } + state.pending_periodic = Some(snapshot); + drop(state); + shared.notify_all(py)?; + } + Ok(()) + } +} + +#[pymethods] +impl DispatcherWorker { + fn __call__(&self, py: Python<'_>) -> PyResult<()> { + match self.role { + WorkerRole::Dispatch => self.run_dispatch(py), + WorkerRole::Notify => self.run_notifications(py), + WorkerRole::Health => self.run_health(py), + } + } +} + +impl InvocationCompletion { + /// A reporter that records into `completion` and ends `invocation`. + fn new(shared: Arc, invocation: u64, completion: &Arc>) -> Self { + Self { + shared, + invocation, + completion: Arc::clone(completion), + } + } +} + +#[pymethods] +impl InvocationCompletion { + #[pyo3(signature = (error, /))] + fn __call__(&self, py: Python<'_>, error: Option<&Bound<'_, PyAny>>) -> PyResult<()> { + if let Some(error) = error { + let error_type: String = error_type_of(py, &PyErr::from_value(error.clone())); + if let Ok(mut completion) = self.completion.lock() { + completion.error_type = Some(error_type); + } + } + let finished = self.shared.invocation_finished(py, self.invocation); + let _held = self.shared.hold(py)?; + if let Ok(mut completion) = self.completion.lock() { + completion.done = true; + } + self.shared.notify_all(py)?; + finished + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs index 218e7fcce1..232b5681e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs @@ -15,7 +15,7 @@ use sqlbuild_analysis::fingerprint_storage::main::selected_positions::selected_p use sqlbuild_analysis::fingerprint_storage::models::{FingerprintIdentity, LatestIndex}; const MODELS_MODULE: &str = "sqlbuild.compiler.fingerprints.models"; -const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.fingerprints.exceptions"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const DATETIME_MODULE: &str = "datetime"; const BINASCII_MODULE: &str = "binascii"; const BASE64_MODULE: &str = "base64"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs index 7012e15f9f..745865ab83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs @@ -11,7 +11,7 @@ use sqlbuild_core::panics::main::is_compiler_panic::is_compiler_panic; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::types::CompilerDetach; -const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.fingerprints.exceptions"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; /// `QueryFingerprintError(message)`, or the compiler error a native panic becomes. fn query_fingerprint_error(py: Python<'_>, message: String) -> PyErr { if is_compiler_panic(&message) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs index 6af55e77bd..851ffca206 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs @@ -12,7 +12,7 @@ use sqlbuild_rules::lint_text::models::FormatSettings; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const LINT_MODELS_MODULE: &str = "sqlbuild.lint.models"; +const LINT_MODELS_MODULE: &str = "sqlbuild._native"; /// Format settings and whether lint suppressions are allowed. pub(crate) struct FormatConfig { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs index 3d07d2ab76..7c4995d502 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::format_command::files::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const FIXES_MODULE: &str = "sqlbuild._native"; const FIXTURE_AUTOFIX_MODULE: &str = "sqlbuild._native"; -const LINT_ERRORS_MODULE: &str = "sqlbuild.lint.exceptions"; +const LINT_ERRORS_MODULE: &str = "sqlbuild._native"; const DESCRIPTION_PRESENT: &str = "description-present"; /// What one format run formats and how. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs index c4f56b8921..c2e5ca131a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs @@ -16,11 +16,13 @@ use crate::bindings::_helpers::freshness::observation::{ }; use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; use crate::bindings::_helpers::freshness::support::{ - ADAPTER_EXCEPTIONS, EXCEPTIONS_MODULE, FRESHNESS_MODELS, OBSERVATION_ERROR, text, + ADAPTER_EXCEPTIONS, EXCEPTIONS_MODULE, OBSERVATION_ERROR, text, }; use crate::bindings::_helpers::planner_identity::graph::build_downstream_model_name_closure; use crate::bindings::_helpers::state_rows::freshness::{age_policy, equivalent}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_RECORDS: &str = "sqlbuild._native"; const ADAPTER_STRATEGY: &str = "adapter"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; @@ -269,7 +271,11 @@ pub(crate) fn build_direct_source_freshness_planning_result<'py>( } let unknown_names: Bound<'py, PyDict> = sorted_dict(py, &unknown_sources)?; call_with( - &imported(py, FRESHNESS_MODELS, "DirectSourceFreshnessPlanningResult")?, + &imported( + py, + NATIVE_RECORDS_MODULE, + "DirectSourceFreshnessPlanningResult", + )?, &[ ( "observed_records", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs index 01e4bdacb1..36a0a2c68d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs @@ -9,7 +9,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::frontier::stage_capture::write_stage_capture; -const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; +const FRONTIER_TYPES_MODULE: &str = "sqlbuild._native"; const RUN_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.classes.macro_bridge"; /// Produce the frontier object for `until`, capturing it on request. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs index b5512b437b..8f83ba02a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs @@ -10,8 +10,9 @@ use pyo3::types::{ }; use sqlbuild_cache::stage_capture::constants::{ - CANONICAL_TYPES, DECODED_SEQUENCES, OMITTED_ATTRIBUTES, PROJECTED_ATTRIBUTES, RESERVED_KEYS, - SHARED_MARKER, SHARED_MIN_CHARS, SKIPPED_SLOTS, UNORDERED_ATTRIBUTES, + CANONICAL_TYPES, DECODED_SEQUENCES, HOSTED_SHAPE_TYPES, OMITTED_ATTRIBUTES, + PROJECTED_ATTRIBUTES, RESERVED_KEYS, SHARED_MARKER, SHARED_MIN_CHARS, SKIPPED_SLOTS, + UNORDERED_ATTRIBUTES, }; use sqlbuild_cache::stage_capture::main::capture_sort_key::capture_sort_key; use sqlbuild_cache::stage_capture::main::compact_capture_json::compact_capture_json; @@ -62,6 +63,12 @@ fn lookup<'a, T: Copy>(table: &'a [(&'a str, T)], name: &str) -> Option { .map(|(_, value)| *value) } +/// The `module:qualname` a captured class is written as, natively hosted shapes keeping their Python name. +fn captured_type_name(class: &Bound<'_, PyAny>) -> PyResult { + let name: String = qualified_name(class)?; + Ok(lookup(&HOSTED_SHAPE_TYPES, &name).map_or(name, str::to_owned)) +} + /// `module:qualname` of a callable or class, as Python's `qualified_name` writes it. pub(crate) fn qualified_name(value: &Bound<'_, PyAny>) -> PyResult { let module: Bound<'_, PyAny> = attribute_or_none(value, "__module__")?; @@ -155,7 +162,7 @@ impl<'py> CaptureEncoder<'py> { "__enum__", JsonValue::String(format!( "{}.{name}", - qualified_name(&value.get_type().into_any())? + captured_type_name(&value.get_type().into_any())? )), )); } @@ -238,7 +245,7 @@ impl<'py> CaptureEncoder<'py> { } fn encode_container(&mut self, value: &Bound<'py, PyAny>) -> PyResult { - let type_name: String = qualified_name(&value.get_type().into_any())?; + let type_name: String = captured_type_name(&value.get_type().into_any())?; if DECODED_SEQUENCES.contains(&type_name.as_str()) && value.is_instance(&self.types.sequence)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs index b47a9508fc..c02ad458ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs @@ -9,8 +9,8 @@ use crate::bindings::_helpers::analysis::explicit_references::{ }; use crate::bindings::_helpers::graph::keys::KeyCache; -const GRAPH_MODELS_MODULE: &str = "sqlbuild.compiler.graph.models"; -const GRAPH_TYPES_MODULE: &str = "sqlbuild.compiler.graph.types"; +const GRAPH_MODELS_MODULE: &str = "sqlbuild._native"; +const GRAPH_TYPES_MODULE: &str = "sqlbuild._native"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs index c3ff893545..480c8e128c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs @@ -1,5 +1,6 @@ //! The executable Python-node graph built from discovered project inputs. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class_at; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -8,9 +9,12 @@ use sqlbuild_analysis::python_selection::models::{InventoryIndexes, InventoryNod use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::discovery::imported_modules::provider_usages; use crate::bindings::_helpers::graph::python_selection::resolve_python_sql_run_selection; use crate::bindings::_helpers::graph::views::compiled_graph_keys; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const USER_INVENTORY_MODULE: &str = "sqlbuild.compiler.python_nodes._helpers.user_inventory"; const NODE_MODELS_MODULE: &str = "sqlbuild.compiler.python_nodes.models"; const NODE_TYPES_MODULE: &str = "sqlbuild.compiler.python_nodes.types"; @@ -30,7 +34,7 @@ struct Vocabulary<'py> { node: Bound<'py, PyAny>, kinds: Bound<'py, PyAny>, sql_ref: Bound<'py, PyAny>, - user_facts: Bound<'py, PyAny>, + identity: Bound<'py, PyAny>, providers: Bound<'py, PyDict>, } @@ -58,7 +62,7 @@ fn kind_parts<'py>( .iter() .map(|field| Ok((*field, get(field)?))) .collect::>()?; - call_with(&imported(py, NODE_MODELS_MODULE, class)?, &pairs) + call_with(&shape_class_at(py, NODE_MODELS_MODULE, class)?, &pairs) }; Ok(match kind { "LOADER" => { @@ -104,7 +108,7 @@ fn kind_parts<'py>( }) } -/// One `DiscoveredPythonNode` with its identity and provider usages read by the user shim. +/// One `DiscoveredPythonNode` with its identity and provider usages read by the user shims. fn python_node<'py>( vocabulary: &Vocabulary<'py>, (kind, documented): (&str, bool), @@ -119,18 +123,17 @@ fn python_node<'py>( } } let kind_member: Bound<'py, PyAny> = vocabulary.kinds.getattr(kind)?; - let (usages, identity): (Bound<'py, PyAny>, Bound<'py, PyAny>) = call_with( - &vocabulary.user_facts, + let usages: Bound<'py, PyAny> = provider_usages(&get("function")?, &vocabulary.providers)?; + let identity: Bound<'py, PyAny> = call_with( + &vocabulary.identity, &[ ("kind", kind_member.getattr("value")?), ("name", get("name")?), ("function", get("function")?), ("file_path", get("file_path")?), - ("provider_by_name", vocabulary.providers.clone().into_any()), ("decorator_config", config.into_any()), ], - )? - .extract()?; + )?; let depends_on: Bound<'py, PyAny> = get("depends_on")?; let mut sql_deps: Vec> = Vec::new(); for dependency in depends_on.try_iter()? { @@ -224,7 +227,7 @@ pub(crate) fn python_node_graph<'py>( node: imported(py, NODE_MODELS_MODULE, "DiscoveredPythonNode")?, kinds: imported(py, NODE_TYPES_MODULE, "PythonNodeKind")?, sql_ref: imported(py, SQL_REF_MODULE, "SqlResourceRef")?, - user_facts: imported(py, USER_INVENTORY_MODULE, "python_node_user_facts")?, + identity: imported(py, USER_INVENTORY_MODULE, "python_node_identity")?, providers, }; let mut nodes: Vec> = Vec::new(); @@ -256,7 +259,7 @@ fn graph_record<'py>( paths, } = indexes; let edge_class: Bound<'py, PyAny> = - imported(py, NODE_MODELS_MODULE, "PythonNodeDependencyEdge")?; + imported(py, NATIVE_RECORDS_MODULE, "PythonNodeDependencyEdge")?; let dependency_edges: Vec> = edges .into_iter() .map(|(upstream_name, downstream_name)| { @@ -327,7 +330,7 @@ pub(crate) fn run_selection_from_inputs<'py>( let (sql_keys, python_node_names) = resolve_python_sql_run_selection(project_graph, &python_graph, select, exclude)?; call_with( - &imported(py, NODE_MODELS_MODULE, "PythonSqlRunSelection")?, + &imported(py, NATIVE_RECORDS_MODULE, "PythonSqlRunSelection")?, &[ ("sql_keys", compiled_graph_keys(py, sql_keys)?.into_any()), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs index 79a3b3be7e..6cd147b959 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -const GRAPH_MODELS_MODULE: &str = "sqlbuild.compiler.graph.models"; +const GRAPH_MODELS_MODULE: &str = "sqlbuild._native"; /// The native graph's `(resource type, name)` pair for `key`. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs index e8320dac1d..159c6369c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs @@ -1,5 +1,6 @@ //! A `CompiledModel` for a model input built outside the native compile-input stage. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; @@ -272,7 +273,7 @@ fn located<'py>( let location_class: Object<'py> = py .import(CONTRACT_MODELS_MODULE)? .getattr("SourceLocation")?; - let phase: Object<'py> = types.getattr("DiagnosticPhase")?.getattr("COMPILE")?; + let phase: Object<'py> = shape_class(types.as_any(), "DiagnosticPhase")?.getattr("COMPILE")?; let model: Object<'py> = types.getattr("CompiledResourceType")?.getattr("MODEL")?; let name: Object<'py> = model_file.getattr("file_path")?.getattr("stem")?; let mut built: Vec> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs new file mode 100644 index 0000000000..4e60e02c91 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs @@ -0,0 +1,528 @@ +//! Delete candidates, skipped relations, blocked schemas and direct-mode archives per schema. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_analysis::janitor_planning::constants::{ + AGE_UNAVAILABLE_REASON, AGE_UNSUPPORTED_REASON, BUILT_IN_EXCLUDE_PATTERNS, + MALFORMED_ARCHIVE_REASON, PROTECTED_REASON, QUERY_DIFF_ARTIFACT_REASON, UNTRACKED_REASON, +}; +use sqlbuild_analysis::janitor_planning::main::build_archive_name::build_archive_name; +use sqlbuild_analysis::janitor_planning::main::case_colliding_names::case_colliding_names; +use sqlbuild_analysis::janitor_planning::main::case_collision_reason::case_collision_reason; +use sqlbuild_analysis::janitor_planning::main::is_archive_lookalike_name::is_archive_lookalike_name; +use sqlbuild_analysis::janitor_planning::main::matching_exclude_pattern::matching_exclude_pattern; +use sqlbuild_analysis::janitor_planning::main::parse_archive_name::parse_archive_name; +use sqlbuild_analysis::janitor_planning::main::query_diff_artifact_parts::query_diff_artifact_parts; +use sqlbuild_analysis::janitor_planning::main::unaddressable_relation_reason::unaddressable_relation_reason; + +use crate::bindings::_helpers::janitor::janitor_facts::{ + Warehouse, WarehouseFacts, skipped_relation, +}; +use crate::bindings::_helpers::janitor::janitor_keys::{ + Folded, Key, SchemaKey, archive_timestamp, days, folded_schema, optional, record, + relation_age_timestamp, sorted_schemas, text, timestamp_fields, tuple, utc_datetime, +}; +use crate::bindings::_helpers::planner_scenario::artifacts::is_scenario_artifact_physical_name; +use crate::bindings::_helpers::relation_naming::fitting::shared_error; + +/// Protection rules for one plan. +pub(crate) struct Protection { + pub(crate) keys: HashSet, + pub(crate) reasons: HashMap, +} + +/// Relation-level settings for one plan. +pub(crate) struct Policy<'a> { + pub(crate) exclude_patterns: &'a [String], + pub(crate) delete_tracked_only: bool, + pub(crate) retention_days: i64, + pub(crate) direct_mode: bool, +} + +/// One stale relation eligible for deletion or archiving. +pub(crate) struct Candidate<'py> { + pub(crate) key: Key, + pub(crate) relation: Bound<'py, PyAny>, + pub(crate) age_timestamp: Option>, +} + +impl<'py> Candidate<'py> { + pub(crate) fn to_python(&self) -> PyResult> { + let py: Python<'py> = self.relation.py(); + record( + py, + "JanitorDeleteCandidate", + &[ + ("key", self.key.to_python(py)?), + ("relation", self.relation.clone()), + ( + "age_timestamp", + age_or_none(py, self.age_timestamp.as_ref()), + ), + ], + ) + } +} + +/// A skipped relation and its key. +pub(crate) struct Skipped<'py> { + pub(crate) key: Key, + pub(crate) record: Bound<'py, PyAny>, +} + +fn skipped<'py>(key: Key, relation: &Bound<'py, PyAny>, reason: &str) -> PyResult> { + Ok(Skipped { + record: skipped_relation(&key, relation, reason)?, + key, + }) +} + +/// A source-overlap schema blocked in direct mode. +pub(crate) struct Blocked<'py> { + pub(crate) schema: SchemaKey, + pub(crate) source_names: Vec, + pub(crate) suppressed: Vec>, +} + +/// Relation decisions across target schemas. +pub(crate) struct SchemaClassification<'py> { + pub(crate) candidates: Vec>, + pub(crate) skipped: Vec>, + pub(crate) skipped_schemas: Vec>, + pub(crate) blocked: Vec>, +} + +fn age_or_none<'py>(py: Python<'py>, value: Option<&Bound<'py, PyAny>>) -> Bound<'py, PyAny> { + value.map_or_else(|| py.None().into_bound(py), Clone::clone) +} + +fn effective_patterns(exclude_patterns: &[String]) -> Vec { + BUILT_IN_EXCLUDE_PATTERNS + .iter() + .map(|pattern| (*pattern).to_owned()) + .chain(exclude_patterns.iter().cloned()) + .collect() +} + +fn exclude_reason(py: Python<'_>, key: &Key, patterns: &[String]) -> PyResult> { + let Some(pattern) = matching_exclude_pattern((&key.name, &key.display_name()), patterns) else { + return Ok(None); + }; + let quoted: String = PyString::new(py, pattern).repr()?.extract()?; + Ok(Some(format!("relation matches exclude pattern {quoted}"))) +} + +fn relation_skip_reason( + py: Python<'_>, + (key, facts): (&Key, &WarehouseFacts<'_>), + (protection, policy): (&Protection, &Policy<'_>), + patterns: &[String], +) -> PyResult> { + if query_diff_artifact_parts(&key.name).is_some() { + return Ok(Some(QUERY_DIFF_ARTIFACT_REASON.to_owned())); + } + let folded: Folded = key.folded(); + if protection.keys.contains(&folded) { + return Ok(Some( + protection + .reasons + .get(&folded) + .cloned() + .unwrap_or_else(|| PROTECTED_REASON.to_owned()), + )); + } + if let Some(reason) = exclude_reason(py, key, patterns)? { + return Ok(Some(reason)); + } + if policy.delete_tracked_only + && !facts.tracked.contains(&folded) + && !is_scenario_artifact_physical_name(&key.name) + { + return Ok(Some(UNTRACKED_REASON.to_owned())); + } + Ok(None) +} + +fn retention_skip_reason( + (age_timestamp, age_supported): (Option<&Bound<'_, PyAny>>, bool), + retention_days: i64, + now: &Bound<'_, PyAny>, +) -> PyResult> { + if !age_supported { + return Ok(Some(AGE_UNSUPPORTED_REASON.to_owned())); + } + let Some(age_timestamp) = age_timestamp else { + return Ok(Some(AGE_UNAVAILABLE_REASON.to_owned())); + }; + if age_timestamp.gt(now.sub(days(now.py(), retention_days)?)?)? { + return Ok(Some(format!( + "relation is newer than {retention_days} days" + ))); + } + Ok(None) +} + +/// Relations with age metadata filled when the adapter supports it. +fn read_ages<'py>( + warehouse: &Warehouse<'py>, + relations: Vec>, +) -> PyResult>> { + if relations.is_empty() || !warehouse.age_metadata_support()? { + return Ok(relations); + } + let py: Python<'py> = warehouse.py(); + warehouse + .call( + "with_relation_age_metadata", + &[("relations", tuple(py, relations)?)], + )? + .try_iter()? + .collect() +} + +/// One schema's relations split into delete candidates and skipped relations. +fn classify_relations<'py>( + (warehouse, facts): (&Warehouse<'py>, &WarehouseFacts<'py>), + schema_relations: &[Bound<'py, PyAny>], + (protection, policy): (&Protection, &Policy<'_>), + now: &Bound<'py, PyAny>, +) -> PyResult<(Vec>, Vec>)> { + let py: Python<'py> = warehouse.py(); + let patterns: Vec = effective_patterns(policy.exclude_patterns); + let keys: Vec = schema_relations + .iter() + .map(Key::of) + .collect::>()?; + let colliding: HashSet = case_colliding_names(keys.iter().map(|key| key.name.as_str())); + let mut skipped_relations: Vec> = Vec::new(); + let mut eligible: Vec> = Vec::new(); + for (key, relation) in keys.into_iter().zip(schema_relations) { + if facts.desired.contains(&key.folded()) { + continue; + } + if policy.direct_mode && is_archive_lookalike_name(&key.name) { + continue; + } + let mut reason: Option = + relation_skip_reason(py, (&key, facts), (protection, policy), &patterns)?; + if reason.is_none() { + reason = if policy.direct_mode { + unaddressable_relation_reason(&key.name, &colliding) + } else { + case_collision_reason(&key.name, &colliding) + } + .map(str::to_owned); + } + match reason { + Some(reason) => skipped_relations.push(skipped(key, relation, &reason)?), + None => eligible.push(relation.clone()), + } + } + let age_supported: bool = warehouse.age_metadata_support()?; + let aged: Vec> = if policy.retention_days > 0 { + read_ages(warehouse, eligible)? + } else { + eligible + }; + let mut candidates: Vec> = Vec::new(); + for relation in aged { + let key: Key = Key::of(&relation)?; + let age_timestamp: Option> = relation_age_timestamp(&relation)?; + let retention_reason: Option = if policy.retention_days > 0 { + retention_skip_reason( + (age_timestamp.as_ref(), age_supported), + policy.retention_days, + now, + )? + } else { + None + }; + if let Some(reason) = retention_reason { + skipped_relations.push(skipped(key, &relation, &reason)?); + continue; + } + candidates.push(Candidate { + key, + relation, + age_timestamp, + }); + } + Ok((candidates, skipped_relations)) +} + +/// All target schemas classified, keeping relations in schemas with configured sources. +pub(crate) fn classify_target_schemas<'py>( + (warehouse, facts): (&Warehouse<'py>, &WarehouseFacts<'py>), + (target_schemas, managed): (&[SchemaKey], &[SchemaKey]), + (protection, policy): (&Protection, &Policy<'_>), + now: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = warehouse.py(); + let mut result = SchemaClassification { + candidates: Vec::new(), + skipped: Vec::new(), + skipped_schemas: Vec::new(), + blocked: Vec::new(), + }; + for schema_key in sorted_schemas(target_schemas) { + let schema_relations: &[Bound<'py, PyAny>] = facts.schema_relations(&schema_key); + let source_names: BTreeSet = facts.source_names(&schema_key); + let folded: SchemaKey = folded_schema(&schema_key); + let blocking: Vec = if managed.iter().any(|key| folded_schema(key) == folded) { + source_names.iter().cloned().collect() + } else { + Vec::new() + }; + if !source_names.is_empty() || !blocking.is_empty() { + if policy.direct_mode && !blocking.is_empty() { + let (suppressed, _) = classify_relations( + (warehouse, facts), + schema_relations, + (protection, policy), + now, + )?; + result.blocked.push(Blocked { + schema: schema_key, + source_names: blocking, + suppressed, + }); + continue; + } + result.skipped_schemas.push(record( + py, + "JanitorSkippedSchema", + &[ + ("database", optional(py, schema_key.0.as_deref())?), + ("schema", optional(py, schema_key.1.as_deref())?), + ( + "source_names", + tuple(py, source_names.iter().map(|name| text(py, name)))?, + ), + ( + "skipped_relations", + tuple(py, schema_relations.iter().cloned())?, + ), + ], + )?); + continue; + } + let (candidates, skipped_relations) = classify_relations( + (warehouse, facts), + schema_relations, + (protection, policy), + now, + )?; + result.candidates.extend(candidates); + result.skipped.extend(skipped_relations); + } + Ok(result) +} + +/// Direct-mode archive renames and name-based archive expiry. +pub(crate) struct ArchivePlanning<'py> { + pub(crate) archive_candidates: Vec>, + pub(crate) deletions: Vec>, + pub(crate) retained: Vec>, + pub(crate) skipped: Vec>, + pub(crate) blocked_schemas: Vec>, +} + +fn archived_relation<'py>( + (key, relation_type): (&Key, Bound<'py, PyAny>), + (archived_at, expires_at): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + original_key: Option<&Key>, +) -> PyResult> { + let py: Python<'py> = archived_at.py(); + let mut pairs: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("key", key.to_python(py)?), + ("relation_type", relation_type), + ("archived_at", archived_at.clone()), + ("expires_at", expires_at.clone()), + ]; + if let Some(original_key) = original_key { + pairs.push(("original_key", original_key.to_python(py)?)); + } + record(py, "JanitorArchivedRelation", &pairs) +} + +fn blocked_record<'py>( + py: Python<'py>, + schema: &Blocked<'py>, + suppressed_archives: &HashMap>>, +) -> PyResult> { + let source_names: Vec> = schema + .source_names + .iter() + .map(|name| text(py, name)) + .collect(); + let suppressed: Vec> = schema + .suppressed + .iter() + .map(Candidate::to_python) + .collect::>()?; + let archives: Vec> = suppressed_archives + .get(&schema.schema) + .cloned() + .unwrap_or_default(); + record( + py, + "JanitorBlockedSchema", + &[ + ("database", optional(py, schema.schema.0.as_deref())?), + ("schema", optional(py, schema.schema.1.as_deref())?), + ("source_names", tuple(py, source_names)?), + ("suppressed_candidates", tuple(py, suppressed)?), + ("suppressed_archive_deletions", tuple(py, archives)?), + ], + ) +} + +fn blocked_records<'py>( + py: Python<'py>, + blocked: &[Blocked<'py>], + suppressed_archives: &HashMap>>, +) -> PyResult>> { + blocked + .iter() + .map(|schema| blocked_record(py, schema, suppressed_archives)) + .collect() +} + +fn archive_candidate<'py>( + candidate: &Candidate<'py>, + (archived_at, expires_at): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + identifier_limit: i64, +) -> PyResult<(Key, Bound<'py, PyAny>)> { + let py: Python<'py> = archived_at.py(); + let archive_key = Key { + database: candidate.key.database.clone(), + schema: candidate.key.schema.clone(), + name: build_archive_name( + (&candidate.key.name, timestamp_fields(archived_at)?), + identifier_limit, + None, + ) + .map_err(|message| shared_error(py, message))?, + }; + let record = record( + py, + "JanitorArchiveCandidate", + &[ + ("key", candidate.key.to_python(py)?), + ("relation", candidate.relation.clone()), + ( + "age_timestamp", + age_or_none(py, candidate.age_timestamp.as_ref()), + ), + ("archive_key", archive_key.to_python(py)?), + ("archived_at", archived_at.clone()), + ("expires_at", expires_at.clone()), + ], + )?; + Ok((archive_key, record)) +} + +/// Archive renames and expiry of existing archives; non-direct mode keeps only blocked schemas. +pub(crate) fn plan_archives<'py>( + (warehouse, facts): (&Warehouse<'py>, &WarehouseFacts<'py>), + (schemas, managed): (&SchemaClassification<'py>, &[SchemaKey]), + (policy, archive_retention_days): (&Policy<'_>, i64), + now: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = warehouse.py(); + if !policy.direct_mode { + return Ok(ArchivePlanning { + archive_candidates: Vec::new(), + deletions: Vec::new(), + retained: Vec::new(), + skipped: Vec::new(), + blocked_schemas: blocked_records(py, &schemas.blocked, &HashMap::new())?, + }); + } + let identifier_limit: i64 = warehouse + .adapter + .call_method0("maximum_identifier_length")? + .extract()?; + let archived_at: Bound<'py, PyAny> = archive_timestamp(now)?; + let retention: Bound<'py, PyAny> = days(py, archive_retention_days)?; + let expires_at: Bound<'py, PyAny> = archived_at.add(&retention)?; + let mut archive_candidates: Vec> = Vec::new(); + let mut same_run: Vec> = Vec::new(); + let same_run_expiry: bool = expires_at.le(&archived_at)?; + for candidate in &schemas.candidates { + let (archive_key, record) = + archive_candidate(candidate, (&archived_at, &expires_at), identifier_limit)?; + archive_candidates.push(record); + if same_run_expiry { + same_run.push(archived_relation( + (&archive_key, candidate.relation.getattr("relation_type")?), + (&archived_at, &expires_at), + Some(&candidate.key), + )?); + } + } + let blocked_keys: Vec<&SchemaKey> = schemas + .blocked + .iter() + .map(|blocked| &blocked.schema) + .collect(); + let patterns: Vec = effective_patterns(policy.exclude_patterns); + let mut deletions: Vec> = Vec::new(); + let mut retained: Vec> = Vec::new(); + let mut skipped_relations: Vec> = Vec::new(); + let mut suppressed: HashMap>> = HashMap::new(); + for schema_key in sorted_schemas(managed) { + let schema_relations: &[Bound<'py, PyAny>] = facts.schema_relations(&schema_key); + let keys: Vec = schema_relations + .iter() + .map(Key::of) + .collect::>()?; + let colliding: HashSet = + case_colliding_names(keys.iter().map(|key| key.name.as_str())); + for (key, relation) in keys.into_iter().zip(schema_relations) { + if facts.desired.contains(&key.folded()) || !is_archive_lookalike_name(&key.name) { + continue; + } + let Some(parsed) = parse_archive_name(&key.name) else { + skipped_relations.push(skipped_relation(&key, relation, MALFORMED_ARCHIVE_REASON)?); + continue; + }; + let reason: Option = match unaddressable_relation_reason(&key.name, &colliding) + { + Some(reason) => Some(reason.to_owned()), + None => exclude_reason(py, &key, &patterns)?, + }; + if let Some(reason) = reason { + skipped_relations.push(skipped_relation(&key, relation, &reason)?); + continue; + } + let parsed_at: Bound<'py, PyAny> = utc_datetime(py, parsed.archived_at)?; + let parsed_expiry: Bound<'py, PyAny> = parsed_at.add(&retention)?; + let archived: Bound<'py, PyAny> = archived_relation( + (&key, relation.getattr("relation_type")?), + (&parsed_at, &parsed_expiry), + None, + )?; + if parsed_expiry.gt(now)? { + retained.push(archived); + } else if blocked_keys.contains(&&schema_key) { + suppressed + .entry(schema_key.clone()) + .or_default() + .push(archived); + } else { + deletions.push(archived); + } + } + } + deletions.extend(same_run); + Ok(ArchivePlanning { + archive_candidates, + deletions, + retained, + skipped: skipped_relations, + blocked_schemas: blocked_records(py, &schemas.blocked, &suppressed)?, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 483a823167..d1f820dbb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -1,11 +1,13 @@ //! `sqb janitor`: preview stale warehouse objects, confirm, and clean them up. +use pyo3::IntoPyObject; use pyo3::exceptions::PyKeyboardInterrupt; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::janitor_planning::main::resolve_retention_days::resolve_retention_days; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_core::janitor_text::errors::JanitorSettingsError; use sqlbuild_core::janitor_text::main::janitor_completion_text::janitor_completion_text; @@ -30,7 +32,6 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild._native"; const PREVIEW_TARGET_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; -const RETENTION_MODULE: &str = "sqlbuild.executor.janitor.main.resolve_retention_days"; const COMMAND_NAME: &str = "janitor"; fn imported<'py>(py: Python<'py>, module: &str, name: &str) -> PyResult> { @@ -125,17 +126,12 @@ fn settings<'py>( request: &Bound<'py, PyAny>, janitor: &Bound<'py, PyAny>, ) -> PyResult> { - let retention_days: Bound<'py, PyAny> = - imported(py, RETENTION_MODULE, "resolve_janitor_retention_days")?.call( - (), - Some(&keywords( - py, - &[ - ("override", &request.getattr("retention_days")?), - ("configured", &janitor.getattr("retention_days")?), - ], - )?), - )?; + let retention_days: Bound<'py, PyAny> = resolve_retention_days( + request.getattr("retention_days")?.extract()?, + janitor.getattr("retention_days")?.extract()?, + ) + .into_pyobject(py)? + .into_any(); validate_janitor_settings(retention_days.extract()?, 0) .map_err(|error| settings_error(py, error))?; let requested: Bound<'py, PyAny> = request.getattr("direct_state_history_versions")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs new file mode 100644 index 0000000000..c3af25ab1a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -0,0 +1,531 @@ +//! Warehouse facts janitor planning reads through adapter callbacks, in the executor's order. + +use std::collections::{BTreeSet, HashSet}; + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyBool; +use sqlbuild_analysis::janitor_planning::constants::UNTRACKED_QUERY_DIFF_REASON; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::janitor::janitor_keys::{ + Folded, Key, SchemaKey, dedupe_schema_keys, folded_schema, integer, optional, record, + sorted_schemas, text, tuple, +}; + +const QUERY_ARTIFACT_LIFECYCLE: (&str, &str) = ( + "sqlbuild.executor.diff.classes.query_artifact_lifecycle", + "QueryDiffArtifactLifecycle", +); +const RELATION_LOOKUP: (&str, &str) = ( + "sqlbuild.adapter.relations.main.relation_lookup", + "build_relation_lookup", +); +const LOG_DEBUG_EVENT: (&str, &str) = ( + "sqlbuild.diagnostics.main.log_debug_event", + "log_debug_event", +); +const READ_LATEST_FINGERPRINTS: (&str, &str) = ("sqlbuild._native", "read_latest_fingerprints"); +const EXECUTION_LOGGER: &str = "sqlbuild.execution"; +const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; +const SOURCE_FRESHNESS_TABLE_NAME: &str = "_sqlbuild_source_freshness"; + +/// The adapter and its open connection. +pub(crate) struct Warehouse<'py> { + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) connection: Bound<'py, PyAny>, +} + +impl<'py> Warehouse<'py> { + pub(crate) fn py(&self) -> Python<'py> { + self.adapter.py() + } + + /// `adapter.(connection=connection, **pairs)`. + pub(crate) fn call( + &self, + method: &str, + pairs: &[(&str, Bound<'py, PyAny>)], + ) -> PyResult> { + let mut arguments: Vec<(&str, Bound<'py, PyAny>)> = + vec![("connection", self.connection.clone())]; + arguments.extend(pairs.iter().cloned()); + call_with(&self.adapter.getattr(method)?, &arguments) + } + + pub(crate) fn age_metadata_support(&self) -> PyResult { + self.adapter + .call_method0("supports_relation_age_metadata")? + .is_truthy() + } +} + +/// Relations listed per target schema, in listing order. +pub(crate) type SchemaRelations<'py> = Vec<(SchemaKey, Vec>)>; + +/// Desired, discovered and tracked relation facts for planning. +pub(crate) struct WarehouseFacts<'py> { + pub(crate) desired: HashSet, + pub(crate) source_schemas: Vec<(SchemaKey, BTreeSet)>, + pub(crate) relations: SchemaRelations<'py>, + pub(crate) tracked: HashSet, +} + +impl<'py> WarehouseFacts<'py> { + /// Relations listed for exactly this schema spelling. + pub(crate) fn schema_relations(&self, key: &SchemaKey) -> &[Bound<'py, PyAny>] { + self.relations + .iter() + .find(|(listed, _)| listed == key) + .map_or(&[], |(_, relations)| relations.as_slice()) + } + + /// Active source names configured in one schema, compared case-insensitively. + pub(crate) fn source_names(&self, key: &SchemaKey) -> BTreeSet { + let folded: SchemaKey = folded_schema(key); + self.source_schemas + .iter() + .filter(|(candidate, _)| folded_schema(candidate) == folded) + .flat_map(|(_, names)| names.iter().cloned()) + .collect() + } +} + +fn destination_keys<'py>( + project: &Bound<'py, PyAny>, + field: &str, +) -> PyResult)>> { + let mut keys: Vec<(Key, Bound<'py, PyAny>)> = Vec::new(); + for node in project.getattr(field)?.try_iter()? { + let node: Bound<'py, PyAny> = node?; + keys.push((Key::of(&node.getattr("destination")?)?, node)); + } + Ok(keys) +} + +/// Model destinations with their models, in project order. +pub(crate) fn model_destinations<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult)>> { + destination_keys(project, "models") +} + +/// Schemas the compiled project writes relations to, one spelling per schema. +pub(crate) fn collect_target_schemas(project: &Bound<'_, PyAny>) -> PyResult> { + let mut schemas: Vec = Vec::new(); + for field in ["models", "seeds"] { + for (key, _) in destination_keys(project, field)? { + schemas.push((key.database, key.schema)); + } + } + Ok(dedupe_schema_keys(schemas)) +} + +fn desired_keys(project: &Bound<'_, PyAny>) -> PyResult> { + let mut keys: HashSet = HashSet::new(); + for field in ["models", "seeds"] { + for (key, _) in destination_keys(project, field)? { + keys.insert(key.folded()); + } + } + Ok(keys) +} + +fn source_schemas( + project: &Bound<'_, PyAny>, + (default_database, default_schema): (Option, Option), +) -> PyResult)>> { + let mut schemas: Vec<(SchemaKey, BTreeSet)> = Vec::new(); + for source in project.getattr("sources")?.try_iter()? { + let source: Bound<'_, PyAny> = source?; + let entry: Bound<'_, PyAny> = source.getattr("source_entry")?; + if !entry.getattr("expression")?.is_none() { + continue; + } + let database: Option = entry + .getattr("database")? + .extract::>()? + .or_else(|| default_database.clone()); + let schema: Option = entry + .getattr("schema")? + .extract::>()? + .or_else(|| default_schema.clone()); + let name: String = source.getattr("name")?.extract()?; + let key: SchemaKey = (database, schema); + match schemas.iter_mut().find(|(listed, _)| *listed == key) { + Some((_, names)) => { + names.insert(name); + } + None => schemas.push((key, BTreeSet::from([name]))), + } + } + Ok(schemas) +} + +/// Relations matched case-insensitively and respelled as their target schema. +fn list_target_schema_relations<'py>( + warehouse: &Warehouse<'py>, + target_schemas: &[SchemaKey], +) -> PyResult> { + let py: Python<'py> = warehouse.py(); + let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; + let mut databases: Vec> = Vec::new(); + for (database, _) in sorted_schemas(target_schemas) { + if !databases.contains(&database) { + databases.push(database); + } + } + let mut result: SchemaRelations<'py> = Vec::new(); + for database in databases { + let mut concrete: Vec = target_schemas + .iter() + .filter(|(listed, _)| *listed == database) + .filter_map(|(_, schema)| schema.clone()) + .collect(); + concrete.sort(); + concrete.dedup(); + let schemas: Bound<'py, PyAny> = if concrete.is_empty() { + py.None().into_bound(py) + } else { + tuple(py, concrete.iter().map(|schema| text(py, schema)))? + }; + let listed: Bound<'py, PyAny> = warehouse.call( + "list_relations", + &[ + ("database", optional(py, database.as_deref())?), + ("schemas", schemas), + ], + )?; + for relation in listed.try_iter()? { + let relation: Bound<'py, PyAny> = relation?; + let folded: SchemaKey = folded_schema(&( + relation.getattr("database")?.extract()?, + relation.getattr("schema")?.extract()?, + )); + let Some(target) = target_schemas + .iter() + .find(|target| folded_schema(target) == folded) + else { + continue; + }; + let respelled: Bound<'py, PyAny> = replace.call( + (relation,), + Some(&keywords( + py, + &[ + ("database", optional(py, target.0.as_deref())?), + ("schema", optional(py, target.1.as_deref())?), + ], + )?), + )?; + match result.iter_mut().find(|(listed, _)| listed == target) { + Some((_, relations)) => relations.push(respelled), + None => result.push((target.clone(), vec![respelled])), + } + } + } + Ok(result) +} + +/// `build_relation_lookup` over the given locations. +fn relation_lookup<'py>( + warehouse: &Warehouse<'py>, + locations: &[(Option, String, &str)], +) -> PyResult> { + let py: Python<'py> = warehouse.py(); + let mut items: Vec> = Vec::new(); + for (database, schema, name) in locations { + items.push(tuple( + py, + [ + optional(py, database.as_deref())?, + text(py, schema), + text(py, name), + ], + )?); + } + call_with( + &imported(py, RELATION_LOOKUP.0, RELATION_LOOKUP.1)?, + &[ + ("adapter", warehouse.adapter.clone()), + ("connection", warehouse.connection.clone()), + ("locations", tuple(py, items)?), + ], + ) +} + +fn lookup_exists( + lookup: &Bound<'_, PyAny>, + (database, schema, name): (Option<&str>, &str, &str), +) -> PyResult { + let py: Python<'_> = lookup.py(); + call_with( + &lookup.getattr("exists")?, + &[ + ("database", optional(py, database)?), + ("schema", text(py, schema)), + ("name", text(py, name)), + ], + )? + .is_truthy() +} + +fn concrete_schemas(target_schemas: &[SchemaKey]) -> Vec<(Option, String)> { + sorted_schemas(target_schemas) + .into_iter() + .filter_map(concrete_schema) + .collect() +} + +fn concrete_schema((database, schema): SchemaKey) -> Option<(Option, String)> { + Some((database, schema?)) +} + +/// Exact physical relation keys recorded in fingerprint metadata. +fn tracked_relation_keys( + warehouse: &Warehouse<'_>, + target_schemas: &[SchemaKey], +) -> PyResult> { + let py: Python<'_> = warehouse.py(); + let schemas: Vec<(Option, String)> = concrete_schemas(target_schemas); + let locations: Vec<(Option, String, &str)> = schemas + .iter() + .map(|(database, schema)| (database.clone(), schema.clone(), FINGERPRINT_TABLE_NAME)) + .collect(); + let lookup: Bound<'_, PyAny> = relation_lookup(warehouse, &locations)?; + let mut tracked: HashSet = HashSet::new(); + for (database, schema) in &schemas { + let exists: bool = lookup_exists( + &lookup, + (database.as_deref(), schema, FINGERPRINT_TABLE_NAME), + )?; + let read: PyResult> = call_with( + &imported(py, READ_LATEST_FINGERPRINTS.0, READ_LATEST_FINGERPRINTS.1)?, + &[ + ("connection", warehouse.connection.clone()), + ("execute", warehouse.adapter.getattr("execute")?), + ( + "table_exists", + PyBool::new(py, exists).to_owned().into_any(), + ), + ("database", optional(py, database.as_deref())?), + ("schema", text(py, schema)), + ( + "render_qualified_name", + warehouse.adapter.getattr("render_qualified_name")?, + ), + ( + "render_read_latest_sql", + warehouse + .adapter + .getattr("render_read_latest_fingerprints_sql")?, + ), + ], + ); + let fingerprint_set: Bound<'_, PyAny> = match read { + Ok(set) => set, + Err(error) if error.is_instance_of::(py) => { + call_with( + &imported(py, LOG_DEBUG_EVENT.0, LOG_DEBUG_EVENT.1)?, + &[ + ( + "logger", + imported(py, "logging", "getLogger")?.call1((EXECUTION_LOGGER,))?, + ), + ( + "message", + text( + py, + "janitor fingerprint tracking read failed; skipping schema", + ), + ), + ("database", optional(py, database.as_deref())?), + ("schema", text(py, schema)), + ("sqlbuild_error", error.value(py).str()?.into_any()), + ], + )?; + continue; + } + Err(error) => return Err(error), + }; + for fingerprint in fingerprint_set + .getattr("fingerprints")? + .call_method0("values")? + .try_iter()? + { + let fingerprint: Bound<'_, PyAny> = fingerprint?; + let target_schema: Option = fingerprint.getattr("target_schema")?.extract()?; + let target_name: Option = fingerprint.getattr("target_name")?.extract()?; + let (Some(target_schema), Some(target_name)) = (target_schema, target_name) else { + continue; + }; + tracked.insert( + Key { + database: fingerprint.getattr("target_database")?.extract()?, + schema: Some(target_schema), + name: target_name, + } + .folded(), + ); + } + } + Ok(tracked) +} + +/// Desired, discovered and tracked relation facts, read in the executor's call order. +pub(crate) fn gather_warehouse_facts<'py>( + (project, warehouse): (&Bound<'py, PyAny>, &Warehouse<'py>), + target_schemas: &[SchemaKey], + delete_tracked_only: bool, +) -> PyResult> { + let desired: HashSet = desired_keys(project)?; + let default_database: Option = warehouse + .adapter + .call_method0("default_database")? + .extract()?; + let default_schema: Option = warehouse + .adapter + .call_method0("default_schema")? + .extract()?; + let source_schemas = source_schemas(project, (default_database, default_schema))?; + let relations: SchemaRelations<'py> = list_target_schema_relations(warehouse, target_schemas)?; + let tracked: HashSet = if delete_tracked_only { + tracked_relation_keys(warehouse, target_schemas)? + } else { + HashSet::new() + }; + Ok(WarehouseFacts { + desired, + source_schemas, + relations, + tracked, + }) +} + +/// Fingerprint and source freshness history prune candidates for target schemas. +pub(crate) fn direct_state_prune_candidates<'py>( + warehouse: &Warehouse<'py>, + target_schemas: &[SchemaKey], + retain_versions: i64, +) -> PyResult>> { + let py: Python<'py> = warehouse.py(); + if retain_versions <= 0 { + return Ok(Vec::new()); + } + let schemas: Vec<(Option, String)> = concrete_schemas(target_schemas); + let mut locations: Vec<(Option, String, &str)> = Vec::new(); + for (database, schema) in &schemas { + locations.push((database.clone(), schema.clone(), FINGERPRINT_TABLE_NAME)); + locations.push(( + database.clone(), + schema.clone(), + SOURCE_FRESHNESS_TABLE_NAME, + )); + } + let lookup: Bound<'py, PyAny> = relation_lookup(warehouse, &locations)?; + let mut candidates: Vec> = Vec::new(); + for (database, schema) in &schemas { + for (table_name, render) in [ + ( + FINGERPRINT_TABLE_NAME, + "render_prune_fingerprint_history_sql", + ), + ( + SOURCE_FRESHNESS_TABLE_NAME, + "render_prune_source_freshness_history_sql", + ), + ] { + if !lookup_exists(&lookup, (database.as_deref(), schema, table_name))? { + continue; + } + let prune_sql: Bound<'py, PyAny> = call_with( + &warehouse.adapter.getattr(render)?, + &[ + ("database", optional(py, database.as_deref())?), + ("schema", text(py, schema)), + ("retain_versions", integer(py, retain_versions)?), + ], + )?; + candidates.push(record( + py, + "JanitorDirectStatePruneCandidate", + &[ + ("database", optional(py, database.as_deref())?), + ("schema", text(py, schema)), + ("table_name", text(py, table_name)), + ("retain_versions", integer(py, retain_versions)?), + ("prune_sql", prune_sql), + ], + )?); + } + } + Ok(candidates) +} + +/// Expired query-diff artifact candidates and untracked reserved names as skipped relations. +pub(crate) type ArtifactCandidates<'py> = (Vec>, Vec>); + +/// Expired owned query-diff artifacts, and untracked reserved names as skipped relations. +pub(crate) fn query_diff_artifact_candidates<'py>( + warehouse: &Warehouse<'py>, + schemas: &[(Option, String)], + now: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = warehouse.py(); + let lifecycle: Bound<'py, PyAny> = + imported(py, QUERY_ARTIFACT_LIFECYCLE.0, QUERY_ARTIFACT_LIFECYCLE.1)?; + let mut sorted: Vec<(Option, String)> = schemas.to_vec(); + sorted.sort_by(|left, right| { + (left.0.as_deref().unwrap_or(""), left.1.as_str()) + .cmp(&(right.0.as_deref().unwrap_or(""), right.1.as_str())) + }); + let mut candidates: Vec> = Vec::new(); + let mut skipped: Vec> = Vec::new(); + for (database, schema) in sorted { + let inspection: Bound<'py, PyAny> = call_with( + &lifecycle.getattr("inspect")?, + &[ + ("adapter", warehouse.adapter.clone()), + ("connection", warehouse.connection.clone()), + ("database", optional(py, database.as_deref())?), + ("schema", text(py, &schema)), + ("now", now.clone()), + ], + )?; + for artifact in inspection.getattr("expired")?.try_iter()? { + let key: Key = Key::of(&artifact?)?; + candidates.push(record( + py, + "JanitorQueryDiffArtifactCandidate", + &[("key", key.to_python(py)?)], + )?); + } + for relation in inspection.getattr("untracked")?.try_iter()? { + let relation: Bound<'py, PyAny> = relation?; + skipped.push(skipped_relation( + &Key::of(&relation)?, + &relation, + UNTRACKED_QUERY_DIFF_REASON, + )?); + } + } + Ok((candidates, skipped)) +} + +/// The executor's `JanitorSkippedRelation`. +pub(crate) fn skipped_relation<'py>( + key: &Key, + relation: &Bound<'py, PyAny>, + reason: &str, +) -> PyResult> { + let py: Python<'py> = relation.py(); + record( + py, + "JanitorSkippedRelation", + &[ + ("key", key.to_python(py)?), + ("relation", relation.clone()), + ("reason", text(py, reason)), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_keys.rs new file mode 100644 index 0000000000..8f18cf46da --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_keys.rs @@ -0,0 +1,228 @@ +//! Janitor relation keys, schema keys, times and the executor records planning builds. + +use std::collections::HashSet; + +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyString, PyTuple}; +use sqlbuild_analysis::janitor_planning::models::ArchiveTimestamp; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +pub(crate) const JANITOR_MODELS: &str = "sqlbuild.executor.janitor.models"; +const DATETIME_MODULE: &str = "datetime"; + +/// `(database, schema)` as spelled. +pub(crate) type SchemaKey = (Option, Option); +/// Case-insensitive relation identity. +pub(crate) type Folded = (Option, Option, String); + +/// One physical relation identity, compared case-insensitively but rendered as spelled. +#[derive(Clone, Debug)] +pub(crate) struct Key { + pub(crate) database: Option, + pub(crate) schema: Option, + pub(crate) name: String, +} + +impl Key { + /// The key of an object with `database`, `schema` and `name` attributes. + pub(crate) fn of(item: &Bound<'_, PyAny>) -> PyResult { + Ok(Self { + database: item.getattr("database")?.extract()?, + schema: item.getattr("schema")?.extract()?, + name: item.getattr("name")?.extract()?, + }) + } + + pub(crate) fn folded(&self) -> Folded { + ( + fold(self.database.as_deref()), + fold(self.schema.as_deref()), + self.name.to_lowercase(), + ) + } + + pub(crate) fn display_name(&self) -> String { + [ + self.database.as_deref(), + self.schema.as_deref(), + Some(&self.name), + ] + .into_iter() + .flatten() + .collect::>() + .join(".") + } + + /// The executor's `JanitorRelationKey`. + pub(crate) fn to_python<'py>(&self, py: Python<'py>) -> PyResult> { + record( + py, + "JanitorRelationKey", + &[ + ("database", optional(py, self.database.as_deref())?), + ("schema", optional(py, self.schema.as_deref())?), + ("name", text(py, &self.name)), + ], + ) + } +} + +pub(crate) fn fold(value: Option<&str>) -> Option { + value.map(str::to_lowercase) +} + +pub(crate) fn folded_schema((database, schema): &SchemaKey) -> SchemaKey { + (fold(database.as_deref()), fold(schema.as_deref())) +} + +/// Schema keys ordered by `(database or "", schema or "")`, as Python sorts them. +pub(crate) fn sorted_schemas(keys: &[SchemaKey]) -> Vec { + let mut sorted: Vec = keys.to_vec(); + sorted.sort_by(|left, right| schema_order(left).cmp(&schema_order(right))); + sorted +} + +fn schema_order((database, schema): &SchemaKey) -> (&str, &str) { + ( + database.as_deref().unwrap_or(""), + schema.as_deref().unwrap_or(""), + ) +} + +/// One spelling per case-insensitive schema, keeping the first spelling met. +pub(crate) fn dedupe_schema_keys(keys: impl IntoIterator) -> Vec { + let mut seen: HashSet = HashSet::new(); + let mut kept: Vec = Vec::new(); + for key in keys { + if seen.insert(folded_schema(&key)) { + kept.push(key); + } + } + kept +} + +/// `keys` followed by each of `extra` whose exact spelling is not already present. +pub(crate) fn union_exact( + keys: Vec, + extra: impl IntoIterator, +) -> Vec { + let mut joined: Vec = keys; + for key in extra { + if !joined.contains(&key) { + joined.push(key); + } + } + joined +} + +pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +pub(crate) fn optional<'py>(py: Python<'py>, value: Option<&str>) -> PyResult> { + Ok(value.into_pyobject(py)?.into_any()) +} + +pub(crate) fn integer(py: Python<'_>, value: i64) -> PyResult> { + Ok(value.into_pyobject(py)?.into_any()) +} + +pub(crate) fn tuple<'py>( + py: Python<'py>, + items: impl IntoIterator>, +) -> PyResult> { + let items: Vec> = items.into_iter().collect(); + Ok(PyTuple::new(py, items)?.into_any()) +} + +/// `sqlbuild.executor.janitor.models.(**pairs)`. +pub(crate) fn record<'py>( + py: Python<'py>, + class_name: &str, + pairs: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with(&imported(py, JANITOR_MODELS, class_name)?, pairs) +} + +fn utc(py: Python<'_>) -> PyResult> { + imported(py, DATETIME_MODULE, "UTC") +} + +/// `datetime.now(UTC)`. +pub(crate) fn now_utc(py: Python<'_>) -> PyResult> { + imported(py, DATETIME_MODULE, "datetime")?.call_method1("now", (utc(py)?,)) +} + +/// `timedelta(days=days)`. +pub(crate) fn days<'py>(py: Python<'py>, days: i64) -> PyResult> { + call_with( + &imported(py, DATETIME_MODULE, "timedelta")?, + &[("days", days.into_pyobject(py)?.into_any())], + ) +} + +/// The value as an aware UTC datetime; naive values are taken as UTC. +pub(crate) fn aware_utc<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + if value.getattr("tzinfo")?.is_none() { + return call_with(&value.getattr("replace")?, &[("tzinfo", utc(py)?)]); + } + value.call_method1("astimezone", (utc(py)?,)) +} + +/// The latest known creation or alteration time of a relation. +pub(crate) fn relation_age_timestamp<'py>( + relation: &Bound<'py, PyAny>, +) -> PyResult>> { + let mut latest: Option> = None; + for field in ["created_at", "last_altered_at"] { + let value: Bound<'py, PyAny> = relation.getattr(field)?; + if value.is_none() { + continue; + } + let aware: Bound<'py, PyAny> = aware_utc(&value)?; + latest = match latest { + Some(current) if !aware.gt(¤t)? => Some(current), + _ => Some(aware), + }; + } + Ok(latest) +} + +/// The UTC second-resolution time embedded in archive names. +pub(crate) fn archive_timestamp<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + call_with( + &aware_utc(value)?.getattr("replace")?, + &[("microsecond", 0_i64.into_pyobject(py)?.into_any())], + ) +} + +/// The calendar fields of a datetime. +pub(crate) fn timestamp_fields(value: &Bound<'_, PyAny>) -> PyResult { + Ok(ArchiveTimestamp { + year: value.getattr("year")?.extract()?, + month: value.getattr("month")?.extract()?, + day: value.getattr("day")?.extract()?, + hour: value.getattr("hour")?.extract()?, + minute: value.getattr("minute")?.extract()?, + second: value.getattr("second")?.extract()?, + }) +} + +/// An aware UTC datetime at the given calendar fields. +pub(crate) fn utc_datetime(py: Python<'_>, value: ArchiveTimestamp) -> PyResult> { + call_with( + &imported(py, DATETIME_MODULE, "datetime")?, + &[ + ("year", value.year.into_pyobject(py)?.into_any()), + ("month", value.month.into_pyobject(py)?.into_any()), + ("day", value.day.into_pyobject(py)?.into_any()), + ("hour", value.hour.into_pyobject(py)?.into_any()), + ("minute", value.minute.into_pyobject(py)?.into_any()), + ("second", value.second.into_pyobject(py)?.into_any()), + ("tzinfo", utc(py)?), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs new file mode 100644 index 0000000000..b891784a0e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs @@ -0,0 +1,387 @@ +//! Compatibility views at old model names and pending migration origins the janitor protects. + +use std::collections::{HashMap, HashSet}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::janitor_planning::main::migration_origin_keys::migration_origin_keys; +use sqlbuild_analysis::janitor_planning::main::old_name_request_matches::old_name_request_matches; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::janitor::janitor_facts::{ + SchemaRelations, Warehouse, model_destinations, +}; +use crate::bindings::_helpers::janitor::janitor_keys::{ + Folded, Key, SchemaKey, optional, record, sorted_schemas, text, tuple, +}; + +const MIGRATION_TYPES: &str = "sqlbuild.compiler.migrations.types"; +const IS_COMPATIBILITY_VIEW: (&str, &str) = ( + "sqlbuild.executor.old_name_views._helpers.janitor", + "is_compatibility_view", +); +const READ_HISTORIES: (&str, &str) = ("sqlbuild._native", "read_old_name_view_histories"); +const STORED_COLUMNS: (&str, &str) = ("sqlbuild._native", "stored_old_name_view_columns"); +const OLD_NAME_VIEW_TABLE_NAME: &str = "_sqlbuild_old_name_views"; +const MIGRATION_TABLE_NAME: &str = "_sqlbuild_migrations"; +const MIGRATE_FROM_CONFIG_KEY: &str = "migrate_from"; + +/// Old-name view planning and the old names it owns. +pub(crate) struct OldNamePlanning<'py> { + pub(crate) planning: Bound<'py, PyAny>, + pub(crate) keys: HashSet, +} + +fn drop_reason<'py>(py: Python<'py>, member: &str) -> PyResult> { + imported(py, MIGRATION_TYPES, "OldNameViewDropReason")?.getattr(member) +} + +fn status<'py>( + history: &Bound<'py, PyAny>, + now: &Bound<'py, PyAny>, +) -> PyResult> { + call_with(&history.getattr("status")?, &[("now", now.clone())]) +} + +fn status_is(py: Python<'_>, value: &Bound<'_, PyAny>, member: &str) -> PyResult { + value.eq(imported(py, MIGRATION_TYPES, "OldNameViewStatus")?.getattr(member)?) +} + +fn read_histories<'py>( + warehouse: &Warehouse<'py>, + managed: &[SchemaKey], + relations: &SchemaRelations<'py>, +) -> PyResult>> { + let py: Python<'py> = warehouse.py(); + let mut histories: Vec> = Vec::new(); + for key in sorted_schemas(managed) { + let names: HashSet = match relations.iter().find(|(listed, _)| *listed == key) { + Some((_, listed)) => listed + .iter() + .map(|relation| { + Ok(relation + .getattr("name")? + .extract::()? + .to_lowercase()) + }) + .collect::>()?, + None => HashSet::new(), + }; + let (database, Some(schema)) = key else { + continue; + }; + if !names.contains(OLD_NAME_VIEW_TABLE_NAME) || !names.contains(MIGRATION_TABLE_NAME) { + continue; + } + let stored_columns: Bound<'py, PyAny> = call_with( + &imported(py, STORED_COLUMNS.0, STORED_COLUMNS.1)?, + &[ + ("adapter", warehouse.adapter.clone()), + ("connection", warehouse.connection.clone()), + ("database", optional(py, database.as_deref())?), + ("schema", text(py, &schema)), + ], + )?; + let read: Bound<'py, PyAny> = call_with( + &imported(py, READ_HISTORIES.0, READ_HISTORIES.1)?, + &[ + ("connection", warehouse.connection.clone()), + ("execute", warehouse.adapter.getattr("execute")?), + ("database", optional(py, database.as_deref())?), + ("schema", text(py, &schema)), + ("stored_columns", stored_columns), + ( + "render_qualified_name", + warehouse.adapter.getattr("render_qualified_name")?, + ), + ], + )?; + for history in read.try_iter()? { + histories.push(history?); + } + } + Ok(histories) +} + +fn old_key(history: &Bound<'_, PyAny>) -> PyResult { + Key::of(&history.getattr("old")?) +} + +/// Each history's listed relation at its old name, listing only schemas not already scanned. +fn physical_old_names<'py>( + warehouse: &Warehouse<'py>, + histories: &[Bound<'py, PyAny>], + relations: &SchemaRelations<'py>, +) -> PyResult>> { + let py: Python<'py> = warehouse.py(); + let mut known: Vec<(String, &Vec>)> = Vec::new(); + for ((_, schema), listed) in relations { + let folded: String = schema.as_deref().unwrap_or("").to_lowercase(); + match known.iter_mut().find(|(name, _)| *name == folded) { + Some(entry) => entry.1 = listed, + None => known.push((folded, listed)), + } + } + let mut unscanned: Vec = Vec::new(); + for history in histories { + let key: Key = old_key(history)?; + let folded: String = key.schema.as_deref().unwrap_or("").to_lowercase(); + if !known.iter().any(|(name, _)| *name == folded) { + unscanned.push(key); + } + } + let mut scanned: Vec> = Vec::new(); + if let Some(first) = unscanned.first() { + let mut schemas: Vec = unscanned + .iter() + .map(|key| key.schema.clone().unwrap_or_default()) + .collect(); + schemas.sort(); + schemas.dedup(); + let mut names: Vec = unscanned.iter().map(|key| key.name.clone()).collect(); + names.sort(); + names.dedup(); + let listed: Bound<'py, PyAny> = warehouse.call( + "list_relations", + &[ + ("database", optional(py, first.database.as_deref())?), + ( + "schemas", + tuple(py, schemas.iter().map(|schema| text(py, schema)))?, + ), + ("names", tuple(py, names.iter().map(|name| text(py, name)))?), + ], + )?; + for relation in listed.try_iter()? { + scanned.push(relation?); + } + } + for (_, listed) in &known { + scanned.extend(listed.iter().cloned()); + } + let mut found: Vec<(Key, Bound<'py, PyAny>)> = Vec::new(); + let mut seen: HashSet = HashSet::new(); + for relation in scanned { + let key: Key = Key::of(&relation)?; + if seen.insert(key.folded()) { + found.push((key, relation)); + } + } + let mut physical: HashMap> = HashMap::new(); + for history in histories { + let key: Key = old_key(history)?; + let exact: Option<&Bound<'py, PyAny>> = found + .iter() + .find(|(candidate, _)| candidate.folded() == key.folded()) + .map(|(_, relation)| relation); + let matched: Option<&Bound<'py, PyAny>> = if exact.is_some() || key.database.is_some() { + exact + } else { + found + .iter() + .find(|(candidate, _)| { + candidate.schema.as_deref().unwrap_or("").to_lowercase() + == key.schema.as_deref().unwrap_or("").to_lowercase() + && candidate.name.to_lowercase() == key.name.to_lowercase() + }) + .map(|(_, relation)| relation) + }; + if let Some(relation) = matched { + physical.insert(key.folded(), relation.clone()); + } + } + Ok(physical) +} + +fn view<'py>( + (key, history): (&Key, &Bound<'py, PyAny>), + reason: Option<&str>, + claimed_by: Option>, +) -> PyResult> { + let py: Python<'py> = history.py(); + let created: Bound<'py, PyAny> = history.getattr("created")?; + let expires_at: Bound<'py, PyAny> = if created.is_none() { + py.None().into_bound(py) + } else { + created.getattr("expires_at")? + }; + let mut pairs: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("key", key.to_python(py)?), + ("history", history.clone()), + ("expires_at", expires_at), + ]; + if let Some(reason) = reason { + pairs.push(("drop_reason", drop_reason(py, reason)?)); + } + if let Some(claimed_by) = claimed_by { + pairs.push(( + "occupied", + pyo3::types::PyBool::new(py, true).to_owned().into_any(), + )); + pairs.push(("claimed_by", optional(py, claimed_by.as_deref())?)); + } + record(py, "JanitorOldNameView", &pairs) +} + +/// Each project model's destination mapped to the model's name. +fn project_destinations<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + let destinations: Bound<'py, PyDict> = PyDict::new(py); + for (key, model) in model_destinations(project)? { + destinations.set_item(key.to_python(py)?, model.getattr("name")?)?; + } + Ok(destinations) +} + +fn claiming_model(destinations: &Bound<'_, PyDict>, key: &Key) -> PyResult> { + let lower = |value: Option<&str>| value.unwrap_or("").to_lowercase(); + for (destination, model_name) in destinations.iter() { + let destination: Key = Key::of(&destination)?; + if lower(destination.schema.as_deref()) == lower(key.schema.as_deref()) + && destination.name.to_lowercase() == key.name.to_lowercase() + && (destination.database.is_none() + || key.database.is_none() + || lower(destination.database.as_deref()) == lower(key.database.as_deref())) + { + return model_name.extract().map(Some); + } + } + Ok(None) +} + +/// Recorded views split into live, droppable and gone; another relation at the name is gone. +pub(crate) fn plan_old_name_views<'py>( + (project, warehouse): (&Bound<'py, PyAny>, &Warehouse<'py>), + (managed, relations): (&[SchemaKey], &SchemaRelations<'py>), + (early_drops, now): (&[String], &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = warehouse.py(); + let target_name: Option = project.getattr("effective_target_name")?.extract()?; + let destinations: Bound<'py, PyDict> = project_destinations(project)?; + let mut histories: Vec> = Vec::new(); + for history in read_histories(warehouse, managed, relations)? { + let recorded: Option = + history.getattr("move")?.getattr("target_name")?.extract()?; + if recorded.is_some() && recorded != target_name { + continue; + } + let current: Bound<'py, PyAny> = status(&history, now)?; + if status_is(py, ¤t, "LIVE")? || status_is(py, ¤t, "EXPIRED")? { + histories.push(history); + } + } + if histories.is_empty() && early_drops.is_empty() { + return Ok(OldNamePlanning { + planning: record(py, "JanitorOldNameViewPlanning", &[])?, + keys: HashSet::new(), + }); + } + let physical: HashMap> = + physical_old_names(warehouse, &histories, relations)?; + let mut requested: Vec<(String, bool)> = Vec::new(); + for name in early_drops { + if !requested.iter().any(|(listed, _)| listed == name) { + requested.push((name.clone(), false)); + } + } + let is_compatibility_view: Bound<'py, PyAny> = + imported(py, IS_COMPATIBILITY_VIEW.0, IS_COMPATIBILITY_VIEW.1)?; + let (mut live, mut drops, mut missing) = (Vec::new(), Vec::new(), Vec::new()); + let mut keys: HashSet = HashSet::new(); + for history in &histories { + let key: Key = old_key(history)?; + keys.insert(key.folded()); + let plain: Bound<'py, PyAny> = view((&key, history), None, None)?; + let mut matched: bool = false; + for (name, found) in &mut requested { + if old_name_request_matches( + name, + (key.database.as_deref(), key.schema.as_deref(), &key.name), + ) { + *found = true; + matched = true; + } + } + let Some(relation) = physical.get(&key.folded()) else { + missing.push(view((&key, history), Some("MISSING"), None)?); + continue; + }; + let compatible: bool = call_with( + &is_compatibility_view, + &[ + ("adapter", warehouse.adapter.clone()), + ("connection", warehouse.connection.clone()), + ("view", plain.clone()), + ("relation", relation.clone()), + ], + )? + .is_truthy()?; + if !compatible { + let claimed_by: Option = claiming_model(&destinations, &key)?; + missing.push(view((&key, history), Some("MISSING"), Some(claimed_by))?); + } else if matched { + drops.push(view((&key, history), Some("EARLY"), None)?); + } else if status_is(py, &status(history, now)?, "EXPIRED")? { + drops.push(view((&key, history), Some("EXPIRED"), None)?); + } else { + live.push(plain); + } + } + let unknown: Vec> = requested + .iter() + .filter(|(_, found)| !found) + .map(|(name, _)| text(py, name)) + .collect(); + let planning: Bound<'py, PyAny> = record( + py, + "JanitorOldNameViewPlanning", + &[ + ("live", tuple(py, live)?), + ("drops", tuple(py, drops)?), + ("missing", tuple(py, missing)?), + ("unknown_requests", tuple(py, unknown)?), + ("project_destinations", destinations.into_any()), + ], + )?; + Ok(OldNamePlanning { planning, keys }) +} + +/// Listed origins of migrations whose destination relation does not exist yet, with reasons. +pub(crate) fn pending_migration_origins( + project: &Bound<'_, PyAny>, + (relations, managed): (&SchemaRelations<'_>, &[SchemaKey]), +) -> PyResult> { + let mut listed: HashSet = HashSet::new(); + for (_, schema_relations) in relations { + for relation in schema_relations { + listed.insert(Key::of(relation)?.folded()); + } + } + let schemas: Vec = sorted_schemas(managed); + let mut pending: Vec<(Folded, String)> = Vec::new(); + for (destination, model) in model_destinations(project)? { + let raw: Bound<'_, PyAny> = model + .getattr("config")? + .getattr("values")? + .call_method1("get", (MIGRATE_FROM_CONFIG_KEY,))?; + let Ok(raw) = raw.extract::() else { + continue; + }; + if listed.contains(&destination.folded()) { + continue; + } + let model_name: String = model.getattr("name")?.extract()?; + for origin in migration_origin_keys(&raw, destination.database.as_deref(), &schemas) { + let folded: Folded = Key { + database: origin.database, + schema: origin.schema, + name: origin.name, + } + .folded(); + if listed.contains(&folded) && !pending.iter().any(|(key, _)| *key == folded) { + pending.push((folded, format!("pending migration origin for {model_name}"))); + } + } + } + Ok(pending) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index e3ce1b35b1..a87458722b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -8,12 +8,14 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods}; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::janitor::janitor_facts::Warehouse; +use crate::bindings::_helpers::janitor::janitor_planning::{ + JanitorSettings, RelationScope, build_plan, +}; const NATIVE_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; -const JANITOR_PLAN_MODULE: &str = "sqlbuild.executor.janitor.main.plan"; const JANITOR_EXECUTE_MODULE: &str = "sqlbuild.executor.janitor.main.execute"; -const JANITOR_MODELS: &str = "sqlbuild.executor.janitor.models"; const ERRORS_MODULE: &str = "sqlbuild._native"; const UNKNOWN_VIEW_CODE: &str = "C503"; const UNKNOWN_VIEW_HELP: &str = @@ -212,38 +214,34 @@ pub(crate) fn build_janitor_execution_plan<'py>( let plan: Bound<'py, PyAny> = within( &lifecycle(py, ("janitor", "janitor_candidate_planning"))?, |lifecycle| { - let direct_settings: Bound<'py, PyAny> = call_with( - &imported(py, JANITOR_MODELS, "JanitorDirectModeSettings")?, - &[ - ("enabled", PyBool::new(py, true).to_owned().into_any()), - ( - "state_history_versions", - settings.getattr("direct_state_history_versions")?, - ), - ( - "archive_retention_days", - settings.getattr("archive_retention_days")?, - ), - ], - )?; - let plan: Bound<'py, PyAny> = call_with( - &imported(py, JANITOR_PLAN_MODULE, "build_janitor_plan")?, - &[ - ("project", compiled.project.clone()), - ("adapter", compiled.adapter.clone()), - ("connection", connection.clone()), - ("retention_days", settings.getattr("retention_days")?), - ( - "delete_tracked_only", - janitor.getattr("delete_tracked_only")?, - ), - ("exclude_patterns", janitor.getattr("exclude_patterns")?), - ("direct_settings", direct_settings), - ( - "early_old_name_view_drops", - settings.getattr("drop_old_name_views")?, - ), - ], + let plan: Bound<'py, PyAny> = build_plan( + ( + &compiled.project, + &Warehouse { + adapter: compiled.adapter.clone(), + connection: connection.clone(), + }, + ), + &JanitorSettings { + retention_days: settings.getattr("retention_days")?.extract()?, + delete_tracked_only: janitor.getattr("delete_tracked_only")?.is_truthy()?, + exclude_patterns: janitor.getattr("exclude_patterns")?.extract()?, + direct_mode: true, + state_history_versions: settings + .getattr("direct_state_history_versions")? + .extract()?, + archive_retention_days: settings + .getattr("archive_retention_days")? + .extract()?, + early_old_name_view_drops: settings + .getattr("drop_old_name_views")? + .extract()?, + }, + &RelationScope { + scan_keys: Vec::new(), + protected_keys: Vec::new(), + reasons: Vec::new(), + }, )?; let metadata: Bound<'py, PyDict> = PyDict::new(py); metadata.set_item("item_count", candidate_count(&plan)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs new file mode 100644 index 0000000000..6ddfb15348 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs @@ -0,0 +1,401 @@ +//! The janitor cleanup plan: warehouse facts through adapter callbacks, decisions in Rust. + +use std::collections::HashSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use sqlbuild_analysis::janitor_planning::constants::OLD_NAME_VIEW_REASON; +use sqlbuild_analysis::janitor_planning::main::build_archive_name::build_archive_name; +use sqlbuild_analysis::janitor_planning::main::query_diff_artifact_parts::query_diff_artifact_parts; + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported, within}; +use crate::bindings::_helpers::janitor::janitor_classification::{ + Candidate, Policy, Protection, SchemaClassification, classify_target_schemas, plan_archives, +}; +use crate::bindings::_helpers::janitor::janitor_facts::{ + Warehouse, WarehouseFacts, collect_target_schemas, direct_state_prune_candidates, + gather_warehouse_facts, query_diff_artifact_candidates, +}; +use crate::bindings::_helpers::janitor::janitor_keys::{ + Folded, Key, SchemaKey, archive_timestamp, dedupe_schema_keys, integer, now_utc, record, + sorted_schemas, timestamp_fields, tuple, union_exact, +}; +use crate::bindings::_helpers::janitor::janitor_old_names::{ + OldNamePlanning, pending_migration_origins, plan_old_name_views, +}; +use crate::bindings::_helpers::relation_naming::fitting::shared_error; + +const LIFECYCLE: (&str, &str) = ( + "sqlbuild.runtime.observability.classes.operation_lifecycle", + "OperationLifecycle", +); + +/// Settings of one janitor plan. +pub(crate) struct JanitorSettings { + pub(crate) retention_days: i64, + pub(crate) delete_tracked_only: bool, + pub(crate) exclude_patterns: Vec, + pub(crate) direct_mode: bool, + pub(crate) state_history_versions: i64, + pub(crate) archive_retention_days: i64, + pub(crate) early_old_name_view_drops: Vec, +} + +/// Relations scanned or protected beyond the project's target schemas. +pub(crate) struct RelationScope { + pub(crate) scan_keys: Vec, + pub(crate) protected_keys: Vec, + pub(crate) reasons: Vec<(Key, String)>, +} + +fn keys_of(items: &Bound<'_, PyAny>) -> PyResult> { + items.try_iter()?.map(|item| Key::of(&item?)).collect() +} + +fn bool_value(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { + PyBool::new(py, value).to_owned().into_any() +} + +fn int_field(py: Python<'_>, value: usize) -> PyResult> { + integer(py, i64::try_from(value).unwrap_or(i64::MAX)) +} + +fn completed(lifecycle: &Bound<'_, PyAny>, item_count: usize) -> PyResult<()> { + let py: Python<'_> = lifecycle.py(); + let metadata: Bound<'_, PyDict> = PyDict::new(py); + metadata.set_item("item_count", item_count)?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("metadata", metadata)?; + lifecycle.call_method("completed", (), Some(&options))?; + Ok(()) +} + +fn protection(scope: &RelationScope) -> Protection { + Protection { + keys: scope.protected_keys.iter().map(Key::folded).collect(), + reasons: scope + .reasons + .iter() + .map(|(key, reason)| (key.folded(), reason.clone())) + .collect(), + } +} + +fn protect( + protection: Protection, + pairs: impl IntoIterator, +) -> Protection { + let Protection { + mut keys, + mut reasons, + } = protection; + for (key, reason) in pairs { + keys.insert(key.clone()); + reasons.insert(key, reason); + } + Protection { keys, reasons } +} + +fn scan_schemas(managed: &[SchemaKey], scope: &RelationScope) -> Vec { + let mut seen: HashSet = HashSet::new(); + let mut scoped_keys: Vec = Vec::new(); + for key in scope.protected_keys.iter().chain(&scope.scan_keys) { + if seen.insert(key.folded()) { + scoped_keys.push((key.database.clone(), key.schema.clone())); + } + } + let scoped: Vec = union_exact(Vec::new(), scoped_keys); + dedupe_schema_keys( + sorted_schemas(managed) + .into_iter() + .chain(sorted_schemas(&scoped)), + ) +} + +fn candidate_records<'py>(candidates: &[Candidate<'py>]) -> PyResult>> { + candidates.iter().map(Candidate::to_python).collect() +} + +/// The plan for a project whose target schemas list nothing beyond query-diff artifacts. +fn artifact_only_plan<'py>( + (project, warehouse): (&Bound<'py, PyAny>, &Warehouse<'py>), + settings: &JanitorSettings, + (artifacts, artifact_skips, scanned): (Vec>, Vec>, usize), + now: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = warehouse.py(); + record( + py, + "JanitorPlan", + &[ + ("target_name", project.getattr("effective_target_name")?), + ("retention_days", integer(py, settings.retention_days)?), + ("direct_mode", bool_value(py, settings.direct_mode)), + ( + "archive_retention_days", + integer(py, settings.archive_retention_days)?, + ), + ("query_diff_artifact_candidates", tuple(py, artifacts)?), + ("direct_state_prune_candidates", tuple(py, [])?), + ("skipped_relations", tuple(py, artifact_skips)?), + ("scanned_schema_count", int_field(py, scanned)?), + ( + "age_metadata_supported", + bool_value(py, warehouse.age_metadata_support()?), + ), + ("planned_at", now.clone()), + ], + ) +} + +/// Build the desired-versus-warehouse cleanup plan for the project's target schemas. +pub(crate) fn build_plan<'py>( + (project, warehouse): (&Bound<'py, PyAny>, &Warehouse<'py>), + settings: &JanitorSettings, + scope: &RelationScope, +) -> PyResult> { + let py: Python<'py> = warehouse.py(); + let managed: Vec = collect_target_schemas(project)?; + let target_schemas: Vec = scan_schemas(&managed, scope); + let mut artifact_schemas: Vec<(Option, String)> = Vec::new(); + let mut push_artifact_schema = |database: Option, schema: String| { + if !artifact_schemas.contains(&(database.clone(), schema.clone())) { + artifact_schemas.push((database, schema)); + } + }; + for (database, schema) in &target_schemas { + if let Some(schema) = schema { + push_artifact_schema(database.clone(), schema.clone()); + } + } + if let Some(schema) = project + .getattr("effective_target_schema")? + .extract::>()? + { + push_artifact_schema( + project.getattr("effective_target_database")?.extract()?, + schema, + ); + } + let now: Bound<'py, PyAny> = now_utc(py)?; + let (artifacts, artifact_skips) = + query_diff_artifact_candidates(warehouse, &artifact_schemas, &now)?; + if target_schemas.is_empty() { + let scanned: usize = artifact_schemas.len(); + return artifact_only_plan( + (project, warehouse), + settings, + (artifacts, artifact_skips, scanned), + &now, + ); + } + let lifecycle: Bound<'py, PyAny> = imported(py, LIFECYCLE.0, LIFECYCLE.1)?; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("operation_kind", "janitor")?; + options.set_item("operation_name", "janitor_warehouse_inspection")?; + let (facts, state_prunes): (WarehouseFacts<'py>, Vec>) = + within(&lifecycle.call((), Some(&options))?, |inspection| { + let facts: WarehouseFacts<'py> = gather_warehouse_facts( + (project, warehouse), + &target_schemas, + settings.delete_tracked_only, + )?; + let prunes: Vec> = direct_state_prune_candidates( + warehouse, + &target_schemas, + settings.state_history_versions, + )?; + completed(inspection, target_schemas.len())?; + Ok((facts, prunes)) + })?; + let old_names: OldNamePlanning<'py> = plan_old_name_views( + (project, warehouse), + (&managed, &facts.relations), + (&settings.early_old_name_view_drops, &now), + )?; + let protection: Protection = protect( + protection(scope), + old_names + .keys + .iter() + .map(|key| (key.clone(), OLD_NAME_VIEW_REASON.to_owned())), + ); + let pending: Vec<(Folded, String)> = + pending_migration_origins(project, (&facts.relations, &managed))? + .into_iter() + .filter(|(key, _)| !old_names.keys.contains(key)) + .collect(); + let protection: Protection = protect(protection, pending); + let age_supported: bool = warehouse.age_metadata_support()?; + let policy = Policy { + exclude_patterns: &settings.exclude_patterns, + delete_tracked_only: settings.delete_tracked_only, + retention_days: settings.retention_days, + direct_mode: settings.direct_mode, + }; + let schemas: SchemaClassification<'py> = classify_target_schemas( + (warehouse, &facts), + (&target_schemas, &managed), + (&protection, &policy), + &now, + )?; + let archives = plan_archives( + (warehouse, &facts), + (&schemas, &managed), + (&policy, settings.archive_retention_days), + &now, + )?; + let mut skipped: Vec> = Vec::new(); + for relation in &schemas.skipped { + if query_diff_artifact_parts(&relation.key.name).is_none() + && !old_names.keys.contains(&relation.key.folded()) + { + skipped.push(relation.record.clone()); + } + } + skipped.extend(archives.skipped); + skipped.extend(artifact_skips); + let scanned: Vec = union_exact( + target_schemas.clone(), + artifact_schemas + .into_iter() + .map(|(database, schema)| (database, Some(schema))), + ); + record( + py, + "JanitorPlan", + &[ + ("target_name", project.getattr("effective_target_name")?), + ("retention_days", integer(py, settings.retention_days)?), + ("direct_mode", bool_value(py, settings.direct_mode)), + ( + "archive_retention_days", + integer(py, settings.archive_retention_days)?, + ), + ( + "candidates", + tuple(py, candidate_records(&schemas.candidates)?)?, + ), + ( + "archive_candidates", + tuple(py, archives.archive_candidates)?, + ), + ( + "archive_deletion_candidates", + tuple(py, archives.deletions)?, + ), + ("retained_archives", tuple(py, archives.retained)?), + ("query_diff_artifact_candidates", tuple(py, artifacts)?), + ("direct_state_prune_candidates", tuple(py, state_prunes)?), + ("skipped_relations", tuple(py, skipped)?), + ("skipped_schemas", tuple(py, schemas.skipped_schemas)?), + ("blocked_schemas", tuple(py, archives.blocked_schemas)?), + ("old_name_views", old_names.planning), + ("scanned_schema_count", int_field(py, scanned.len())?), + ("age_metadata_supported", bool_value(py, age_supported)), + ("planned_at", now), + ], + ) +} + +fn reasons_of(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(Vec::new()); + } + let mut reasons: Vec<(Key, String)> = Vec::new(); + for item in value.call_method0("items")?.try_iter()? { + let (key, reason): (Bound<'_, PyAny>, String) = item?.extract()?; + reasons.push((Key::of(&key)?, reason)); + } + Ok(reasons) +} + +fn optional_keys(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(Vec::new()); + } + keys_of(value) +} + +fn value_or<'py, T: pyo3::FromPyObject<'py>>( + arguments: &KeywordArguments<'py>, + name: &str, + default: T, +) -> PyResult { + if arguments.passed(name)? { + return arguments.value(name)?.extract(); + } + Ok(default) +} + +/// Build the janitor cleanup plan; adapter reads stay adapter callbacks in the executor's order. +#[pyfunction] +#[pyo3(name = "build_janitor_plan", signature = (**options))] +pub(crate) fn build_janitor_plan_binding<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_janitor_plan", + options, + ( + &["project", "adapter", "connection", "retention_days"], + &[ + "delete_tracked_only", + "exclude_patterns", + "direct_mode", + "state_history_versions", + "archive_retention_days", + "early_old_name_view_drops", + "scan_relation_keys", + "protected_relation_keys", + "protected_relation_reasons", + ], + ), + )?; + let scope = RelationScope { + scan_keys: optional_keys(&arguments.value("scan_relation_keys")?)?, + protected_keys: optional_keys(&arguments.value("protected_relation_keys")?)?, + reasons: reasons_of(&arguments.value("protected_relation_reasons")?)?, + }; + let settings = JanitorSettings { + retention_days: arguments.value("retention_days")?.extract()?, + delete_tracked_only: value_or(&arguments, "delete_tracked_only", true)?, + exclude_patterns: value_or(&arguments, "exclude_patterns", Vec::new())?, + direct_mode: value_or(&arguments, "direct_mode", true)?, + state_history_versions: value_or(&arguments, "state_history_versions", 20)?, + archive_retention_days: value_or(&arguments, "archive_retention_days", 14)?, + early_old_name_view_drops: value_or(&arguments, "early_old_name_view_drops", Vec::new())?, + }; + let warehouse = Warehouse { + adapter: arguments.value("adapter")?, + connection: arguments.value("connection")?, + }; + build_plan( + (&arguments.value("project")?, &warehouse), + &settings, + &scope, + ) +} + +/// A strict archive name the janitor expires by its embedded UTC timestamp. +#[pyfunction] +#[pyo3(signature = (*, original_name, archived_at, identifier_limit, kind = None))] +pub(crate) fn build_janitor_archive_name( + original_name: &str, + archived_at: &Bound<'_, PyAny>, + identifier_limit: i64, + kind: Option<&str>, +) -> PyResult { + build_archive_name( + ( + original_name, + timestamp_fields(&archive_timestamp(archived_at)?)?, + ), + identifier_limit, + kind, + ) + .map_err(|message| shared_error(archived_at.py(), message)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs index b3e8c36dc2..9f9557101e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs @@ -1,4 +1,9 @@ +pub(crate) mod janitor_classification; pub(crate) mod janitor_command; +pub(crate) mod janitor_facts; +pub(crate) mod janitor_keys; +pub(crate) mod janitor_old_names; pub(crate) mod janitor_phases; pub(crate) mod janitor_plan; +pub(crate) mod janitor_planning; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs index 4bbcf0e7e1..d68f7e7003 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs @@ -1,10 +1,16 @@ -//! Register the native `sqb janitor` command and its plan rendering. +//! Register the native `sqb janitor` command, its planning and its plan rendering. -use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; +use crate::bindings::_helpers::janitor::janitor_planning::{ + build_janitor_archive_name, build_janitor_plan_binding, +}; use crate::bindings::_helpers::janitor::{janitor_command, janitor_plan}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { janitor_command::register(module)?; - janitor_plan::register(module) + janitor_plan::register(module)?; + module.add_function(wrap_pyfunction!(build_janitor_plan_binding, module)?)?; + module.add_function(wrap_pyfunction!(build_janitor_archive_name, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index f644e44552..698fbfa063 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -39,6 +39,8 @@ use crate::bindings::_helpers::lineage::lineage_cache::{ use crate::bindings::_helpers::lineage::lineage_index::{graph_index, index_views, lineage_keys}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; @@ -48,7 +50,6 @@ const SELECTION_MODULE: &str = "sqlbuild._native"; const SELECTOR_TOKENS_MODULE: &str = "sqlbuild._native"; const SELECTOR_PARSE_MODULE: &str = "sqlbuild._native"; const SELECTOR_MATCH_MODULE: &str = "sqlbuild._native"; -const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; const UNKNOWN_SELECTOR_ERROR_CODE: &str = "S007"; const RICH_MODE: &str = "rich"; @@ -485,7 +486,7 @@ fn selector_anchors( ) -> PyResult { let parse: Bound<'_, PyAny> = imported(py, SELECTOR_PARSE_MODULE, "parse_project_selector")?; let matcher: Bound<'_, PyAny> = imported(py, SELECTOR_MATCH_MODULE, "match_project_selector")?; - let path_class: Bound<'_, PyAny> = imported(py, PLANNER_MODELS, "PathSelector")?; + let path_class: Bound<'_, PyAny> = imported(py, NATIVE_RECORDS_MODULE, "PathSelector")?; let names: HashMap<&str, &LineageKey> = name_lookup(index); let named = |name: String| -> PyResult { names diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs index 3ddf618a35..5b18e3cbef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs @@ -7,7 +7,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::lineage_command::models::{LineageIndex, LineageKey, LineageNode}; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; -const GRAPH_MODELS: &str = "sqlbuild.compiler.graph.models"; +const GRAPH_MODELS: &str = "sqlbuild._native"; /// A Python `CompiledObjectKey` as a lineage key. pub(crate) fn lineage_key(key: &Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs index 45a01e499d..31b7d98744 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs @@ -1,10 +1,11 @@ //! The Python lineage models and enums native lineage builds. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyResult, Python}; use pyo3::types::{PyDict, PyTuple}; use sqlbuild_analysis::lineage::models::{LineageColumn, LineageSource}; -const LINEAGE_MODELS_MODULE: &str = "sqlbuild.compiler.lineage.models"; +const LINEAGE_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; @@ -48,12 +49,12 @@ impl<'py> LineageClasses<'py> { column_lineage: models.getattr("ColumnLineage")?, model_lineage: models.getattr("ModelColumnLineage")?, semantic_use: models.getattr("DirectSemanticColumnUse")?, - column_fact: compile_models.getattr("CompiledLineageColumnFact")?, + column_fact: shape_class(compile_models.as_any(), "CompiledLineageColumnFact")?, source_fact: compile_models.getattr("CompiledLineageSourceFact")?, compact_facts: compile_models.getattr("CompactLineageFacts")?, - transform_kind: types.getattr("ColumnTransformKind")?, - confidence: types.getattr("ColumnLineageConfidence")?, - analysis_dialect: types.getattr("PolyglotAnalysisDialect")?, + transform_kind: shape_class(types.as_any(), "ColumnTransformKind")?, + confidence: shape_class(types.as_any(), "ColumnLineageConfidence")?, + analysis_dialect: shape_class(types.as_any(), "PolyglotAnalysisDialect")?, model_resource: resource_type.getattr("MODEL")?, unknown_nullability: nullability.getattr("UNKNOWN")?, nullability, @@ -72,8 +73,11 @@ impl<'py> LineageClasses<'py> { resource_name: &str, column_name: &str, ) -> PyResult> { - self.qualified_column - .call1((resource_type, resource_name, column_name)) + self.qualified_column.call1(( + self.resource_type.call1((resource_type,))?, + resource_name, + column_name, + )) } /// `ColumnLineageEdge(source, target, transform_kind, confidence)`. @@ -104,15 +108,11 @@ impl<'py> LineageClasses<'py> { &self, column: &LineageColumn, nullability: &Bound<'py, PyAny>, - resource_enum: bool, ) -> PyResult> { let mut sources: Vec> = Vec::with_capacity(column.upstream_columns.len()); for source in &column.upstream_columns { - let resource_type: Bound<'py, PyAny> = if resource_enum { - self.resource_type.call1((source.resource_type.as_str(),))? - } else { - pyo3::types::PyString::new(self.py(), source.resource_type.as_str()).into_any() - }; + let resource_type: Bound<'py, PyAny> = + self.resource_type.call1((source.resource_type.as_str(),))?; sources.push(self.lineage_source.call1(( resource_type, &source.resource_name, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs index 93d1b1f820..470d1f1b80 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::lineage::project_schema::{ use crate::bindings::models::ProjectCatalog; use crate::bindings::types::CompilerDetach; -const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.lineage.exceptions"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const FAST_MODE: &str = "fast"; const RICH_MODE: &str = "rich"; const GENERIC_DIALECT: &str = "generic"; @@ -221,7 +221,7 @@ fn fast_lineage<'py>( Some(FastLineageOutcome::Built { columns, has_star }) => { let lineage_columns: Vec> = columns .iter() - .map(|column| classes.column(column, &classes.unknown_nullability, false)) + .map(|column| classes.column(column, &classes.unknown_nullability)) .collect::>()?; built.set_item( &model.name, @@ -307,7 +307,7 @@ fn rich_lineage<'py>( for column in &columns { let nullability: Bound<'py, PyAny> = classes.nullability.call1((column.nullability.as_str(),))?; - lineage_columns.push(classes.column(&column.column, &nullability, true)?); + lineage_columns.push(classes.column(&column.column, &nullability)?); } built.set_item(&name, classes.model(&name, lineage_columns, has_star)?)?; } @@ -331,7 +331,12 @@ fn rich_lineage<'py>( let transform: Bound<'py, PyAny> = column.getattr("transform_kind")?; let confidence: Bound<'py, PyAny> = column.getattr("confidence")?; for upstream in column.getattr("upstream_columns")?.try_iter()? { - let source: Bound<'py, PyAny> = upstream?.call_method0("as_qualified_column")?; + let upstream: Bound<'py, PyAny> = upstream?; + let source: Bound<'py, PyAny> = classes.qualified_column.call1(( + upstream.getattr("resource_type")?, + upstream.getattr("resource_name")?, + upstream.getattr("column_name")?, + ))?; edges.push(classes.edge(source, &target, &transform, &confidence)?); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs index d2bfcafadb..d78fbea497 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs @@ -20,7 +20,7 @@ use crate::bindings::_helpers::manifest::rows::{ ProjectRows, plan_input, project_rows, python_hooks, }; -const FINGERPRINT_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.fingerprints.exceptions"; +const FINGERPRINT_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; const DATETIME_MODULE: &str = "datetime"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 3277ea6a3a..f59238f6b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -16,7 +16,9 @@ pub(crate) mod compile_attachment; pub(crate) mod compile_command; pub(crate) mod compile_driver; pub(crate) mod compile_outputs; +pub(crate) mod compile_records; pub(crate) mod compiled_project; +pub(crate) mod compiler_records; pub(crate) mod compute_logs; pub(crate) mod contract_command; pub(crate) mod contracts; @@ -25,6 +27,7 @@ pub(crate) mod cost_report; pub(crate) mod cost_telemetry; pub(crate) mod cursor; pub(crate) mod dag_artifact; +pub(crate) mod dagster_invocation; pub(crate) mod dbt_command; pub(crate) mod dbt_events; pub(crate) mod dbt_interop; @@ -32,6 +35,7 @@ pub(crate) mod debug_command; pub(crate) mod diagnostics; pub(crate) mod diff_report; pub(crate) mod discovery; +pub(crate) mod event_exporting; pub(crate) mod execution_history; pub(crate) mod fingerprints; pub(crate) mod format_command; @@ -46,6 +50,7 @@ pub(crate) mod load; pub(crate) mod macro_bridge; pub(crate) mod manifest; pub(crate) mod model_config; +pub(crate) mod native_threads; pub(crate) mod output_capture; pub(crate) mod plan_command; pub(crate) mod plan_output; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs index 4727631155..48f6bfb0f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs @@ -6,12 +6,12 @@ use pyo3::{Py, PyErr, pymethods}; use sqlbuild_model_config::errors::{ConfigError, ErrorClass}; use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::model_config::config_values::config_value_type_error; use crate::bindings::_helpers::model_config::errors::NativeConfigError; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; -const IDENTITY_ERRORS_MODULE: &str = "sqlbuild.compiler.resource_names.exceptions"; -const CONTRACT_ERRORS_MODULE: &str = "sqlbuild.spec.contracts.exceptions"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; +const IDENTITY_ERRORS_MODULE: &str = "sqlbuild._native"; const EXPECTED_STRING: &str = "a string"; const COMPILE_INPUT_CLASS: &str = "compile_input"; @@ -74,12 +74,11 @@ impl NativeConfigError { if self.class_name == CONFIG_VALUE_TYPE_CLASS && let (Some(key), Some(values)) = (&self.key, values) { - let keywords: Bound<'py, PyDict> = PyDict::new(py); - keywords.set_item("key", key)?; - keywords.set_item("expected", EXPECTED_STRING)?; - keywords.set_item("actual_type", values.get_item(key)?.get_type())?; - return imported(py, CONTRACT_ERRORS_MODULE, "ConfigValueTypeError")? - .call((), Some(&keywords)); + return config_value_type_error( + key, + EXPECTED_STRING, + &values.get_item(key)?.get_type().into_any(), + ); } let keywords: Bound<'py, PyDict> = PyDict::new(py); keywords.set_item("help", &self.help)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs index 46cab3f137..f5ae424dd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs @@ -6,7 +6,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -const EXCEPTIONS_MODULE: &str = "sqlbuild.spec.contracts.exceptions"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const DATETIME_MODULE: &str = "datetime"; /// `values.get(key)`, or `None` when the value is absent or `None`. @@ -17,24 +17,38 @@ fn authored<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult) -> PyErr { - let py = value.py(); - let raised: PyResult> = - imported(py, EXCEPTIONS_MODULE, "ConfigValueTypeError").and_then(|class| { - call_with( - &class, - &[ - ("key", PyString::new(py, key).into_any()), - ("expected", PyString::new(py, expected).into_any()), - ("actual_type", value.get_type().into_any()), - ], - ) - }); - match raised { + match config_value_type_error(key, expected, &value.get_type().into_any()) { Ok(error) => PyErr::from_value(error), Err(error) => error, } } +/// The `ConfigValueTypeError` naming `key`, the `expected` kind and the value's `actual_type`. +pub(crate) fn config_value_type_error<'py>( + key: &str, + expected: &str, + actual_type: &Bound<'py, PyAny>, +) -> PyResult> { + let py = actual_type.py(); + let type_name: String = actual_type.getattr("__name__")?.extract()?; + call_with( + &imported(py, EXCEPTIONS_MODULE, "ConfigValueTypeError")?, + &[ + ( + "message", + PyString::new( + py, + &format!("config key '{key}' expected {expected}, got {type_name}"), + ) + .into_any(), + ), + ("key", PyString::new(py, key).into_any()), + ("expected", PyString::new(py, expected).into_any()), + ("actual_type", actual_type.clone()), + ], + ) +} + /// An authored string, preserving a present empty string. #[pyfunction(name = "get_config_str")] #[pyo3(signature = (*, values, key))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs index 3c8b08c2a9..574276f280 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::model_config::config_errors::native_config_error; use crate::bindings::_helpers::model_config::contract_classes::contract_classes; use crate::bindings::_helpers::model_config::header_metadata::{ContractClasses, column_tuple}; -const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; const DECLARATION_ERROR_CLASS: &str = "DeclarationParseError"; /// `(file_path, label)`: the path and label every message names. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/constants.rs new file mode 100644 index 0000000000..8de268820c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/constants.rs @@ -0,0 +1,4 @@ +//! Thread attributes native capture and delivery code share. + +/// Marks a thread whose writes command-output capture ignores, such as one running a command-output sink. +pub(crate) const OUTPUT_SUPPRESSED_ATTRIBUTE: &str = "_sqlbuild_command_output_suppressed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/exit_signal.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/exit_signal.rs new file mode 100644 index 0000000000..dbe63d9e8c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/exit_signal.rs @@ -0,0 +1,30 @@ +//! The interpreter-exit signal native waits poll so Python daemon threads leave native frames before finalization. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +static EXIT_SIGNAL: PyOnceLock> = PyOnceLock::new(); + +/// The process-wide `threading.Event` set once interpreter exit begins. +fn exit_signal(py: Python<'_>) -> PyResult> { + EXIT_SIGNAL + .get_or_try_init(py, || { + imported(py, "threading", "Event")? + .call0() + .map(Bound::unbind) + }) + .map(|signal| signal.bind(py).clone()) +} + +/// Whether an exit hook has started stopping native daemon threads. +pub(crate) fn interpreter_exiting(py: Python<'_>) -> PyResult { + exit_signal(py)?.call_method0("is_set")?.is_truthy() +} + +/// Tell every native wait to stop waiting on Python work that may outlive the interpreter. +pub(crate) fn signal_interpreter_exit(py: Python<'_>) -> PyResult<()> { + exit_signal(py)?.call_method0("set")?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/mod.rs new file mode 100644 index 0000000000..9d58356de9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod constants; +pub(crate) mod exit_signal; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs b/crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs index d7bd60f0a6..37ae8d21a6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs @@ -7,7 +7,7 @@ use std::time::Instant; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, pyclass, - pymethods, + pyfunction, pymethods, wrap_pyfunction, }; use pyo3::types::{PyDict, PyString, PyTuple}; use sqlbuild_core::command_output::constants::{ @@ -23,12 +23,18 @@ use sqlbuild_core::command_output::models::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::native_threads::constants::OUTPUT_SUPPRESSED_ATTRIBUTE; +use crate::bindings::_helpers::native_threads::exit_signal::signal_interpreter_exit; use crate::bindings::_helpers::output_capture::records::{MODELS_MODULE, TYPES_MODULE}; use crate::bindings::_helpers::output_capture::values::{ freeze, object, output_error, pass_text, python_text, unencodable, }; const THREAD_NAME: &str = "sqlbuild-output-exporter"; +/// Attribute naming a capture thread's worker, for the exit hook. +const WORKER_ATTRIBUTE: &str = "_sqlbuild_output_capture_worker"; +/// How long interpreter exit waits for each stopped capture worker. +const EXIT_JOIN_SECONDS: f64 = 1.0; const DEFAULT_QUEUE_CAPACITY: i64 = 1024; const DEFAULT_BATCH_SIZE: i64 = 64; const DEFAULT_MAX_RECORD_BYTES: i64 = 64 * 1024; @@ -104,6 +110,50 @@ fn thread_ident(py: Python<'_>) -> PyResult { imported(py, "threading", "get_ident")?.call0()?.extract() } +/// Whether the current thread runs a command-output sink, whose writes are never captured. +fn delivering_thread(py: Python<'_>) -> PyResult { + imported(py, "threading", "current_thread")? + .call0()? + .hasattr(OUTPUT_SUPPRESSED_ATTRIBUTE) +} + +/// Stop every capture worker and join it before finalization ends it inside native code. +#[pyfunction] +fn stop_output_capture_threads(py: Python<'_>) -> PyResult<()> { + signal_interpreter_exit(py)?; + let mut threads: Vec> = Vec::new(); + for thread in imported(py, "threading", "enumerate")? + .call0()? + .try_iter()? + { + let thread: Bound<'_, PyAny> = thread?; + let Ok(worker) = thread.getattr(WORKER_ATTRIBUTE) else { + continue; + }; + if let Ok(worker) = worker.downcast::() { + let shared = &worker.get().shared; + let _held = shared.hold(py)?; + { + let mut state = shared.locked()?; + state.accepting = false; + state.stopping = true; + state.force_stop = true; + } + shared.notify(py, "notify_all")?; + threads.push(thread); + } + } + for thread in threads { + if thread.call_method0("is_alive")?.is_truthy()? { + call_with( + &thread.getattr("join")?, + &[("timeout", object(py, EXIT_JOIN_SECONDS)?)], + )?; + } + } + Ok(()) +} + impl Shared { fn locked(&self) -> PyResult> { self.state @@ -387,14 +437,16 @@ impl OutputCaptureDispatcher { shared: Arc::clone(&shared), }, )?; + let worker = worker.into_bound(py).into_any(); let thread = call_with( &imported(py, "threading", "Thread")?, &[ - ("target", worker.into_bound(py).into_any()), + ("target", worker.clone()), ("name", object(py, THREAD_NAME)?), ("daemon", object(py, true)?), ], )?; + thread.setattr(WORKER_ATTRIBUTE, worker)?; thread.call_method0("start")?; Ok(Self { shared, @@ -412,7 +464,10 @@ impl OutputCaptureDispatcher { ) -> PyResult<()> { let shared = &self.shared; let thread = thread_ident(py)?; - if text.is_empty()? || shared.locked()?.suppressed_threads.contains(&thread) { + if text.is_empty()? + || shared.locked()?.suppressed_threads.contains(&thread) + || delivering_thread(py)? + { return Ok(()); } let stream = match stream.getattr("value")?.extract::()?.as_str() { @@ -506,7 +561,11 @@ impl OutputCaptureDispatcher { } } +/// Add the capture classes and run `stop_output_capture_threads` at interpreter exit. pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let hook = wrap_pyfunction!(stop_output_capture_threads, module)?; + module.add_function(hook.clone())?; + imported(module.py(), "atexit", "register")?.call1((hook,))?; module.add_class::()?; module.add_class::() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index d9a739d4db..bd714e7a32 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -12,11 +12,12 @@ use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_conn use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; -const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; -const PRESENTATION_MODELS: &str = "sqlbuild.presentation.models"; +const PRESENTATION_MODELS: &str = "sqlbuild._native"; const COMMAND_NAME: &str = "plan"; const DEFAULT_SECTION_ENTRIES: i64 = 50; const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; @@ -212,7 +213,7 @@ fn compile_pipeline<'py>( )?; options.set_item("record_migration_fingerprints", false)?; let pipeline_options: Bound<'py, PyAny> = - imported(py, PIPELINE_MODELS, "CompilePipelineOptions")?.call((), Some(&options))?; + imported(py, NATIVE_RECORDS_MODULE, "CompilePipelineOptions")?.call((), Some(&options))?; call_with( &imported(py, NATIVE_MODULE, "run_compile_pipeline")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs index d2ebdc8e5f..30084fa8d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs @@ -28,7 +28,7 @@ use crate::bindings::_helpers::plan_output::python_identity::{ }; use crate::bindings::_helpers::plan_output::values::{items, optional_text, text, texts}; -const DISPLAY_MODELS_MODULE: &str = "sqlbuild.presentation.models"; +const DISPLAY_MODELS_MODULE: &str = "sqlbuild._native"; const SKIP_ACTION: &str = "skip"; const QUERY_CHANGED: &str = "query_changed"; const CONFIG_CHANGED: &str = "config_changed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index fdedc2eb8d..ee974e5a4e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, within, }; +use crate::bindings::_helpers::native_threads::exit_signal::signal_interpreter_exit; use crate::bindings::_helpers::planner_entries::values::{ COMPILE_TYPES, boolean, native, replaced, string, }; @@ -16,13 +17,44 @@ const INSPECTION_CATALOG: (&str, &str) = ( "open_inspection_catalog", ); const TEST_PLANNING_THREAD: &str = "sqlbuild-test-planning"; +/// How long interpreter exit waits for each SQL test planning thread still running native code. +const PLANNING_EXIT_JOIN_SECONDS: f64 = 5.0; + +/// The planned SQL tests' future and, when planning runs in the background, its daemon thread. +type TestPlanning<'py> = (Bound<'py, PyAny>, Option>); + +/// `thread.join(timeout)` for a live thread; no timeout when `timeout` is `None`. +fn join_thread(thread: &Bound<'_, PyAny>, timeout: Option) -> PyResult<()> { + if thread.call_method0("is_alive")?.is_truthy()? { + let options = PyDict::new(thread.py()); + options.set_item("timeout", timeout)?; + thread.call_method("join", (), Some(&options))?; + } + Ok(()) +} + +/// Join SQL test planning threads at exit: finalization ending one inside native planning aborts. +#[pyfunction] +fn join_sql_test_planning_threads(py: Python<'_>) -> PyResult<()> { + signal_interpreter_exit(py)?; + for thread in imported(py, "threading", "enumerate")? + .call0()? + .try_iter()? + { + let thread: Bound<'_, PyAny> = thread?; + if thread.getattr("name")?.eq(TEST_PLANNING_THREAD)? { + join_thread(&thread, Some(PLANNING_EXIT_JOIN_SECONDS))?; + } + } + Ok(()) +} /// A future resolving to the planned SQL tests, planned on a daemon thread when enabled. fn sql_test_planning<'py>( (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), (selected_keys, sql_test_selection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), enabled: bool, -) -> PyResult> { +) -> PyResult> { let py = project.py(); let future = imported(py, "concurrent.futures", "Future")?.call0()?; if !enabled { @@ -37,7 +69,7 @@ fn sql_test_planning<'py>( ], )?,), )?; - return Ok(future); + return Ok((future, None)); } let kwargs = PyDict::new(py); kwargs.set_item("future", &future)?; @@ -45,7 +77,7 @@ fn sql_test_planning<'py>( kwargs.set_item("adapter", adapter)?; kwargs.set_item("selected_keys", selected_keys)?; kwargs.set_item("sql_test_selection", sql_test_selection)?; - call_with( + let thread = call_with( &imported(py, "threading", "Thread")?, &[ ( @@ -56,9 +88,9 @@ fn sql_test_planning<'py>( ("name", string(py, TEST_PLANNING_THREAD)), ("daemon", boolean(py, true)?), ], - )? - .call_method0("start")?; - Ok(future) + )?; + thread.call_method0("start")?; + Ok((future, Some(thread))) } /// Selected scopes, and the overrides with cursor overrides typed for the selected models. @@ -365,7 +397,7 @@ fn build_execution_plan<'py>( )?; let (scopes, overrides) = scopes_and_cursor_overrides(&args)?; let inspection = scopes.getattr("inspection_scope")?; - let test_planning = sql_test_planning( + let (test_planning, planning_thread) = sql_test_planning( (&project, &adapter), ( &inspection.getattr("selected_keys")?, @@ -375,13 +407,21 @@ fn build_execution_plan<'py>( .getattr("plan_sql_tests")? .is_truthy()?, )?; - let catalog = call_with( - &imported(py, INSPECTION_CATALOG.0, INSPECTION_CATALOG.1)?, - &[("adapter", adapter), ("connection", connection)], - )?; - within(&catalog, |_| { - planned_output(&args, &runtime, (&scopes, &overrides), &test_planning) - }) + let planned = (|| { + let catalog = call_with( + &imported(py, INSPECTION_CATALOG.0, INSPECTION_CATALOG.1)?, + &[("adapter", adapter), ("connection", connection)], + )?; + within(&catalog, |_| { + planned_output(&args, &runtime, (&scopes, &overrides), &test_planning) + }) + })(); + let joined = planning_thread + .as_ref() + .map_or(Ok(()), |thread| join_thread(thread, None)); + let planned = planned?; + joined?; + Ok(planned) } /// Start SQL test planning on a daemon thread, or resolve to no tests when disabled. @@ -406,17 +446,23 @@ fn start_sql_test_planning<'py>( &[], ), )?; - sql_test_planning( + let (future, _) = sql_test_planning( (&args.value("project")?, &args.value("adapter")?), ( &args.value("selected_keys")?, &args.value("sql_test_selection")?, ), args.flag("enabled")?, - ) + )?; + Ok(future) } +/// Add the execution plan entry points and join SQL test planning threads at interpreter exit. pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(build_execution_plan, module)?)?; - module.add_function(wrap_pyfunction!(start_sql_test_planning, module)?) + module.add_function(wrap_pyfunction!(start_sql_test_planning, module)?)?; + let hook = wrap_pyfunction!(join_sql_test_planning_threads, module)?; + module.add_function(hook.clone())?; + imported(module.py(), "atexit", "register")?.call1((hook,))?; + Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs index 2da73008f8..ca8c68b3eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs @@ -1,6 +1,7 @@ //! Python boundary values shared by plan entry and plan output assembly. use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class_at; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyString; @@ -31,7 +32,7 @@ pub(super) fn planner_model<'py>( name: &str, pairs: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { - call_with(&imported(py, PLANNER_MODELS, name)?, pairs) + call_with(&shape_class_at(py, PLANNER_MODELS, name)?, pairs) } /// One member of a planner enumeration. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs index 138507dbd4..2a358d1938 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs @@ -12,6 +12,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; use sqlbuild_analysis::planner_migrations::main::column_decision::column_decision; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + pub(super) fn decide<'py>( runtime: &Bound<'py, PyAny>, candidate: &ColumnCandidate<'py>, @@ -57,7 +59,7 @@ pub(super) fn decide<'py>( } let relation = imported( py, - "sqlbuild.compiler.migrations.main.relation_for_location", + "sqlbuild.executor.migrations._helpers.relation_for_location", "migration_relation_for_location", )? .call1((candidate.model.getattr("destination")?,))?; @@ -111,7 +113,8 @@ pub(super) fn decide<'py>( ("destination_column", destination), ( "discovery", - imported(py, MIGRATION_TYPES, "MigrationDiscovery")?.call1((discovery,))?, + imported(py, NATIVE_RECORDS_MODULE, "MigrationDiscovery")? + .call1((discovery,))?, ), ("decision", decision), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs index b9aa0e3417..743e4061b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs @@ -205,12 +205,7 @@ fn result<'py>( status: &str, findings: Bound<'py, PyTuple>, ) -> PyResult> { - let status = imported( - py, - "sqlbuild.compiler.migrations.types", - "MigrationCompatibility", - )? - .call1((status,))?; + let status = imported(py, "sqlbuild._native", "MigrationCompatibility")?.call1((status,))?; call_with( &imported(py, "sqlbuild._native", "MigrationCompatibilityResult")?, &[("status", status), ("findings", findings.into_any())], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs index 5810366377..b4e58f3c62 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs @@ -22,7 +22,7 @@ pub(super) fn newest<'py>( ) -> PyResult> { let relation = imported( location.py(), - "sqlbuild.compiler.migrations.main.relation_for_location", + "sqlbuild.executor.migrations._helpers.relation_for_location", "migration_relation_for_location", )? .call1((location,))?; @@ -32,7 +32,7 @@ pub(super) fn newest<'py>( pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { imported( location.py(), - "sqlbuild.compiler.migrations.main.relation_for_location", + "sqlbuild.executor.migrations._helpers.relation_for_location", "migration_relation_for_location", )? .call1((location,)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs index 01a9e160b6..5ef0c9fcb1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs @@ -145,12 +145,8 @@ pub(super) fn automatic<'py>( ("model", item.clone()), ( "discovery", - imported( - py, - "sqlbuild.compiler.migrations.types", - "MigrationDiscovery", - )? - .getattr("AUTOMATIC")?, + imported(py, "sqlbuild._native", "MigrationDiscovery")? + .getattr("AUTOMATIC")?, ), ("origin_location", location), ("origin_model", origin_model), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs index 4142649951..ac1fb2aa3d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs @@ -15,10 +15,12 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyBool, PyTuple}; use sqlbuild_analysis::planner_migrations::main::model_decision::model_decision; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { imported( location.py(), - "sqlbuild.compiler.migrations.main.relation_for_location", + "sqlbuild.executor.migrations._helpers.relation_for_location", "migration_relation_for_location", )? .call1((location,)) @@ -106,7 +108,8 @@ pub(super) fn decide<'py>(args: &KeywordArguments<'py>) -> PyResult(item: &Bound<'py, PyAny>) -> PyResult>> { let py = item.py(); let values = item.getattr("config")?.getattr("values")?; @@ -26,7 +27,7 @@ fn declared<'py>(item: &Bound<'py, PyAny>) -> PyResult> ("model", item.clone()), ( "discovery", - imported(py, MIGRATION_TYPES, "MigrationDiscovery")?.getattr("MANUAL")?, + imported(py, NATIVE_RECORDS_MODULE, "MigrationDiscovery")?.getattr("MANUAL")?, ), ("raw_origin", origin.call_method0("strip")?), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs index b594f187b8..e45c22183f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs @@ -363,10 +363,5 @@ impl<'py> OldNamePlanner<'py> { } fn action<'py>(runtime: &Bound<'py, PyAny>, name: &str) -> PyResult> { - imported( - runtime.py(), - "sqlbuild.compiler.migrations.types", - "OldNameViewAction", - )? - .getattr(name) + imported(runtime.py(), "sqlbuild._native", "OldNameViewAction")?.getattr(name) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs index c9911903b6..9c8a9026c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs @@ -8,7 +8,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { imported( location.py(), - "sqlbuild.compiler.migrations.main.relation_for_location", + "sqlbuild.executor.migrations._helpers.relation_for_location", "migration_relation_for_location", )? .call1((location,)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs index 41beda161f..bcba81d3e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs @@ -563,7 +563,7 @@ planner_record! { /// SQL test entries planned for one selection; reusable only for the same selection. PlannedSqlTests, false, [ (selected_keys = "selected_keys", FieldDefault::Required, true, true), - (sql_test_selection = "sql_test_selection", FieldDefault::Class(("sqlbuild.compiler.planner.models", "SqlTestSelection")), true, true), + (sql_test_selection = "sql_test_selection", FieldDefault::Class(("sqlbuild._native", "SqlTestSelection")), true, true), (entries = "entries", FieldDefault::EmptyTuple, true, true), (warnings = "warnings", FieldDefault::EmptyTuple, true, true), ] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs index fcbd7bb887..c928d53cfa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs @@ -1,6 +1,6 @@ //! Python boundaries for native scenario planning. -mod artifacts; +pub(crate) mod artifacts; mod cli; mod constants; mod entries; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs index 52d74f9dc4..0a194a1d52 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs @@ -5,7 +5,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::planner_scope::loaders::expand_selected_loader_dependencies; -use crate::bindings::_helpers::planner_scope::values::{MODELS, NATIVE, edges, keys}; +use crate::bindings::_helpers::planner_scope::values::{NATIVE, edges, keys}; use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::PyDictMethods; @@ -16,6 +16,8 @@ use sqlbuild_analysis::planner_scope::main::{ }; use std::collections::HashSet; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + #[pyfunction] #[pyo3(signature = (*, selected_keys, upstream_deps, project))] pub(crate) fn managed_source_upstream_keys<'py>( @@ -133,10 +135,10 @@ pub(crate) fn build_planner_scope<'py>( } let mut tests = args.value("sql_test_selection")?; if !tests.is_truthy()? { - tests = imported(py, MODELS, "SqlTestSelection")?.call0()?; + tests = imported(py, NATIVE_RECORDS_MODULE, "SqlTestSelection")?.call0()?; } call_with( - &imported(py, MODELS, "PlannerScope")?, + &imported(py, NATIVE_RECORDS_MODULE, "PlannerScope")?, &[ ("upstream_deps", upstream), ("downstream_deps", downstream), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs index 2694a72991..ea833aa748 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs @@ -23,8 +23,8 @@ use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; use crate::bindings::_helpers::graph::views::{GraphViewMappings, graph_from_view_mappings}; use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; -const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; -const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const VIEW_OPTIONS: &[&str] = &[ "all_keys", "upstream_deps", @@ -46,11 +46,11 @@ pub(crate) fn parsed_selector<'py>( upstream, downstream, } => call_with( - &imported(py, PLANNER_MODELS_MODULE, "ParsedSelector")?, + &imported(py, NATIVE_RECORDS_MODULE, "ParsedSelector")?, &[ ( "kind", - imported(py, PLANNER_TYPES_MODULE, "SelectorKind")?.call1((kind,))?, + imported(py, NATIVE_RECORDS_MODULE, "SelectorKind")?.call1((kind,))?, ), ("value", carrier.back(&value)?.into_any()), ("upstream", PyBool::new(py, upstream).to_owned().into_any()), @@ -66,7 +66,7 @@ pub(crate) fn parsed_selector<'py>( upstream, downstream, } => call_with( - &imported(py, PLANNER_MODELS_MODULE, "PathSelector")?, + &imported(py, NATIVE_RECORDS_MODULE, "PathSelector")?, &[ ("start_name", carrier.back(&start)?.into_any()), ("end_name", carrier.back(&end)?.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs index c914f16529..8f063da418 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs @@ -13,7 +13,8 @@ use sqlbuild_analysis::planner_selection::models::{ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; -const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const RESOURCE_ENTRY_FILES: [(&str, &str); 3] = [ ("schema_files", "model_entries"), ("schema_files", "seed_entries"), @@ -110,7 +111,7 @@ fn split_python<'py>( ) .map_err(|error| carrier.error(error))?; let selection: Bound<'py, PyAny> = call_with( - &imported(py, PLANNER_MODELS_MODULE, "SqlTestSelection")?, + &imported(py, NATIVE_RECORDS_MODULE, "SqlTestSelection")?, &[ ("names", PyFrozenSet::new(py, &split.test_names)?.into_any()), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs index d4d74b7dde..5b1a3d6b84 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs @@ -177,9 +177,11 @@ pub(crate) fn audit_is_selected( let kind = audit.getattr("attached_target_kind")?; let name = audit.getattr("attached_target_name")?; if !kind.is_none() && !name.is_none() { - let resource_type = imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")? + let target_value = imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")? .call1((kind,))? - .getattr("resource_type")?; + .getattr("value")?; + let resource_type = + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.call1((target_value,))?; let key = call_with( &imported(py, COMPILE_MODELS_MODULE, "CompiledObjectKey")?, &[("resource_type", resource_type), ("name", name)], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs index 674d1d0a2a..6a4b719603 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs @@ -126,7 +126,7 @@ fn lineage_reads(model: &Bound<'_, PyAny>) -> PyResult> { let fact = fact?; if !fact.getattr("confidence")?.eq(model .py() - .import("sqlbuild.compiler.lineage.types")? + .import("sqlbuild._native")? .getattr("ColumnLineageConfidence")? .getattr("HIGH")?)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs index 482e9284a2..ff677ebe85 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs @@ -82,7 +82,7 @@ fn projections(model: &Bound<'_, PyAny>) -> PyResult> } let direct = model .py() - .import("sqlbuild.compiler.lineage.types")? + .import("sqlbuild._native")? .getattr("ColumnTransformKind")? .getattr("DIRECT")?; for column in lineage.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 28796ee67a..f86c402d4d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -1,5 +1,9 @@ //! The layouts of every native record class. +use crate::bindings::_helpers::compile_records::constants::COMPILE_VALUE_RECORDS; +use crate::bindings::_helpers::compiler_records::constants::{ + COMPILER_ENUMS, COMPILER_ERRORS, COMPILER_RECORDS, +}; use crate::bindings::_helpers::freshness::observation::normalize_observation; use crate::bindings::_helpers::records::models::{ ClassValue, EnumSpec, ErrorSpec, FieldDefault, FieldSpec, RecordSpec, @@ -24,9 +28,15 @@ pub(crate) const ERROR_SPEC_ATTRIBUTE: &str = "__native_error__"; pub(crate) const EXCEPTION_PARENT: &str = "Exception"; /// The `parent` of an error class deriving directly from `ValueError`. pub(crate) const VALUE_ERROR_PARENT: &str = "ValueError"; +/// Prefix of a `parent` naming a builtin exception kept with its own constructor. +pub(crate) const BUILTIN_PARENT_PREFIX: &str = "builtins."; +pub(crate) const BUILTINS_MODULE: &str = "builtins"; + +/// Every native error group, in registration order. +pub(crate) const ERRORS: &[&[ErrorSpec]] = &[CLI_ERRORS, COMPILER_ERRORS]; -/// Every native expected-error class, each after its parent. -pub(crate) const ERRORS: &[ErrorSpec] = &[ +/// The expected CLI and migration-state error classes, each after its parent. +const CLI_ERRORS: &[ErrorSpec] = &[ ErrorSpec { name: "CliUserError", parent: EXCEPTION_PARENT, @@ -106,8 +116,13 @@ pub(crate) const ERRORS: &[ErrorSpec] = &[ ]; /// Every native string enum group, registered before the records. -pub(crate) const ENUMS: &[&[EnumSpec]] = - &[FRESHNESS_ENUMS, CLI_ENUMS, DBT_ENUMS, PRESENTATION_ENUMS]; +pub(crate) const ENUMS: &[&[EnumSpec]] = &[ + FRESHNESS_ENUMS, + CLI_ENUMS, + DBT_ENUMS, + PRESENTATION_ENUMS, + COMPILER_ENUMS, +]; /// Every native record group, in registration order. pub(crate) const RECORDS: &[&[RecordSpec]] = &[ @@ -120,6 +135,8 @@ pub(crate) const RECORDS: &[&[RecordSpec]] = &[ COST_RECORDS, SQL_VALUE_RECORDS, DIAGNOSTIC_RECORDS, + COMPILER_RECORDS, + COMPILE_VALUE_RECORDS, ]; /// Records whose bound values are normalized before the record is created. @@ -745,11 +762,7 @@ const LINEAGE_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("exclude", FieldDefault::EmptyTuple), FieldSpec::new( "lineage_mode", - FieldDefault::Member( - "sqlbuild.compiler.lineage.types", - "ColumnLineageMode", - "RICH", - ), + FieldDefault::Member("sqlbuild._native", "ColumnLineageMode", "RICH"), ), FieldSpec::new("include_uses", FieldDefault::False), FieldSpec::new("cli_vars", FieldDefault::None), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs index 6c95b0f2a7..c5d83e70b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs @@ -126,7 +126,11 @@ fn initialize( error.setattr("message", message)?; match code { Some(code) if !code.is_none() => error.setattr("code", code)?, - _ => error.setattr("code", class.getattr("code")?)?, + _ => { + if let Ok(code) = class.getattr("code") { + error.setattr("code", code)?; + } + } } error.setattr( "help", @@ -152,7 +156,9 @@ fn initialize( fn error_spec(index: &usize) -> PyResult<&'static ErrorSpec> { ERRORS - .get(*index) + .iter() + .flat_map(|group| group.iter()) + .nth(*index) .ok_or_else(|| PyTypeError::new_err("unknown native error class")) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs index 6bd817393d..3705508d4e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs @@ -5,9 +5,9 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use crate::bindings::_helpers::records::constants::{ - DATACLASS_FIELD_KIND, DATACLASS_FIELDS_ATTRIBUTE, DATACLASSES_MODULE, ENUM_MODULE, ENUMS, - ERROR_SPEC_ATTRIBUTE, ERRORS, EXCEPTION_PARENT, NATIVE_MODULE, RECORD_SPEC_ATTRIBUTE, STR_ENUM, - VALUE_ERROR_PARENT, + BUILTIN_PARENT_PREFIX, BUILTINS_MODULE, DATACLASS_FIELD_KIND, DATACLASS_FIELDS_ATTRIBUTE, + DATACLASSES_MODULE, ENUM_MODULE, ENUMS, ERROR_SPEC_ATTRIBUTE, ERRORS, EXCEPTION_PARENT, + NATIVE_MODULE, RECORD_SPEC_ATTRIBUTE, STR_ENUM, VALUE_ERROR_PARENT, }; use crate::bindings::_helpers::records::error_classes::{ BoundErrorInitializer, ErrorInitializer, class_value, @@ -29,11 +29,14 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { } module.add_class::()?; module.add_class::()?; - for (index, spec) in ERRORS.iter().enumerate() { + for (index, spec) in ERRORS.iter().flat_map(|group| group.iter()).enumerate() { let parent: Bound<'_, PyAny> = match spec.parent { EXCEPTION_PARENT => py.get_type::().into_any(), VALUE_ERROR_PARENT => py.get_type::().into_any(), - parent => module.getattr(parent)?, + parent => match parent.strip_prefix(BUILTIN_PARENT_PREFIX) { + Some(builtin) => py.import(BUILTINS_MODULE)?.getattr(builtin)?, + None => module.getattr(parent)?, + }, }; module.add(spec.name, error_class(&parent, (index, spec))?)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs index 595e088c2d..a8063b2f4d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs @@ -9,8 +9,8 @@ use sqlbuild_core::text::main::python_casefold::python_casefold; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::type_system::equality::types_equal as native_types_equal; -const ADOPTION_MODELS_MODULE: &str = "sqlbuild.compiler.contract_adoption.models"; -const ADOPTION_TYPES_MODULE: &str = "sqlbuild.compiler.contract_adoption.types"; +const ADOPTION_MODELS_MODULE: &str = "sqlbuild._native"; +const ADOPTION_TYPES_MODULE: &str = "sqlbuild._native"; const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const DYNAMIC_FAMILIES_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs index 80af619584..141dbbc21c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs @@ -16,9 +16,9 @@ use sqlbuild_refactor::refactoring::types::PhysicalColumn; use crate::bindings::_helpers::boundary::panics::compiler_guard; -const ADOPTION_ERRORS_MODULE: &str = "sqlbuild.compiler.contract_adoption.exceptions"; -const ADOPTION_MODELS_MODULE: &str = "sqlbuild.compiler.contract_adoption.models"; -const ADOPTION_TYPES_MODULE: &str = "sqlbuild.compiler.contract_adoption.types"; +const ADOPTION_ERRORS_MODULE: &str = "sqlbuild._native"; +const ADOPTION_MODELS_MODULE: &str = "sqlbuild._native"; +const ADOPTION_TYPES_MODULE: &str = "sqlbuild._native"; /// Finding kinds an edit resolves without overwrite, then the ones overwrite also resolves. const RESOLVED_KINDS: ([&str; 1], [&str; 2]) = ( ["missing_declaration"], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs index 6bb3e8ecb9..1aaca63c39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs @@ -2,22 +2,25 @@ use std::path::Path; +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class_at; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyString}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const COMPILE_ERRORS: &str = "sqlbuild.compiler.compile.exceptions"; const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; const RULES_ERRORS: &str = "sqlbuild.rule_engine.exceptions"; -const SPEC_ERRORS: &str = "sqlbuild.spec.contracts.exceptions"; +const SPEC_ERRORS: &str = "sqlbuild._native"; const ENTRY_MODELS: &str = "sqlbuild._native"; const COMMAND_TYPES: &str = "sqlbuild._native"; -const REFACTOR_MODELS: &str = "sqlbuild.compiler.refactoring.models"; +const REFACTOR_MODELS: &str = "sqlbuild._native"; const DEFAULT_ERROR_CODE: &str = "E001"; /// One offline compile's project facts, or `None`, and every error that stopped or followed it. @@ -27,7 +30,7 @@ pub(crate) struct OfflineCompile<'py> { } fn enum_member<'py>(py: Python<'py>, class: &str, member: &str) -> PyResult> { - imported(py, COMPILE_TYPES, class)?.getattr(member) + shape_class_at(py, COMPILE_TYPES, class)?.getattr(member) } fn matches_any_class(py: Python<'_>, error: &PyErr, classes: &[(&str, &str)]) -> PyResult { @@ -102,7 +105,7 @@ fn analyzed<'py>( no_cache: bool, ) -> PyResult> { let selection: Bound<'py, PyAny> = call_with( - &imported(py, COMPILE_MODELS, "CompileAnalysisSelection")?, + &imported(py, NATIVE_RECORDS_MODULE, "CompileAnalysisSelection")?, &[("no_cache", PyBool::new(py, no_cache).to_owned().into_any())], )?; call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/python_strings.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/python_strings.rs index 50d2e4a214..b28bc6ecb7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/python_strings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/python_strings.rs @@ -10,7 +10,7 @@ use sqlbuild_pysource::string_literals::models::StringQuery; use crate::bindings::_helpers::boundary::keyword_request::Request; -const REFACTORING_MODELS_MODULE: &str = "sqlbuild.compiler.refactoring.models"; +const REFACTORING_MODELS_MODULE: &str = "sqlbuild._native"; /// The discovered Python declarations whose files a refactoring searches. const PYTHON_DECLARATIONS: [&str; 8] = [ "loader_functions", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs index 69b2026012..854a7e8d8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs @@ -42,7 +42,7 @@ use crate::bindings::_helpers::refactor::refactor_facts::{ }; const ERRORS_MODULE: &str = "sqlbuild._native"; -const REFACTOR_ERRORS_MODULE: &str = "sqlbuild.compiler.refactoring.exceptions"; +const REFACTOR_ERRORS_MODULE: &str = "sqlbuild._native"; const STAGING_PREFIX: &str = "sqb-refactor-"; const MOVE_COMMAND: &str = "mv"; const EDIT_ERROR_CODE: &str = "C957"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs index 86f0264adf..4aa6770210 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs @@ -11,7 +11,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; const SHARED_EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; -fn shared_error(py: Python<'_>, message: String) -> PyErr { +pub(crate) fn shared_error(py: Python<'_>, message: String) -> PyErr { imported(py, SHARED_EXCEPTIONS_MODULE, "SharedInputError") .and_then(|class| class.call1((message,))) .map_or_else(|error| error, PyErr::from_value) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs index 38926e328f..fcfa29b3a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs @@ -19,6 +19,8 @@ use crate::bindings::_helpers::render::macro_expansion_result::expand_sql_macros use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, interpolation_batch}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; @@ -65,7 +67,7 @@ fn substituted<'py>( /// `ExpansionSpan` objects for span rows, in order. fn span_objects<'py>(py: Python<'py>, spans: &[SpanRow]) -> PyResult> { - let span_class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS_MODULE, "ExpansionSpan")?; + let span_class: Bound<'py, PyAny> = imported(py, NATIVE_RECORDS_MODULE, "ExpansionSpan")?; let objects: Vec> = spans .iter() .map(|span| span_class.call1(*span)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs index 9396908efc..1d40296f1f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs @@ -9,13 +9,15 @@ use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const NATIVE_MODULE: &str = "sqlbuild._native"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; const BUILTINS_MODULE: &str = "builtins"; const NO_DECLARATION_KIND: u8 = 255; /// Declaration kinds by their native code: enum 0, constant 1, macro 2. @@ -138,7 +140,7 @@ pub(crate) fn declaration_scope_resolver<'py>( lookup.clone() }; let projection: Bound<'py, PyAny> = call_with( - &imported(py, COMPILE_MODELS_MODULE, "DeclarationRuntimeProjection")?, + &imported(py, NATIVE_RECORDS_MODULE, "DeclarationRuntimeProjection")?, &[( "declarations", imported(py, TYPES_MODULE, "MappingProxyType")?.call1((&declarations,))?, @@ -147,7 +149,7 @@ pub(crate) fn declaration_scope_resolver<'py>( let native_contexts: Bound<'py, PyAny> = native_declaration_contexts(&lookup, declarations.as_any())?; call_with( - &imported(py, COMPILE_MODELS_MODULE, "DeclarationScopeResolver")?, + &imported(py, NATIVE_RECORDS_MODULE, "DeclarationScopeResolver")?, &[ ("project_dir", discovered.getattr("project_dir")?), ("lookup", lookup), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index a3882b0bcf..957e496407 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -16,10 +16,12 @@ use crate::bindings::_helpers::render::macro_expansion::{ }; use crate::bindings::_helpers::sqltext::sql_references::extract_references; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const FRONTIER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.frontier.exceptions"; +const FRONTIER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const SCOPES_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const SCOPES_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const REF_MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; @@ -199,7 +201,7 @@ fn result_model<'py>( pairs: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { call_with( - &imported(py, COMPILE_MODELS_MODULE, "MacroExpansionResult")?, + &imported(py, NATIVE_RECORDS_MODULE, "MacroExpansionResult")?, pairs, ) } @@ -239,7 +241,7 @@ fn expansion_result<'py>(request: ExpansionRequest<'py>) -> PyResult = imported(py, ACTIVE_BRIDGE_MODULE, "active_macro_bridge")?.call0()?; let output: ExpansionOutput = expanded(&expansion, request.sql.to_str()?, &bridge)?; - let span_class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS_MODULE, "ExpansionSpan")?; + let span_class: Bound<'py, PyAny> = imported(py, NATIVE_RECORDS_MODULE, "ExpansionSpan")?; let mut spans: Vec> = Vec::with_capacity(output.spans.len()); for span in output.spans { let (source_start, source_end, output_start, output_end) = span_row(span); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs index abeb1410a4..aef30876a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs @@ -1,5 +1,6 @@ //! The built-in rule catalogue returned as the Python `Rule` models. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -62,7 +63,7 @@ fn rule<'py>( .map(|declared| option(&option_class, declared)) .collect::>()?; let guidance: Bound<'py, PyAny> = match metadata.guidance { - Some(guidance) => models.getattr("RuleGuidance")?.call( + Some(guidance) => shape_class(models.as_any(), "RuleGuidance")?.call( (), Some(&keywords( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs index ca56e429af..76232eeaa2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs @@ -24,8 +24,9 @@ use crate::bindings::_helpers::rules::custom_hosts::{ }; use crate::bindings::_helpers::rules::custom_rule_cache::CustomRuleCache; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; -const RULES_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; const RULE_CONTEXT_MODULE: &str = "sqlbuild.rule_engine.classes.rule_context"; @@ -179,7 +180,7 @@ pub(crate) fn outcome<'py>( (custom_ms, custom_cpu_ms): (i64, i64), ) -> PyResult> { call_with( - &imported(py, RULES_MODELS_MODULE, "CustomRulesOutcome")?, + &imported(py, NATIVE_RECORDS_MODULE, "CustomRulesOutcome")?, &[ ("findings", PyTuple::new(py, findings)?.into_any()), ("cache_hits", hits.into_bound_py_any(py)?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs index bd153bbdd6..e7af596e65 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs @@ -22,9 +22,9 @@ use sqlbuild_rules::dsl_headers::types::HeaderParser; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const LINT_MODELS_MODULE: &str = "sqlbuild.lint.models"; -const LINT_TYPES_MODULE: &str = "sqlbuild.lint.types"; -const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const LINT_MODELS_MODULE: &str = "sqlbuild._native"; +const LINT_TYPES_MODULE: &str = "sqlbuild._native"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; const PATHLIB_MODULE: &str = "pathlib"; const SENTINEL_PATH: &str = ""; const ENGINE: &str = "sqlbuild"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index 46e1d43542..f372189d29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -12,11 +12,14 @@ use sqlbuild_rules::rule_cache::main::rules_bulk_cache_path::rules_bulk_cache_pa use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const RULE_THREADS_MODULE: &str = "sqlbuild._native"; -const RULES_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; -const LINT_MODELS_MODULE: &str = "sqlbuild.lint.models"; -const LINT_CONSTANTS_MODULE: &str = "sqlbuild.lint.constants"; +const LINT_MODELS_MODULE: &str = "sqlbuild._native"; +/// The lint thresholds used when the project leaves them unset. +const DEFAULT_MAX_LITERAL_LENGTH: i64 = 100; +const DEFAULT_MAX_RANKING_ORDER_BY: i64 = 5; const FUTURES_MODULE: &str = "concurrent.futures"; const THREADING_MODULE: &str = "threading"; const LINT_THREAD_PREFIX: &str = "sqlbuild-lint"; @@ -40,8 +43,8 @@ pub(crate) fn sql_lint_config<'py>( (relation_keys, allow_suppressions): (Bound<'py, PyAny>, Bound<'py, PyAny>), ) -> PyResult> { let py: Python<'py> = dialect.py(); - let threshold = |key: &str, default: &str| -> PyResult> { - thresholds.call_method1("get", (key, imported(py, LINT_CONSTANTS_MODULE, default)?)) + let threshold = |key: &str, default: i64| -> PyResult> { + thresholds.call_method1("get", (key, default)) }; call_with( &imported(py, LINT_MODELS_MODULE, "LintConfig")?, @@ -51,11 +54,11 @@ pub(crate) fn sql_lint_config<'py>( ("header_rules_enabled", false.into_bound_py_any(py)?), ( "max_literal_length", - threshold("max_literal_length", "DEFAULT_MAX_LITERAL_LENGTH")?, + threshold("max_literal_length", DEFAULT_MAX_LITERAL_LENGTH)?, ), ( "max_ranking_order_by", - threshold("max_ranking_order_by", "DEFAULT_MAX_RANKING_ORDER_BY")?, + threshold("max_ranking_order_by", DEFAULT_MAX_RANKING_ORDER_BY)?, ), ("relation_keys", relation_keys), ("allow_suppressions", allow_suppressions), @@ -265,7 +268,7 @@ impl EarlySqlLint { declaration_scope } else { call_with( - &imported(py, RULES_MODELS_MODULE, "SqlExpansionReuse")?, + &imported(py, NATIVE_RECORDS_MODULE, "SqlExpansionReuse")?, &[ ("discovered_inputs", inputs.getattr("discovered_inputs")?), ("declaration_scope", declaration_scope), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_facts.rs index 8fe0ca9604..43c7b97d6e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_facts.rs @@ -6,7 +6,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -const LINT_MODELS: &str = "sqlbuild.lint.models"; +const LINT_MODELS: &str = "sqlbuild._native"; /// Index a compilation by model key and diagnostic owner without changing its fact views. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index 4ccc3d3135..3e9a741619 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -19,10 +19,10 @@ use crate::bindings::_helpers::rules::fix_facts::compile_facts; use crate::bindings::_helpers::rules::fix_verdict::fix_verdict; const NATIVE: &str = "sqlbuild._native"; -const MODELS: &str = "sqlbuild.lint.models"; +const MODELS: &str = "sqlbuild._native"; const ADAPTER: &str = "sqlbuild.adapter.contract.classes.base_adapter"; const COMPILE_ERRORS: &str = "sqlbuild.compiler.compile.exceptions"; -const LINT_ERRORS: &str = "sqlbuild.lint.exceptions"; +const LINT_ERRORS: &str = "sqlbuild._native"; const VERIFICATION_FAILURE: &str = "Compilation could not prove unchanged output columns, types, dependencies and lineage"; const APPLIED_REASON: &str = "Meaning-preserving rewrite; compiler invariants verified"; @@ -268,7 +268,7 @@ impl<'py> FixSession<'py> { while !accepted.is_empty() { let verdict = self.verdict(&edited_files(&self.unblocked(current)?, &accepted)?)?; if let Some(verdict) = &verdict - && verdict.getattr("verified")?.is_truthy()? + && verified(verdict)? { confirmed = true; break; @@ -286,10 +286,7 @@ impl<'py> FixSession<'py> { let candidate = edited_files(&self.unblocked(current)?, &accepted)?; if !accepted.is_empty() && !confirmed { let verdict = self.verdict(&candidate)?; - if !verdict - .as_ref() - .map_or(Ok(false), |value| value.getattr("verified")?.is_truthy())? - { + if !verdict.as_ref().map_or(Ok(false), verified)? { return Ok(None); } } @@ -319,7 +316,7 @@ impl<'py> FixSession<'py> { single.set_item(&path, &path_edits)?; let verdict = self.verdict(&edited_files(&base, &single)?)?; if let Some(verdict) = &verdict - && verdict.getattr("verified")?.is_truthy()? + && verified(verdict)? { accepted.set_item(path, path_edits)?; } else { @@ -669,3 +666,9 @@ fn result<'py>( )? .into_any()) } + +/// Whether a fix verdict proved every edit: nothing failed and nothing was unattributed. +fn verified(verdict: &Bound<'_, PyAny>) -> PyResult { + Ok(!verdict.getattr("failing")?.is_truthy()? + && !verdict.getattr("unattributed")?.is_truthy()?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs index e6ebf6a74b..4c88ccdba1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs @@ -10,7 +10,7 @@ use sqlbuild_rules::fix_verification::models::{DiagnosticComparison, ModelCompar use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -const LINT_MODELS: &str = "sqlbuild.lint.models"; +const LINT_MODELS: &str = "sqlbuild._native"; /// Compare compile facts using native decisions and the existing Python boundary models. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs index 59eac5be48..3e835c3457 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs @@ -15,8 +15,8 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.adapter.discovery.main.resolve_adapter"; const ADAPTER_EXCEPTIONS_MODULE: &str = "sqlbuild.adapter.contract.exceptions"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const DISCOVERY_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; -const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild.lint.exceptions"; +const DISCOVERY_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; +const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const COMPILE_FIRST_MESSAGE: &str = "compiler-integrated Rules check the SQL your project \ actually produces, so the project must compile first"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs index 930aeaec07..06c6b34fd9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs @@ -9,7 +9,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const NATIVE: &str = "sqlbuild._native"; -const MODELS: &str = "sqlbuild.lint.models"; +const MODELS: &str = "sqlbuild._native"; /// Classify a lint path, retaining the caller's supplied relative path when available. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs index 15c2c97235..44023b8dd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs @@ -12,11 +12,12 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::rules::lint_headers::{lint_body_ranges, lint_file_role}; use crate::bindings::_helpers::rules::lint_stop::check_stop; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE: &str = "sqlbuild._native"; -const MODELS: &str = "sqlbuild.lint.models"; -const EXPANSION_MODELS: &str = "sqlbuild.compiler.compile.models"; +const MODELS: &str = "sqlbuild._native"; const COMPILE_ERRORS: &str = "sqlbuild.compiler.compile.exceptions"; -const LINT_ERRORS: &str = "sqlbuild.lint.exceptions"; +const LINT_ERRORS: &str = "sqlbuild._native"; /// Expand a single body and finish its native SQL preparation. #[pyfunction] @@ -204,7 +205,7 @@ fn sentinel_spans<'py>(sites: &Bound<'py, PyAny>) -> PyResult> for site in sites.try_iter()? { let site = site?; spans.push(call_with( - &imported(py, EXPANSION_MODELS, "ExpansionSpan")?, + &imported(py, NATIVE_RECORDS_MODULE, "ExpansionSpan")?, &[ ("source_start", site.getattr("original_start")?), ("source_end", site.getattr("original_end")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs index 921104c611..cd59b9bc4a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs @@ -12,8 +12,8 @@ use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::types::CompilerDetach; -const LINT_MODELS_MODULE: &str = "sqlbuild.lint.models"; -const LINT_TYPES_MODULE: &str = "sqlbuild.lint.types"; +const LINT_MODELS_MODULE: &str = "sqlbuild._native"; +const LINT_TYPES_MODULE: &str = "sqlbuild._native"; const NATIVE_ENGINE: &str = "native"; fn spans(passes: &Bound<'_, PyAny>) -> PyResult>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs index 2dd45f982e..6c0a379e09 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs @@ -11,7 +11,7 @@ use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{context_variable, imported}; -const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild.lint.exceptions"; +const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const STOP_VARIABLE: &str = "active_lint_stop"; static ACTIVE_STOP: PyOnceLock> = PyOnceLock::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs index db2392bac6..a748bb5224 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs @@ -26,8 +26,8 @@ use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::types::CompilerDetach; -const LINT_MODELS_MODULE: &str = "sqlbuild.lint.models"; -const LINT_TYPES_MODULE: &str = "sqlbuild.lint.types"; +const LINT_MODELS_MODULE: &str = "sqlbuild._native"; +const LINT_TYPES_MODULE: &str = "sqlbuild._native"; const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const NATIVE_ENGINE: &str = "native"; const FORMAT_UNSAFE_CODE: &str = "format-unsafe"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index bc993b60e1..0c8e28ffef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -38,8 +38,9 @@ use crate::bindings::_helpers::rules::sql_test_objects::{scenario_fact, sql_test use crate::bindings::_helpers::type_system::equality::types_equal; use crate::bindings::types::CompilerDetach; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; -const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const DATACLASSES_MODULE: &str = "dataclasses"; const NATIVE_RULES_CACHE_FILE: &str = "native.json"; const NATIVE_RULES_MEMO_FILE: &str = "native-response.json"; @@ -242,7 +243,7 @@ fn evaluated<'py>( .map(|fault| classes.finding_of_fault(fault, false)) .collect::>()?; call_with( - &imported(py, MODELS_MODULE, "NativeRulesEvaluation")?, + &imported(py, NATIVE_RECORDS_MODULE, "NativeRulesEvaluation")?, &[ ("findings", PyTuple::new(py, findings)?.into_any()), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index 04271a8ec9..0d6f2b81a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -23,6 +23,8 @@ use crate::bindings::_helpers::rules::custom_evaluation::{ }; use crate::bindings::_helpers::rules::custom_hosts::HostCancel; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const RULES_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; @@ -369,7 +371,7 @@ pub(crate) fn prepare_sql_rules<'py>( Some(&work), )?; call_with( - &imported(py, RULES_MODELS_MODULE, "PreparedSqlLint")?, + &imported(py, NATIVE_RECORDS_MODULE, "PreparedSqlLint")?, &[ ("codes", codes), ("project_dir", project_dir), @@ -417,7 +419,7 @@ fn prepare_early_sql_lint<'py>( ], )?; call_with( - &imported(py, RULES_MODELS_MODULE, "PreparedSqlLintResult")?, + &imported(py, NATIVE_RECORDS_MODULE, "PreparedSqlLintResult")?, &[("result", result), ("context", context)], ) }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs index 64d111f236..b7f202a1c5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -12,10 +12,11 @@ use crate::bindings::_helpers::rules::catalogue::{ }; use crate::bindings::_helpers::rules::rule_run::freed_memory_released; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const CUSTOM_PAYLOADS_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.custom_payloads"; const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; -const RULES_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const DATACLASSES_MODULE: &str = "dataclasses"; const PATHLIB_MODULE: &str = "pathlib"; @@ -101,7 +102,7 @@ fn plan_rule_run<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult( (integer(result, field)? + integer(sql_result, field)?).into_bound_py_any(py) }; call_with( - &imported(py, MODELS_MODULE, "RulesRunResult")?, + &imported(py, NATIVE_RECORDS_MODULE, "RulesRunResult")?, &[ ("findings", PyTuple::new(py, findings)?.into_any()), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index f70369c857..a428a3a4c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -16,9 +16,13 @@ use crate::bindings::_helpers::boundary::panics::compiler_guard; const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const NATIVE_MODULE: &str = "sqlbuild._native"; +/// The rules configuration models that stay Python for the custom-rule API. +const PYTHON_CLASSES: &[&str] = &["RulesConfig", "LayoutConfig"]; struct Models<'py> { module: Bound<'py, PyModule>, + native: Bound<'py, PyModule>, } impl<'py> Models<'py> { @@ -31,7 +35,12 @@ impl<'py> Models<'py> { for (name, value) in fields { arguments.set_item(name, value)?; } - self.module.getattr(class)?.call((), Some(&arguments)) + let module: &Bound<'py, PyModule> = if PYTHON_CLASSES.contains(&class) { + &self.module + } else { + &self.native + }; + module.getattr(class)?.call((), Some(&arguments)) } } @@ -121,6 +130,7 @@ fn tables<'py, T>( fn config_object<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult> { let models = Models { module: py.import(MODELS_MODULE)?, + native: py.import(NATIVE_MODULE)?, }; let threshold_overrides = tables(py, &config.threshold_overrides, |item| { models.build( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs index 8c4bad037c..74cc787ae7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs @@ -21,7 +21,7 @@ use crate::bindings::_helpers::rules::lint_stop::check_stop; const NATIVE_MODULE: &str = "sqlbuild._native"; const EXPANSION_MODULE: &str = "sqlbuild._native"; -const LINT_MODELS_MODULE: &str = "sqlbuild.lint.models"; +const LINT_MODELS_MODULE: &str = "sqlbuild._native"; fn resolved(path: &Path) -> PathBuf { fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index 878902bc88..6c483d569b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -15,9 +15,11 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::early_sql_lint::{sql_lint_config, sql_rule_cache_path}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const RULES_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; -const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild.lint.exceptions"; +const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; const BUILTINS_MODULE: &str = "builtins"; @@ -110,7 +112,7 @@ fn run_planned_sql_rules<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult< evaluated_sql_rules(&run, (&lint_key, identities), project_files)?; let sql_ms: i64 = (started.elapsed().as_secs_f64() * 1000.0).round_ties_even() as i64; call_with( - &imported(py, RULES_MODELS_MODULE, "SqlRulesEvaluation")?, + &imported(py, NATIVE_RECORDS_MODULE, "SqlRulesEvaluation")?, &[ ("findings", findings), ("cache_hits", hits.into_bound_py_any(py)?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs index 899fc77974..cd2e98c6ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::compiled_project::records::plain_typed_value; use crate::bindings::_helpers::compiled_project::values::plain_value; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const MODEL_PAYLOAD_CLASS: &str = "CompiledModelSqlTestPayload"; const DESCRIPTION_KEY: &str = "description"; @@ -89,7 +89,7 @@ fn with_payload(test: &Bound<'_, PyAny>, mut fact: SqlTestFact) -> PyResult = test.getattr("payload")?; let model_payload: Bound<'_, PyAny> = test .py() - .import(COMPILE_MODELS_MODULE)? + .import(NATIVE_RECORDS_MODULE)? .getattr(MODEL_PAYLOAD_CLASS)?; if !payload.is_instance(&model_payload)? { return Ok(fact); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index aee03fa27c..2dfdfeddb7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -25,13 +25,14 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::discovery::guidance::sql_analysis_guidance; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SCENARIO_CONFIG_MODULE: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; -const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const SCENARIO_MODELS: &str = "sqlbuild.executor.scenario.models"; const METADATA_MODULE: &str = "importlib.metadata"; @@ -304,7 +305,7 @@ pub(crate) fn compile<'py>( let mut pairs: Vec<(&str, Bound<'py, PyAny>)> = options.to_vec(); pairs.push(("external_sql_reference_resolver", resolver)); let options: Bound<'py, PyAny> = call_with( - &imported(py, PIPELINE_MODELS, "CompilePipelineOptions")?, + &imported(py, NATIVE_RECORDS_MODULE, "CompilePipelineOptions")?, &pairs, )?; let lifecycle: Bound<'py, PyAny> = call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs index ec8592a9ca..40e1cd71cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -20,7 +20,7 @@ use sqlbuild_scopes::scope_index::models::{ use crate::bindings::_helpers::compiled_project::model_loop_named::scope_path_message; -const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const COLLECTOR_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs index 6c12986892..dcfde4fa8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs @@ -16,7 +16,8 @@ use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use crate::bindings::types::CompilerDetach; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; /// One scan's error, or its values. @@ -112,7 +113,7 @@ fn extract_sql_test_ctes<'py>( } }; let cte_class: Bound<'py, PyAny> = py - .import(COMPILE_MODELS_MODULE)? + .import(NATIVE_RECORDS_MODULE)? .getattr("CompileSqlTestCte")?; let mut built: Vec> = Vec::with_capacity(ctes.len()); for (name, body) in ctes { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs index d72e06f2ea..742be7540d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs @@ -30,7 +30,9 @@ use crate::bindings::_helpers::scopes::scope_objects::{ use crate::bindings::_helpers::scopes::scope_validation::replace; use crate::bindings::types::CompilerDetach; -const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +/// Why `UnindexableScopeError` is raised; discovery only yields normalized project paths. +const UNINDEXABLE_SCOPE_MESSAGE: &str = "Native scope indexing could not place a discovered path or resource kind. Discovery always yields normalized project-relative paths, so this is a SQLBuild bug; please file an issue with the project layout"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const INSPECT_MODULE: &str = "inspect"; const CODE_TYPE_MODULE: &str = "types"; const PATHLIB_MODULE: &str = "pathlib"; @@ -603,7 +605,7 @@ fn unindexable(py: Python<'_>) -> PyResult { Ok(PyErr::from_value( py.import(EXCEPTIONS_MODULE)? .getattr("UnindexableScopeError")? - .call0()?, + .call1((UNINDEXABLE_SCOPE_MESSAGE,))?, )) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index 50cae7b73c..a0b8ce015d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -20,6 +20,8 @@ use crate::bindings::_helpers::scopes::scope_tolerant::scope_tolerant_index; use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const SCOPE_CACHE_DIRECTORY_NAME: &str = "declaration-scopes-v2"; const SCOPE_CACHE_FILENAME: &str = "scope-index.json"; @@ -27,8 +29,7 @@ const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models" const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const COLLECTION_RENDERING_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.adapter.discovery.main.resolve_adapter"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; +const FRONTIER_TYPES_MODULE: &str = "sqlbuild._native"; const MACRO_INVENTORY_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; /// Where the scope index cache file of a project lives: `(directory, file name)`. @@ -90,7 +91,7 @@ fn compiled_index<'py>( ], )?; let adapter_context: Bound<'py, PyAny> = call_with( - &imported(py, COMPILE_MODELS_MODULE, "CompileAdapterContext")?, + &imported(py, NATIVE_RECORDS_MODULE, "CompileAdapterContext")?, &[ ("value_renderer", adapter.clone()), ("collection_rendering", collection_rendering), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs index a93b2a472f..2c489b1889 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs @@ -24,7 +24,7 @@ use crate::bindings::_helpers::scopes::scope_objects::{ use crate::bindings::types::CompilerDetach; const TYPES_MODULE: &str = "types"; -const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; /// `(declarations, canonical positions)` matched, else identity and path error messages. pub(crate) type TargetRows = (bool, Vec, Option, Option); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs index b4dc0d9265..75b24cbe18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs @@ -58,7 +58,7 @@ pub(crate) type ListArguments<'py> = ( bool, ); -const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const CURRENT_FOLDER: &str = "."; /// One lookup's native lookup and every declaration's values in input order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs index 5d969e577f..c63490bee2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs @@ -26,7 +26,7 @@ use crate::bindings::types::CompilerDetach; const DATACLASSES_MODULE: &str = "dataclasses"; const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; -const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; const RUNTIME_USAGE: usize = 2; /// Compile input groups whose items carry declaration usages, after the model inputs. const COMPILE_USAGE_GROUPS: [&str; 5] = [ @@ -391,10 +391,18 @@ pub(crate) fn validate_scope_index(index: &Bound<'_, PyAny>) -> Result<(), PyErr if errors.is_empty() { return Ok(()); } + let mut message: String = String::from("Invalid declaration scope index:"); + for item in &errors { + let code: String = item.getattr("code")?.getattr("value")?.extract()?; + let text: String = item.getattr("message")?.extract()?; + message.push_str(&format!("\n[{code}] {text}")); + } + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("diagnostics", PyTuple::new(py, errors)?)?; Err(PyErr::from_value( py.import(SCOPE_ERRORS_MODULE)? .getattr("ScopeValidationError")? - .call1((PyTuple::new(py, errors)?,))?, + .call((message,), Some(&options))?, )) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs index 91425ea463..3d77e5e4dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::scopes::scope_objects::{ type PyObject = Py; -const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const CURRENT_PATH: &str = "."; fn native_lookup(lookup: &Bound<'_, PyAny>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 28e5f913bd..2e907c9cca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -18,10 +18,11 @@ use crate::bindings::_helpers::command_runtime::invocation::resolve_command_invo use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; -const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const SEED_PLAN_MODULE: &str = "sqlbuild._native"; const SEED_STATE_MODULE: &str = "sqlbuild._native"; const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; @@ -201,7 +202,7 @@ fn compile_seeds<'py>( ], )?; let options: Bound<'py, PyAny> = call_with( - &imported(py, PIPELINE_MODELS, "CompilePipelineOptions")?, + &imported(py, NATIVE_RECORDS_MODULE, "CompilePipelineOptions")?, &[ ("selected_target", request.getattr("selected_target")?), ("select", request.getattr("select")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs index 273c987b1a..0c06652ffe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs @@ -12,7 +12,7 @@ use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, f use crate::bindings::models::ProjectCatalog; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; /// The column names a metadata config value lists: a name, or the names of a list or tuple. pub(crate) fn column_names(value: &Bound<'_, PyAny>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs index 9161c7c459..fe679be210 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs @@ -1,10 +1,12 @@ //! The Python models, enums and compile helpers the semantic completion stage reads and builds. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString}; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE: &str = "P009"; const SQL_ANALYSIS_CONFIG_KEY: &str = "sql_analysis"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; @@ -43,7 +45,7 @@ impl<'py> StageClasses<'py> { pub(crate) fn new(py: Python<'py>) -> PyResult { let models = py.import(COMPILE_MODELS_MODULE)?; let types = py.import(COMPILE_TYPES_MODULE)?; - let severity = types.getattr("DiagnosticSeverity")?; + let severity = shape_class(types.as_any(), "DiagnosticSeverity")?; let resource = types.getattr("CompiledResourceType")?; let kinds = py .import(REFERENCE_TYPES_MODULE)? @@ -53,8 +55,8 @@ impl<'py> StageClasses<'py> { location: py .import(CONTRACT_MODELS_MODULE)? .getattr("SourceLocation")?, - model_test_payload: models.getattr("CompiledModelSqlTestPayload")?, - compile_phase: types.getattr("DiagnosticPhase")?.getattr("COMPILE")?, + model_test_payload: shape_class(models.as_any(), "CompiledModelSqlTestPayload")?, + compile_phase: shape_class(types.as_any(), "DiagnosticPhase")?.getattr("COMPILE")?, error: severity.getattr("ERROR")?, warning: severity.getattr("WARNING")?, model_resource: resource.getattr("MODEL")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs index b705744b91..db1e80a2bd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs @@ -2,6 +2,7 @@ use std::collections::HashSet; +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -19,10 +20,11 @@ use crate::bindings::_helpers::semantic_shapes::shapes::{ model_inputs, project_shapes, published_shape, }; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; + const NATIVE_MODULE: &str = "sqlbuild._native"; -const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const BINDING_UNKNOWN_TABLE_INTERNAL_CODE: &str = "B000"; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -308,8 +310,8 @@ fn validated_diagnostics<'py>( let models_module = py.import(COMPILE_MODELS_MODULE)?; let types = py.import(COMPILE_TYPES_MODULE)?; let diagnostic_class: Object<'py> = models_module.getattr("CompilerDiagnostic")?; - let severity_class: Object<'py> = types.getattr("DiagnosticSeverity")?; - let phase: Object<'py> = types.getattr("DiagnosticPhase")?.getattr("COMPILE")?; + let severity_class: Object<'py> = shape_class(types.as_any(), "DiagnosticSeverity")?; + let phase: Object<'py> = shape_class(types.as_any(), "DiagnosticPhase")?.getattr("COMPILE")?; let model_type: Object<'py> = types.getattr("CompiledResourceType")?.getattr("MODEL")?; let mut diagnostics: Vec> = Vec::new(); for ((model, request), result) in models.iter().zip(&requests).zip(results.try_iter()?) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 3a6644efe0..2649457aa3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -32,10 +32,11 @@ use crate::bindings::_helpers::compile_outputs::runtime_target::write_test_runti use crate::bindings::_helpers::result_output::integration_values::json_value; use crate::bindings::_helpers::result_output::resource_ids::sql_test_identity; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const NATIVE_MODULE: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; -const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const TEST_PLAN_MODULE: &str = "sqlbuild._native"; const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; const EXECUTOR_PIPELINE_MODELS: &str = "sqlbuild.executor.pipeline.models"; @@ -323,7 +324,7 @@ fn compile_tests<'py>( ], )?; let options: Bound<'py, PyAny> = call_with( - &imported(py, PIPELINE_MODELS, "CompilePipelineOptions")?, + &imported(py, NATIVE_RECORDS_MODULE, "CompilePipelineOptions")?, &[ ("selected_target", request.getattr("selected_target")?), ("no_sql_validation", request.getattr("no_sql_validation")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs index 0bad28def1..96bbfa3179 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs @@ -1,5 +1,6 @@ //! Compiled SQL tests, their keyed P013 diagnostics and assembly failures, built natively. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; @@ -11,7 +12,7 @@ use crate::bindings::_helpers::sql_test_glue::assembly::{ const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const GLUE_MODELS_MODULE: &str = "sqlbuild.compiler.sql_test_glue.models"; +const GLUE_MODELS_MODULE: &str = "sqlbuild._native"; const SPEC_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; /// Fields a direct-logic compiled payload copies from its input payload, in order. const DIRECT_PAYLOAD_FIELDS: [&str; 5] = [ @@ -66,10 +67,10 @@ impl<'py> Classes<'py> { let types: Bound<'py, PyModule> = py.import(COMPILE_TYPES_MODULE)?; let glue: Bound<'py, PyModule> = py.import(GLUE_MODELS_MODULE)?; Ok(Self { - direct_input: models.getattr("CompileDirectLogicSqlTestInputPayload")?, + direct_input: shape_class(&models, "CompileDirectLogicSqlTestInputPayload")?, resource_type: types.getattr("CompiledResourceType")?, - phase: types.getattr("DiagnosticPhase")?.getattr("COMPILE")?, - severity: types.getattr("DiagnosticSeverity")?.getattr("ERROR")?, + phase: shape_class(&types, "DiagnosticPhase")?.getattr("COMPILE")?, + severity: shape_class(&types, "DiagnosticSeverity")?.getattr("ERROR")?, assembly: glue.getattr("NativeSqlTestAssembly")?, failure: glue.getattr("NativeSqlTestFailure")?, location: py.import(SPEC_MODELS_MODULE)?.getattr("SourceLocation")?, @@ -83,7 +84,7 @@ impl<'py> Classes<'py> { } fn model(&self, name: &str) -> PyResult> { - self.models.getattr(name) + shape_class(&self.models, name) } fn object_key(&self, resource_type: &str, name: &str) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs index 28afc8a867..5b41fbb0ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs @@ -3,7 +3,6 @@ pub(super) const UDF: &str = "udf"; pub(super) const TABLE_FN: &str = "table_fn"; pub(super) const ERROR_SEVERITY: &str = "error"; -pub(super) const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; pub(super) const PLAN_MODELS: &str = "sqlbuild.compiler.planner.models"; pub(super) const NATIVE_PLANNING: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs index 1754ec8725..92028ee3d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs @@ -17,11 +17,7 @@ pub(super) fn chains<'py>( return Ok(PyTuple::empty(py).into_any()); } let request = call_with( - &imported( - py, - "sqlbuild.compiler.sql_test_glue.models", - "NativeSqlTestChainRequest", - )?, + &imported(py, "sqlbuild._native", "NativeSqlTestChainRequest")?, &[ ("models", project.getattr("models")?), ("tests", tests.clone()), @@ -79,11 +75,7 @@ fn planned<'py>(args: &KeywordArguments<'py>, artifact: bool) -> PyResult(row: &Bound<'py, PyAny>, chain: bool) -> PyResult }, )?; } - imported( + shape_class_at( py, PLAN_MODELS, if chain { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs index 1a3617db47..dcdcf61198 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs @@ -5,7 +5,7 @@ use pyo3::types::PyString; use crate::bindings::_helpers::compile_driver::python_calls::imported; -use crate::bindings::_helpers::sql_test_glue::constants::COMPILE_MODELS; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; pub(super) fn string<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() @@ -14,7 +14,7 @@ pub(super) fn string<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { pub(super) fn model_payload(test: &Bound<'_, PyAny>) -> PyResult { test.getattr("payload")?.is_instance(&imported( test.py(), - COMPILE_MODELS, + NATIVE_RECORDS_MODULE, "CompiledModelSqlTestPayload", )?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs index 7fa8ceaa39..f8a5141d1d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs @@ -1,5 +1,6 @@ //! Authored header values as the Python header objects: scalars, sets, constants and hooks. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class_at; use pyo3::prelude::{IntoPyObject, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; @@ -30,9 +31,7 @@ fn instance( (module, class): (&str, &str), arguments: &pyo3::Bound<'_, PyDict>, ) -> PyResult> { - Ok(py - .import(module)? - .getattr(class)? + Ok(shape_class_at(py, module, class)? .call((), Some(arguments))? .unbind()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs index 8fd76caa2a..5a1046c80b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs @@ -1,5 +1,6 @@ //! Logical SQL references extracted natively under one adapter's lexical rules. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -15,6 +16,8 @@ use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; +const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; + const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; @@ -116,7 +119,7 @@ fn scanned<'py>( let reference_type: Bound<'py, PyAny> = imported(py, COMPILE_MODELS_MODULE, "CompileSqlReference")?; let invalid_type: Bound<'py, PyAny> = - imported(py, COMPILE_MODELS_MODULE, "InvalidSqlReferenceCall")?; + imported(py, NATIVE_RECORDS_MODULE, "InvalidSqlReferenceCall")?; let references: Vec> = scan .references .into_iter() @@ -165,7 +168,7 @@ fn scan_sql_reference_calls<'py>( let py: Python<'py> = syntax.py(); let (references, invalid_calls) = scanned(py, sql, syntax)?; call_with( - &imported(py, COMPILE_MODELS_MODULE, "SqlReferenceScan")?, + &imported(py, NATIVE_RECORDS_MODULE, "SqlReferenceScan")?, &[ ("references", PyTuple::new(py, references)?.into_any()), ("invalid_calls", PyTuple::new(py, invalid_calls)?.into_any()), @@ -192,11 +195,11 @@ pub(crate) fn extract_references<'py>( &[ ( "phase", - types.getattr("DiagnosticPhase")?.getattr("COMPILE")?, + shape_class(types.as_any(), "DiagnosticPhase")?.getattr("COMPILE")?, ), ( "severity", - types.getattr("DiagnosticSeverity")?.getattr("ERROR")?, + shape_class(types.as_any(), "DiagnosticSeverity")?.getattr("ERROR")?, ), ("code", text(py, REFERENCE_CALL_SYNTAX_CODE)?), ("message", call.getattr("message")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs index 04c014d2d3..9cbe53d4e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs @@ -1,5 +1,6 @@ //! Create, read and append SQL of migration state tables, and stored rows decoded to events. +use crate::bindings::_helpers::compiler_records::shape_classes::shape_class_at; use pyo3::exceptions::{PyKeyError, PyValueError}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -258,7 +259,7 @@ fn enumerated<'py>( class: &str, value: Bound<'py, PyAny>, ) -> PyResult> { - imported(py, MIGRATION_TYPES, class)?.call1((value,)) + shape_class_at(py, MIGRATION_TYPES, class)?.call1((value,)) } fn known_member<'py>( diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index da39e9d00a..141ff1ece5 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -18,13 +18,6 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { records::registration::register(module)?; rules::registration::register(module)?; cli::registration::register(module)?; - module.add( - "NativeCompilerError", - module.py().get_type::(), - )?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; analysis::normalization::register(module)?; analysis::queries::register(module)?; analysis::sql_tests::register(module)?; @@ -43,6 +36,13 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { } fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add( + "NativeCompilerError", + module.py().get_type::(), + )?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; cache::registration::register(module)?; attachments::registration::register(module)?; refactor::registration::register(module)?; diff --git a/fensu.toml b/fensu.toml index bf472a55e7..141966664d 100644 --- a/fensu.toml +++ b/fensu.toml @@ -58,6 +58,11 @@ rule = "FFR309" path = "src/sqlbuild/compiler/planner/__init__.py" reason = "Native bindings own planning, plan entries, plan output, command plans and the planner value records; this package only declares the plan models, types, constants and errors the executor, integrations and pipeline import." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/migrations/__init__.py" +reason = "Native bindings own migration planning and state rows; this package only declares the migration models, decisions and table names the executor imports." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/source_freshness/__init__.py" @@ -476,5 +481,5 @@ reason = "Old-name steps re-read facts and physical state per move so a retry ne [[targets.python.rule_exceptions]] rule = "XSB051" path = "src/sqlbuild/executor/old_name_views/_helpers/janitor.py" -symbols = ["plan_old_name_views", "apply_old_name_view_drops"] +symbols = ["apply_old_name_view_drops"] reason = "Janitor reads each compatibility view's stored definition right before deciding to drop it, so a relation that replaced the view is never dropped; a project has few live old names." diff --git a/python-boundary.toml b/python-boundary.toml index ec3df3e61e..8667730cda 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 9546 +remaining_lines = 5847 [[stays]] path = "executor" @@ -295,9 +295,64 @@ reason = "public-api" note = "Rule 3 shapes: format version, retention and read-limit constants re-exported by sqlbuild.compute_logs." [[stays]] -path = "runtime/event_exporting" -reason = "executor" -note = "Exports execution events while the executor runs." +path = "runtime/event_exporting/__init__.py" +reason = "public-api" +note = "Package marker of the public sink declarations below; the dispatcher, queue, accounting and command scope are native EventExporterDispatcher/EventExporterCommandScope." + +[[stays]] +path = "runtime/event_exporting/main/__init__.py" +reason = "public-api" +note = "Package marker of the public sink decorators and the user-sink invocation shims below." + +[[stays]] +path = "runtime/event_exporting/main/lifecycle_event_sink.py" +reason = "public-api" +note = "Rule 3 facade: the @lifecycle_event_sink decorator re-exported by sqlbuild.sinks, run when user sink modules are imported." + +[[stays]] +path = "runtime/event_exporting/main/command_output_sink.py" +reason = "public-api" +note = "Rule 3 facade: the @command_output_sink decorator re-exported by sqlbuild.sinks, run when user sink modules are imported." + +[[stays]] +path = "runtime/event_exporting/main/get_lifecycle_event_sink_definition.py" +reason = "public-api" +note = "Rule 3 facade: reads the decorator's definition off a user function; re-exported by sqlbuild.sinks." + +[[stays]] +path = "runtime/event_exporting/main/get_command_output_sink_definition.py" +reason = "public-api" +note = "Rule 3 facade: reads the decorator's definition off a user function; re-exported by sqlbuild.sinks." + +[[stays]] +path = "runtime/event_exporting/main/_invoke_lifecycle_event_sink.py" +reason = "user-code" +note = "Rule 2 shim: the daemon delivery thread's target that calls one user lifecycle sink and reports how it ended to the native dispatcher, so user code never runs above native frames that interpreter finalization would abort (event_exporting/workers.rs)." + +[[stays]] +path = "runtime/event_exporting/main/_run_command_output_sinks.py" +reason = "user-code" +note = "Rule 2 shim: target of each dispatcher's single command-output sink thread; waits on a Python queue and calls user sinks in order on that one thread (provider thread affinity), reporting each outcome through a Python queue, so no native frame is on that thread when finalization can end it (event_exporting/dispatcher.rs)." + +[[stays]] +path = "runtime/event_exporting/models.py" +reason = "public-api" +note = "Rule 3 shape: LifecycleEventSinkDefinition re-exported by sqlbuild.sinks." + +[[stays]] +path = "runtime/event_exporting/types.py" +reason = "public-api" +note = "Rule 3 shape: LifecycleEventKind re-exported by sqlbuild.sinks." + +[[stays]] +path = "runtime/event_exporting/exceptions.py" +reason = "public-api" +note = "Rule 3 shapes: errors the public decorators raise and the native dispatcher raises." + +[[stays]] +path = "runtime/event_exporting/constants.py" +reason = "public-api" +note = "Rule 3 shapes: decorator attribute names, sink kinds and severities the public decorators and discovery read." [[stays]] path = "runtime/output_capture/classes" @@ -337,7 +392,7 @@ note = "Rule 3 shapes: the invocation-context variable and wire constants used b [[stays]] path = "integrations/dagster" reason = "public-api" -note = "Documented orchestrator integration." +note = "Documented orchestrator integration and rule 4 Dagster glue: context reads, subprocess streaming and Dagster event construction; CLI arguments, selector logs, payload and failure parsing are native." [[stays]] path = "integrations/rivers" @@ -347,12 +402,12 @@ note = "Documented orchestrator integration." [[stays]] path = "integrations/dlt" reason = "user-code" -note = "Loader runtime that runs inside user loaders." +note = "Loader runtime that runs inside user loaders; synthetic loader record building and naming are native." [[stays]] path = "integrations/ingestr" reason = "user-code" -note = "Loader runtime that runs inside the executor load phase." +note = "Loader runtime that runs inside the executor load phase; synthetic loader record building and naming are native." [[stays]] path = "integrations/dbt/_helpers/profile/load.py" @@ -437,12 +492,12 @@ note = "Tracks the loaded user modules macro call results depend on, and runs a [[stays]] path = "compiler/python_nodes/_helpers/user_inventory.py" reason = "user-code" -note = "Reads loaded user node functions: provider parameter annotations, identities over live functions and decorated dependency names; graph construction is native." +note = "Reads loaded user node functions: identities over live functions and decorated dependency names; graph construction is native." [[stays]] -path = "compiler/discovery/_helpers/filesystem/core.py" +path = "compiler/discovery/_helpers/filesystem/project_modules.py" reason = "user-code" -note = "Imports project Python node, provider, sink and materialization modules and reads their definitions." +note = "Rule 2 shim: imports project Python node, hook, provider, sink and materialization modules and reads live members, signatures, provider parameter annotations and source lines; record building, provider naming and validation, settings errors, factory results and duplicate checks are native." [[stays]] path = "compiler/discovery/_helpers/validation/discovery.py" @@ -614,6 +669,141 @@ path = "tasks.py" reason = "public-api" note = "Public facade." +[[stays]] +path = "cost/models.py" +reason = "executor" +note = "Shape rule: data records only (StatementExecutionTelemetry, CostResourceContext, RunCostSummary), no logic; consumed by adapters/snowflake/classes/snowflake_adapter.py, adapters/snowflake/classes/snowflake_cursor.py, executor/build/classes/build_scheduler.py and 2 more." + +[[stays]] +path = "cli/compile_reuse/models.py" +reason = "user-code" +note = "Shape rule: data records only (CompileReuseRequest, SettingsEnvironmentInputs, SettingsInputsResult), no logic; consumed by cli/compile_reuse/_helpers/provider_settings.py, cli/compile_reuse/_helpers/runtime_identity.py." + +[[stays]] +path = "compiler/source_freshness/constants.py" +reason = "adapter" +note = "Shape rule: data constants only (SOURCE_FRESHNESS_TABLE_NAME, COLUMN_SOURCE_NAME, COLUMN_TARGET_DATABASE, COLUMN_TARGET_SCHEMA, ...), no logic; consumed by adapter/contract/classes/duckdb_backed_adapter.py, adapter/state_sql/_helpers/source_freshness.py, adapters/bigquery/classes/bigquery_adapter.py and 6 more." + +[[stays]] +path = "compiler/auditing/types.py" +reason = "executor" +note = "Shape rule: data enums and aliases only (AuditSeverity, AuditRunScope, AuditAttachmentKind, AuditEvaluationMode, ...), no logic; consumed by audits.py, compiler/auditing/main/above.py, compiler/auditing/main/below.py and 17 more." + +[[stays]] +path = "compiler/python_nodes/types.py" +reason = "executor" +note = "Shape rule: data enums and aliases only (PythonNodeKind, PythonNodeStatus, SkipMode, PythonNodeFanInAction), no logic; consumed by assets.py, assets.pyi, executor/load/_helpers/dag_runtime.py and 16 more." + +[[stays]] +path = "compiler/macro_bridge/constants.py" +reason = "user-code" +note = "Shape rule: data constants only (ACTIVE_MACRO_BRIDGE, COMPILES_STARTED, STORE_MODULE_PATHS, MACRO_CALL_STORE_FILE_NAME, ...), no logic; consumed by compiler/macro_bridge/_helpers/store_environment.py, compiler/macro_bridge/classes/macro_bridge.py, compiler/macro_bridge/main/_claim_first_compile.py and 2 more." + +[[stays]] +path = "compiler/references/models.py" +reason = "executor" +note = "Shape rule: data records only (RelationName, ProjectRelation, ProjectRelationIndex), no logic; consumed by executor/python_nodes/classes/runtime_relation_guard.py." + +[[stays]] +path = "runtime/contracts/types.py" +reason = "executor" +note = "Shape rule: data enums and aliases only (ExecutionResourceKind, ConnectionElapsedCallback, NodeStartCallback), no logic; consumed by executor/build/_helpers/scheduler.py, executor/build/_helpers/source_node.py, executor/build/classes/build_scheduler.py and 15 more." + +[[stays]] +path = "errors/contracts/exceptions.py" +reason = "executor" +note = "Shape rule: data exceptions only (SharedInputError, ExecutorInputError), no logic; consumed by executor/audit_results/_helpers/sql.py, executor/build/_helpers/external_source_loads.py, executor/build/_helpers/retention.py and 53 more." + +[[stays]] +path = "compiler/fingerprints/constants.py" +reason = "executor" +note = "Shape rule: data constants only (FINGERPRINT_TABLE_NAME, COLUMN_NODE_TYPE, COLUMN_NODE_NAME, COLUMN_TARGET_DATABASE, ...), no logic; consumed by adapter/contract/classes/duckdb_backed_adapter.py, adapters/bigquery/classes/bigquery_adapter.py, adapters/databricks/classes/databricks_adapter.py and 13 more." + +[[stays]] +path = "diagnostics/models.py" +reason = "public-api" +note = "Shape rule: data records only (ProcessResourceUsage, DiagnosticRoutingOptions), no logic; consumed by cli/commands/main/entrypoint/_dispatch_with_compute_logs.py, diagnostics/classes/invocation_diagnostic_routing.py, diagnostics/main/_log_process_resources.py and 3 more." + +[[stays]] +path = "compiler/dag/types.py" +reason = "public-api" +note = "Shape rule: data enums and aliases only (NodeKind), no logic; consumed by integrations/dagster/constants.py, integrations/rivers/constants.py." + +[[stays]] +path = "runtime/contracts/models.py" +reason = "executor" +note = "Shape rule: data records only (ConnectionHooks), no logic; consumed by executor/pipeline/_helpers/scenario.py." + +[[stays]] +path = "cursor_algebra/types.py" +reason = "executor" +note = "Shape rule: data enums and aliases only (CursorScalar, BoundSentinel, Bound), no logic; consumed by executor/load/_helpers/cursors.py, executor/load/main/_execute.py, executor/run/_helpers/materializations/microbatch.py and 2 more." + +[[stays]] +path = "cost/types.py" +reason = "adapter" +note = "Shape rule: data enums and aliases only (CostCapability, CostStatus), no logic; consumed by adapters/snowflake/classes/snowflake_adapter.py." + +[[stays]] +path = "compiler/lineage/types.py" +reason = "adapter" +note = "Shape rule: data enums and aliases only (InferredNullability), no logic; consumed by adapter/contract/types.py, adapter/type_system/_helpers/inference_rules.py, adapter/type_system/main/conditional_result_nullability.py and 2 more." + +[[stays]] +path = "compiler/macro_bridge/models.py" +reason = "user-code" +note = "Shape rule: data records only (ModuleSources), no logic; consumed by compiler/macro_bridge/_helpers/store_environment.py, compiler/macro_bridge/classes/macro_bridge.py." + +[[stays]] +path = "integrations/dbt/exceptions.py" +reason = "user-code" +note = "Shape rule: data exceptions only (DbtProfileError), no logic; consumed by integrations/dbt/_helpers/profile/load.py, integrations/dbt/_helpers/profile/render.py." + +[[stays]] +path = "compiler/migrations/constants.py" +reason = "executor" +note = "Shape rule: data constants only (MIGRATION_TABLE_NAME, MIGRATION_WRITE_ATTEMPTS, COLUMN_MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME), no logic; consumed by adapters/sqlserver/classes/sqlserver_adapter.py, executor/janitor/constants.py, executor/migrations/_helpers/columns.py and 4 more." + +[[stays]] +path = "diagnostics/constants.py" +reason = "public-api" +note = "Shape rule: data constants only (LOGGER_ROOT_NAME, LOGGER_PREFIX, SQL_DIGEST_FIELD, SQL_TEXT_FIELD, ...), no logic; consumed by diagnostics/_helpers/logging.py, diagnostics/_helpers/redaction.py, diagnostics/classes/invocation_diagnostic_routing.py and 2 more." + +[[stays]] +path = "cursor_algebra/exceptions.py" +reason = "executor" +note = "Shape rule: data exceptions only (CursorAlgebraError), no logic; consumed by executor/run/models.py." + +[[stays]] +path = "spec/contracts/constants.py" +reason = "adapter" +note = "Shape rule: data constants only (DEFAULT_SEED_CSV_SETTINGS, ZERO_DAY_CURSOR_DURATION), no logic; consumed by adapter/contract/classes/base_adapter.py, adapter/contract/classes/duckdb_backed_adapter.py, adapter/contract/classes/materialization.py and 6 more." + +[[stays]] +path = "cli/entry/constants.py" +reason = "public-api" +note = "Shape rule: data constants only (NATIVE_ALLOCATOR_PURGE_DELAY_VARIABLE, NATIVE_ALLOCATOR_PURGE_DELAY_MILLISECONDS, OUTPUT_STREAM_ENCODING, OUTPUT_STREAM_ERRORS, ...), no logic; consumed by cli/entry/_helpers/streams.py, cli/entry/main/entry.py, cli/entry/main/launched.py." + +[[stays]] +path = "cli/compile_reuse/constants.py" +reason = "user-code" +note = "Shape rule: data constants only (REUSE_FORMAT_VERSION, OUTPUT_FILE_DESCRIPTORS), no logic; consumed by cli/compile_reuse/_helpers/runtime_identity.py." + +[[stays]] +path = "sql_values/exceptions.py" +reason = "adapter" +note = "Shape rule: data exceptions only (SqlValueRenderingError), no logic; consumed by adapter/contract/classes/base_adapter.py, adapter/contract/exceptions.py." + +[[stays]] +path = "cli/output/constants.py" +reason = "public-api" +note = "Shape rule: data constants only (INTEGRATION_RESULT_PATH_ENV), no logic; consumed by integrations/dagster/_helpers/invocation_factory.py." + +[[stays]] +path = "compiler/compile/types.py" +reason = "executor" +note = "Shape rule: typing protocols, enums and aliases only (CompiledProjectView, CompiledModelView, CompiledResourceType, AuditInput, ...), no logic; consumed by executor/janitor/_helpers/plan.py, executor/pipeline/_helpers/testing.py, executor/scenario/_helpers/local/execution.py and project audit factories that annotate AuditInput." + [[handled_native_errors]] location = "compiler/planner/models.py::CursorBounds._parse_inferred" note = "A bound that is not integer text is parsed as a timestamp instead; CursorAlgebraError is an input error, and native compiler failures are not caught." diff --git a/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py b/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py index 8ca95c915d..7041ada59b 100644 --- a/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py +++ b/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py @@ -11,7 +11,7 @@ import sqlbuild._native as _native from scripts.cold_compile_performance.exceptions import CompileBenchmarkFixtureError -from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic +from sqlbuild._native import SqlBindingDiagnostic if TYPE_CHECKING: from sqlbuild._native import NativeDiagnosticRow, NativePositionsModule diff --git a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py index 8fbbbcf835..55b090329d 100644 --- a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py +++ b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py @@ -39,10 +39,9 @@ FailureCase, ProjectRun, ) +from sqlbuild._native import DiscoveryError, ResourceIdentityError from sqlbuild.compiler.compile import constants as compile_constants from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.discovery.exceptions import DiscoveryError -from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError from sqlbuild.compiler.scopes.types import ScopeDiagnosticCode _SOURCE_DIRECTORY: str = "source" diff --git a/scripts/compiler_differential/_helpers/coverage/render.py b/scripts/compiler_differential/_helpers/coverage/render.py index 637a7900d4..8f314e0bce 100644 --- a/scripts/compiler_differential/_helpers/coverage/render.py +++ b/scripts/compiler_differential/_helpers/coverage/render.py @@ -31,8 +31,7 @@ RENDER_NON_COLLECTION_FIELDS, RENDER_STAGE_CAPTURE_SUFFIX, ) -from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES -from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES, CompileProjectInputs _MACRO_CALL: re.Pattern[str] = re.compile(r"@([A-Za-z_]\w*)\(") _REFERENCE_CALL: re.Pattern[str] = re.compile(r"__(ref|source|seed)\(\s*\"([^\"]+)\"\s*\)") diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index 0347c73e70..0c6594ebd2 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -260,7 +260,7 @@ EVENT_EXPORTER_INTEGRATION_PATHS: frozenset[str] = frozenset( { "src/sqlbuild/cli/commands/_helpers/entry/observability.py", - "src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py", + "src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py", "src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py", "src/sqlbuild/compiler/discovery/_helpers/yml/project.py", "src/sqlbuild/compiler/discovery/main/runtime_extensions.py", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 6afbf93f48..4a5eeb5dc1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,6 +1,7 @@ """Private SQLBuild native engine bindings.""" import threading +from collections import Counter from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence, Set from concurrent.futures import Executor, Future from contextlib import AbstractContextManager @@ -14,8 +15,12 @@ from threading import Event from types import TracebackType from typing import Any, ClassVar, Literal, Protocol, Self, TextIO, TypedDict -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter +from sqlbuild.adapter.contract.classes.base_adapter import ( + BaseAdapter, +) +from sqlbuild.adapter.contract.classes.strict_adapter import ( + StrictAdapter, +) from sqlbuild.adapter.contract.models import ( ColumnInfo, ExpressionInferenceProfile, @@ -45,49 +50,23 @@ from sqlbuild.compiler.auditing.types import ( AuditSeverity, ) from sqlbuild.compiler.compile.models import ( - AnalysisCacheContext, - CompileAdapterContext, - CompileAnalysisSelection, CompiledAudit, - CompiledFunction, - CompiledLineageColumnFact, CompiledLineageSourceFact, CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompiledSeed, - CompiledSource, - CompiledSqlExpansion, CompiledSqlScenario, CompiledSqlTest, - CompileModelConfig, CompilerDiagnostic, CompileSqlReference, - CompileSqlScenarioInput, - CompileSqlTestCte, - CompileSqlTestInput, - CursorInputRoles, DeclarationResolutionContext, - DeclarationScopeBuild, - DeclarationScopeResolver, - DynamicColumnContractProof, - ExpansionSpan, - FixtureColumnInference, - FunctionArgument, - FunctionReturnColumn, InferredColumn, LoadedMacro, MacroContext, - MacroExpansionResult, - MappedOffset, ModelInputBuildContext, - NativeModelConfigInputs, - NativeModelConfigSession, - PythonSqlReferenceReport, - SqlExpansionContext, - SqlReferenceScan, StaticMacroExport, + _native, ) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, @@ -99,54 +78,37 @@ from sqlbuild.compiler.compile.types import ( CompiledResourceType, CompiledSeedView, CompiledSourceView, - CompileInputsView, - FunctionInput, FunctionLanguage, - ModelInput, - SeedInput, + RelationKeyRow, SourceInput, + SqlTestMode, TypedSqlValueRenderer, ) -from sqlbuild.compiler.contract_adoption.models import ContractEvidence, ContractFinding -from sqlbuild.compiler.contracts.models import ContractValidationResult from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, - DiscoveredAdapterFile, - DiscoveredAuditBlock, - DiscoveredAuditFile, DiscoveredCheckFunction, - DiscoveredConstantFile, - DiscoveredEnumFile, + DiscoveredCommandOutputSink, + DiscoveredEventExporter, + DiscoveredEventExporterDeclaration, DiscoveredHookFunction, DiscoveredLoaderFunction, DiscoveredMacroFile, DiscoveredMaterializationFile, - DiscoveredModelSchemaFile, DiscoveredProjectInputs, - DiscoveredPythonFunctionFile, - DiscoveredSchemaFile, - DiscoveredSeedFile, + DiscoveredProvider, + DiscoveredProviderUsage, + DiscoveredPythonNodeFunctions, DiscoveredSourceFile, - DiscoveredSqlFunctionFile, - DiscoveredSqlHookFile, DiscoveredSqlScenarioFile, - DiscoveredSqlTestBlock, DiscoveredSqlTestFile, - DiscoveryFileFault, EnumDeclaration, - ModelSchemaDeclaration, SqlTestParameterDeclaration, - TolerantScopeDiscovery, ) +from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet -from sqlbuild.compiler.frontier.types import CompilerStage -from sqlbuild.compiler.graph.models import AttachedAuditGateEdge, HookReadEdge, LineageGraphViews -from sqlbuild.compiler.lineage.models import ( - ColumnLineageEdge, - DirectSemanticColumnUse, - ModelColumnLineage, +from sqlbuild.compiler.lineage.types import ( + InferredNullability, ) -from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from sqlbuild.compiler.migrations.models import ( ColumnMigrationEvent, MigrationEvent, @@ -155,23 +117,13 @@ from sqlbuild.compiler.migrations.models import ( OldNameViewHistory, ) from sqlbuild.compiler.migrations.types import ( - MigrationCompatibility, MigrationDecision, - MigrationDiscovery, MigrationPromotion, OldNameViewEventType, ) from sqlbuild.compiler.pipeline.models import ( - ClonePipelineConnection, - ClonePipelineOptions, - ClonePipelineResult, - CompiledProjectPhaseResult, - CompilePipelineOptions, CompilePipelineResult, ProjectGraph, - PythonPlanEntry, - PythonRunPlanOutputs, - StaticCommandContext, ) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( @@ -183,7 +135,6 @@ from sqlbuild.compiler.planner.models import ( CursorInputEvidence, CursorInputRelation, CursorOverrides, - DeferralInputs, EmptyCursorInputDecision, FunctionPlanEntry, FutureCursorSafetyEvidence, @@ -194,13 +145,6 @@ from sqlbuild.compiler.planner.models import ( ModelPlanEntry, OldNameView, OldNameViewPlanEntry, - ParsedSelector, - PathSelector, - PlannerOverrides, - PlannerPolicies, - PlannerRelationsContext, - PlannerScope, - PlannerSelection, PlanOutput, RetentionPlanEntry, ScenarioExecutionPlan, @@ -210,9 +154,7 @@ from sqlbuild.compiler.planner.models import ( SchemaFinding, SeedPlanEntry, SourceLoadPlanEntry, - SqlTestAssertionStep, SqlTestPlanEntry, - SqlTestSelection, TableTypePlanEntry, ) from sqlbuild.compiler.planner.types import ( @@ -228,11 +170,9 @@ from sqlbuild.compiler.planner.types import ( from sqlbuild.compiler.python_nodes.models import ( PythonNodeGraph, PythonSqlRunLifecyclePlan, - PythonSqlRunSelection, ) -from sqlbuild.compiler.refactoring.models import ManualLocation +from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.compiler.references.models import ( - LiteralSqlRelation, ProjectRelation, ProjectRelationIndex, RelationName, @@ -240,6 +180,7 @@ from sqlbuild.compiler.references.models import ( from sqlbuild.compiler.references.types import ( ExternalSqlReferenceResolver, HardCodedRelationOwnerKind, + SqlReferenceKind, ) from sqlbuild.compiler.scopes.models import ( DeclarationExplanation, @@ -257,90 +198,78 @@ from sqlbuild.compiler.scopes.models import ( ScopeReportFilters, ScopeTargetQuery, UsageRecord, + VisibilityRecord, VisibilityResolution, ) -from sqlbuild.compiler.scopes.types import DeclarationKind +from sqlbuild.compiler.scopes.types import ( + DeclarationKind, + ScopeKind, +) from sqlbuild.compiler.source_freshness.models import ( - DirectSourceFreshnessPlanningResult, SourceFreshnessRecord, ) from sqlbuild.compiler.sql_analysis.models import ( - SqlBindingResult, SqlLexicalSyntax, - SqlSchemaValidationRequest, -) -from sqlbuild.compiler.sql_test_glue.models import ( - NativeSqlTestAssembly, - NativeSqlTestChainRequest, - NativeSqlTestPlanningRequest, ) from sqlbuild.cost.models import CostResourceContext, RunCostSummary, StatementExecutionTelemetry from sqlbuild.cost.types import CostStatus from sqlbuild.cursor_algebra.models import AlignedInterval, DateValue, Duration, TimestampValue from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar from sqlbuild.diagnostics.models import ProcessResourceUsage -from sqlbuild.executor.auditing.models import AuditExecutionResult +from sqlbuild.executor.auditing.models import ( + AuditExecutionResult, +) from sqlbuild.executor.build.models import ( BuildExecutionResult, FunctionExecutionResult, SchedulerState, SeedExecutionResult, ) -from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult +from sqlbuild.executor.clone.models import ( + CloneExecutionResult, + CloneItemResult, +) from sqlbuild.executor.diff.models import DiffExecutionResult, RowDiffSamplingOverride -from sqlbuild.executor.janitor.models import JanitorExecutionResult, JanitorPlan +from sqlbuild.executor.janitor.models import ( + JanitorExecutionResult, + JanitorPlan, + JanitorRelationKey, +) from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.python_nodes.models import ( PythonCheckExecutionResult, PythonNodeExecutionResult, ) -from sqlbuild.executor.run.models import ModelExecutionResult -from sqlbuild.executor.scenario.models import ScenarioRunResult, ScenarioSnapshotCaptureRunResult -from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult +from sqlbuild.executor.run.models import ( + ModelExecutionResult, +) +from sqlbuild.executor.scenario.models import ( + ScenarioRunResult, + ScenarioSnapshotCaptureRunResult, +) +from sqlbuild.executor.testing.models import ( + SqlTestExecutionResult, + StepResult, +) from sqlbuild.integrations.dbt.models import ( - DbtCliOptions, - DbtCommandResult, - DbtInitRequest, - DbtInitResult, - DbtLsResult, - DbtNodeExecutionResult, - NormalizedDbtProfileConnection, ResolvedDbtProfileOutput, ) from sqlbuild.integrations.dbt.types import DbtInvoker -from sqlbuild.lint.models import ( - CompileFacts, - ExpandedLintFile, - FixVerdict, - HeaderSpan, - LintBody, - LintConfig, - LintFileRole, - LintRunResult, - LintViolation, - PendingLintBody, - PreparedLintFiles, - RuleFixResult, -) -from sqlbuild.lint.types import RelationKeyRow from sqlbuild.microbatches.models import MicrobatchInterval -from sqlbuild.presentation.models import DisplayOptions from sqlbuild.presentation.types import TransientLineOwner from sqlbuild.provider.classes.container import ProviderContainer -from sqlbuild.python_nodes.models import SqlResourceRef +from sqlbuild.provider.classes.session import ProviderSession +from sqlbuild.python_nodes.models import ( + ColumnLineageRef, + RetryPolicy, + SqlResourceRef, +) +from sqlbuild.python_nodes.types import PythonCheckSeverity from sqlbuild.rule_engine.models import ( - CustomRulesOutcome, Finding, - NativeRulesEvaluation, - PreparedSqlLint, - PreparedSqlLintResult, Rule, - RuleRunPlan, RulesConfig, RulesResult, - RulesRunResult, - SqlExpansionReuse, - SqlRulesEvaluation, ) from sqlbuild.runtime.compute_logs.models import ( CaptureByteCounts, @@ -369,6 +298,7 @@ from sqlbuild.runtime.execution_history.types import ( from sqlbuild.runtime.observability.classes.event_dispatcher import EventDispatcher from sqlbuild.runtime.observability.models import DiagnosticLog, LifecycleEvent from sqlbuild.runtime.output_capture.models import ( + BoundCommandOutputSink, CommandOutputCaptureSummary, CommandOutputRecord, OutputCaptureContext, @@ -384,6 +314,8 @@ from sqlbuild.spec.contracts.models import ( LocalConfig, LocalDbtConfig, ProjectConfig, + ResolvedTableType, + ResolvedTimeTravelRetention, ScenarioConfig, SchemaAuditInstance, SchemaColumn, @@ -392,6 +324,7 @@ from sqlbuild.spec.contracts.models import ( SchemaSeedEntry, SettingsConfig, SnapshotsConfig, + SourceColumnEntry, SourceEntry, SourceFreshnessAgePolicy, SourceLocation, @@ -399,14 +332,18 @@ from sqlbuild.spec.contracts.models import ( TargetConfig, ) from sqlbuild.spec.contracts.types import ( + EventExportSeverity, MicrobatchLimitAction, MissingMigrationOriginPolicy, SourceFreshnessStrategy, SourceFreshnessValueKind, + SourceWriteStrategy, WarehouseGroup, ) from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import CollectionRendering, StateSqlValueType +from sqlbuild.sql_values.types import ( + StateSqlValueType, +) BUILD_IDENTITY: str @@ -668,6 +605,178 @@ class NativeProjectFacts: type FixtureKey = tuple[CompiledResourceType, str] type InputColumns = Callable[[str, str], frozenset[str] | None] type FixtureGroups = tuple[tuple[CompiledResourceType, dict[str, str]], ...] + +class SeedInput(Protocol): + """One seed paired with its declaration, as the native owner view holds it.""" + + @property + def seed_file(self) -> DiscoveredSeedFile: ... + @property + def schema_entry(self) -> SchemaSeedEntry: ... + @property + def schema_file(self) -> DiscoveredSchemaFile: ... + +class FunctionInput(Protocol): + """One attached function, as the native owner view holds it.""" + + @property + def function_file(self) -> DiscoveredSqlFunctionFile | DiscoveredPythonFunctionFile: ... + @property + def name(self) -> str: ... + @property + def arguments(self) -> tuple[FunctionArgument, ...]: ... + @property + def returns(self) -> str: ... + @property + def body_sql(self) -> str: ... + @property + def return_columns(self) -> tuple[FunctionReturnColumn, ...]: ... + @property + def references(self) -> tuple[CompileSqlReference, ...]: ... + @property + def language(self) -> FunctionLanguage: ... + @property + def runtime_version(self) -> str | None: ... + @property + def entry_point(self) -> str | None: ... + @property + def packages(self) -> tuple[str, ...]: ... + @property + def tags(self) -> tuple[str, ...]: ... + @property + def description(self) -> str | None: ... + @property + def database(self) -> str | None: ... + @property + def schema(self) -> str | None: ... + @property + def logical_database(self) -> str | None: ... + @property + def logical_schema(self) -> str | None: ... + @property + def fingerprint_database(self) -> str | None: ... + @property + def fingerprint_schema(self) -> str | None: ... + @property + def fingerprint_logical_database(self) -> str | None: ... + @property + def fingerprint_logical_schema(self) -> str | None: ... + @property + def declaration_usages(self) -> tuple[UsageRecord, ...]: ... + @property + def assembly_row( + self, + ) -> tuple[str, str, list[str], list[tuple[str, str, str | None]]]: ... + +class CompileInputsView(Protocol): + """A compile's attached inputs: a native owner view, or hand-built `CompileProjectInputs`.""" + + @property + def project_config(self) -> ProjectConfig: ... + @property + def local_config(self) -> LocalConfig: ... + @property + def discovered_inputs(self) -> DiscoveredProjectInputs: ... + @property + def sql_lexical_syntax(self) -> SqlLexicalSyntax: ... + @property + def run_id(self) -> str: ... + @property + def effective_target_name(self) -> str | None: ... + @property + def effective_target(self) -> TargetConfig | None: ... + @property + def compile_cache_dir(self) -> Path | None: ... + @property + def effective_connection(self) -> dict[str, object]: ... + @property + def effective_settings(self) -> SettingsConfig: ... + @property + def no_sql_validation(self) -> bool: ... + @property + def effective_vars(self) -> dict[str, object]: ... + @property + def macro_context(self) -> MacroContext | None: ... + @property + def loaded_macros(self) -> dict[str, LoadedMacro]: ... + @property + def public_enums(self) -> dict[str, EnumDeclaration]: ... + @property + def public_constants(self) -> dict[str, ConstantDeclaration]: ... + @property + def model_inputs(self) -> tuple[ModelInput, ...]: ... + @property + def seed_inputs(self) -> tuple[SeedInput, ...]: ... + @property + def source_inputs(self) -> tuple[SourceInput, ...]: ... + @property + def sql_function_inputs(self) -> tuple[FunctionInput, ...]: ... + @property + def test_inputs(self) -> tuple[CompileSqlTestInput, ...]: ... + @property + def scenario_inputs(self) -> tuple[CompileSqlScenarioInput, ...]: ... + @property + def audit_inputs(self) -> tuple[AuditInput, ...]: ... + @property + def diagnostics(self) -> tuple[CompilerDiagnostic, ...]: ... + @property + def external_sql_reference_resolver(self) -> ExternalSqlReferenceResolver | None: ... + @property + def scope_index(self) -> ScopeIndex: ... + @property + def declaration_scope(self) -> DeclarationScopeBuild | None: ... + @property + def native_project(self) -> _native.NativeCompiledProject | None: ... + +class ModelInput(Protocol): + """One model's compile input: a native owner view, or a hand-built `CompileModelInput`.""" + + @property + def model_file(self) -> DiscoveredSqlModelFile: ... + @property + def config(self) -> CompileModelConfig: ... + @property + def query_sql(self) -> str: ... + @property + def macro_source_sql(self) -> str: ... + @property + def references(self) -> tuple[CompileSqlReference, ...]: ... + @property + def schema_entry(self) -> SchemaModelEntry | None: ... + @property + def schema_file(self) -> DiscoveredSchemaFile | None: ... + @property + def sql_validation_enabled(self) -> bool: ... + @property + def rejected_sql_analysis_opt_out(self) -> SourceLocation | None: ... + @property + def enum_declarations(self) -> tuple[EnumDeclaration, ...]: ... + @property + def constant_declarations(self) -> tuple[ConstantDeclaration, ...]: ... + @property + def enum_columns(self) -> dict[str, EnumDeclaration]: ... + @property + def macro_deps(self) -> tuple[str, ...]: ... + @property + def macro_usages(self) -> tuple[UsageRecord, ...]: ... + @property + def declaration_usages(self) -> tuple[UsageRecord, ...]: ... + @property + def sql_expansion(self) -> CompiledSqlExpansion | None: ... + @property + def lint_expansion(self) -> CompiledSqlExpansion | None: ... + @property + def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: ... + @property + def attached_schema_name(self) -> str | None: ... + @property + def relation_key_row(self) -> RelationKeyRow: ... + +class CompileInputsReadyCallback(Protocol): + """Invocation-local observer for the completed pre-semantic input snapshot.""" + + def __call__(self, *, inputs: CompileInputsView) -> None: ... + type _GraphKey = tuple[str, str] type _GraphEdges = list[tuple[_GraphKey, list[_GraphKey]]] type _GraphIndexes = tuple[ @@ -3471,6 +3580,188 @@ class CliObservabilityScope: def cli_observability_scope(*, args: CliNamespace, project_dir: Path) -> CliObservabilityScope: ... +class EventExporterCounts: + """Frozen attempt counters for one exporter or their aggregate.""" + + @property + def accepted(self) -> int: ... + @property + def filtered(self) -> int: ... + @property + def delivered(self) -> int: ... + @property + def failed(self) -> int: ... + @property + def dropped(self) -> int: ... + +class EventExporterAccounting: + """Frozen accounting snapshot for one named exporter.""" + + @property + def exporter_name(self) -> str: ... + @property + def counts(self) -> EventExporterCounts: ... + +class EventExportSummary: + """Final or point-in-time best-effort delivery accounting.""" + + @property + def aggregate(self) -> EventExporterCounts: ... + @property + def per_exporter(self) -> tuple[EventExporterAccounting, ...]: ... + @property + def queue_depth(self) -> int: ... + @property + def queue_capacity(self) -> int: ... + @property + def flush_complete(self) -> bool: ... + @property + def accepted(self) -> int: ... + @property + def filtered(self) -> int: ... + @property + def delivered(self) -> int: ... + @property + def failed(self) -> int: ... + @property + def dropped(self) -> int: ... + +class EventExporterFailure: + """Sanitized exporter failure information safe for diagnostics.""" + + @property + def exporter_name(self) -> str: ... + @property + def error_type(self) -> str: ... + @property + def event_kind(self) -> str: ... + @property + def event_severity(self) -> EventExportSeverity: ... + +class BoundEventExporter: + """One lifecycle-event sink bound to command-scoped provider instances.""" + + def __init__( + self, + name: str, + function: Callable[..., object], + provider_arguments: Mapping[str, object], + event_kinds: Iterable[str] | None = None, + min_severity: EventExportSeverity | str | None = None, + ) -> None: ... + @property + def name(self) -> str: ... + @property + def function(self) -> Callable[..., object]: ... + @property + def provider_arguments(self) -> Mapping[str, object]: ... + @property + def event_kinds(self) -> frozenset[str]: ... + @property + def min_severity(self) -> EventExportSeverity: ... + +class EventExporterDispatcher: + """Deliver filtered lifecycle events off execution threads with bounded memory.""" + + def __init__( + self, + *, + exporters: Iterable[BoundEventExporter] | None = None, + queue_capacity: int = 1024, + shutdown_timeout_seconds: float = 2.0, + invocation_timeout_seconds: float = 1.0, + notification_queue_capacity: int = 256, + health_interval_seconds: float | None = None, + failure_callback: Callable[[EventExporterFailure], object] | None = None, + summary_callback: Callable[[EventExportSummary], object] | None = None, + ) -> None: ... + @property + def thread(self) -> threading.Thread: ... + @property + def notification_thread(self) -> threading.Thread | None: ... + def enqueue(self, event: LifecycleEvent, /) -> None: ... + def bind(self, exporters: Iterable[BoundEventExporter], /) -> None: ... + def bind_command_output_sinks(self, sinks: Iterable[BoundCommandOutputSink], /) -> None: ... + def export_output(self, records: Iterable[CommandOutputRecord]) -> None: ... + def shutdown(self) -> EventExportSummary: ... + def finalize_when_idle(self, finalizer: Callable[[], object], /) -> None: ... + def summary(self) -> EventExportSummary: ... + +class DispatcherWorker: + def __call__(self) -> None: ... + +class InvocationCompletion: + def __call__(self, error: BaseException | None, /) -> None: ... + +def stop_event_exporter_threads() -> None: ... + +class EventExporterCommandScope: + """Bind discovered typed sinks and own their command-scoped provider session.""" + + def __init__(self, *, dispatcher: EventExporterDispatcher) -> None: ... + @property + def dispatcher(self) -> EventExporterDispatcher: ... + @property + def provider_session(self) -> ProviderSession | None: ... + @property + def project_dir(self) -> Path | None: ... + @property + def providers(self) -> tuple[DiscoveredProvider, ...]: ... + @property + def event_exporters(self) -> tuple[DiscoveredEventExporter, ...]: ... + @property + def command_output_sinks(self) -> tuple[DiscoveredCommandOutputSink, ...]: ... + def configure(self, discovered_inputs: DiscoveredProjectInputs, /) -> None: ... + def configure_extensions( + self, + *, + project_dir: Path | None = None, + providers: tuple[DiscoveredProvider, ...], + event_exporters: tuple[DiscoveredEventExporter, ...], + command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] | None = None, + ) -> None: ... + def cached_extensions( + self, *, project_dir: Path + ) -> ( + tuple[ + tuple[DiscoveredProvider, ...], + tuple[DiscoveredEventExporter, ...], + tuple[DiscoveredCommandOutputSink, ...], + ] + | None + ): ... + def close(self) -> EventExportSummary: ... + def export_output(self, records: tuple[CommandOutputRecord, ...]) -> None: ... + +class ProviderSessionCloser: + def __call__(self) -> None: ... + +class EventExporterScopeContext: + def __enter__(self) -> EventExporterCommandScope: ... + def __exit__(self, *exc_info: object) -> bool: ... + +def event_exporter_command_scope( + scope: EventExporterCommandScope, / +) -> EventExporterScopeContext: ... +def current_event_exporter_command_scope() -> EventExporterCommandScope | None: ... +def cached_event_exporter_extensions( + *, project_dir: Path +) -> ( + tuple[ + tuple[DiscoveredProvider, ...], + tuple[DiscoveredEventExporter, ...], + tuple[DiscoveredCommandOutputSink, ...], + ] + | None +): ... +def lifecycle_export_policy_catalog() -> dict[str, tuple[str, EventExportSeverity, int]]: ... +def event_export_severity_at_least( + *, severity: EventExportSeverity | str, minimum: EventExportSeverity | str +) -> bool: ... +def stricter_event_export_severity( + *, first: EventExportSeverity | str, second: EventExportSeverity | str | None +) -> EventExportSeverity: ... + class OutputCaptureDispatcher: """Turn stream fragments into bounded text chunks and export them off execution threads.""" @@ -3494,6 +3785,8 @@ class OutputCaptureDispatcher: class OutputDeliveryWorker: def __call__(self) -> None: ... +def stop_output_capture_threads() -> None: ... + class CommandOutputContext: def __enter__(self) -> None: ... def __exit__(self, *exc_info: object) -> bool: ... @@ -5993,8 +6286,94 @@ def observe_source_freshness_json( previous_records_by_source_name: Mapping[str, SourceFreshnessRecord] | None = None, ) -> str: ... def render_janitor_plan(*, plan: JanitorPlan, use_color: bool = False) -> str: ... -def render_janitor_completion( - *, result: JanitorExecutionResult, use_color: bool = False +def discover_python_node_functions( + *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] | None = None +) -> DiscoveredPythonNodeFunctions: ... +def discover_hook_functions( + *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] | None = None +) -> tuple[DiscoveredHookFunction, ...]: ... +def discover_materialization_files( + *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] | None = None +) -> tuple[DiscoveredMaterializationFile, ...]: ... +def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, ...]: ... +def discover_event_exporter_declarations( + *, project_dir: Path +) -> tuple[DiscoveredEventExporterDeclaration, ...]: ... +def bind_event_exporter_declarations( + *, + declarations: tuple[DiscoveredEventExporterDeclaration, ...], + providers: tuple[DiscoveredProvider, ...], + project_dir: Path, +) -> tuple[DiscoveredEventExporter, ...]: ... +def discover_event_exporter_functions( + *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] | None = None +) -> tuple[DiscoveredEventExporter, ...]: ... +def dagster_cli_arguments( + *, + args: list[str], + dag: Mapping[str, Any] | None, + has_context: bool, + read_selection: Callable[[], tuple[list[tuple[str, ...]], list[tuple[tuple[str, ...], str]]]], +) -> dict[str, Any]: ... +def dagster_selector_log_lines(*, selectors: list[str]) -> list[str]: ... +def dagster_payload_results( + *, + dag: Mapping[str, Any], + payload: Mapping[str, Any], + command: list[str], + selected_asset_paths: list[tuple[str, ...]], + selected_check_keys: list[tuple[tuple[str, ...], str]], + check_selection_is_explicit: bool, +) -> list[tuple[Any, ...]]: ... +def dagster_integration_results( + *, + dag: Mapping[str, Any], + envelope: IntegrationResultEnvelope, + command: list[str], + selected_asset_paths: list[tuple[str, ...]], + selected_check_keys: list[tuple[tuple[str, ...], str]], + check_selection_is_explicit: bool, + emitted_asset_paths: list[tuple[str, ...]], +) -> list[tuple[Any, ...]]: ... +def dagster_selected_asset_results( + *, dag: Mapping[str, Any], command: list[str], selected_asset_paths: list[tuple[str, ...]] +) -> list[tuple[Any, ...]]: ... +def dagster_execution_payload(*, text: str) -> Mapping[str, Any] | None: ... +def dagster_failure( + *, + command: list[str], + project_dir: Path, + selection: list[str], + selector_file: str, + returncode: int, + execution_payload: Mapping[str, Any] | None, + stdout: str, + stderr: str, +) -> tuple[str, dict[str, Any]]: ... +def dagster_live_failure( + *, asset: IntegrationAssetResult, envelope: IntegrationResultEnvelope, command: list[str] +) -> tuple[list[str], dict[str, Any]]: ... +def build_janitor_plan( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + connection: Any, + retention_days: int, + delete_tracked_only: bool = True, + exclude_patterns: Sequence[str] = (), + direct_mode: bool = True, + state_history_versions: int = 20, + archive_retention_days: int = 14, + early_old_name_view_drops: Sequence[str] = (), + scan_relation_keys: Iterable[JanitorRelationKey] | None = None, + protected_relation_keys: Iterable[JanitorRelationKey] | None = None, + protected_relation_reasons: Mapping[JanitorRelationKey, str] | None = None, +) -> JanitorPlan: ... +def build_janitor_archive_name( + *, original_name: str, archived_at: datetime, identifier_limit: int, kind: str | None = None +) -> str: ... +def render_janitor_completion( + *, result: JanitorExecutionResult, use_color: bool = False ) -> str: ... # Native analysis: SQL test planning glue. @@ -6524,6 +6903,7 @@ def start_sql_test_planning( sql_test_selection: SqlTestSelection, enabled: bool, ) -> Future[PlannedSqlTests]: ... +def join_sql_test_planning_threads() -> None: ... def build_display_only_sqlbuild_plan( *, project: CompiledProjectView, selected_model_names: tuple[str, ...], full_refresh: bool ) -> PlanOutput: ... @@ -9896,3 +10276,3089 @@ class MigrationStateError(ValueError): def __init__( self, message: str, *, code: str | None = None, help: str | None = None ) -> None: ... + +class ContractFindingKind(StrEnum): + """Stable physical-to-authored contract difference categories.""" + + MISSING_DECLARATION = "missing_declaration" + MISSING_PHYSICAL_COLUMN = "missing_physical_column" + TYPE_MISMATCH = "type_mismatch" + RELATION_MISSING = "relation_missing" + INSPECTION_UNAVAILABLE = "inspection_unavailable" + OWNERSHIP_CONFLICT = "ownership_conflict" + +class CompilerStage(StrEnum): + """Frontier objects handed from one compiler stage to the next, in pipeline order.""" + + DISCOVERED_PROJECT_INPUTS = "discovered_project_inputs" + COMPILE_PROJECT_INPUTS = "compile_project_inputs" + COMPILED_PROJECT = "compiled_project" + +class HookReadType(StrEnum): + """Kind of model hook that declares or writes a read.""" + + PYTHON = "Python" + SQL = "SQL" + INLINE_SQL = "inline SQL" + +class LintSeverity(StrEnum): + """Severity assigned to one lint violation.""" + + FAULT = "fault" + WARNING = "warning" + +class DiscoveryError(ValueError): + """Base error for expected discovery failures.""" + + code: str + message: str + help: str | None + def __init__( + self, message: str, *, code: str | None = None, help: str | None = None + ) -> None: ... + +class ProjectConfigError(DiscoveryError): + """Raised when shared or local project config files are invalid.""" + +class ModelSqlParseError(DiscoveryError): + """Raised when an authored SQL model file has an invalid shape.""" + +class SqlTestParseError(DiscoveryError): + """Raised when an authored SQL-native test file has an invalid shape.""" + +class SqlAuditParseError(DiscoveryError): + """Raised when an authored SQL audit file has an invalid shape.""" + +class SchemaParseError(DiscoveryError): + """Raised when a schema.yml file has an invalid shape.""" + +class SourceParseError(DiscoveryError): + """Raised when a sources/*.yml file has an invalid shape.""" + +class DiscoveryConflictError(DiscoveryError): + """Raised when discovered project inputs conflict across files.""" + +class SeedDiscoveryError(DiscoveryError): + """Raised when declared seed metadata does not match local seed files.""" + +class SqlScenarioParseError(DiscoveryError): + """Raised when an authored SQL scenario file has an invalid shape.""" + +class PythonNodeDiscoveryError(DiscoveryError): + """Raised when project Python nodes cannot be discovered.""" + +class ProviderDiscoveryError(DiscoveryError): + """Raised when project providers cannot be discovered.""" + +class DeclarationParseError(DiscoveryError): + """Raised when an authored public SQL declaration is invalid.""" + +class SqlHookParseError(DiscoveryError): + """Raised when an authored SQL hook resource has an invalid shape.""" + +class EventExporterDiscoveryError(DiscoveryError): + """Raised when project lifecycle event exporters are invalid.""" + +class ProjectPythonPathError(DiscoveryError): + """Raised when project-owned Python lives outside a supported extension root.""" + +class ProjectPathError(DiscoveryError): + """Raised when an authored project path cannot be read as valid UTF-8.""" + +class ResourceIdentityError(DiscoveryError): + """Raised when an authored resource identity is not canonical snake_case.""" + +class MeasurementAuditError(ValueError): + """Raised when a measurement audit contract or value is invalid.""" + +class ContractAdoptionError(RuntimeError): + """Raised when safe contract source editing cannot continue.""" + +class FingerprintInputError(ValueError): + """Raised when fingerprint storage inputs or rows are invalid.""" + + code: str + message: str + help: str | None + def __init__( + self, message: str, *, code: str | None = None, help: str | None = None + ) -> None: ... + +class QueryFingerprintError(ValueError): + """Raised when SQL cannot be tokenized into a change fingerprint.""" + +class NativeStageMismatchError(RuntimeError): + """Raised when native state the compiler replays is inconsistent with its inputs.""" + +class UnknownLineageDialectError(ValueError): + """Raised, as Python's parser raised `ValueError`, for a dialect Polyglot does not know.""" + +class RefactorValueError(ValueError): + """An unhandled value failure the Python planner also leaves to the caller.""" + +class RefactorIOError(OSError): + """A file read or write failure the Python planner also leaves to the caller.""" + +class ScopeError(ValueError): + """Base error for invalid scope-domain input.""" + + code: str + message: str + help: str | None + def __init__( + self, message: str, *, code: str | None = None, help: str | None = None + ) -> None: ... + +class InvalidScopePathError(ScopeError): + """Raised when a project-relative scope path is invalid.""" + +class InvalidQualifiedIdentityError(ScopeError): + """Raised when a qualified scope identity cannot be parsed.""" + +class UnindexableScopeError(ScopeError): + """Raised when the native scope index cannot place a discovered path or resource kind.""" + +class ScopeValidationError(ScopeError): + """Aggregate error raised only when strict scope validation is requested.""" + + diagnostics: Any + +class UnsupportedLexicalSyntaxError(ValueError): + """Raised when an adapter declares lexical rules the native SQL scanners do not support.""" + +class LintError(Exception): + """A lint or format run could not proceed.""" + + code: str + message: str + help: str | None + def __init__( + self, message: str, *, code: str | None = None, help: str | None = None + ) -> None: ... + +class NativeLintError(LintError): + """The SQLBuild-owned native lint engine rejected a request or response.""" + +class ProjectCompileError(LintError): + """The project could not be compiled, so its SQL cannot be linted.""" + +class LintStoppedError(Exception): + """A speculative lint run was stopped because its result is no longer needed.""" + +class SpecConfigError(RuntimeError): + """Raised when project specification configuration is invalid.""" + +class ConfigValueTypeError(ValueError): + """Raised when an authored config value has the wrong runtime type.""" + + code: str + message: str + help: str | None + def __init__( + self, message: str, *, code: str | None = None, help: str | None = None + ) -> None: ... + key: Any + expected: Any + actual_type: Any + +class NativeModelAnalysisRequest(NativeRecord): + """The Python model analysis's arguments plus the facts dynamic pivot proofs need.""" + + known_functions: tuple[str, ...] + known_types: tuple[str, ...] + model_inputs: tuple[ModelInput, ...] + column_nullability_by_table: dict[str, dict[str, InferredNullability]] + column_types_by_table: dict[str, dict[str, str]] + inference_profile: ExpressionInferenceProfile + allow_compact_analysis: bool + rich_type_inference: bool + analysis_cache: AnalysisCacheContext | None + complete_binding_schemas: dict[str, dict[str, str]] + dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] + def __init__( + self, + known_functions: tuple[str, ...], + known_types: tuple[str, ...], + model_inputs: tuple[ModelInput, ...], + column_nullability_by_table: dict[str, dict[str, InferredNullability]], + column_types_by_table: dict[str, dict[str, str]], + inference_profile: ExpressionInferenceProfile, + allow_compact_analysis: bool, + rich_type_inference: bool, + analysis_cache: AnalysisCacheContext | None, + complete_binding_schemas: dict[str, dict[str, str]], + dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], + ) -> None: ... + +class NativeModelAnalyses(NativeRecord): + """Each analysed model's position in the finished session that keeps its outcome.""" + + positions: dict[str, int] + succeeded: frozenset[str] + session: Any | None + def __init__( + self, + positions: dict[str, int], + succeeded: frozenset[str], + session: Any | None, + ) -> None: ... + +class NativePivotTables(NativeRecord): + """The relation facts Python's dynamic pivot proofs read, for proofs outside a session.""" + + dialect: str | None + column_types_by_table: dict[str, dict[str, str]] + authoritative_column_types_by_table: dict[str, dict[str, str]] + column_nullability_by_table: dict[str, dict[str, InferredNullability]] + dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] + def __init__( + self, + dialect: str | None, + column_types_by_table: dict[str, dict[str, str]], + authoritative_column_types_by_table: dict[str, dict[str, str]], + column_nullability_by_table: dict[str, dict[str, InferredNullability]], + dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], + ) -> None: ... + +class NativeColumnQuery(NativeRecord): + """One query outside the session whose output columns Python's analysis inferred.""" + + sql: str + mode: Literal["batch", "reanalysis", "parse", "legacy"] + placeholders: dict[str, str] | None + references: tuple[CompileSqlReference, ...] + recover_cte_facts: bool + def __init__( + self, + sql: str, + mode: Literal["batch", "reanalysis", "parse", "legacy"], + placeholders: dict[str, str] | None = ..., + references: tuple[CompileSqlReference, ...] = ..., + recover_cte_facts: bool = ..., + ) -> None: ... + +class NativeQueryColumns(NativeRecord): + """One query's analysis: whether it succeeded, its columns and whether it projects a star.""" + + succeeded: bool + columns: tuple[InferredColumn, ...] | None + has_star: bool + def __init__( + self, + succeeded: bool, + columns: tuple[InferredColumn, ...] | None, + has_star: bool, + ) -> None: ... + +class ContractFinding(NativeRecord): + resource_type: CompiledResourceType + resource_name: str + kind: ContractFindingKind + message: str + column_name: str | None + declared_type: str | None + physical_type: str | None + def __init__( + self, + resource_type: CompiledResourceType, + resource_name: str, + kind: ContractFindingKind, + message: str, + column_name: str | None = ..., + declared_type: str | None = ..., + physical_type: str | None = ..., + ) -> None: ... + +class ContractEvidence(NativeRecord): + resource_type: CompiledResourceType + resource_name: str + database: str | None + schema: str | None + relation: str + declared_columns: tuple[ColumnInfo, ...] + physical_columns: tuple[ColumnInfo, ...] | None + source_path: Path | None + dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] + findings: tuple[ContractFinding, ...] + def __init__( + self, + resource_type: CompiledResourceType, + resource_name: str, + database: str | None, + schema: str | None, + relation: str, + declared_columns: tuple[ColumnInfo, ...], + physical_columns: tuple[ColumnInfo, ...] | None, + source_path: Path | None, + dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = ..., + findings: tuple[ContractFinding, ...] = ..., + ) -> None: ... + +class ContractAdoptionResult(NativeRecord): + from_target: str + evidence: tuple[ContractEvidence, ...] + written_paths: tuple[Path, ...] + def __init__( + self, + from_target: str, + evidence: tuple[ContractEvidence, ...], + written_paths: tuple[Path, ...] = ..., + ) -> None: ... + +class ContractValidationResult(NativeRecord): + """Diagnostics emitted by model contract validation.""" + + diagnostics: tuple[CompilerDiagnostic, ...] + def __init__( + self, + diagnostics: tuple[CompilerDiagnostic, ...] = ..., + ) -> None: ... + +class LineageGraphViews(NativeRecord): + """A native project graph's lineage edges and selector indexes as Python dicts.""" + + upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] + downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] + tag_index: dict[str, frozenset[CompiledObjectKey]] + path_index: dict[CompiledObjectKey, str] + all_keys: dict[str, CompiledObjectKey] + def __init__( + self, + upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + tag_index: dict[str, frozenset[CompiledObjectKey]], + path_index: dict[CompiledObjectKey, str], + all_keys: dict[str, CompiledObjectKey], + ) -> None: ... + +class AttachedAuditGateEdge(NativeRecord): + """One ordering edge: the gated node waits for a resource its attached audit reads.""" + + audit_name: str + target: CompiledObjectKey + gated: CompiledObjectKey + read: CompiledObjectKey + def __init__( + self, + audit_name: str, + target: CompiledObjectKey, + gated: CompiledObjectKey, + read: CompiledObjectKey, + ) -> None: ... + +class HookReadEdge(NativeRecord): + """One ordering edge: a model waits for a resource one of its hooks reads.""" + + hook_name: str + gated: CompiledObjectKey + read: CompiledObjectKey + hook_type: HookReadType + hook_path: Path | None + def __init__( + self, + hook_name: str, + gated: CompiledObjectKey, + read: CompiledObjectKey, + hook_type: HookReadType = ..., + hook_path: Path | None = ..., + ) -> None: ... + +class QualifiedLineageColumn(NativeRecord): + """A resource-qualified column in the collapsed SQLBuild lineage graph.""" + + resource_type: CompiledResourceType | str + resource_name: str + column_name: str + def __init__( + self, + resource_type: CompiledResourceType | str, + resource_name: str, + column_name: str, + ) -> None: ... + +class ColumnLineageSource(NativeRecord): + """One upstream column dependency for an output column.""" + + resource_type: CompiledResourceType | str + resource_name: str + column_name: str + def __init__( + self, + resource_type: CompiledResourceType | str, + resource_name: str, + column_name: str, + ) -> None: ... + +class ColumnLineageEdge(NativeRecord): + """One collapsed SQLBuild lineage graph edge.""" + + source: QualifiedLineageColumn + target: QualifiedLineageColumn + transform_kind: ColumnTransformKind + confidence: ColumnLineageConfidence + def __init__( + self, + source: QualifiedLineageColumn, + target: QualifiedLineageColumn, + transform_kind: ColumnTransformKind = ..., + confidence: ColumnLineageConfidence = ..., + ) -> None: ... + +class DirectSemanticColumnUse(NativeRecord): + """One direct non-projection use of a resolved resource column.""" + + consumer_model: str + context: str + expression_sql: str + source: QualifiedLineageColumn + confidence: ColumnLineageConfidence + line: int | None + column: int | None + def __init__( + self, + consumer_model: str, + context: str, + expression_sql: str, + source: QualifiedLineageColumn, + confidence: ColumnLineageConfidence = ..., + line: int | None = ..., + column: int | None = ..., + ) -> None: ... + +class ColumnLineage(NativeRecord): + """Lineage for one output column in one model.""" + + output_column: str + transform_kind: ColumnTransformKind + expression_sql: str | None + upstream_columns: tuple[ColumnLineageSource, ...] + nullability: InferredNullability + confidence: ColumnLineageConfidence + def __init__( + self, + output_column: str, + transform_kind: ColumnTransformKind, + expression_sql: str | None, + upstream_columns: tuple[ColumnLineageSource, ...], + nullability: InferredNullability = ..., + confidence: ColumnLineageConfidence = ..., + ) -> None: ... + +class ModelColumnLineage(NativeRecord): + """Column lineage extracted for one compiled model.""" + + model_name: str + columns: tuple[ColumnLineage, ...] + has_star: bool + def __init__( + self, + model_name: str, + columns: tuple[ColumnLineage, ...], + has_star: bool = ..., + ) -> None: ... + +class RefactorProject(NativeRecord): + """Compiler facts of one project, as the refactoring planner reads them.""" + + project_dir: Path + graph: ProjectGraph + discovered: DiscoveredProjectInputs + def __init__( + self, + project_dir: Path, + graph: ProjectGraph, + discovered: DiscoveredProjectInputs, + ) -> None: ... + +class ManualLocation(NativeRecord): + """A location a refactoring cannot rewrite safely.""" + + path: str + line: int | None + column: int | None + reason: str + def __init__( + self, + path: str, + line: int | None, + column: int | None, + reason: str, + ) -> None: ... + +class NativeSqlTestPlanningRequest(NativeRecord): + """Compiled models and tests to plan, with the adapter facts native planning cannot derive.""" + + models: tuple[CompiledModelView, ...] + rendered_model_sql: dict[int, str] + functions: tuple[tuple[str, str, str, str, str], ...] + tests: tuple[CompiledSqlTest, ...] + rendered_test_overrides: dict[int, dict[str, str]] + sql_analysis_enabled: bool + sql_analysis_dialect: str | None + set_difference_operator: str + requires_derived_table_aliases: bool + lexical_syntax: dict[str, object] + render_sql: bool + include_plan: bool + def __init__( + self, + models: tuple[CompiledModelView, ...], + rendered_model_sql: dict[int, str], + functions: tuple[tuple[str, str, str, str, str], ...], + tests: tuple[CompiledSqlTest, ...], + rendered_test_overrides: dict[int, dict[str, str]], + sql_analysis_enabled: bool, + sql_analysis_dialect: str | None, + set_difference_operator: str, + requires_derived_table_aliases: bool, + lexical_syntax: dict[str, object], + render_sql: bool, + include_plan: bool, + ) -> None: ... + +class NativeSqlTestChainRequest(NativeRecord): + """Compiled models and the tests whose unmocked model chains to resolve.""" + + models: tuple[CompiledModelView, ...] + tests: tuple[CompiledSqlTest, ...] + lexical_syntax: dict[str, object] + def __init__( + self, + models: tuple[CompiledModelView, ...], + tests: tuple[CompiledSqlTest, ...], + lexical_syntax: dict[str, object], + ) -> None: ... + +class NativeSqlTestFailure(NativeRecord): + """The error one test's assembly raises: its kind and message.""" + + kind: Literal["input", "internal", "decimal_overflow"] + message: str + def __init__( + self, + kind: Literal["input", "internal", "decimal_overflow"], + message: str, + ) -> None: ... + +class NativeSqlTestAssembly(NativeRecord): + """One natively assembled test, its keyed diagnostics, and the error its assembly raises.""" + + test: CompiledSqlTest | None + diagnostics: tuple[tuple[tuple[str, ...], CompilerDiagnostic], ...] + failure: NativeSqlTestFailure | None + def __init__( + self, + test: CompiledSqlTest | None, + diagnostics: tuple[tuple[tuple[str, ...], CompilerDiagnostic], ...], + failure: NativeSqlTestFailure | None = ..., + ) -> None: ... + +class HeaderSpan(NativeRecord): + """A located DSL header region inside one file.""" + + kind: str + start: int + end: int + def __init__( + self, + kind: str, + start: int, + end: int, + ) -> None: ... + +class LintFileRole(NativeRecord): + """Path-derived classification of one lint input, derived once per file.""" + + declaration_kind: DeclarationKind | None + in_project: bool + in_hook_directory: bool + def __init__( + self, + declaration_kind: DeclarationKind | None = ..., + in_project: bool = ..., + in_hook_directory: bool = ..., + ) -> None: ... + +class InterpolationSite(NativeRecord): + """One sqlbuild interpolation occurrence replaced by a unique sentinel.""" + + sentinel: str + neutralized_start: int + neutralized_end: int + original_start: int + original_end: int + original_text: str + def __init__( + self, + sentinel: str, + neutralized_start: int, + neutralized_end: int, + original_start: int, + original_end: int, + original_text: str, + ) -> None: ... + +class LintBody(NativeRecord): + """One authored SQL body prepared for linting, with its expansion spans.""" + + file_path: Path + body_start: int + body_end: int + lint_text: str + passes: tuple[tuple[ExpansionSpan, ...], ...] + external_identifiers: tuple[str, ...] + dependency_identifiers: tuple[str, ...] + externally_referenced_ctes: tuple[str, ...] + allows_ceremonial_select: bool + allows_dynamic_output_star: bool + allows_empty_fixture_star: bool + dependency_relations: tuple[tuple[str, str], ...] + def __init__( + self, + file_path: Path, + body_start: int, + body_end: int, + lint_text: str, + passes: tuple[tuple[ExpansionSpan, ...], ...], + external_identifiers: tuple[str, ...] = ..., + dependency_identifiers: tuple[str, ...] = ..., + externally_referenced_ctes: tuple[str, ...] = ..., + allows_ceremonial_select: bool = ..., + allows_dynamic_output_star: bool = ..., + allows_empty_fixture_star: bool = ..., + dependency_relations: tuple[tuple[str, str], ...] = ..., + ) -> None: ... + +class PendingLintBody(NativeRecord): + """One expanded authored body awaiting its native lexical preparation.""" + + file_path: Path + body_start: int + body_end: int + expansion_input: str + expanded: str + expansion_passes: tuple[tuple[ExpansionSpan, ...], ...] + pre_expansion_sites: tuple[InterpolationSite, ...] + external_identifiers: tuple[str, ...] + allows_ceremonial_select: bool + allows_dynamic_output_star: bool + allows_empty_fixture_star: bool + request: NativeLintPreparationRequest + def __init__( + self, + file_path: Path, + body_start: int, + body_end: int, + expansion_input: str, + expanded: str, + expansion_passes: tuple[tuple[ExpansionSpan, ...], ...], + pre_expansion_sites: tuple[InterpolationSite, ...], + external_identifiers: tuple[str, ...], + allows_ceremonial_select: bool, + allows_dynamic_output_star: bool, + allows_empty_fixture_star: bool, + request: NativeLintPreparationRequest, + ) -> None: ... + +class ExpandedLintFile(NativeRecord): + """The expanded bodies of one file, up to the first body that could not be expanded.""" + + file_path: Path + bodies: tuple[PendingLintBody, ...] + failure: ProjectCompileError | None + def __init__( + self, + file_path: Path, + bodies: tuple[PendingLintBody, ...], + failure: ProjectCompileError | None, + ) -> None: ... + +class PreparedLintFiles(NativeRecord): + """Lint bodies of every expandable file, and why the skipped files could not expand.""" + + bodies: tuple[LintBody, ...] + unexpandable: dict[Path, str] + def __init__( + self, + bodies: tuple[LintBody, ...], + unexpandable: dict[Path, str], + ) -> None: ... + +class LintEdit(NativeRecord): + """One deterministic authored-source edit proposed by a lint rule.""" + + file_path: Path + code: str + start: int + end: int + replacement: str + def __init__( + self, + file_path: Path, + code: str, + start: int, + end: int, + replacement: str, + ) -> None: ... + +class LintViolation(NativeRecord): + """One lint diagnostic reported against an authored file.""" + + file_path: Path + line: int + column: int + code: str + message: str + severity: LintSeverity + engine: str + end_line: int | None + end_column: int | None + remediation: str | None + fix: LintEdit | None + fix_unavailable_reason: str | None + def __init__( + self, + file_path: Path, + line: int, + column: int, + code: str, + message: str, + severity: LintSeverity, + engine: str, + end_line: int | None = ..., + end_column: int | None = ..., + remediation: str | None = ..., + fix: LintEdit | None = ..., + fix_unavailable_reason: str | None = ..., + ) -> None: ... + +class RuleFixResult(NativeRecord): + """One applied, refused, or unavailable semantic Rule fix.""" + + file_path: Path + code: str + line: int + status: RuleFixStatus + reason: str + def __init__( + self, + file_path: Path, + code: str, + line: int, + status: RuleFixStatus, + reason: str, + ) -> None: ... + +class LintConfig(NativeRecord): + """Resolved lint and format configuration for one run.""" + + native_enabled: bool + max_description_lines: int + line_width: int + dialect: str + enabled_native_rules: tuple[str, ...] | None + ignored_native_rules: tuple[str, ...] + header_rules_enabled: bool + max_literal_length: int + max_ranking_order_by: int + allow_suppressions: bool + relation_keys: Mapping[str, tuple[tuple[str, ...], ...]] + def __init__( + self, + native_enabled: bool = ..., + max_description_lines: int = ..., + line_width: int = ..., + dialect: str = ..., + enabled_native_rules: tuple[str, ...] | None = ..., + ignored_native_rules: tuple[str, ...] = ..., + header_rules_enabled: bool = ..., + max_literal_length: int = ..., + max_ranking_order_by: int = ..., + allow_suppressions: bool = ..., + relation_keys: Mapping[str, tuple[tuple[str, ...], ...]] = ..., + ) -> None: ... + +class LintRunResult(NativeRecord): + """Aggregated outcome of one lint or format run.""" + + files_checked: int + violations: tuple[LintViolation, ...] + formatted_files: tuple[Path, ...] + source_texts: Mapping[Path, str] + unexpandable: Mapping[Path, str] + def __init__( + self, + files_checked: int, + violations: tuple[LintViolation, ...], + formatted_files: tuple[Path, ...], + source_texts: Mapping[Path, str] = ..., + unexpandable: Mapping[Path, str] = ..., + ) -> None: ... + +class CompileFacts(NativeRecord): + """Per-model outputs and per-owner diagnostics of one compilation.""" + + models: dict[CompiledObjectKey, CompiledModelView] + model_paths: dict[CompiledObjectKey, Path] + diagnostics: dict[Path, Counter[DiagnosticIdentity]] + def __init__( + self, + models: dict[CompiledObjectKey, CompiledModelView], + model_paths: dict[CompiledObjectKey, Path], + diagnostics: dict[Path, Counter[DiagnosticIdentity]], + ) -> None: ... + +class FixVerdict(NativeRecord): + """Edited files that fail verification, and whether any change is unattributable.""" + + failing: dict[Path, str] + unattributed: bool + def __init__( + self, + failing: dict[Path, str], + unattributed: bool, + ) -> None: ... + +class DisplayOptions(NativeRecord): + """Options controlling bounded human output.""" + + max_entries_per_section: int | None + overflow_flag: str + def __init__( + self, + max_entries_per_section: int | None = ..., + overflow_flag: str = ..., + ) -> None: ... + +type RuleFixStatus = Literal["applied", "refused", "unavailable"] +type DiagnosticIdentity = tuple[str, str, str, str] +type NativeLintPreparationRequest = LintPreparationRequest + +class DiagnosticPhase(StrEnum): + """Phase that produced a compiler diagnostic.""" + + COMPILE = "compile" + CONTRACT = "contract" + RULE = "rule" + PLAN = "plan" + BUILD = "build" + AUDIT = "audit" + TEST = "test" + CONNECTION = "connection" + +class DiagnosticSeverity(StrEnum): + """Severity for a compiler diagnostic.""" + + ERROR = "error" + WARNING = "warning" + INFO = "info" + +class ColumnTransformKind(StrEnum): + """High-level transform classification for one output column.""" + + DIRECT = "direct" + CAST = "cast" + EXPRESSION = "expression" + AGGREGATION = "aggregation" + STAR = "star" + CONSTANT = "constant" + UNKNOWN = "unknown" + +class ColumnLineageConfidence(StrEnum): + """Coarse confidence that a lineage edge is fully understood.""" + + HIGH = "high" + MEDIUM = "medium" + LOW = "low" + UNKNOWN = "unknown" + +class ColumnLineageMode(StrEnum): + """Column lineage analyzer mode.""" + + RICH = "rich" + FAST = "fast" + +class PolyglotAnalysisDialect(StrEnum): + """Dialect names accepted by Polyglot analyze_query.""" + + GENERIC = "generic" + POSTGRESQL = "postgresql" + MYSQL = "mysql" + BIGQUERY = "bigquery" + SNOWFLAKE = "snowflake" + DUCKDB = "duckdb" + SQLITE = "sqlite" + HIVE = "hive" + SPARK = "spark" + TRINO = "trino" + PRESTO = "presto" + REDSHIFT = "redshift" + TSQL = "tsql" + ORACLE = "oracle" + CLICKHOUSE = "clickhouse" + DATABRICKS = "databricks" + ATHENA = "athena" + TERADATA = "teradata" + DORIS = "doris" + STARROCKS = "starrocks" + MATERIALIZE = "materialize" + RISINGWAVE = "risingwave" + SINGLESTORE = "singlestore" + COCKROACHDB = "cockroachdb" + TIDB = "tidb" + DRUID = "druid" + SOLR = "solr" + TABLEAU = "tableau" + DUNE = "dune" + FABRIC = "fabric" + DRILL = "drill" + DREMIO = "dremio" + EXASOL = "exasol" + DATAFUSION = "datafusion" + +class MigrationDiscovery(StrEnum): + """How a migration was requested.""" + + MANUAL = "manual" + AUTOMATIC = "automatic" + +class MigrationCompatibility(StrEnum): + """Outcome of checking the origin relation against the destination model.""" + + COMPATIBLE = "compatible" + INCOMPATIBLE = "incompatible" + NOT_CHECKED = "not_checked" + +class OldNameViewAction(StrEnum): + """What a build does at a migrated model's old name.""" + + ARCHIVE_AND_VIEW = "archive_and_view" + VIEW_ONLY = "view_only" + LIVE = "live" + NONE = "none" + +class SelectorKind(StrEnum): + NAME = "name" + SEED = "seed" + SOURCE = "source" + TASK = "task" + ASSET = "asset" + LOADER = "loader" + CHECK = "check" + TAG = "tag" + PATH = "path" + TEST = "test" + +class PythonRunPhase(StrEnum): + """Lifecycle-aware Python execution phase.""" + + PRE_SQL_INGRESS = "pre_sql_ingress" + READ_SIDE = "read_side" + +class PythonIdentityStatus(StrEnum): + """Read-only identity diff status for Python plan display.""" + + NEW = "new" + CHANGED = "changed" + UNCHANGED = "unchanged" + UNKNOWN = "unknown" + +class CollectionRendering(StrEnum): + """Physical rendering choices for homogeneous SQL collections.""" + + VALUE_LIST = "value_list" + ARRAY = "array" + +class DbtInteropError(ValueError): + """Base class for expected dbt interop errors.""" + + code: str + message: str + help: str | None + def __init__( + self, message: str, *, code: str | None = None, help: str | None = None + ) -> None: ... + +class DbtInteropConfigError(DbtInteropError): + """Raised when dbt project configuration is missing or invalid.""" + +class DbtInteropRuntimeError(DbtInteropError): + """Raised when dbt command execution fails.""" + +class DbtInteropArgumentError(DbtInteropError): + """Raised when `sqb dbt` arguments cannot be routed safely.""" + +class RelatedLocation(NativeRecord): + """A secondary authored location that adds context to a diagnostic.""" + + label: str + location: SourceLocation + message: str | None + def __init__( + self, + label: str, + location: SourceLocation, + message: str | None = ..., + ) -> None: ... + +class FixtureColumnInference(NativeRecord): + """Inferred fixture columns plus outputs that are explicitly bare NULL literals.""" + + columns: tuple[InferredColumn, ...] + null_literal_names: frozenset[str] + quoted_names: frozenset[str] + def __init__( + self, + columns: tuple[InferredColumn, ...], + null_literal_names: frozenset[str], + quoted_names: frozenset[str], + ) -> None: ... + +class DynamicColumnFamilyProof(NativeRecord): + """Compiler evidence for one declared runtime-generated column family.""" + + name: str + inferred_type: str | None + def __init__( + self, + name: str, + inferred_type: str | None = ..., + ) -> None: ... + +class DynamicColumnContractProof(NativeRecord): + """Closed output-shape evidence for a dynamic column contract.""" + + output_proven: bool + fixed_columns: tuple[InferredColumn, ...] + families: tuple[DynamicColumnFamilyProof, ...] + input_relations: tuple[str, ...] + failure_reason: str | None + bare_dynamic_pivot: bool + def __init__( + self, + output_proven: bool, + fixed_columns: tuple[InferredColumn, ...] = ..., + families: tuple[DynamicColumnFamilyProof, ...] = ..., + input_relations: tuple[str, ...] = ..., + failure_reason: str | None = ..., + bare_dynamic_pivot: bool = ..., + ) -> None: ... + +class CompileModelConfig(NativeRecord): + """Pre-semantic effective config layers attached to a model input.""" + + values: dict[str, object] + model_header_keys: tuple[str, ...] + matched_path_default: str | None + logical_schema: str | None + layer_schema: str | None + logical_database: str | None + time_travel_retention: ResolvedTimeTravelRetention + table_type: ResolvedTableType + def __init__( + self, + values: dict[str, object] = ..., + model_header_keys: tuple[str, ...] = ..., + matched_path_default: str | None = ..., + logical_schema: str | None = ..., + layer_schema: str | None = ..., + logical_database: str | None = ..., + time_travel_retention: ResolvedTimeTravelRetention = ..., + table_type: ResolvedTableType = ..., + ) -> None: ... + +class CursorInputRoles(NativeRecord): + """Effective filter and watermark inputs for one compiled model.""" + + filter_inputs: dict[str, str] + watermark_inputs: dict[str, str] + def __init__( + self, + filter_inputs: dict[str, str], + watermark_inputs: dict[str, str], + ) -> None: ... + +class DeclarationRuntimeProjection(NativeRecord): + """Process-only declaration values keyed by canonical static identity.""" + + declarations: Mapping[DeclarationIdentity, EnumDeclaration | ConstantDeclaration | LoadedMacro] + def __init__( + self, + declarations: Mapping[ + DeclarationIdentity, EnumDeclaration | ConstantDeclaration | LoadedMacro + ] = ..., + ) -> None: ... + +class DeclarationScopeResolver(NativeRecord): + """Process-only lookup state used to project visible runtime declarations.""" + + project_dir: Path | None + lookup: ScopeLookup + projection: DeclarationRuntimeProjection + native_contexts: _native.NativeDeclarationContexts + resource_specific: frozenset[ResourceIdentity] | None + contexts_by_directory: dict[tuple[str, str], DeclarationResolutionContext] + def __init__( + self, + project_dir: Path | None, + lookup: ScopeLookup, + projection: DeclarationRuntimeProjection, + native_contexts: _native.NativeDeclarationContexts, + resource_specific: frozenset[ResourceIdentity] | None = ..., + contexts_by_directory: dict[tuple[str, str], DeclarationResolutionContext] = ..., + ) -> None: ... + +class DeclarationScopeBuild(NativeRecord): + """Static artifact and process-local values shared by compile attachment phases.""" + + loaded_macros: dict[str, LoadedMacro] + index: ScopeIndex + resolver: DeclarationScopeResolver + sql_test_scans: SqlTestScanCache + def __init__( + self, + loaded_macros: dict[str, LoadedMacro], + index: ScopeIndex, + resolver: DeclarationScopeResolver, + sql_test_scans: SqlTestScanCache, + ) -> None: ... + +class MacroExpansionResult(NativeRecord): + """Expanded SQL and resolved macro usage facts from one authored string.""" + + sql: str + spans: tuple[ExpansionSpan, ...] + dependencies: tuple[DeclarationIdentity, ...] + usages: tuple[UsageRecord, ...] + argument_references: tuple[CompileSqlReference, ...] + def __init__( + self, + sql: str, + spans: tuple[ExpansionSpan, ...] = ..., + dependencies: tuple[DeclarationIdentity, ...] = ..., + usages: tuple[UsageRecord, ...] = ..., + argument_references: tuple[CompileSqlReference, ...] = ..., + ) -> None: ... + +class VisibleModelDeclarations(NativeRecord): + """One model's visible declarations and macros, as the native model loop resolves them.""" + + local_enums: dict[str, EnumDeclaration] + local_constants: dict[str, ConstantDeclaration] + enums: dict[str, EnumDeclaration] + constants: dict[str, ConstantDeclaration] + inaccessible_enums: dict[str, DeclarationRecord] + inaccessible_constants: dict[str, DeclarationRecord] + enum_visibility: dict[str, tuple[VisibilityRecord, ...]] + constant_visibility: dict[str, tuple[VisibilityRecord, ...]] + macros: dict[str, LoadedMacro] + macro_records: dict[str, DeclarationRecord] + inaccessible_macros: dict[str, DeclarationRecord] + def __init__( + self, + local_enums: dict[str, EnumDeclaration], + local_constants: dict[str, ConstantDeclaration], + enums: dict[str, EnumDeclaration], + constants: dict[str, ConstantDeclaration], + inaccessible_enums: dict[str, DeclarationRecord], + inaccessible_constants: dict[str, DeclarationRecord], + enum_visibility: dict[str, tuple[VisibilityRecord, ...]], + constant_visibility: dict[str, tuple[VisibilityRecord, ...]], + macros: dict[str, LoadedMacro], + macro_records: dict[str, DeclarationRecord], + inaccessible_macros: dict[str, DeclarationRecord], + ) -> None: ... + +class DeclarationExpansionContext(NativeRecord): + """Declarations and adapter rendering used by one authored SQL owner.""" + + declarations: DeclarationResolutionContext + value_renderer: TypedSqlValueRenderer + collection_rendering: CollectionRendering + resolver: DeclarationScopeResolver | None + def __init__( + self, + declarations: DeclarationResolutionContext, + value_renderer: TypedSqlValueRenderer, + collection_rendering: CollectionRendering, + resolver: DeclarationScopeResolver | None = ..., + ) -> None: ... + +class CompileAdapterContext(NativeRecord): + """Cycle-free adapter behavior required while building compile inputs.""" + + value_renderer: TypedSqlValueRenderer + collection_rendering: CollectionRendering + python_functions_inherit_default_namespace: bool + sql_lexical_syntax: SqlLexicalSyntax + def __init__( + self, + value_renderer: TypedSqlValueRenderer, + collection_rendering: CollectionRendering, + python_functions_inherit_default_namespace: bool, + sql_lexical_syntax: SqlLexicalSyntax, + ) -> None: ... + +class InvalidSqlReferenceCall(NativeRecord): + """One reference call whose arguments compile could not replace with a relation.""" + + ref_kind: SqlReferenceKind + call: str + start: int + message: str + help: str + corrected_call: str + def __init__( + self, + ref_kind: SqlReferenceKind, + call: str, + start: int, + message: str, + help: str, + corrected_call: str, + ) -> None: ... + +class SqlReferenceScan(NativeRecord): + """Valid references and rejected reference calls found in one SQL text.""" + + references: tuple[CompileSqlReference, ...] + invalid_calls: tuple[InvalidSqlReferenceCall, ...] + def __init__( + self, + references: tuple[CompileSqlReference, ...], + invalid_calls: tuple[InvalidSqlReferenceCall, ...] = ..., + ) -> None: ... + +class CompiledLineageColumnFact(NativeRecord): + """Compact output column lineage fact extracted during SQL analysis.""" + + output_column: str + upstream_columns: tuple[CompiledLineageSourceFact, ...] + transform_kind: ColumnTransformKind + confidence: ColumnLineageConfidence + def __init__( + self, + output_column: str, + upstream_columns: tuple[CompiledLineageSourceFact, ...] = ..., + transform_kind: ColumnTransformKind = ..., + confidence: ColumnLineageConfidence = ..., + ) -> None: ... + +class PolyglotAnalysisResult(NativeRecord): + """Outcome of one Polyglot column and lineage analysis pass.""" + + analysis_succeeded: bool + columns: tuple[InferredColumn, ...] | None + lineage_columns: Sequence[CompiledLineageColumnFact] + has_star: bool + star_resolved: bool + binding_diagnostics: tuple[SqlBindingDiagnostic, ...] + binding_validated: bool + def __init__( + self, + analysis_succeeded: bool, + columns: tuple[InferredColumn, ...] | None = ..., + lineage_columns: Sequence[CompiledLineageColumnFact] = ..., + has_star: bool = ..., + star_resolved: bool = ..., + binding_diagnostics: tuple[SqlBindingDiagnostic, ...] = ..., + binding_validated: bool = ..., + ) -> None: ... + +class AnalysisCacheContext(NativeRecord): + """Shared project-local analysis cache identity for one compile invocation.""" + + root: Path + shared_fingerprint: str + signature_namespace: str + def __init__( + self, + root: Path, + shared_fingerprint: str, + signature_namespace: str = ..., + ) -> None: ... + +class CompileAnalysisSelection(NativeRecord): + """Deep-analysis selection and invocation-local preparation observer.""" + + select: tuple[str, ...] + exclude: tuple[str, ...] + auto_load_sources: bool + no_cache: bool + on_inputs_ready: CompileInputsReadyCallback | None + def __init__( + self, + select: tuple[str, ...] = ..., + exclude: tuple[str, ...] = ..., + auto_load_sources: bool = ..., + no_cache: bool = ..., + on_inputs_ready: CompileInputsReadyCallback | None = ..., + ) -> None: ... + +class CompileSqlScenarioCte(NativeRecord): + """One top-level SQL-native scenario CTE extracted after macro expansion.""" + + name: str + sql_body: str + def __init__( + self, + name: str, + sql_body: str, + ) -> None: ... + +class FunctionArgument(NativeRecord): + """One SQL function argument with an adapter-native type string.""" + + name: str + type: str + def __init__( + self, + name: str, + type: str, + ) -> None: ... + +class FunctionReturnColumn(NativeRecord): + """One named table-function return column with an adapter-native type string.""" + + name: str + type: str + def __init__( + self, + name: str, + type: str, + ) -> None: ... + +class HandBuiltAnalysis(NativeRecord): + """A hand-built model input's session position, fallback columns and standalone proof.""" + + session: tuple[Any, int] | None + fallback: list[tuple[str, str | None, str]] | None + standalone_proof: ( + tuple[ + bool, + list[tuple[str, str | None, str]], + list[tuple[str, str | None]], + list[str], + str | None, + bool, + ] + | None + ) + def __init__( + self, + session: tuple[Any, int] | None, + fallback: list[tuple[str, str | None, str]] | None, + standalone_proof: tuple[ + bool, + list[tuple[str, str | None, str]], + list[tuple[str, str | None]], + list[str], + str | None, + bool, + ] + | None, + ) -> None: ... + +class ModelSqlAnalysis(NativeRecord): + """Completed model analysis and its in-memory binding proof.""" + + polyglot_analysis: PolyglotAnalysisResult + placeholders: dict[str, str] | None + cached: bool + fused_binding_validated: bool + cleaned_sql: str | None + validated_schema: dict[str, dict[str, str]] | None + dynamic_column_contract: DynamicColumnContractProof | None + def __init__( + self, + polyglot_analysis: PolyglotAnalysisResult, + placeholders: dict[str, str] | None, + cached: bool = ..., + fused_binding_validated: bool = ..., + cleaned_sql: str | None = ..., + validated_schema: dict[str, dict[str, str]] | None = ..., + dynamic_column_contract: DynamicColumnContractProof | None = ..., + ) -> None: ... + +class CompileSqlScenarioInput(NativeRecord): + """One discovered SQL-native scenario with compile-time SQL expansion applied.""" + + scenario_file: DiscoveredSqlScenarioFile + sql_body: str + authored_ctes: tuple[CompileSqlScenarioCte, ...] + expected_ctes: tuple[CompileSqlScenarioCte, ...] + assertion_ctes: tuple[CompileSqlScenarioCte, ...] + source_fixture_names: tuple[str, ...] + ref_fixture_names: tuple[str, ...] + seed_fixture_names: tuple[str, ...] + dbt_ref_fixture_names: tuple[str, ...] + expected_model_names: tuple[str, ...] + assertion_names: tuple[str, ...] + assertion_target_model_names: tuple[str, ...] + target_model_names: tuple[str, ...] + declaration_usages: tuple[UsageRecord, ...] + def __init__( + self, + scenario_file: DiscoveredSqlScenarioFile, + sql_body: str, + authored_ctes: tuple[CompileSqlScenarioCte, ...] = ..., + expected_ctes: tuple[CompileSqlScenarioCte, ...] = ..., + assertion_ctes: tuple[CompileSqlScenarioCte, ...] = ..., + source_fixture_names: tuple[str, ...] = ..., + ref_fixture_names: tuple[str, ...] = ..., + seed_fixture_names: tuple[str, ...] = ..., + dbt_ref_fixture_names: tuple[str, ...] = ..., + expected_model_names: tuple[str, ...] = ..., + assertion_names: tuple[str, ...] = ..., + assertion_target_model_names: tuple[str, ...] = ..., + target_model_names: tuple[str, ...] = ..., + declaration_usages: tuple[UsageRecord, ...] = ..., + ) -> None: ... + +class CompileProjectInputs(NativeRecord): + """Attached project view used as the first pre-semantic compile input snapshot.""" + + project_config: ProjectConfig + local_config: LocalConfig + discovered_inputs: DiscoveredProjectInputs + sql_lexical_syntax: SqlLexicalSyntax + run_id: str + effective_target_name: str | None + effective_target: TargetConfig | None + compile_cache_dir: Path | None + effective_connection: dict[str, object] + effective_settings: SettingsConfig + no_sql_validation: bool + effective_vars: dict[str, object] + macro_context: MacroContext | None + loaded_macros: dict[str, LoadedMacro] + public_enums: dict[str, EnumDeclaration] + public_constants: dict[str, ConstantDeclaration] + model_inputs: tuple[ModelInput, ...] + seed_inputs: tuple[SeedInput, ...] + source_inputs: tuple[SourceInput, ...] + sql_function_inputs: tuple[FunctionInput, ...] + test_inputs: tuple[CompileSqlTestInput, ...] + scenario_inputs: tuple[CompileSqlScenarioInput, ...] + audit_inputs: tuple[AuditInput, ...] + diagnostics: tuple[CompilerDiagnostic, ...] + external_sql_reference_resolver: ExternalSqlReferenceResolver | None + scope_index: ScopeIndex + declaration_scope: DeclarationScopeBuild | None + native_project: _native.NativeCompiledProject | None + def __init__( + self, + project_config: ProjectConfig, + local_config: LocalConfig, + discovered_inputs: DiscoveredProjectInputs, + sql_lexical_syntax: SqlLexicalSyntax, + run_id: str = ..., + effective_target_name: str | None = ..., + effective_target: TargetConfig | None = ..., + compile_cache_dir: Path | None = ..., + effective_connection: dict[str, object] = ..., + effective_settings: SettingsConfig = ..., + no_sql_validation: bool = ..., + effective_vars: dict[str, object] = ..., + macro_context: MacroContext | None = ..., + loaded_macros: dict[str, LoadedMacro] = ..., + public_enums: dict[str, EnumDeclaration] = ..., + public_constants: dict[str, ConstantDeclaration] = ..., + model_inputs: tuple[ModelInput, ...] = ..., + seed_inputs: tuple[SeedInput, ...] = ..., + source_inputs: tuple[SourceInput, ...] = ..., + sql_function_inputs: tuple[FunctionInput, ...] = ..., + test_inputs: tuple[CompileSqlTestInput, ...] = ..., + scenario_inputs: tuple[CompileSqlScenarioInput, ...] = ..., + audit_inputs: tuple[AuditInput, ...] = ..., + diagnostics: tuple[CompilerDiagnostic, ...] = ..., + external_sql_reference_resolver: ExternalSqlReferenceResolver | None = ..., + scope_index: ScopeIndex = ..., + declaration_scope: DeclarationScopeBuild | None = ..., + native_project: _native.NativeCompiledProject | None = ..., + ) -> None: ... + +class CompiledSource(NativeRecord): + """Planner-ready compiled source metadata.""" + + key: CompiledObjectKey + deps: tuple[CompiledObjectKey, ...] + name: str + source_entry: SourceEntry + source_file: DiscoveredSourceFile + def __init__( + self, + key: CompiledObjectKey, + deps: tuple[CompiledObjectKey, ...], + name: str, + source_entry: SourceEntry, + source_file: DiscoveredSourceFile, + ) -> None: ... + +class CompiledSeed(NativeRecord): + """Planner-ready compiled seed metadata.""" + + key: CompiledObjectKey + deps: tuple[CompiledObjectKey, ...] + name: str + seed_file: DiscoveredSeedFile + schema_entry: SchemaSeedEntry + schema_file: DiscoveredSchemaFile + destination: CompiledRelationLocation + external: bool + def __init__( + self, + key: CompiledObjectKey, + deps: tuple[CompiledObjectKey, ...], + name: str, + seed_file: DiscoveredSeedFile, + schema_entry: SchemaSeedEntry, + schema_file: DiscoveredSchemaFile, + destination: CompiledRelationLocation, + external: bool = ..., + ) -> None: ... + +class CompiledFunction(NativeRecord): + """Planner-ready compiled SQL function metadata.""" + + key: CompiledObjectKey + deps: tuple[CompiledObjectKey, ...] + name: str + relative_path: Path + arguments: tuple[FunctionArgument, ...] + returns: str + body_sql: str + destination: CompiledRelationLocation + fingerprint_destination: CompiledRelationLocation + return_columns: tuple[FunctionReturnColumn, ...] + references: tuple[CompileSqlReference, ...] + language: FunctionLanguage + source_file_path: Path | None + runtime_version: str | None + entry_point: str | None + packages: tuple[str, ...] + tags: tuple[str, ...] + description: str | None + def __init__( + self, + key: CompiledObjectKey, + deps: tuple[CompiledObjectKey, ...], + name: str, + relative_path: Path, + arguments: tuple[FunctionArgument, ...], + returns: str, + body_sql: str, + destination: CompiledRelationLocation, + fingerprint_destination: CompiledRelationLocation, + return_columns: tuple[FunctionReturnColumn, ...] = ..., + references: tuple[CompileSqlReference, ...] = ..., + language: FunctionLanguage = ..., + source_file_path: Path | None = ..., + runtime_version: str | None = ..., + entry_point: str | None = ..., + packages: tuple[str, ...] = ..., + tags: tuple[str, ...] = ..., + description: str | None = ..., + ) -> None: ... + +class PythonSqlReferenceReport(NativeRecord): + """P008 diagnostics plus literal SQL relations that named no project relation.""" + + diagnostics: tuple[CompilerDiagnostic, ...] + unmatched: tuple[LiteralSqlRelation, ...] + def __init__( + self, + diagnostics: tuple[CompilerDiagnostic, ...] = ..., + unmatched: tuple[LiteralSqlRelation, ...] = ..., + ) -> None: ... + +class CompileSqlTestCte(NativeRecord): + """One top-level SQL-native test CTE extracted after macro expansion.""" + + name: str + sql_body: str + def __init__( + self, + name: str, + sql_body: str, + ) -> None: ... + +class CompileModelSqlTestInputPayload(NativeRecord): + """Model-mode SQL test compile payload.""" + + authored_ctes: tuple[CompileSqlTestCte, ...] + macro_mocks: dict[str, str] + mock_model_names: tuple[str, ...] + mock_source_names: tuple[str, ...] + mock_seed_names: tuple[str, ...] + mock_dbt_ref_names: tuple[str, ...] + mock_table_function_names: tuple[str, ...] + expected_ctes: tuple[CompileSqlTestCte, ...] + expected_model_names: tuple[str, ...] + assertion_ctes: tuple[CompileSqlTestCte, ...] + assertion_names: tuple[str, ...] + assertion_target_model_names: tuple[str, ...] + read_helper_names: tuple[str, ...] + reference_target_model_names: tuple[str, ...] + def __init__( + self, + authored_ctes: tuple[CompileSqlTestCte, ...] = ..., + macro_mocks: dict[str, str] = ..., + mock_model_names: tuple[str, ...] = ..., + mock_source_names: tuple[str, ...] = ..., + mock_seed_names: tuple[str, ...] = ..., + mock_dbt_ref_names: tuple[str, ...] = ..., + mock_table_function_names: tuple[str, ...] = ..., + expected_ctes: tuple[CompileSqlTestCte, ...] = ..., + expected_model_names: tuple[str, ...] = ..., + assertion_ctes: tuple[CompileSqlTestCte, ...] = ..., + assertion_names: tuple[str, ...] = ..., + assertion_target_model_names: tuple[str, ...] = ..., + read_helper_names: tuple[str, ...] = ..., + reference_target_model_names: tuple[str, ...] = ..., + ) -> None: ... + +class CompileDirectLogicSqlTestInputPayload(NativeRecord): + """Direct-logic SQL test compile payload.""" + + actual_cte: CompileSqlTestCte + expected_cte: CompileSqlTestCte + mode: SqlTestMode + helper_ctes: tuple[CompileSqlTestCte, ...] + tested_resource_names: tuple[str, ...] + def __init__( + self, + actual_cte: CompileSqlTestCte, + expected_cte: CompileSqlTestCte, + mode: SqlTestMode, + helper_ctes: tuple[CompileSqlTestCte, ...] = ..., + tested_resource_names: tuple[str, ...] = ..., + ) -> None: ... + +class CompileSqlTestInput(NativeRecord): + """One discovered SQL-native test block with compile-time SQL expansion applied.""" + + test_file: DiscoveredSqlTestFile + test_block: DiscoveredSqlTestBlock + sql_body: str + mode: SqlTestMode + payload: CompileModelSqlTestInputPayload | CompileDirectLogicSqlTestInputPayload + declaration_usages: tuple[UsageRecord, ...] + parent_name: str | None + case_name: str | None + case_index: int | None + parameter_schema: tuple[SqlTestParameterDeclaration, ...] + parameter_values: tuple[tuple[str, SqlValue], ...] + def __init__( + self, + test_file: DiscoveredSqlTestFile, + test_block: DiscoveredSqlTestBlock, + sql_body: str, + mode: SqlTestMode = ..., + payload: CompileModelSqlTestInputPayload | CompileDirectLogicSqlTestInputPayload = ..., + declaration_usages: tuple[UsageRecord, ...] = ..., + parent_name: str | None = ..., + case_name: str | None = ..., + case_index: int | None = ..., + parameter_schema: tuple[SqlTestParameterDeclaration, ...] = ..., + parameter_values: tuple[tuple[str, SqlValue], ...] = ..., + ) -> None: ... + +class CompiledModelSqlTestPayload(NativeRecord): + """Compiled model-mode SQL test payload.""" + + authored_ctes: tuple[CompileSqlTestCte, ...] + macro_mocks: dict[str, str] + model_query_overrides: dict[str, str] + mock_model_names: tuple[str, ...] + mock_source_names: tuple[str, ...] + mock_seed_names: tuple[str, ...] + mock_dbt_ref_names: tuple[str, ...] + mock_table_function_names: tuple[str, ...] + expected_ctes: tuple[CompileSqlTestCte, ...] + expected_model_names: tuple[str, ...] + assertion_ctes: tuple[CompileSqlTestCte, ...] + assertion_names: tuple[str, ...] + def __init__( + self, + authored_ctes: tuple[CompileSqlTestCte, ...] = ..., + macro_mocks: dict[str, str] = ..., + model_query_overrides: dict[str, str] = ..., + mock_model_names: tuple[str, ...] = ..., + mock_source_names: tuple[str, ...] = ..., + mock_seed_names: tuple[str, ...] = ..., + mock_dbt_ref_names: tuple[str, ...] = ..., + mock_table_function_names: tuple[str, ...] = ..., + expected_ctes: tuple[CompileSqlTestCte, ...] = ..., + expected_model_names: tuple[str, ...] = ..., + assertion_ctes: tuple[CompileSqlTestCte, ...] = ..., + assertion_names: tuple[str, ...] = ..., + ) -> None: ... + +class SqlExpansionContext(NativeRecord): + """Everything needed to expand one project's authored SQL bodies.""" + + effective_vars: dict[str, object] + loaded_macros: dict[str, LoadedMacro] + macro_context: MacroContext + enums: dict[str, EnumDeclaration] + constants: dict[str, ConstantDeclaration] + value_renderer: TypedSqlValueRenderer + collection_rendering: CollectionRendering + local_declarations: dict[Path, DeclarationResolutionContext] + declaration_resolver: DeclarationScopeResolver | None + def __init__( + self, + effective_vars: dict[str, object], + loaded_macros: dict[str, LoadedMacro], + macro_context: MacroContext, + enums: dict[str, EnumDeclaration], + constants: dict[str, ConstantDeclaration], + value_renderer: TypedSqlValueRenderer, + collection_rendering: CollectionRendering, + local_declarations: dict[Path, DeclarationResolutionContext] = ..., + declaration_resolver: DeclarationScopeResolver | None = ..., + ) -> None: ... + +class ExpansionSpan(NativeRecord): + """One substituted region, pairing its source range with its rendered range.""" + + source_start: int + source_end: int + output_start: int + output_end: int + def __init__( + self, + source_start: int, + source_end: int, + output_start: int, + output_end: int, + ) -> None: ... + +class CompiledSqlExpansion(NativeRecord): + """Process-local authored-to-expanded SQL evidence shared with compiler checks.""" + + authored_sql: str + expanded_sql: str + passes: tuple[tuple[ExpansionSpan, ...], ...] + def __init__( + self, + authored_sql: str, + expanded_sql: str, + passes: tuple[tuple[ExpansionSpan, ...], ...], + ) -> None: ... + +class MappedOffset(NativeRecord): + """An offset in rendered SQL resolved back onto the text that produced it.""" + + offset: int + generated: bool + def __init__( + self, + offset: int, + generated: bool, + ) -> None: ... + +class CompiledDirectLogicSqlTestPayload(NativeRecord): + """Compiled direct-logic SQL test payload.""" + + actual_cte: CompileSqlTestCte + expected_cte: CompileSqlTestCte + mode: SqlTestMode + helper_ctes: tuple[CompileSqlTestCte, ...] + tested_resource_names: tuple[str, ...] + def __init__( + self, + actual_cte: CompileSqlTestCte, + expected_cte: CompileSqlTestCte, + mode: SqlTestMode = ..., + helper_ctes: tuple[CompileSqlTestCte, ...] = ..., + tested_resource_names: tuple[str, ...] = ..., + ) -> None: ... + +class CompiledSqlTestResource(NativeRecord): + """One direct SQL test resource with its authored resource kind.""" + + kind: SqlTestMode + name: str + def __init__( + self, + kind: SqlTestMode, + name: str, + ) -> None: ... + +class NativeModelConfigSession(NativeRecord): + """The native config builder, validator, and inherited storage policies for one compile.""" + + path_defaults: dict[str, dict[str, object]] + builder: _native.NativeModelConfigBuilder + validator: _native.NativeModelValidator + inherited_storage: dict[str | None, tuple[ResolvedTimeTravelRetention, ResolvedTableType]] + def __init__( + self, + path_defaults: dict[str, dict[str, object]], + builder: _native.NativeModelConfigBuilder, + validator: _native.NativeModelValidator, + inherited_storage: dict[str | None, tuple[ResolvedTimeTravelRetention, ResolvedTableType]], + ) -> None: ... + +class NativeModelConfigInputs(NativeRecord): + """The project and run inputs every model's native config build reads.""" + + project_config: ProjectConfig + target_config: TargetConfig | None + effective_vars: dict[str, object] + effective_target_name: str | None + run_id: str + microbatch_concurrency: bool + def __init__( + self, + project_config: ProjectConfig, + target_config: TargetConfig | None, + effective_vars: dict[str, object], + effective_target_name: str | None, + run_id: str, + microbatch_concurrency: bool, + ) -> None: ... + +class ModelValidationRequest(NativeRecord): + """One model's effective config and the facts its validators read.""" + + model_file: DiscoveredSqlModelFile + config: CompileModelConfig + references: tuple[CompileSqlReference, ...] + declared_columns: tuple[SchemaColumn, ...] | None + query_sql: str + def __init__( + self, + model_file: DiscoveredSqlModelFile, + config: CompileModelConfig, + references: tuple[CompileSqlReference, ...], + declared_columns: tuple[SchemaColumn, ...] | None, + query_sql: str, + ) -> None: ... + +class NamedSqlHookEntry(NativeRecord): + """An unresolved invocation of a discovered SQL hook resource.""" + + name: str + kwargs: dict[str, object] + def __init__( + self, + name: str, + kwargs: dict[str, object], + ) -> None: ... + +class DiscoveredSqlFunctionFile(NativeRecord): + """A discovered SQL function file and its raw contents.""" + + file_path: Path + relative_path: Path + contents: str + header_values: dict[str, object] + body_sql: str + def __init__( + self, + file_path: Path, + relative_path: Path, + contents: str, + header_values: dict[str, object], + body_sql: str, + ) -> None: ... + +class DiscoveredSqlHookFile(NativeRecord): + """A discovered named SQL lifecycle hook resource.""" + + file_path: Path + relative_path: Path + contents: str + header_values: dict[str, object] + sql_body: str + name: str + description: str | None + scope_kind: ScopeKind + ownership_root: Path | None + owning_path: Path | None + declaration_root: Path | None + def __init__( + self, + file_path: Path, + relative_path: Path, + contents: str, + header_values: dict[str, object], + sql_body: str, + name: str, + description: str | None = ..., + scope_kind: ScopeKind = ..., + ownership_root: Path | None = ..., + owning_path: Path | None = ..., + declaration_root: Path | None = ..., + ) -> None: ... + +class EnumMember(NativeRecord): + """One named member in a SQLBuild enum declaration.""" + + name: str + value: str | int + def __init__( + self, + name: str, + value: str | int, + ) -> None: ... + +class DiscoveredEnumFile(NativeRecord): + """A public SQL file containing one or more enum declarations.""" + + file_path: Path + relative_path: Path + contents: str + declarations: tuple[EnumDeclaration, ...] + scope_kind: ScopeKind + ownership_root: Path | None + owning_path: Path | None + declaration_root: Path | None + def __init__( + self, + file_path: Path, + relative_path: Path, + contents: str, + declarations: tuple[EnumDeclaration, ...], + scope_kind: ScopeKind = ..., + ownership_root: Path | None = ..., + owning_path: Path | None = ..., + declaration_root: Path | None = ..., + ) -> None: ... + +class DiscoveredConstantFile(NativeRecord): + """A public SQL file containing one or more constant declarations.""" + + file_path: Path + relative_path: Path + contents: str + declarations: tuple[ConstantDeclaration, ...] + scope_kind: ScopeKind + ownership_root: Path | None + owning_path: Path | None + declaration_root: Path | None + def __init__( + self, + file_path: Path, + relative_path: Path, + contents: str, + declarations: tuple[ConstantDeclaration, ...], + scope_kind: ScopeKind = ..., + ownership_root: Path | None = ..., + owning_path: Path | None = ..., + declaration_root: Path | None = ..., + ) -> None: ... + +class ModelSchemaDeclaration(NativeRecord): + """One public reusable model column schema declaration.""" + + name: str + columns: tuple[SchemaColumn, ...] + relative_path: Path + extends: str | None + description: str | None + def __init__( + self, + name: str, + columns: tuple[SchemaColumn, ...], + relative_path: Path, + extends: str | None = ..., + description: str | None = ..., + ) -> None: ... + +class DiscoveredModelSchemaFile(NativeRecord): + """A public SQL file containing reusable model schemas.""" + + file_path: Path + relative_path: Path + contents: str + declarations: tuple[ModelSchemaDeclaration, ...] + scope_kind: ScopeKind + ownership_root: Path | None + owning_path: Path | None + declaration_root: Path | None + def __init__( + self, + file_path: Path, + relative_path: Path, + contents: str, + declarations: tuple[ModelSchemaDeclaration, ...], + scope_kind: ScopeKind = ..., + ownership_root: Path | None = ..., + owning_path: Path | None = ..., + declaration_root: Path | None = ..., + ) -> None: ... + +class DiscoveredPythonFunctionFile(NativeRecord): + """A discovered Python function file and its parsed UDF metadata.""" + + file_path: Path + relative_path: Path + contents: str + header_values: dict[str, object] + entry_point: str + body_python: str + def __init__( + self, + file_path: Path, + relative_path: Path, + contents: str, + header_values: dict[str, object], + entry_point: str, + body_python: str, + ) -> None: ... + +class DiscoveredSchemaFile(NativeRecord): + """A discovered schema.yml file and its raw contents.""" + + file_path: Path + relative_path: Path + contents: str + model_entries: tuple[SchemaModelEntry, ...] + seed_entries: tuple[SchemaSeedEntry, ...] + def __init__( + self, + file_path: Path, + relative_path: Path, + contents: str, + model_entries: tuple[SchemaModelEntry, ...], + seed_entries: tuple[SchemaSeedEntry, ...], + ) -> None: ... + +class DiscoveredSeedFile(NativeRecord): + """A discovered seed file.""" + + file_path: Path + relative_path: Path + def __init__( + self, + file_path: Path, + relative_path: Path, + ) -> None: ... + +class DiscoveredSqlTestCase(NativeRecord): + """One ordered named case belonging to a SQL-native test template.""" + + name: str + values: tuple[tuple[str, SqlValue], ...] + case_index: int + def __init__( + self, + name: str, + values: tuple[tuple[str, SqlValue], ...], + case_index: int, + ) -> None: ... + +class DiscoveredSqlTestBlock(NativeRecord): + """One raw TEST(...) block discovered from a SQL-native test file.""" + + test_index: int + header_values: dict[str, object] + sql_body: str + name: str | None + mode: SqlTestMode + parameters: tuple[SqlTestParameterDeclaration, ...] + cases: tuple[DiscoveredSqlTestCase, ...] + cursor_start: str | None + cursor_end: str | None + sql_body_span: tuple[int, int] | None + def __init__( + self, + test_index: int, + header_values: dict[str, object], + sql_body: str, + name: str | None = ..., + mode: SqlTestMode = ..., + parameters: tuple[SqlTestParameterDeclaration, ...] = ..., + cases: tuple[DiscoveredSqlTestCase, ...] = ..., + cursor_start: str | None = ..., + cursor_end: str | None = ..., + sql_body_span: tuple[int, int] | None = ..., + ) -> None: ... + +class DiscoveredAuditFile(NativeRecord): + """A discovered audit SQL file and its raw contents.""" + + file_path: Path + relative_path: Path + contents: str + blocks: tuple[DiscoveredAuditBlock, ...] + declaration_kind: DeclarationKind + scope_kind: ScopeKind + ownership_root: Path | None + owning_path: Path | None + declaration_root: Path | None + def __init__( + self, + file_path: Path, + relative_path: Path, + contents: str, + blocks: tuple[DiscoveredAuditBlock, ...], + declaration_kind: DeclarationKind = ..., + scope_kind: ScopeKind = ..., + ownership_root: Path | None = ..., + owning_path: Path | None = ..., + declaration_root: Path | None = ..., + ) -> None: ... + +class DiscoveredAuditBlock(NativeRecord): + """One raw AUDIT(...) block discovered from a SQL audit file.""" + + audit_index: int + header_values: dict[str, object] + sql_body: str + name: str | None + evaluation_mode: AuditEvaluationMode + measurement_contract: MeasurementContract | None + measure_sql: str | None + evidence_sql: str | None + def __init__( + self, + audit_index: int, + header_values: dict[str, object], + sql_body: str, + name: str | None = ..., + evaluation_mode: AuditEvaluationMode = ..., + measurement_contract: MeasurementContract | None = ..., + measure_sql: str | None = ..., + evidence_sql: str | None = ..., + ) -> None: ... + +class DiscoveredAdapterFile(NativeRecord): + """A detected project adapter Python file.""" + + file_path: Path + relative_path: Path + def __init__( + self, + file_path: Path, + relative_path: Path, + ) -> None: ... + +class DiscoveryFileFault(NativeRecord): + """One project-relative authored file fault captured during tolerant discovery.""" + + path: Path | None + message: str + def __init__( + self, + path: Path | None, + message: str, + ) -> None: ... + +class TolerantScopeDiscovery(NativeRecord): + """Parsed scope inputs and independently categorized tolerant discovery faults.""" + + discovered_inputs: DiscoveredProjectInputs + resource_faults: tuple[DiscoveryFileFault, ...] + declaration_faults: tuple[DiscoveryFileFault, ...] + relationship_faults: tuple[DiscoveryFileFault, ...] + config_faults: tuple[DiscoveryFileFault, ...] + def __init__( + self, + discovered_inputs: DiscoveredProjectInputs, + resource_faults: tuple[DiscoveryFileFault, ...] = ..., + declaration_faults: tuple[DiscoveryFileFault, ...] = ..., + relationship_faults: tuple[DiscoveryFileFault, ...] = ..., + config_faults: tuple[DiscoveryFileFault, ...] = ..., + ) -> None: ... + +class CompilePipelineOptions(NativeRecord): + """Selection, deferral, and planning options for one compile pipeline run.""" + + selected_target: str | None + no_sql_validation: bool + defer_to: str | None + defer_sources_to: str | None + source_deferral_enabled: bool + select: tuple[str, ...] + exclude: tuple[str, ...] + cursor_overrides: CursorOverrides | None + full_refresh: bool + auto_load_sources: bool + reload_sources: bool + connection_config: dict[str, object] | None + cli_vars: dict[str, object] | None + external_sql_reference_resolver: ExternalSqlReferenceResolver | None + resolve_python_run_selectors: bool + no_cache: bool + max_microbatches: int | None + selection_diagnostics: bool + plan_sql_tests: bool + record_migration_fingerprints: bool + accepts_unit_test_selectors: bool + sql_test_selection: SqlTestSelection + def __init__( + self, + selected_target: str | None = ..., + no_sql_validation: bool = ..., + defer_to: str | None = ..., + defer_sources_to: str | None = ..., + source_deferral_enabled: bool = ..., + select: tuple[str, ...] = ..., + exclude: tuple[str, ...] = ..., + cursor_overrides: CursorOverrides | None = ..., + full_refresh: bool = ..., + auto_load_sources: bool = ..., + reload_sources: bool = ..., + connection_config: dict[str, object] | None = ..., + cli_vars: dict[str, object] | None = ..., + external_sql_reference_resolver: ExternalSqlReferenceResolver | None = ..., + resolve_python_run_selectors: bool = ..., + no_cache: bool = ..., + max_microbatches: int | None = ..., + selection_diagnostics: bool = ..., + plan_sql_tests: bool = ..., + record_migration_fingerprints: bool = ..., + accepts_unit_test_selectors: bool = ..., + sql_test_selection: SqlTestSelection = ..., + ) -> None: ... + +class PythonPlanEntry(NativeRecord): + """Display-ready Python node entry for plan output.""" + + name: str + kind: PythonNodeKind + phase: PythonRunPhase + identity_status: PythonIdentityStatus + current_definition_json: str | None + previous_definition_json: str | None + current_metadata_json: str | None + previous_metadata_json: str | None + provider_usages: tuple[DiscoveredProviderUsage, ...] + def __init__( + self, + name: str, + kind: PythonNodeKind, + phase: PythonRunPhase, + identity_status: PythonIdentityStatus = ..., + current_definition_json: str | None = ..., + previous_definition_json: str | None = ..., + current_metadata_json: str | None = ..., + previous_metadata_json: str | None = ..., + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + ) -> None: ... + +class PythonRunPlanOutputs(NativeRecord): + """Plan output and Python entries after python-aware run selection.""" + + plan_output: PlanOutput + python_plan_entries: tuple[PythonPlanEntry, ...] + selected_python_node_names: frozenset[str] + def __init__( + self, + plan_output: PlanOutput, + python_plan_entries: tuple[PythonPlanEntry, ...], + selected_python_node_names: frozenset[str], + ) -> None: ... + +class CompiledProjectPhaseResult(NativeRecord): + """Canonical project compilation output before command-specific planning.""" + + project: CompiledProjectView + connection_config: dict[str, object] + compile_seconds: float + def __init__( + self, + project: CompiledProjectView, + connection_config: dict[str, object], + compile_seconds: float, + ) -> None: ... + +class StaticCommandContext(NativeRecord): + """Shared compile, selection, and relation phases for focused commands.""" + + project: CompiledProjectView + scope: PlannerScope + relations: PlannerRelationsContext + connection_config: dict[str, object] + compile_seconds: float + def __init__( + self, + project: CompiledProjectView, + scope: PlannerScope, + relations: PlannerRelationsContext, + connection_config: dict[str, object], + compile_seconds: float, + ) -> None: ... + +class ClonePipelineOptions(NativeRecord): + """Compilation and selection options for one clone pipeline.""" + + no_sql_validation: bool + select: tuple[str, ...] + exclude: tuple[str, ...] + cli_vars: dict[str, object] | None + no_cache: bool + def __init__( + self, + no_sql_validation: bool = ..., + select: tuple[str, ...] = ..., + exclude: tuple[str, ...] = ..., + cli_vars: dict[str, object] | None = ..., + no_cache: bool = ..., + ) -> None: ... + +class ClonePipelineConnection(NativeRecord): + """Resolved destination connection used throughout clone compilation and planning.""" + + config: dict[str, object] + handle: Any + def __init__( + self, + config: dict[str, object], + handle: Any, + ) -> None: ... + +class ClonePipelineResult(NativeRecord): + """Prepared clone inputs for origin and destination target environments.""" + + origin_project: CompiledProjectView + destination_project: CompiledProjectView + clone_plan: PlanOutput + destination_source_entries: tuple[CloneSourcePlanEntry, ...] + destination_model_entries: tuple[ModelPlanEntry, ...] + destination_seed_entries: tuple[SeedPlanEntry, ...] + destination_function_entries: tuple[FunctionPlanEntry, ...] + origin_model_entries: tuple[ModelPlanEntry, ...] + origin_seed_entries: tuple[SeedPlanEntry, ...] + origin_source_entries: tuple[CloneSourcePlanEntry, ...] + def __init__( + self, + origin_project: CompiledProjectView, + destination_project: CompiledProjectView, + clone_plan: PlanOutput, + destination_source_entries: tuple[CloneSourcePlanEntry, ...] = ..., + destination_model_entries: tuple[ModelPlanEntry, ...] = ..., + destination_seed_entries: tuple[SeedPlanEntry, ...] = ..., + destination_function_entries: tuple[FunctionPlanEntry, ...] = ..., + origin_model_entries: tuple[ModelPlanEntry, ...] = ..., + origin_seed_entries: tuple[SeedPlanEntry, ...] = ..., + origin_source_entries: tuple[CloneSourcePlanEntry, ...] = ..., + ) -> None: ... + +class ParsedSelector(NativeRecord): + """One parsed selector token before graph resolution.""" + + kind: SelectorKind + value: str + upstream: bool + downstream: bool + def __init__( + self, + kind: SelectorKind, + value: str, + upstream: bool = ..., + downstream: bool = ..., + ) -> None: ... + +class PathSelector(NativeRecord): + """A directed path selector between two model names with optional endpoint expansion.""" + + start_name: str + end_name: str + upstream: bool + downstream: bool + def __init__( + self, + start_name: str, + end_name: str, + upstream: bool = ..., + downstream: bool = ..., + ) -> None: ... + +class SqlTestSelection(NativeRecord): + """Unit tests selected or excluded by name, beside the tests of the selected resources.""" + + names: frozenset[str] + excluded_names: frozenset[str] + models_selected: bool + def __init__( + self, + names: frozenset[str] = ..., + excluded_names: frozenset[str] = ..., + models_selected: bool = ..., + ) -> None: ... + +class PlannerScope(NativeRecord): + """Resolved graph scope for one planner invocation.""" + + upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] + downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] + all_keys: dict[str, CompiledObjectKey] + models_by_name: dict[str, CompiledModelView] + selected_keys: frozenset[CompiledObjectKey] + execution_order: tuple[CompiledObjectKey, ...] + user_selected_keys: frozenset[CompiledObjectKey] + python_read_source_names: frozenset[str] + sql_test_selection: SqlTestSelection + def __init__( + self, + upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + all_keys: dict[str, CompiledObjectKey], + models_by_name: dict[str, CompiledModelView], + selected_keys: frozenset[CompiledObjectKey], + execution_order: tuple[CompiledObjectKey, ...], + user_selected_keys: frozenset[CompiledObjectKey] = ..., + python_read_source_names: frozenset[str] = ..., + sql_test_selection: SqlTestSelection = ..., + ) -> None: ... + +class PlannerRelationsContext(NativeRecord): + """Resolved relation and source inputs for plan entry construction.""" + + model_locations: dict[str, CompiledRelationLocation] + seed_locations: dict[str, CompiledRelationLocation] + function_locations: dict[str, CompiledRelationLocation] + source_map: dict[str, SourceEntry] + source_read_map: dict[str, SourceEntry] + python_source_read_map: dict[str, SourceEntry] + source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]] + star_exclude_keyword: str + listed_source_names: frozenset[str] + def __init__( + self, + model_locations: dict[str, CompiledRelationLocation], + seed_locations: dict[str, CompiledRelationLocation], + function_locations: dict[str, CompiledRelationLocation], + source_map: dict[str, SourceEntry], + source_read_map: dict[str, SourceEntry], + python_source_read_map: dict[str, SourceEntry], + source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]], + star_exclude_keyword: str, + listed_source_names: frozenset[str] = ..., + ) -> None: ... + +class SqlTestAssertionStep(NativeRecord): + """One zero-row assertion step in a SQL-native unit test.""" + + name: str + resolved_sql: str + lifted_ctes: tuple[tuple[str, str], ...] + comparison_body_sql: str | None + def __init__( + self, + name: str, + resolved_sql: str, + lifted_ctes: tuple[tuple[str, str], ...] = ..., + comparison_body_sql: str | None = ..., + ) -> None: ... + +class PlannerSelection(NativeRecord): + """User-facing selection inputs for one planner invocation.""" + + select: tuple[str, ...] + exclude: tuple[str, ...] + selected_keys: frozenset[CompiledObjectKey] | None + python_read_source_names: frozenset[str] + sql_test_selection: SqlTestSelection + def __init__( + self, + select: tuple[str, ...] = ..., + exclude: tuple[str, ...] = ..., + selected_keys: frozenset[CompiledObjectKey] | None = ..., + python_read_source_names: frozenset[str] = ..., + sql_test_selection: SqlTestSelection = ..., + ) -> None: ... + +class PlannerOverrides(NativeRecord): + """Explicit user overrides that force or bound planner decisions.""" + + cursor_overrides: CursorOverrides | None + full_refresh: bool + reload_sources: bool + forced_stale_model_names: tuple[str, ...] + external_blocked_model_names: tuple[str, ...] + max_microbatches: int | None + def __init__( + self, + cursor_overrides: CursorOverrides | None = ..., + full_refresh: bool = ..., + reload_sources: bool = ..., + forced_stale_model_names: tuple[str, ...] = ..., + external_blocked_model_names: tuple[str, ...] = ..., + max_microbatches: int | None = ..., + ) -> None: ... + +class DeferralInputs(NativeRecord): + """Deferred relation and source-deferral inputs for planning.""" + + deferred_locations: dict[str, CompiledRelationLocation] | None + deferred_relations: dict[str, RelationInfo] | None + defer_sources_to: str | None + source_deferral_enabled: bool + def __init__( + self, + deferred_locations: dict[str, CompiledRelationLocation] | None = ..., + deferred_relations: dict[str, RelationInfo] | None = ..., + defer_sources_to: str | None = ..., + source_deferral_enabled: bool = ..., + ) -> None: ... + +class PlannerPolicies(NativeRecord): + """Behavior policies selected by the caller for one planner invocation.""" + + auto_load_sources: bool + selection_diagnostics: bool + plan_sql_tests: bool + record_migration_fingerprints: bool + def __init__( + self, + auto_load_sources: bool = ..., + selection_diagnostics: bool = ..., + plan_sql_tests: bool = ..., + record_migration_fingerprints: bool = ..., + ) -> None: ... + +class DiscoveredPythonLoaderMetadata(NativeRecord): + """Loader-specific metadata carried by an internal Python loader node.""" + + destination: str | None + write_strategy: SourceWriteStrategy | None + cursor_column: str | None + unique_key: tuple[str, ...] + columns: tuple[SourceColumnEntry, ...] + contract: str | None + connection_mode: LoaderConnectionMode + def __init__( + self, + destination: str | None = ..., + write_strategy: SourceWriteStrategy | None = ..., + cursor_column: str | None = ..., + unique_key: tuple[str, ...] = ..., + columns: tuple[SourceColumnEntry, ...] = ..., + contract: str | None = ..., + connection_mode: LoaderConnectionMode = ..., + ) -> None: ... + +class DiscoveredPythonTaskMetadata(NativeRecord): + """Task-specific metadata carried by an internal Python task node.""" + + retry: RetryPolicy | None + def __init__( + self, + retry: RetryPolicy | None = ..., + ) -> None: ... + +class DiscoveredPythonAssetMetadata(NativeRecord): + """Asset-specific metadata carried by an internal Python asset node.""" + + columns: tuple[SourceColumnEntry, ...] + column_lineage: dict[str, tuple[ColumnLineageRef, ...]] | None + retry: RetryPolicy | None + def __init__( + self, + columns: tuple[SourceColumnEntry, ...] = ..., + column_lineage: dict[str, tuple[ColumnLineageRef, ...]] | None = ..., + retry: RetryPolicy | None = ..., + ) -> None: ... + +class DiscoveredPythonCheckMetadata(NativeRecord): + """Check-specific metadata carried by an internal Python check node.""" + + severity: PythonCheckSeverity + def __init__( + self, + severity: PythonCheckSeverity = ..., + ) -> None: ... + +class PythonNodeDependencyEdge(NativeRecord): + """Internal dependency edge between two discovered Python nodes.""" + + upstream_name: str + downstream_name: str + def __init__( + self, + upstream_name: str, + downstream_name: str, + ) -> None: ... + +class PythonSqlRunSelection(NativeRecord): + """Unified run-command selection excluding Python checks.""" + + sql_keys: frozenset[CompiledObjectKey] + python_node_names: frozenset[str] + def __init__( + self, + sql_keys: frozenset[CompiledObjectKey], + python_node_names: frozenset[str], + ) -> None: ... + +class LiteralSqlRelation(NativeRecord): + """A relation named in literal SQL in Python code that matched no project relation.""" + + owner_label: str + owner_kind: HardCodedRelationOwnerKind + relative_path: Path + line: int + method: str + relation: RelationName + def __init__( + self, + owner_label: str, + owner_kind: HardCodedRelationOwnerKind, + relative_path: Path, + line: int, + method: str, + relation: RelationName, + ) -> None: ... + +class DirectSourceFreshnessPlanningResult(NativeRecord): + """Direct planning-time source freshness observations and comparisons.""" + + observed_records: tuple[SourceFreshnessRecord, ...] + previous_records: tuple[SourceFreshnessRecord, ...] + changed_identities: frozenset[SourceFreshnessIdentity] + unchanged_identities: frozenset[SourceFreshnessIdentity] + unknown_source_names: tuple[str, ...] + unknown_sources: dict[str, SourceFreshnessUnknown] + age_statuses: dict[SourceFreshnessIdentity, SourceFreshnessAgeStatus] + propagation: DirectSourceFreshnessPropagationResult | None + def __init__( + self, + observed_records: tuple[SourceFreshnessRecord, ...] = ..., + previous_records: tuple[SourceFreshnessRecord, ...] = ..., + changed_identities: frozenset[SourceFreshnessIdentity] = ..., + unchanged_identities: frozenset[SourceFreshnessIdentity] = ..., + unknown_source_names: tuple[str, ...] = ..., + unknown_sources: dict[str, SourceFreshnessUnknown] = ..., + age_statuses: dict[SourceFreshnessIdentity, SourceFreshnessAgeStatus] = ..., + propagation: DirectSourceFreshnessPropagationResult | None = ..., + ) -> None: ... + +class SqlBindingDiagnostic(NativeRecord): + """One native binding error or non-blocking runtime-conversion warning.""" + + code: str + message: str + line: int | None + column: int | None + start: int | None + end: int | None + severity: str + def __init__( + self, + code: str, + message: str, + line: int | None = ..., + column: int | None = ..., + start: int | None = ..., + end: int | None = ..., + severity: str = ..., + ) -> None: ... + +class SqlBindingResult(NativeRecord): + """Stable result of schema-aware semantic SQL validation.""" + + diagnostics: tuple[SqlBindingDiagnostic, ...] + def __init__( + self, + diagnostics: tuple[SqlBindingDiagnostic, ...] = ..., + ) -> None: ... + +class SqlSchemaValidationRequest(NativeRecord): + """One expanded SQL query and its complete relation schemas.""" + + sql: str + dialect: str | None + schema: Mapping[str, Mapping[str, str]] + known_functions: tuple[str, ...] + known_types: tuple[str, ...] + quoted_identifiers_ignore_case: bool + catalog: Any | None + def __init__( + self, + sql: str, + dialect: str | None, + schema: Mapping[str, Mapping[str, str]], + known_functions: tuple[str, ...] = ..., + known_types: tuple[str, ...] = ..., + quoted_identifiers_ignore_case: bool = ..., + catalog: Any | None = ..., + ) -> None: ... + +class DbtCliOptions(NativeRecord): + """Resolved dbt CLI options shared by dbt commands.""" + + project_dir: Path | None + profiles_dir: Path | None + target: str | None + target_path: Path | None + vars: str | None + state: Path | None + defer: bool + def __init__( + self, + project_dir: Path | None = ..., + profiles_dir: Path | None = ..., + target: str | None = ..., + target_path: Path | None = ..., + vars: str | None = ..., + state: Path | None = ..., + defer: bool = ..., + ) -> None: ... + +class NormalizedDbtProfileConnection(NativeRecord): + """Rendered dbt profile output normalized for SQLBuild.""" + + adapter: str + connection: dict[str, object] + target_schema: str | None + target_database: str | None + warnings: tuple[str, ...] + def __init__( + self, + adapter: str, + connection: dict[str, object], + target_schema: str | None = ..., + target_database: str | None = ..., + warnings: tuple[str, ...] = ..., + ) -> None: ... + +class DbtInitProgressCallbacks(NativeRecord): + """Optional progress hooks for `sqb dbt init` phases.""" + + start: Callable[[str], None] | None + complete: Callable[[str], None] | None + def __init__( + self, + start: Callable[[str], None] | None = ..., + complete: Callable[[str], None] | None = ..., + ) -> None: ... + +class DbtInitRequest(NativeRecord): + """Request to initialize a SQLBuild project from a dbt project.""" + + cwd: Path + dbt_project_dir: Path + profiles_dir: Path | None + profile_name: str | None + target_name: str | None + sqb_output_dir: Path | None + dry_run: bool + overwrite: bool + skip_dbt_debug: bool + progress_callbacks: DbtInitProgressCallbacks + def __init__( + self, + cwd: Path, + dbt_project_dir: Path, + profiles_dir: Path | None, + profile_name: str | None, + target_name: str | None, + sqb_output_dir: Path | None, + dry_run: bool = ..., + overwrite: bool = ..., + skip_dbt_debug: bool = ..., + progress_callbacks: DbtInitProgressCallbacks = ..., + ) -> None: ... + +class DbtInitResult(NativeRecord): + """Result from `sqb dbt init`.""" + + output_dir: Path + project_file: Path + project_name: str + adapter: str + target_name: str + profile_name: str + toml: str + warnings: tuple[str, ...] + dry_run: bool + def __init__( + self, + output_dir: Path, + project_file: Path, + project_name: str, + adapter: str, + target_name: str, + profile_name: str, + toml: str, + warnings: tuple[str, ...] = ..., + dry_run: bool = ..., + ) -> None: ... + +class DbtCommandResult(NativeRecord): + """Completed dbt command output.""" + + argv: tuple[str, ...] + returncode: int + stdout: str + stderr: str + def __init__( + self, + argv: tuple[str, ...], + returncode: int, + stdout: str = ..., + stderr: str = ..., + ) -> None: ... + +class DbtNodeExecutionResult(NativeRecord): + """One dbt node execution result parsed from JSON logs.""" + + unique_id: str + resource_type: str + node_name: str + status: str + index: int | None + total: int | None + execution_time: float | None + materialized: str | None + relation_name: str | None + database: str | None + schema: str | None + node_checksum: str | None + messages: tuple[DbtNodeMessage, ...] + def __init__( + self, + unique_id: str, + resource_type: str, + node_name: str, + status: str, + index: int | None, + total: int | None, + execution_time: float | None, + materialized: str | None = ..., + relation_name: str | None = ..., + database: str | None = ..., + schema: str | None = ..., + node_checksum: str | None = ..., + messages: tuple[DbtNodeMessage, ...] = ..., + ) -> None: ... + +class DbtLsResult(NativeRecord): + """Parsed result from one dbt ls invocation.""" + + nodes: tuple[DbtLsNode, ...] + command: DbtCommandResult + def __init__( + self, + nodes: tuple[DbtLsNode, ...], + command: DbtCommandResult, + ) -> None: ... + +class RuleGuidance(NativeRecord): + """Structured authoring guidance shared by diagnostics and generated documentation.""" + + good_example: str + anti_tautology: str + mutation_check: str + def __init__( + self, + good_example: str, + anti_tautology: str, + mutation_check: str, + ) -> None: ... + +class RuleExemption(NativeRecord): + rule: str + path: str + reason: str + def __init__( + self, + rule: str, + path: str, + reason: str, + ) -> None: ... + +class GraphEdgeExclusion(NativeRecord): + """One exact, temporary internal-layer dependency exception.""" + + consumer: str + dependency: str + reason: str + def __init__( + self, + consumer: str, + dependency: str, + reason: str, + ) -> None: ... + +class RuleIgnore(NativeRecord): + rules: tuple[str, ...] + paths: tuple[str, ...] + reason: str + selectors: tuple[str, ...] + def __init__( + self, + rules: tuple[str, ...], + paths: tuple[str, ...], + reason: str, + selectors: tuple[str, ...] = ..., + ) -> None: ... + +class SelectStarAllow(NativeRecord): + paths: tuple[str, ...] + reason: str + def __init__( + self, + paths: tuple[str, ...], + reason: str, + ) -> None: ... + +class ThresholdOverride(NativeRecord): + paths: tuple[str, ...] + thresholds: dict[str, int] + reason: str + def __init__( + self, + paths: tuple[str, ...], + thresholds: dict[str, int], + reason: str, + ) -> None: ... + +class RulesCacheConfig(NativeRecord): + enabled: bool + def __init__( + self, + enabled: bool = ..., + ) -> None: ... + +class SqlTestRulesConfig(NativeRecord): + pipeline_directory: str + def __init__( + self, + pipeline_directory: str = ..., + ) -> None: ... + +class PreparedSqlLintResult(NativeRecord): + """Reusable expansion context and completed preliminary SQL checks.""" + + result: LintRunResult + context: SqlExpansionContext + def __init__( + self, + result: LintRunResult, + context: SqlExpansionContext, + ) -> None: ... + +class SqlExpansionReuse(NativeRecord): + """Compiler declaration scope for post-assembly SQL checks; assembly still runs its macros.""" + + discovered_inputs: DiscoveredProjectInputs + declaration_scope: DeclarationScopeBuild + def __init__( + self, + discovered_inputs: DiscoveredProjectInputs, + declaration_scope: DeclarationScopeBuild, + ) -> None: ... + +class PreparedSqlLint(NativeRecord): + """Process-local SQL checks started after compiler expansion has completed.""" + + codes: tuple[str, ...] + future: Future[PreparedSqlLintResult] + project_dir: Path | None + def __init__( + self, + codes: tuple[str, ...], + future: Future[PreparedSqlLintResult], + project_dir: Path | None = ..., + ) -> None: ... + +class SqlRulesEvaluation(NativeRecord): + """SQL rule findings of one run, their cache accounting and elapsed time.""" + + findings: tuple[Finding, ...] + cache_hits: int + cache_misses: int + sql_ms: int + def __init__( + self, + findings: tuple[Finding, ...], + cache_hits: int, + cache_misses: int, + sql_ms: int = ..., + ) -> None: ... + +class RuleRunPlan(NativeRecord): + """The catalogue, selected rules, project slice and settings one rules run evaluates.""" + + effective_config: RulesConfig + project_dir: Path + catalogue: tuple[Rule, ...] + custom_payloads: list[dict[str, object]] + selected: tuple[Rule, ...] + skipped_type_proof_rules: tuple[str, ...] + model_paths: frozenset[str] | None + project: CompiledProjectView + config: RulesConfig + def __init__( + self, + effective_config: RulesConfig, + project_dir: Path, + catalogue: tuple[Rule, ...], + custom_payloads: list[dict[str, object]], + selected: tuple[Rule, ...], + skipped_type_proof_rules: tuple[str, ...], + model_paths: frozenset[str] | None, + project: CompiledProjectView, + config: RulesConfig, + ) -> None: ... + +class RulesRunResult(NativeRecord): + """Unified findings and phase measurements for configured rules.""" + + findings: tuple[Finding, ...] + evaluated_models: int + built_in_ms: int + custom_ms: int + custom_cpu_ms: int + unevaluated_resources: int + cache_hits: int + cache_misses: int + skipped_type_proof_rules: tuple[str, ...] + def __init__( + self, + findings: tuple[Finding, ...], + evaluated_models: int, + built_in_ms: int, + custom_ms: int, + custom_cpu_ms: int = ..., + unevaluated_resources: int = ..., + cache_hits: int = ..., + cache_misses: int = ..., + skipped_type_proof_rules: tuple[str, ...] = ..., + ) -> None: ... + +class NativeRulesEvaluation(NativeRecord): + """The built-in rules response, decoded natively.""" + + findings: tuple[Finding, ...] + selected_codes: tuple[str, ...] + evaluated_models: int + cache_hits: int + cache_misses: int + built_in_ms: int + reused: bool + def __init__( + self, + findings: tuple[Finding, ...], + selected_codes: tuple[str, ...], + evaluated_models: int, + cache_hits: int, + cache_misses: int, + built_in_ms: int, + reused: bool, + ) -> None: ... + +class CustomRulesOutcome(NativeRecord): + """Custom-rule findings in canonical order and their cache accounting.""" + + findings: tuple[Finding, ...] + cache_hits: int + cache_misses: int + custom_ms: int + custom_cpu_ms: int + def __init__( + self, + findings: tuple[Finding, ...], + cache_hits: int, + cache_misses: int, + custom_ms: int, + custom_cpu_ms: int = ..., + ) -> None: ... diff --git a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py index 8d34692f81..53387605c0 100644 --- a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py +++ b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py @@ -7,16 +7,14 @@ from sqlbuild._native import ( CostContext, + DirectSourceFreshnessPlanningResult, SourceFreshnessIdentity, SourceFreshnessRenderers, write_source_freshness_records, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput -from sqlbuild.compiler.source_freshness.models import ( - DirectSourceFreshnessPlanningResult, - SourceFreshnessRecord, -) +from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/compiler/analysis_session/models.py b/src/sqlbuild/compiler/analysis_session/models.py deleted file mode 100644 index c2084e97c1..0000000000 --- a/src/sqlbuild/compiler/analysis_session/models.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Inputs handed to the native model analysis session.""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any, Literal - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import ( - AnalysisCacheContext, - CompileSqlReference, - InferredColumn, -) -from sqlbuild.compiler.compile.types import ModelInput -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -@dataclass(frozen=True) -class NativeModelAnalysisRequest: - """The Python model analysis's arguments plus the facts dynamic pivot proofs need.""" - - known_functions: tuple[str, ...] - known_types: tuple[str, ...] - model_inputs: tuple[ModelInput, ...] - column_nullability_by_table: dict[str, dict[str, InferredNullability]] - column_types_by_table: dict[str, dict[str, str]] - inference_profile: ExpressionInferenceProfile - allow_compact_analysis: bool - rich_type_inference: bool - analysis_cache: AnalysisCacheContext | None - complete_binding_schemas: dict[str, dict[str, str]] - dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] - - -@dataclass(frozen=True, slots=True) -class NativeModelAnalyses: - """Each analysed model's position in the finished session that keeps its outcome.""" - - positions: dict[str, int] - succeeded: frozenset[str] - session: Any | None - - -@dataclass(frozen=True, slots=True) -class NativePivotTables: - """The relation facts Python's dynamic pivot proofs read, for proofs outside a session.""" - - dialect: str | None - column_types_by_table: dict[str, dict[str, str]] - authoritative_column_types_by_table: dict[str, dict[str, str]] - column_nullability_by_table: dict[str, dict[str, InferredNullability]] - dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] - - -@dataclass(frozen=True, slots=True) -class NativeColumnQuery: - """One query outside the session whose output columns Python's analysis inferred.""" - - sql: str - mode: Literal["batch", "reanalysis", "parse", "legacy"] - placeholders: dict[str, str] | None = None - references: tuple[CompileSqlReference, ...] = () - recover_cte_facts: bool = True - - -@dataclass(frozen=True, slots=True) -class NativeQueryColumns: - """One query's analysis: whether it succeeded, its columns and whether it projects a star.""" - - succeeded: bool - columns: tuple[InferredColumn, ...] | None - has_star: bool diff --git a/src/sqlbuild/compiler/auditing/exceptions.py b/src/sqlbuild/compiler/auditing/exceptions.py deleted file mode 100644 index f8bd16ebfd..0000000000 --- a/src/sqlbuild/compiler/auditing/exceptions.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Compiler auditing exceptions.""" - - -class MeasurementAuditError(ValueError): - """Raised when a measurement audit contract or value is invalid.""" diff --git a/src/sqlbuild/compiler/auditing/models.py b/src/sqlbuild/compiler/auditing/models.py index ebfb9ef125..ef187e02d3 100644 --- a/src/sqlbuild/compiler/auditing/models.py +++ b/src/sqlbuild/compiler/auditing/models.py @@ -6,7 +6,7 @@ from math import isfinite from typing import cast -from sqlbuild.compiler.auditing.exceptions import MeasurementAuditError +from sqlbuild._native import MeasurementAuditError from sqlbuild.compiler.auditing.types import AuditSeverity, ThresholdOperator diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 748300918b..453c264aba 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -10,94 +10,77 @@ from typing import Any, overload import sqlbuild._native as _native -from sqlbuild._native import SqlTestScanCache +from sqlbuild._native import ( + CollectionRendering, + ColumnLineageConfidence, + ColumnTransformKind, + CompiledDirectLogicSqlTestPayload, + CompiledLineageColumnFact, + CompiledModelSqlTestPayload, + CompiledSqlExpansion, + CompiledSqlTestResource, + CompileModelConfig, + CompileSqlScenarioCte, + DeclarationExpansionContext, + DeclarationScopeResolver, + DiagnosticPhase, + DiagnosticSeverity, + DiscoveredAuditBlock, + DiscoveredAuditFile, + DiscoveredSchemaFile, + DiscoveredSqlHookFile, + DiscoveredSqlTestBlock, + DynamicColumnContractProof, + LiteralSqlRelation, + ModelSchemaDeclaration, + RelatedLocation, +) from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE, MACRO_CONTEXT_PARAMETER_NAME from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, - AuditInput, CompiledAuditView, CompiledFunctionView, CompiledModelView, CompiledResourceType, CompiledSeedView, CompiledSourceView, - CompileInputsReadyCallback, - DiagnosticPhase, - DiagnosticSeverity, - FunctionInput, - FunctionLanguage, - ModelInput, - SeedInput, - SourceInput, + RelationKeyRow, SqlTestMode, TypedSqlValueRenderer, ) from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, - DiscoveredAuditBlock, - DiscoveredAuditFile, DiscoveredHookFunction, DiscoveredLoaderFunction, DiscoveredMaterializationFile, - DiscoveredProjectInputs, - DiscoveredSchemaFile, - DiscoveredSeedFile, - DiscoveredSourceFile, - DiscoveredSqlHookFile, DiscoveredSqlModelFile, DiscoveredSqlScenarioFile, - DiscoveredSqlTestBlock, DiscoveredSqlTestFile, EnumDeclaration, - ModelSchemaDeclaration, SqlTestParameterDeclaration, ) -from sqlbuild.compiler.lineage.types import ( - ColumnLineageConfidence, - ColumnTransformKind, - InferredNullability, -) -from sqlbuild.compiler.references.models import LiteralSqlRelation +from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, ResourceIdentity, ScopeIndex, - ScopeLookup, UsageRecord, VisibilityRecord, ) -from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic, SqlLexicalSyntax -from sqlbuild.lint.types import RelationKeyRow +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - ResolvedTableType, - ResolvedTimeTravelRetention, ScenarioConfig, - SchemaColumn, SchemaModelEntry, - SchemaSeedEntry, SettingsConfig, - SourceEntry, SourceLocation, TargetConfig, ) from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import CollectionRendering - - -@dataclass(frozen=True) -class RelatedLocation: - """A secondary authored location that adds context to a diagnostic.""" - - label: str - location: SourceLocation - message: str | None = None @dataclass(frozen=True) @@ -161,59 +144,6 @@ class InferredColumn: nullability: InferredNullability = InferredNullability.UNKNOWN -@dataclass(frozen=True) -class FixtureColumnInference: - """Inferred fixture columns plus outputs that are explicitly bare NULL literals.""" - - columns: tuple[InferredColumn, ...] - null_literal_names: frozenset[str] - quoted_names: frozenset[str] - - -@dataclass(frozen=True) -class DynamicColumnFamilyProof: - """Compiler evidence for one declared runtime-generated column family.""" - - name: str - inferred_type: str | None = None - - -@dataclass(frozen=True) -class DynamicColumnContractProof: - """Closed output-shape evidence for a dynamic column contract.""" - - output_proven: bool - fixed_columns: tuple[InferredColumn, ...] = field(default_factory=tuple) - families: tuple[DynamicColumnFamilyProof, ...] = field(default_factory=tuple) - input_relations: tuple[str, ...] = field(default_factory=tuple) - failure_reason: str | None = None - bare_dynamic_pivot: bool = False - - -@dataclass(frozen=True) -class CompileModelConfig: - """Pre-semantic effective config layers attached to a model input.""" - - values: dict[str, object] = field(default_factory=dict) - model_header_keys: tuple[str, ...] = field(default_factory=tuple) - matched_path_default: str | None = None - logical_schema: str | None = None - layer_schema: str | None = None - logical_database: str | None = None - time_travel_retention: ResolvedTimeTravelRetention = field( - default_factory=ResolvedTimeTravelRetention - ) - table_type: ResolvedTableType = field(default_factory=ResolvedTableType) - - -@dataclass(frozen=True) -class CursorInputRoles: - """Effective filter and watermark inputs for one compiled model.""" - - filter_inputs: dict[str, str] - watermark_inputs: dict[str, str] - - @dataclass(frozen=True) class LoadedMacro: """One loaded project macro available for compile-time SQL expansion.""" @@ -268,39 +198,6 @@ class StaticMacroInventory: faults: tuple[StaticMacroFault, ...] = field(default_factory=tuple) -@dataclass(frozen=True) -class DeclarationRuntimeProjection: - """Process-only declaration values keyed by canonical static identity.""" - - declarations: Mapping[ - DeclarationIdentity, EnumDeclaration | ConstantDeclaration | LoadedMacro - ] = field(default_factory=dict) - - -@dataclass(frozen=True) -class DeclarationScopeResolver: - """Process-only lookup state used to project visible runtime declarations.""" - - project_dir: Path | None - lookup: ScopeLookup - projection: DeclarationRuntimeProjection - native_contexts: _native.NativeDeclarationContexts = field(compare=False, repr=False) - resource_specific: frozenset[ResourceIdentity] | None = None - contexts_by_directory: dict[tuple[str, str], DeclarationResolutionContext] = field( - default_factory=dict, compare=False, repr=False - ) - - -@dataclass(frozen=True) -class DeclarationScopeBuild: - """Static artifact and process-local values shared by compile attachment phases.""" - - loaded_macros: dict[str, LoadedMacro] - index: ScopeIndex - resolver: DeclarationScopeResolver - sql_test_scans: SqlTestScanCache - - @dataclass(frozen=True) class MacroContext: """Expose target settings and caller-visible declarations to a Python SQL macro.""" @@ -350,34 +247,6 @@ def render_enum_member(self, *, enum_name: str, member_name: str) -> str: return render_enum_member_value(value=value) -@dataclass(frozen=True) -class MacroExpansionResult: - """Expanded SQL and resolved macro usage facts from one authored string.""" - - sql: str - spans: tuple[ExpansionSpan, ...] = field(default_factory=tuple) - dependencies: tuple[DeclarationIdentity, ...] = field(default_factory=tuple) - usages: tuple[UsageRecord, ...] = field(default_factory=tuple) - argument_references: tuple[CompileSqlReference, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class VisibleModelDeclarations: - """One model's visible declarations and macros, as the native model loop resolves them.""" - - local_enums: dict[str, EnumDeclaration] - local_constants: dict[str, ConstantDeclaration] - enums: dict[str, EnumDeclaration] - constants: dict[str, ConstantDeclaration] - inaccessible_enums: dict[str, DeclarationRecord] - inaccessible_constants: dict[str, DeclarationRecord] - enum_visibility: dict[str, tuple[VisibilityRecord, ...]] - constant_visibility: dict[str, tuple[VisibilityRecord, ...]] - macros: dict[str, LoadedMacro] - macro_records: dict[str, DeclarationRecord] - inaccessible_macros: dict[str, DeclarationRecord] - - @dataclass(frozen=True) class DeclarationResolutionContext: """Declarations visible during one authored SQL expansion.""" @@ -395,26 +264,6 @@ class DeclarationResolutionContext: consumer: ResourceIdentity | DeclarationIdentity | None = None -@dataclass(frozen=True) -class DeclarationExpansionContext: - """Declarations and adapter rendering used by one authored SQL owner.""" - - declarations: DeclarationResolutionContext - value_renderer: TypedSqlValueRenderer - collection_rendering: CollectionRendering - resolver: DeclarationScopeResolver | None = None - - -@dataclass(frozen=True) -class CompileAdapterContext: - """Cycle-free adapter behavior required while building compile inputs.""" - - value_renderer: TypedSqlValueRenderer - collection_rendering: CollectionRendering - python_functions_inherit_default_namespace: bool - sql_lexical_syntax: SqlLexicalSyntax - - @dataclass(frozen=True) class ModelInputBuildContext: """Run-constant config and macros for building model compile inputs.""" @@ -462,26 +311,6 @@ def __post_init__(self) -> None: object.__setattr__(self, "ref_kind", SqlReferenceKind(self.ref_kind)) -@dataclass(frozen=True) -class InvalidSqlReferenceCall: - """One reference call whose arguments compile could not replace with a relation.""" - - ref_kind: SqlReferenceKind - call: str - start: int - message: str - help: str - corrected_call: str - - -@dataclass(frozen=True) -class SqlReferenceScan: - """Valid references and rejected reference calls found in one SQL text.""" - - references: tuple[CompileSqlReference, ...] - invalid_calls: tuple[InvalidSqlReferenceCall, ...] = () - - @dataclass(frozen=True) class CompiledLineageSourceFact: """Compact upstream column fact extracted during SQL analysis.""" @@ -494,16 +323,6 @@ def __post_init__(self) -> None: object.__setattr__(self, "resource_type", CompiledResourceType(self.resource_type)) -@dataclass(frozen=True) -class CompiledLineageColumnFact: - """Compact output column lineage fact extracted during SQL analysis.""" - - output_column: str - upstream_columns: tuple[CompiledLineageSourceFact, ...] = field(default_factory=tuple) - transform_kind: ColumnTransformKind = ColumnTransformKind.UNKNOWN - confidence: ColumnLineageConfidence = ColumnLineageConfidence.UNKNOWN - - @dataclass(frozen=True, eq=False) class CompactLineageFacts(Sequence[CompiledLineageColumnFact]): """Indexed native lineage rows with lazy object projection.""" @@ -595,41 +414,6 @@ def confidence(code: int) -> ColumnLineageConfidence: )[code] -@dataclass(frozen=True) -class PolyglotAnalysisResult: - """Outcome of one Polyglot column and lineage analysis pass.""" - - analysis_succeeded: bool - columns: tuple[InferredColumn, ...] | None = None - lineage_columns: Sequence[CompiledLineageColumnFact] = field(default_factory=tuple) - has_star: bool = False - star_resolved: bool = False - binding_diagnostics: tuple[SqlBindingDiagnostic, ...] = field(default_factory=tuple) - binding_validated: bool = False - - -@dataclass(frozen=True) -class AnalysisCacheContext: - """Shared project-local analysis cache identity for one compile invocation.""" - - root: Path - shared_fingerprint: str - signature_namespace: str = "default" - - -@dataclass(frozen=True) -class CompileAnalysisSelection: - """Deep-analysis selection and invocation-local preparation observer.""" - - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - auto_load_sources: bool = False - no_cache: bool = False - on_inputs_ready: CompileInputsReadyCallback | None = field( - default=None, compare=False, repr=False - ) - - @dataclass(frozen=True) class CompiledObjectKey: """Stable logical identity for one compiled resource or external dependency.""" @@ -641,30 +425,6 @@ def __post_init__(self) -> None: object.__setattr__(self, "resource_type", CompiledResourceType(self.resource_type)) -@dataclass(frozen=True) -class CompileSqlScenarioCte: - """One top-level SQL-native scenario CTE extracted after macro expansion.""" - - name: str - sql_body: str - - -@dataclass(frozen=True) -class FunctionArgument: - """One SQL function argument with an adapter-native type string.""" - - name: str - type: str - - -@dataclass(frozen=True) -class FunctionReturnColumn: - """One named table-function return column with an adapter-native type string.""" - - name: str - type: str - - @dataclass(frozen=True) class CompileModelInput: """One discovered model file with its attached schema metadata, if any.""" @@ -713,94 +473,6 @@ def attached_schema_name(self) -> str | None: """The compile's whole expansion, variables included, which early SQL lint reuses.""" -@dataclass(frozen=True) -class HandBuiltAnalysis: - """A hand-built model input's session position, fallback columns and standalone proof.""" - - session: tuple[Any, int] | None - fallback: list[tuple[str, str | None, str]] | None - standalone_proof: ( - tuple[ - bool, - list[tuple[str, str | None, str]], - list[tuple[str, str | None]], - list[str], - str | None, - bool, - ] - | None - ) - - -@dataclass(frozen=True) -class ModelSqlAnalysis: - """Completed model analysis and its in-memory binding proof.""" - - polyglot_analysis: PolyglotAnalysisResult - placeholders: dict[str, str] | None - cached: bool = False - fused_binding_validated: bool = False - cleaned_sql: str | None = None - validated_schema: dict[str, dict[str, str]] | None = None - dynamic_column_contract: DynamicColumnContractProof | None = None - - -@dataclass(frozen=True) -class CompileSqlScenarioInput: - """One discovered SQL-native scenario with compile-time SQL expansion applied.""" - - scenario_file: DiscoveredSqlScenarioFile - sql_body: str - authored_ctes: tuple[CompileSqlScenarioCte, ...] = field(default_factory=tuple) - expected_ctes: tuple[CompileSqlScenarioCte, ...] = field(default_factory=tuple) - assertion_ctes: tuple[CompileSqlScenarioCte, ...] = field(default_factory=tuple) - source_fixture_names: tuple[str, ...] = field(default_factory=tuple) - ref_fixture_names: tuple[str, ...] = field(default_factory=tuple) - seed_fixture_names: tuple[str, ...] = field(default_factory=tuple) - dbt_ref_fixture_names: tuple[str, ...] = field(default_factory=tuple) - expected_model_names: tuple[str, ...] = field(default_factory=tuple) - assertion_names: tuple[str, ...] = field(default_factory=tuple) - assertion_target_model_names: tuple[str, ...] = field(default_factory=tuple) - target_model_names: tuple[str, ...] = field(default_factory=tuple) - declaration_usages: tuple[UsageRecord, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class CompileProjectInputs: - """Attached project view used as the first pre-semantic compile input snapshot.""" - - project_config: ProjectConfig - local_config: LocalConfig - discovered_inputs: DiscoveredProjectInputs - sql_lexical_syntax: SqlLexicalSyntax - run_id: str = "" - effective_target_name: str | None = None - effective_target: TargetConfig | None = None - compile_cache_dir: Path | None = None - effective_connection: dict[str, object] = field(default_factory=dict) - effective_settings: SettingsConfig = field(default_factory=SettingsConfig) - no_sql_validation: bool = False - effective_vars: dict[str, object] = field(default_factory=dict) - macro_context: MacroContext | None = field(default=None, repr=False, compare=False) - loaded_macros: dict[str, LoadedMacro] = field(default_factory=dict) - public_enums: dict[str, EnumDeclaration] = field(default_factory=dict) - public_constants: dict[str, ConstantDeclaration] = field(default_factory=dict) - model_inputs: tuple[ModelInput, ...] = field(default_factory=tuple) - seed_inputs: tuple[SeedInput, ...] = field(default_factory=tuple) - source_inputs: tuple[SourceInput, ...] = field(default_factory=tuple) - sql_function_inputs: tuple[FunctionInput, ...] = field(default_factory=tuple) - test_inputs: tuple[CompileSqlTestInput, ...] = field(default_factory=tuple) - scenario_inputs: tuple[CompileSqlScenarioInput, ...] = field(default_factory=tuple) - audit_inputs: tuple[AuditInput, ...] = field(default_factory=tuple) - diagnostics: tuple[CompilerDiagnostic, ...] = field(default_factory=tuple) - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None - scope_index: ScopeIndex = field(default_factory=ScopeIndex) - declaration_scope: DeclarationScopeBuild | None = field(default=None, repr=False, compare=False) - native_project: _native.NativeCompiledProject | None = field( - default=None, repr=False, compare=False - ) - - @dataclass(frozen=True) class CompiledRelationLocation: """Logical and physical relation location resolved during compile.""" @@ -910,55 +582,6 @@ def output_column_count(self) -> int: return len(self.schema_entry.columns) if self.schema_entry is not None else 0 -@dataclass(frozen=True) -class CompiledSource: - """Planner-ready compiled source metadata.""" - - key: CompiledObjectKey - deps: tuple[CompiledObjectKey, ...] - name: str - source_entry: SourceEntry - source_file: DiscoveredSourceFile - - -@dataclass(frozen=True) -class CompiledSeed: - """Planner-ready compiled seed metadata.""" - - key: CompiledObjectKey - deps: tuple[CompiledObjectKey, ...] - name: str - seed_file: DiscoveredSeedFile - schema_entry: SchemaSeedEntry - schema_file: DiscoveredSchemaFile - destination: CompiledRelationLocation - external: bool = False - - -@dataclass(frozen=True) -class CompiledFunction: - """Planner-ready compiled SQL function metadata.""" - - key: CompiledObjectKey - deps: tuple[CompiledObjectKey, ...] - name: str - relative_path: Path - arguments: tuple[FunctionArgument, ...] - returns: str - body_sql: str - destination: CompiledRelationLocation - fingerprint_destination: CompiledRelationLocation - return_columns: tuple[FunctionReturnColumn, ...] = field(default_factory=tuple) - references: tuple[CompileSqlReference, ...] = field(default_factory=tuple) - language: FunctionLanguage = FunctionLanguage.SQL - source_file_path: Path | None = None - runtime_version: str | None = None - entry_point: str | None = None - packages: tuple[str, ...] = field(default_factory=tuple) - tags: tuple[str, ...] = field(default_factory=tuple) - description: str | None = None - - @dataclass(frozen=True) class CompiledAudit: """Compiled audit metadata selected by scope dependencies.""" @@ -1024,14 +647,6 @@ def __post_init__(self) -> None: ) -@dataclass(frozen=True) -class PythonSqlReferenceReport: - """P008 diagnostics plus literal SQL relations that named no project relation.""" - - diagnostics: tuple[CompilerDiagnostic, ...] = () - unmatched: tuple[LiteralSqlRelation, ...] = () - - @dataclass(frozen=True) class CompiledProject: """Planner-ready whole-project compile output.""" @@ -1127,143 +742,6 @@ def _graph_tags(value: object) -> list[str]: return [] -@dataclass(frozen=True) -class CompileSqlTestCte: - """One top-level SQL-native test CTE extracted after macro expansion.""" - - name: str - sql_body: str - - -@dataclass(frozen=True) -class CompileModelSqlTestInputPayload: - """Model-mode SQL test compile payload.""" - - authored_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) - macro_mocks: dict[str, str] = field(default_factory=dict) - mock_model_names: tuple[str, ...] = field(default_factory=tuple) - mock_source_names: tuple[str, ...] = field(default_factory=tuple) - mock_seed_names: tuple[str, ...] = field(default_factory=tuple) - mock_dbt_ref_names: tuple[str, ...] = field(default_factory=tuple) - mock_table_function_names: tuple[str, ...] = field(default_factory=tuple) - expected_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) - expected_model_names: tuple[str, ...] = field(default_factory=tuple) - assertion_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) - assertion_names: tuple[str, ...] = field(default_factory=tuple) - assertion_target_model_names: tuple[str, ...] = field(default_factory=tuple) - read_helper_names: tuple[str, ...] = field(default_factory=tuple) - reference_target_model_names: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class CompileDirectLogicSqlTestInputPayload: - """Direct-logic SQL test compile payload.""" - - actual_cte: CompileSqlTestCte - expected_cte: CompileSqlTestCte - mode: SqlTestMode - helper_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) - tested_resource_names: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class CompileSqlTestInput: - """One discovered SQL-native test block with compile-time SQL expansion applied.""" - - test_file: DiscoveredSqlTestFile - test_block: DiscoveredSqlTestBlock - sql_body: str - mode: SqlTestMode = DEFAULT_SQL_TEST_MODE - payload: CompileModelSqlTestInputPayload | CompileDirectLogicSqlTestInputPayload = field( - default_factory=CompileModelSqlTestInputPayload - ) - declaration_usages: tuple[UsageRecord, ...] = field(default_factory=tuple) - parent_name: str | None = None - case_name: str | None = None - case_index: int | None = None - parameter_schema: tuple[SqlTestParameterDeclaration, ...] = field(default_factory=tuple) - parameter_values: tuple[tuple[str, SqlValue], ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class CompiledModelSqlTestPayload: - """Compiled model-mode SQL test payload.""" - - authored_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) - macro_mocks: dict[str, str] = field(default_factory=dict) - model_query_overrides: dict[str, str] = field(default_factory=dict) - mock_model_names: tuple[str, ...] = field(default_factory=tuple) - mock_source_names: tuple[str, ...] = field(default_factory=tuple) - mock_seed_names: tuple[str, ...] = field(default_factory=tuple) - mock_dbt_ref_names: tuple[str, ...] = field(default_factory=tuple) - mock_table_function_names: tuple[str, ...] = field(default_factory=tuple) - expected_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) - expected_model_names: tuple[str, ...] = field(default_factory=tuple) - assertion_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) - assertion_names: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class SqlExpansionContext: - """Everything needed to expand one project's authored SQL bodies.""" - - effective_vars: dict[str, object] - loaded_macros: dict[str, LoadedMacro] - macro_context: MacroContext - enums: dict[str, EnumDeclaration] - constants: dict[str, ConstantDeclaration] - value_renderer: TypedSqlValueRenderer - collection_rendering: CollectionRendering - local_declarations: dict[Path, DeclarationResolutionContext] = field(default_factory=dict) - declaration_resolver: DeclarationScopeResolver | None = None - - -@dataclass(frozen=True) -class ExpansionSpan: - """One substituted region, pairing its source range with its rendered range.""" - - source_start: int - source_end: int - output_start: int - output_end: int - - -@dataclass(frozen=True) -class CompiledSqlExpansion: - """Process-local authored-to-expanded SQL evidence shared with compiler checks.""" - - authored_sql: str - expanded_sql: str - passes: tuple[tuple[ExpansionSpan, ...], ...] - - -@dataclass(frozen=True) -class MappedOffset: - """An offset in rendered SQL resolved back onto the text that produced it.""" - - offset: int - generated: bool - - -@dataclass(frozen=True) -class CompiledDirectLogicSqlTestPayload: - """Compiled direct-logic SQL test payload.""" - - actual_cte: CompileSqlTestCte - expected_cte: CompileSqlTestCte - mode: SqlTestMode = DEFAULT_SQL_TEST_MODE - helper_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) - tested_resource_names: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class CompiledSqlTestResource: - """One direct SQL test resource with its authored resource kind.""" - - kind: SqlTestMode - name: str - - @dataclass(frozen=True) class CompiledSqlTest: """Compiled SQL-native unit test metadata selected by expected targets.""" @@ -1311,36 +789,3 @@ def __post_init__(self) -> None: "parent_name", self.test_block.name or self.test_file.relative_path.stem, ) - - -@dataclass(frozen=True) -class NativeModelConfigSession: - """The native config builder, validator, and inherited storage policies for one compile.""" - - path_defaults: dict[str, dict[str, object]] - builder: _native.NativeModelConfigBuilder - validator: _native.NativeModelValidator - inherited_storage: dict[str | None, tuple[ResolvedTimeTravelRetention, ResolvedTableType]] - - -@dataclass(frozen=True) -class NativeModelConfigInputs: - """The project and run inputs every model's native config build reads.""" - - project_config: ProjectConfig - target_config: TargetConfig | None - effective_vars: dict[str, object] - effective_target_name: str | None - run_id: str - microbatch_concurrency: bool - - -@dataclass(frozen=True) -class ModelValidationRequest: - """One model's effective config and the facts its validators read.""" - - model_file: DiscoveredSqlModelFile - config: CompileModelConfig - references: tuple[CompileSqlReference, ...] - declared_columns: tuple[SchemaColumn, ...] | None - query_sql: str diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index 65da2263d0..23f8069960 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -12,65 +12,68 @@ from pathlib import Path import sqlbuild._native as _native + from sqlbuild._native import ( + CompiledLineageColumnFact, + CompiledSeed, + CompiledSource, + CompiledSqlExpansion, + CompileModelConfig, + DiscoveredAuditBlock, + DiscoveredAuditFile, + DiscoveredSchemaFile, + DiscoveredSeedFile, + DiscoveredSqlHookFile, + DynamicColumnContractProof, + FunctionArgument, + FunctionReturnColumn, + LiteralSqlRelation, + ) from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity from sqlbuild.compiler.compile.models import ( - CompiledLineageColumnFact, CompiledObjectKey, CompiledRelationLocation, - CompiledSeed, - CompiledSource, - CompiledSqlExpansion, CompiledSqlScenario, CompiledSqlTest, - CompileModelConfig, CompilerDiagnostic, CompileSqlReference, - CompileSqlScenarioInput, - CompileSqlTestInput, - DeclarationScopeBuild, - DynamicColumnContractProof, - FunctionArgument, - FunctionReturnColumn, InferredColumn, LoadedMacro, - MacroContext, ) from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, - DiscoveredAuditBlock, - DiscoveredAuditFile, DiscoveredHookFunction, DiscoveredLoaderFunction, DiscoveredMaterializationFile, - DiscoveredProjectInputs, - DiscoveredPythonFunctionFile, - DiscoveredSchemaFile, - DiscoveredSeedFile, DiscoveredSourceFile, - DiscoveredSqlFunctionFile, - DiscoveredSqlHookFile, - DiscoveredSqlModelFile, EnumDeclaration, ) - from sqlbuild.compiler.references.models import LiteralSqlRelation from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.compiler.scopes.models import ScopeIndex, UsageRecord from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - from sqlbuild.lint.types import RelationKeyRow from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, ScenarioConfig, SchemaModelEntry, SchemaSeedEntry, SettingsConfig, SourceEntry, SourceLocation, - TargetConfig, ) +type RelationKeyRow = tuple[ + str, + str, + str, + str, + str, + bool, + list[str] | None, + list[tuple[str, list[str], bool]], + list[tuple[str, str | None]], +] + + class SourceInput(Protocol): """One attached source, as the native owner view holds it.""" @@ -99,17 +102,6 @@ def source_entry(self) -> SourceEntry: ... def source_file(self) -> DiscoveredSourceFile: ... -class SeedInput(Protocol): - """One seed paired with its declaration, as the native owner view holds it.""" - - @property - def seed_file(self) -> DiscoveredSeedFile: ... - @property - def schema_entry(self) -> SchemaSeedEntry: ... - @property - def schema_file(self) -> DiscoveredSchemaFile: ... - - class CompiledSeedView(Protocol): """One assembled seed: a native owner view, or a hand-built `CompiledSeed`.""" @@ -131,59 +123,6 @@ def destination(self) -> CompiledRelationLocation: ... def external(self) -> bool: ... -class FunctionInput(Protocol): - """One attached function, as the native owner view holds it.""" - - @property - def function_file(self) -> DiscoveredSqlFunctionFile | DiscoveredPythonFunctionFile: ... - @property - def name(self) -> str: ... - @property - def arguments(self) -> tuple[FunctionArgument, ...]: ... - @property - def returns(self) -> str: ... - @property - def body_sql(self) -> str: ... - @property - def return_columns(self) -> tuple[FunctionReturnColumn, ...]: ... - @property - def references(self) -> tuple[CompileSqlReference, ...]: ... - @property - def language(self) -> FunctionLanguage: ... - @property - def runtime_version(self) -> str | None: ... - @property - def entry_point(self) -> str | None: ... - @property - def packages(self) -> tuple[str, ...]: ... - @property - def tags(self) -> tuple[str, ...]: ... - @property - def description(self) -> str | None: ... - @property - def database(self) -> str | None: ... - @property - def schema(self) -> str | None: ... - @property - def logical_database(self) -> str | None: ... - @property - def logical_schema(self) -> str | None: ... - @property - def fingerprint_database(self) -> str | None: ... - @property - def fingerprint_schema(self) -> str | None: ... - @property - def fingerprint_logical_database(self) -> str | None: ... - @property - def fingerprint_logical_schema(self) -> str | None: ... - @property - def declaration_usages(self) -> tuple[UsageRecord, ...]: ... - @property - def assembly_row( - self, - ) -> tuple[str, str, list[str], list[tuple[str, str, str | None]]]: ... - - class CompiledFunctionView(Protocol): """One assembled function: a native owner view, or a hand-built `CompiledFunction`.""" @@ -225,55 +164,6 @@ def destination(self) -> CompiledRelationLocation: ... def fingerprint_destination(self) -> CompiledRelationLocation: ... -class AuditInput(Protocol): - """One attached audit, as the native owner view holds it.""" - - @property - def audit_file(self) -> DiscoveredAuditFile: ... - @property - def audit_block(self) -> DiscoveredAuditBlock: ... - @property - def sql_body(self) -> str: ... - @property - def evaluation_mode(self) -> AuditEvaluationMode: ... - @property - def measurement_contract(self) -> MeasurementContract | None: ... - @property - def thresholds(self) -> MeasurementThresholds | None: ... - @property - def minimum_samples(self) -> int | None: ... - @property - def measure_sql(self) -> str | None: ... - @property - def evidence_sql(self) -> str | None: ... - @property - def evidence_limit(self) -> int | None: ... - @property - def references(self) -> tuple[CompileSqlReference, ...]: ... - @property - def attached_target_kind(self) -> AttachedAuditTargetKind | None: ... - @property - def attached_target_name(self) -> str | None: ... - @property - def attached_column_name(self) -> str | None: ... - @property - def severity(self) -> AuditSeverity | None: ... - @property - def run_scope(self) -> str | None: ... - @property - def always_run(self) -> bool: ... - @property - def description(self) -> str | None: ... - @property - def declaration_usages(self) -> tuple[UsageRecord, ...]: ... - @property - def name(self) -> str | None: ... - @property - def assembly_row( - self, - ) -> tuple[list[tuple[str, str, str | None]], tuple[str, str] | None]: ... - - class CompiledAuditView(Protocol): """One assembled audit: a native owner view, or a hand-built `CompiledAudit`.""" @@ -323,67 +213,6 @@ def always_run(self) -> bool: ... def description(self) -> str | None: ... -class CompileInputsView(Protocol): - """A compile's attached inputs: a native owner view, or hand-built `CompileProjectInputs`.""" - - @property - def project_config(self) -> ProjectConfig: ... - @property - def local_config(self) -> LocalConfig: ... - @property - def discovered_inputs(self) -> DiscoveredProjectInputs: ... - @property - def sql_lexical_syntax(self) -> SqlLexicalSyntax: ... - @property - def run_id(self) -> str: ... - @property - def effective_target_name(self) -> str | None: ... - @property - def effective_target(self) -> TargetConfig | None: ... - @property - def compile_cache_dir(self) -> Path | None: ... - @property - def effective_connection(self) -> dict[str, object]: ... - @property - def effective_settings(self) -> SettingsConfig: ... - @property - def no_sql_validation(self) -> bool: ... - @property - def effective_vars(self) -> dict[str, object]: ... - @property - def macro_context(self) -> MacroContext | None: ... - @property - def loaded_macros(self) -> dict[str, LoadedMacro]: ... - @property - def public_enums(self) -> dict[str, EnumDeclaration]: ... - @property - def public_constants(self) -> dict[str, ConstantDeclaration]: ... - @property - def model_inputs(self) -> tuple[ModelInput, ...]: ... - @property - def seed_inputs(self) -> tuple[SeedInput, ...]: ... - @property - def source_inputs(self) -> tuple[SourceInput, ...]: ... - @property - def sql_function_inputs(self) -> tuple[FunctionInput, ...]: ... - @property - def test_inputs(self) -> tuple[CompileSqlTestInput, ...]: ... - @property - def scenario_inputs(self) -> tuple[CompileSqlScenarioInput, ...]: ... - @property - def audit_inputs(self) -> tuple[AuditInput, ...]: ... - @property - def diagnostics(self) -> tuple[CompilerDiagnostic, ...]: ... - @property - def external_sql_reference_resolver(self) -> ExternalSqlReferenceResolver | None: ... - @property - def scope_index(self) -> ScopeIndex: ... - @property - def declaration_scope(self) -> DeclarationScopeBuild | None: ... - @property - def native_project(self) -> _native.NativeCompiledProject | None: ... - - class CompiledProjectView(Protocol): """One compiled project: a native owner view, or a hand-built `CompiledProject`.""" @@ -534,51 +363,6 @@ def has_dynamic_columns(self) -> bool: ... def output_column_count(self) -> int: ... -class ModelInput(Protocol): - """One model's compile input: a native owner view, or a hand-built `CompileModelInput`.""" - - @property - def model_file(self) -> DiscoveredSqlModelFile: ... - @property - def config(self) -> CompileModelConfig: ... - @property - def query_sql(self) -> str: ... - @property - def macro_source_sql(self) -> str: ... - @property - def references(self) -> tuple[CompileSqlReference, ...]: ... - @property - def schema_entry(self) -> SchemaModelEntry | None: ... - @property - def schema_file(self) -> DiscoveredSchemaFile | None: ... - @property - def sql_validation_enabled(self) -> bool: ... - @property - def rejected_sql_analysis_opt_out(self) -> SourceLocation | None: ... - @property - def enum_declarations(self) -> tuple[EnumDeclaration, ...]: ... - @property - def constant_declarations(self) -> tuple[ConstantDeclaration, ...]: ... - @property - def enum_columns(self) -> dict[str, EnumDeclaration]: ... - @property - def macro_deps(self) -> tuple[str, ...]: ... - @property - def macro_usages(self) -> tuple[UsageRecord, ...]: ... - @property - def declaration_usages(self) -> tuple[UsageRecord, ...]: ... - @property - def sql_expansion(self) -> CompiledSqlExpansion | None: ... - @property - def lint_expansion(self) -> CompiledSqlExpansion | None: ... - @property - def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: ... - @property - def attached_schema_name(self) -> str | None: ... - @property - def relation_key_row(self) -> RelationKeyRow: ... - - class TypedSqlValueRenderer(Protocol): """Adapter rendering operations consumed by authored SQL expansion.""" @@ -599,12 +383,6 @@ class AttachedAuditTargetKind(StrEnum): SOURCE = "source" SEED = "seed" - @property - def resource_type(self) -> CompiledResourceType: - """Return the compiled resource type of the attached target.""" - - return CompiledResourceType(self.value) - class CompiledResourceType(StrEnum): MODEL = "model" @@ -618,27 +396,6 @@ class CompiledResourceType(StrEnum): SQL_SCENARIO = "sql_scenario" -class DiagnosticPhase(StrEnum): - """Phase that produced a compiler diagnostic.""" - - COMPILE = "compile" - CONTRACT = "contract" - RULE = "rule" - PLAN = "plan" - BUILD = "build" - AUDIT = "audit" - TEST = "test" - CONNECTION = "connection" - - -class DiagnosticSeverity(StrEnum): - """Severity for a compiler diagnostic.""" - - ERROR = "error" - WARNING = "warning" - INFO = "info" - - class FunctionLanguage(StrEnum): SQL = "sql" PYTHON = "python" @@ -654,7 +411,50 @@ class SqlTestMode(StrEnum): type PythonResourceRow = tuple[str, str, str | None, Callable[[], SourceLocation], str] -class CompileInputsReadyCallback(Protocol): - """Invocation-local observer for the completed pre-semantic input snapshot.""" +class AuditInput(Protocol): + """One attached audit, as the native owner view holds it.""" - def __call__(self, *, inputs: CompileInputsView) -> None: ... + @property + def audit_file(self) -> DiscoveredAuditFile: ... + @property + def audit_block(self) -> DiscoveredAuditBlock: ... + @property + def sql_body(self) -> str: ... + @property + def evaluation_mode(self) -> AuditEvaluationMode: ... + @property + def measurement_contract(self) -> MeasurementContract | None: ... + @property + def thresholds(self) -> MeasurementThresholds | None: ... + @property + def minimum_samples(self) -> int | None: ... + @property + def measure_sql(self) -> str | None: ... + @property + def evidence_sql(self) -> str | None: ... + @property + def evidence_limit(self) -> int | None: ... + @property + def references(self) -> tuple[CompileSqlReference, ...]: ... + @property + def attached_target_kind(self) -> AttachedAuditTargetKind | None: ... + @property + def attached_target_name(self) -> str | None: ... + @property + def attached_column_name(self) -> str | None: ... + @property + def severity(self) -> AuditSeverity | None: ... + @property + def run_scope(self) -> str | None: ... + @property + def always_run(self) -> bool: ... + @property + def description(self) -> str | None: ... + @property + def declaration_usages(self) -> tuple[UsageRecord, ...]: ... + @property + def name(self) -> str | None: ... + @property + def assembly_row( + self, + ) -> tuple[list[tuple[str, str, str | None]], tuple[str, str] | None]: ... diff --git a/src/sqlbuild/compiler/contract_adoption/exceptions.py b/src/sqlbuild/compiler/contract_adoption/exceptions.py deleted file mode 100644 index 9f35d99458..0000000000 --- a/src/sqlbuild/compiler/contract_adoption/exceptions.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Contract adoption failures.""" - - -class ContractAdoptionError(RuntimeError): - """Raised when safe contract source editing cannot continue.""" diff --git a/src/sqlbuild/compiler/contract_adoption/models.py b/src/sqlbuild/compiler/contract_adoption/models.py deleted file mode 100644 index 2600c8c5d4..0000000000 --- a/src/sqlbuild/compiler/contract_adoption/models.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Contract adoption domain models.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from pathlib import Path - -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.contract_adoption.types import ContractFindingKind -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -@dataclass(frozen=True) -class ContractFinding: - resource_type: CompiledResourceType - resource_name: str - kind: ContractFindingKind - message: str - column_name: str | None = None - declared_type: str | None = None - physical_type: str | None = None - - -@dataclass(frozen=True) -class ContractEvidence: - resource_type: CompiledResourceType - resource_name: str - database: str | None - schema: str | None - relation: str - declared_columns: tuple[ColumnInfo, ...] - physical_columns: tuple[ColumnInfo, ...] | None - source_path: Path | None - dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = field(default_factory=tuple) - findings: tuple[ContractFinding, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class ContractAdoptionResult: - from_target: str - evidence: tuple[ContractEvidence, ...] - written_paths: tuple[Path, ...] = field(default_factory=tuple) - - @property - def findings(self) -> tuple[ContractFinding, ...]: - findings: list[ContractFinding] = [] - for item in self.evidence: - findings.extend(item.findings) - return tuple(findings) diff --git a/src/sqlbuild/compiler/contract_adoption/types.py b/src/sqlbuild/compiler/contract_adoption/types.py deleted file mode 100644 index 55abc7c162..0000000000 --- a/src/sqlbuild/compiler/contract_adoption/types.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Contract adoption type-layer declarations.""" - -from __future__ import annotations - -from enum import StrEnum - - -class ContractFindingKind(StrEnum): - """Stable physical-to-authored contract difference categories.""" - - MISSING_DECLARATION = "missing_declaration" - MISSING_PHYSICAL_COLUMN = "missing_physical_column" - TYPE_MISMATCH = "type_mismatch" - RELATION_MISSING = "relation_missing" - INSPECTION_UNAVAILABLE = "inspection_unavailable" - OWNERSHIP_CONFLICT = "ownership_conflict" diff --git a/src/sqlbuild/compiler/contracts/models.py b/src/sqlbuild/compiler/contracts/models.py deleted file mode 100644 index 70d9863a9d..0000000000 --- a/src/sqlbuild/compiler/contracts/models.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Contract validation result models.""" - -from __future__ import annotations - -from dataclasses import dataclass, field - -from sqlbuild.compiler.compile.models import CompilerDiagnostic - - -@dataclass(frozen=True) -class ContractValidationResult: - """Diagnostics emitted by model contract validation.""" - - diagnostics: tuple[CompilerDiagnostic, ...] = field(default_factory=tuple) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py index 7295e669ad..b961b51995 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py @@ -9,12 +9,11 @@ from typing import get_type_hints import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( - _load_sink_module, - _provider_by_name, - _provider_usages, +from sqlbuild._native import EventExporterDiscoveryError +from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( + load_sink_module, + provider_usages, ) -from sqlbuild.compiler.discovery.exceptions import EventExporterDiscoveryError from sqlbuild.compiler.discovery.models import ( DiscoveredCommandOutputSink, DiscoveredCommandOutputSinkDeclaration, @@ -34,10 +33,6 @@ def discover_command_output_sink_functions( ) -> tuple[DiscoveredCommandOutputSink, ...]: """Discover validated command-output sink declarations under sinks/.""" - from sqlbuild.runtime.event_exporting.main.cached_event_exporter_extensions import ( - cached_event_exporter_extensions, - ) - cached: ( tuple[ tuple[DiscoveredProvider, ...], @@ -45,7 +40,7 @@ def discover_command_output_sink_functions( tuple[DiscoveredCommandOutputSink, ...], ] | None - ) = cached_event_exporter_extensions(project_dir=project_dir) + ) = _native.cached_event_exporter_extensions(project_dir=project_dir) if cached is not None: return cached[2] declarations: tuple[DiscoveredCommandOutputSinkDeclaration, ...] = ( @@ -71,7 +66,7 @@ def discover_command_output_sink_declarations( for file_path in _native.discover_python_module_files( project_dir=project_dir, root="sinks", public_only=True ): - module: ModuleType = _load_sink_module(file_path=file_path, project_dir=project_dir) + module: ModuleType = load_sink_module(file_path=file_path, project_dir=project_dir) seen_function_ids: set[int] = set() for _, value in inspect.getmembers(module, inspect.isfunction): if value.__module__ != module.__name__: @@ -119,7 +114,9 @@ def bind_command_output_sink_declarations( ) -> tuple[DiscoveredCommandOutputSink, ...]: """Validate command-output sink provider parameters.""" - provider_by_name: dict[str, DiscoveredProvider] = _provider_by_name(providers) + provider_by_name: dict[str, DiscoveredProvider] = { + provider.name: provider for provider in providers + } bound: list[DiscoveredCommandOutputSink] = [] for declaration in declarations: usages: tuple[DiscoveredProviderUsage, ...] = _bind_sink_provider_usages( @@ -259,4 +256,4 @@ def _bind_sink_provider_usages( f"{sink_label} '{sink_name}' in {relative_path} provider parameter " f"'{parameter.name}' must be unannotated or exactly {provider_class_name}" ) - return _provider_usages(function=function, provider_by_name=provider_by_name) + return provider_usages(function=function, provider_by_name=provider_by_name) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py deleted file mode 100644 index 89d26cad35..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ /dev/null @@ -1,1153 +0,0 @@ -"""Filesystem-backed discovery helpers for project inputs.""" - -from __future__ import annotations - -import ast -import importlib.util -import inspect -import sys -from collections.abc import Callable -from dataclasses import dataclass, field -from importlib.machinery import ModuleSpec -from pathlib import Path -from types import ModuleType -from typing import TYPE_CHECKING, Any, get_type_hints - -import sqlbuild._native as _native -from sqlbuild._native import ( - project_relative_path, - record_settings_class_read, - validate_resource_identity, -) -from sqlbuild.compiler.discovery.constants import ( - PYTHON_INIT_MODULE_STEM, - PYTHON_NODE_ROOT, -) -from sqlbuild.compiler.discovery.exceptions import ( - EventExporterDiscoveryError, - ProviderDiscoveryError, - PythonNodeDiscoveryError, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredAssetFunction, - DiscoveredAuditFactory, - DiscoveredCheckFunction, - DiscoveredCommandOutputSink, - DiscoveredEventExporter, - DiscoveredEventExporterDeclaration, - DiscoveredHookFunction, - DiscoveredLoaderFunction, - DiscoveredMaterializationFile, - DiscoveredProvider, - DiscoveredProviderUsage, - DiscoveredPythonNodeFunctions, - DiscoveredTaskFunction, -) -from sqlbuild.provider.exceptions import ProviderInputError -from sqlbuild.python_nodes.main.read_asset_definition import read_asset_definition -from sqlbuild.python_nodes.main.read_audit_factory_definition import ( - read_audit_factory_definition, -) -from sqlbuild.python_nodes.main.read_check_definition import read_check_definition -from sqlbuild.python_nodes.main.read_factory_definition import read_factory_definition -from sqlbuild.python_nodes.main.read_hook_definition import read_hook_definition -from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition -from sqlbuild.python_nodes.main.read_task_definition import read_task_definition -from sqlbuild.python_nodes.models import ( - AssetDefinition, - AuditCase, - AuditFactoryDefinition, - CheckDefinition, - FactoryDefinition, - HookDefinition, - LoaderDefinition, - TaskDefinition, -) -from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORTER_EVENT_PARAMETER_NAME -from sqlbuild.runtime.event_exporting.main.get_lifecycle_event_sink_definition import ( - get_lifecycle_event_sink_definition, -) -from sqlbuild.runtime.event_exporting.models import LifecycleEventSinkDefinition -from sqlbuild.runtime.observability.models import LifecycleEvent - -if TYPE_CHECKING: - from pydantic import ValidationError - - from sqlbuild.provider.classes.provider import Provider - -_PROJECT_PYTHON_PACKAGE_MARKER: str = "__sqlbuild_project_python__" - - -@dataclass -class _PythonNodeDiscoveryBucket: - loaders: list[DiscoveredLoaderFunction] = field(default_factory=list) - tasks: list[DiscoveredTaskFunction] = field(default_factory=list) - assets: list[DiscoveredAssetFunction] = field(default_factory=list) - checks: list[DiscoveredCheckFunction] = field(default_factory=list) - audit_factories: list[DiscoveredAuditFactory] = field(default_factory=list) - - def add_loader(self, item: DiscoveredLoaderFunction) -> None: - self.loaders.append(item) - - def add_task(self, item: DiscoveredTaskFunction) -> None: - self.tasks.append(item) - - def add_asset(self, item: DiscoveredAssetFunction) -> None: - self.assets.append(item) - - def add_check(self, item: DiscoveredCheckFunction) -> None: - self.checks.append(item) - - def add_audit_factory(self, item: DiscoveredAuditFactory) -> None: - self.audit_factories.append(item) - - -def discover_materialization_files( - *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () -) -> tuple[DiscoveredMaterializationFile, ...]: - """Discover custom materialization Python files under materializations/.""" - - materializations_root: Path = project_dir / "materializations" - if not materializations_root.is_dir(): - return () - - provider_by_name: dict[str, DiscoveredProvider] = _provider_by_name(providers) - discovered_files: list[DiscoveredMaterializationFile] = [] - file_path: Path - for file_path in _native.discover_python_module_files( - project_dir=project_dir, root="materializations", public_only=False - ): - materialize_fn: Callable[..., object] | None = _load_materialize_function( - file_path=file_path, - project_dir=project_dir, - ) - discovered_files.append( - DiscoveredMaterializationFile( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - name=file_path.stem, - provider_usages=( - _provider_usages(function=materialize_fn, provider_by_name=provider_by_name) - if materialize_fn is not None - else () - ), - ) - ) - return tuple(discovered_files) - - -def discover_python_node_functions( - *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () -) -> DiscoveredPythonNodeFunctions: - """Discover decorated Python DAG node functions under python/.""" - - bucket: _PythonNodeDiscoveryBucket = _discover_python_node_functions( - project_dir=project_dir, - providers=providers, - ) - return DiscoveredPythonNodeFunctions( - loaders=tuple(bucket.loaders), - tasks=tuple(bucket.tasks), - assets=tuple(bucket.assets), - checks=tuple(bucket.checks), - audit_factories=tuple(bucket.audit_factories), - ) - - -def discover_hook_functions( - *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () -) -> tuple[DiscoveredHookFunction, ...]: - """Discover decorated model lifecycle hook functions under hooks/python/.""" - - discovered_hooks: list[DiscoveredHookFunction] = [] - seen_names: dict[str, Path] = {} - provider_by_name: dict[str, DiscoveredProvider] = _provider_by_name(providers) - hook_file: dict[str, Any] - for hook_file in _native.discover_python_hook_files(project_dir=project_dir): - file_path: Path = hook_file["file_path"] - module: ModuleType = _load_python_node_module( - file_path=file_path, - project_dir=project_dir, - ) - for _, value in inspect.getmembers(module, inspect.isfunction): - if value.__module__ != module.__name__: - continue - hook_definition: HookDefinition | None = read_hook_definition(value) - if hook_definition is None: - continue - existing_path: Path | None = seen_names.get(hook_definition.name) - if existing_path is not None: - raise PythonNodeDiscoveryError( - f"Duplicate hook name '{hook_definition.name}' found in " - f"{existing_path.relative_to(project_dir)} and " - f"{file_path.relative_to(project_dir)}" - ) - seen_names[hook_definition.name] = file_path - discovered_hooks.append( - DiscoveredHookFunction( - **hook_file, - name=hook_definition.name, - function=value, - description=hook_definition.description, - reads=hook_definition.reads, - provider_usages=_provider_usages( - function=value, - provider_by_name=provider_by_name, - ), - ) - ) - return tuple(discovered_hooks) - - -def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, ...]: - """Discover provider classes under providers/.""" - - from sqlbuild.runtime.event_exporting.main.cached_event_exporter_extensions import ( - cached_event_exporter_extensions, - ) - - cached: ( - tuple[ - tuple[DiscoveredProvider, ...], - tuple[DiscoveredEventExporter, ...], - tuple[DiscoveredCommandOutputSink, ...], - ] - | None - ) = cached_event_exporter_extensions(project_dir=project_dir) - if cached is not None: - for cached_provider in cached[0]: - record_settings_class_read(cached_provider.provider_class) - return cached[0] - providers_root: Path = project_dir / "providers" - if not providers_root.is_dir(): - return () - from sqlbuild.provider.classes.provider import Provider - - discovered_providers: list[DiscoveredProvider] = [] - seen_names: dict[str, Path] = {} - file_path: Path - for file_path in _native.discover_python_module_files( - project_dir=project_dir, root="providers", public_only=True - ): - module: ModuleType = _load_provider_module(file_path=file_path, project_dir=project_dir) - for _, value in inspect.getmembers(module, inspect.isclass): - if value.__module__ != module.__name__: - continue - if value is Provider or not issubclass(value, Provider) or inspect.isabstract(value): - continue - provider_class: type[Provider] = value - provider_name: str = _provider_name( - provider_class=provider_class, - file_path=file_path, - project_dir=project_dir, - ) - existing_path: Path | None = seen_names.get(provider_name) - if existing_path is not None: - raise ProviderDiscoveryError( - f"Duplicate provider name '{provider_name}' found in " - f"{existing_path.relative_to(project_dir)} and " - f"{file_path.relative_to(project_dir)}" - ) - seen_names[provider_name] = file_path - record_settings_class_read(provider_class) - discovered_providers.append( - DiscoveredProvider( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - name=provider_name, - provider_class=provider_class, - settings=_provider_instance( - provider_class=provider_class, - provider_name=provider_name, - file_path=file_path, - project_dir=project_dir, - ), - ) - ) - return tuple(discovered_providers) - - -def discover_event_exporter_functions( - *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () -) -> tuple[DiscoveredEventExporter, ...]: - """Discover validated lifecycle-event sink declarations under sinks/.""" - - from sqlbuild.runtime.event_exporting.main.cached_event_exporter_extensions import ( - cached_event_exporter_extensions, - ) - - cached: ( - tuple[ - tuple[DiscoveredProvider, ...], - tuple[DiscoveredEventExporter, ...], - tuple[DiscoveredCommandOutputSink, ...], - ] - | None - ) = cached_event_exporter_extensions(project_dir=project_dir) - if cached is not None: - return cached[1] - declarations: tuple[DiscoveredEventExporterDeclaration, ...] = ( - discover_event_exporter_declarations(project_dir=project_dir) - ) - return bind_event_exporter_declarations( - declarations=declarations, - providers=providers, - project_dir=project_dir, - ) - - -def discover_event_exporter_declarations( - *, project_dir: Path -) -> tuple[DiscoveredEventExporterDeclaration, ...]: - """Import sink modules and collect lifecycle declarations without discovering providers.""" - - if (project_dir / "event_exporters").exists(): - raise EventExporterDiscoveryError( - "event_exporters/ was replaced by sinks/ and @lifecycle_event_sink" - ) - exporters_root: Path = project_dir / "sinks" - if not exporters_root.is_dir(): - return () - discovered: list[DiscoveredEventExporterDeclaration] = [] - seen_names: dict[str, Path] = {} - for file_path in _native.discover_python_module_files( - project_dir=project_dir, root="sinks", public_only=True - ): - module: ModuleType = _load_sink_module(file_path=file_path, project_dir=project_dir) - seen_function_ids: set[int] = set() - for _, value in inspect.getmembers(module, inspect.isfunction): - if value.__module__ != module.__name__: - continue - function_id: int = id(value) - if function_id in seen_function_ids: - continue - seen_function_ids.add(function_id) - definition: LifecycleEventSinkDefinition | None = get_lifecycle_event_sink_definition( - value - ) - if definition is None: - continue - existing_path: Path | None = seen_names.get(definition.name) - if existing_path is not None: - raise EventExporterDiscoveryError( - f"Duplicate event exporter name '{definition.name}' found in " - f"{existing_path.relative_to(project_dir)} and " - f"{file_path.relative_to(project_dir)}" - ) - _validate_event_exporter_declaration_signature( - function=value, - exporter_name=definition.name, - file_path=file_path, - project_dir=project_dir, - ) - seen_names[definition.name] = file_path - discovered.append( - DiscoveredEventExporterDeclaration( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - name=definition.name, - function=value, - event_kinds=definition.event_kinds, - min_severity=definition.min_severity, - ) - ) - return tuple(discovered) - - -def bind_event_exporter_declarations( - *, - declarations: tuple[DiscoveredEventExporterDeclaration, ...], - providers: tuple[DiscoveredProvider, ...], - project_dir: Path, -) -> tuple[DiscoveredEventExporter, ...]: - """Validate declaration provider parameters against discovered project providers.""" - - provider_by_name: dict[str, DiscoveredProvider] = _provider_by_name(providers) - bound: list[DiscoveredEventExporter] = [] - for declaration in declarations: - usages: tuple[DiscoveredProviderUsage, ...] = _bind_event_exporter_provider_usages( - function=declaration.function, - exporter_name=declaration.name, - file_path=declaration.file_path, - project_dir=project_dir, - provider_by_name=provider_by_name, - ) - bound.append( - DiscoveredEventExporter( - file_path=declaration.file_path, - relative_path=declaration.relative_path, - name=declaration.name, - function=declaration.function, - event_kinds=declaration.event_kinds, - min_severity=declaration.min_severity, - provider_usages=usages, - ) - ) - return tuple(bound) - - -def _validate_event_exporter_declaration_signature( - *, - function: Callable[..., object], - exporter_name: str, - file_path: Path, - project_dir: Path, -) -> None: - relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) - if inspect.iscoroutinefunction(function): - raise EventExporterDiscoveryError( - f"Event exporter '{exporter_name}' in {relative_path} must be synchronous" - ) - parameters: tuple[inspect.Parameter, ...] = tuple( - inspect.signature(function).parameters.values() - ) - if not parameters or parameters[0].name != EVENT_EXPORTER_EVENT_PARAMETER_NAME: - raise EventExporterDiscoveryError( - f"Event exporter '{exporter_name}' in {relative_path} must declare event first" - ) - if any( - parameter.kind - in { - inspect.Parameter.POSITIONAL_ONLY, - inspect.Parameter.VAR_POSITIONAL, - inspect.Parameter.VAR_KEYWORD, - } - for parameter in parameters - ): - raise EventExporterDiscoveryError( - f"Event exporter '{exporter_name}' in {relative_path} must use named parameters" - ) - if any(parameter.default is not inspect.Parameter.empty for parameter in parameters): - raise EventExporterDiscoveryError( - f"Event exporter '{exporter_name}' in {relative_path} parameters must not have defaults" - ) - try: - type_hints: dict[str, object] = get_type_hints(function) - except (NameError, TypeError) as error: - raise EventExporterDiscoveryError( - f"Event exporter '{exporter_name}' in {relative_path} has invalid annotations" - ) from error - event_annotation: object = type_hints.get("event", parameters[0].annotation) - if event_annotation not in {inspect.Parameter.empty, LifecycleEvent}: - raise EventExporterDiscoveryError( - f"Event exporter '{exporter_name}' in {relative_path} event must be LifecycleEvent" - ) - return_annotation: object = type_hints.get( - "return", inspect.signature(function).return_annotation - ) - if return_annotation not in {inspect.Signature.empty, None, type(None)}: - raise EventExporterDiscoveryError( - f"Event exporter '{exporter_name}' in {relative_path} return annotation must be None" - ) - - -def _bind_event_exporter_provider_usages( - *, - function: Callable[..., object], - exporter_name: str, - file_path: Path, - project_dir: Path, - provider_by_name: dict[str, DiscoveredProvider], -) -> tuple[DiscoveredProviderUsage, ...]: - relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) - parameters: tuple[inspect.Parameter, ...] = tuple( - inspect.signature(function).parameters.values() - ) - type_hints: dict[str, object] = get_type_hints(function) - for parameter in parameters[1:]: - provider: DiscoveredProvider | None = provider_by_name.get(parameter.name) - if provider is None: - raise EventExporterDiscoveryError( - f"Event exporter '{exporter_name}' in {relative_path} requires unknown provider " - f"'{parameter.name}'" - ) - annotation: object = type_hints.get(parameter.name, parameter.annotation) - if annotation is not inspect.Parameter.empty and annotation is not provider.provider_class: - provider_class_name: str = provider.provider_class.__name__ - raise EventExporterDiscoveryError( - f"Event exporter '{exporter_name}' in {relative_path} provider parameter " - f"'{parameter.name}' must be unannotated or exactly {provider_class_name}" - ) - return _provider_usages(function=function, provider_by_name=provider_by_name) - - -def _provider_name(*, provider_class: type[Provider], file_path: Path, project_dir: Path) -> str: - relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) - explicit_name: str | None = provider_class.provider_name - if explicit_name is not None: - validate_resource_identity( - name=explicit_name, - kind="provider", - path=relative_path, - ) - try: - provider_name: str = provider_class.name() - except ProviderInputError as error: - raise ProviderDiscoveryError( - f"Provider class {provider_class.__name__} in {relative_path} " - f"has an invalid provider name: {error}" - ) from error - validate_resource_identity( - name=provider_name, - kind="provider", - path=relative_path, - ) - return provider_name - - -def _provider_instance( - *, provider_class: type[Provider], provider_name: str, file_path: Path, project_dir: Path -) -> Provider: - from pydantic import ValidationError - - try: - return provider_class() - except ValidationError as error: - relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) - raise ProviderDiscoveryError( - f"Provider '{provider_name}' in {relative_path} has invalid settings:\n" - f"{_format_provider_validation_error(error)}" - ) from error - - -def _format_provider_validation_error(error: ValidationError) -> str: - details: list[str] = [] - for item in error.errors(include_input=False): - location: object = item.get("loc", ()) - location_text: str = ".".join(str(part) for part in location) if location else "" - message: object = item.get("msg", "invalid value") - error_type: object = item.get("type", "validation_error") - details.append(f"{location_text}: {message} [{error_type}]") - return "\n".join(details) or "invalid provider settings" - - -def _provider_by_name(providers: tuple[DiscoveredProvider, ...]) -> dict[str, DiscoveredProvider]: - return {provider.name: provider for provider in providers} - - -def _provider_usages( - *, function: Callable[..., object], provider_by_name: dict[str, DiscoveredProvider] -) -> tuple[DiscoveredProviderUsage, ...]: - try: - type_hints: dict[str, object] = get_type_hints(function) - except TypeError: - type_hints = {} - usages: list[DiscoveredProviderUsage] = [] - parameter: inspect.Parameter - for parameter in inspect.signature(function).parameters.values(): - discovered_provider: DiscoveredProvider | None = provider_by_name.get(parameter.name) - if discovered_provider is None: - continue - from sqlbuild.provider.classes.provider import Provider - - annotation: object = type_hints.get(parameter.name, parameter.annotation) - annotation_class_name: str | None = None - annotation_module: str | None = None - if isinstance(annotation, type) and issubclass(annotation, Provider): - annotation_class_name = annotation.__name__ - annotation_module = annotation.__module__ - usages.append( - DiscoveredProviderUsage( - provider_name=discovered_provider.name, - parameter_name=parameter.name, - annotation_class_name=annotation_class_name, - annotation_module=annotation_module, - ) - ) - return tuple(usages) - - -def _load_materialize_function( - *, file_path: Path, project_dir: Path -) -> Callable[..., object] | None: - module: ModuleType = _load_materialization_module( - file_path=file_path, - project_dir=project_dir, - ) - materialize_fn: object = getattr(module, "materialize", None) - return materialize_fn if callable(materialize_fn) else None - - -def _discover_python_node_functions( - *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () -) -> _PythonNodeDiscoveryBucket: - bucket: _PythonNodeDiscoveryBucket = _PythonNodeDiscoveryBucket() - provider_by_name: dict[str, DiscoveredProvider] = _provider_by_name(providers) - node_root: Path = project_dir / PYTHON_NODE_ROOT - if not node_root.is_dir(): - return bucket - saved_modules: dict[str, ModuleType] = _install_python_root_package( - node_root=node_root, project_dir=project_dir - ) - try: - modules: tuple[tuple[Path, ModuleType], ...] = tuple( - (file_path, _import_python_root_module(file_path=file_path, project_dir=project_dir)) - for file_path in _native.discover_python_module_files( - project_dir=project_dir, root=PYTHON_NODE_ROOT, public_only=False - ) - ) - direct_functions: dict[int, Callable[..., object]] = {} - file_path: Path - module: ModuleType - for file_path, module in modules: - direct_functions.update( - (id(function), function) - for function in _append_module_direct_nodes( - bucket=bucket, - module=module, - file_path=file_path, - project_dir=project_dir, - provider_by_name=provider_by_name, - ) - ) - for file_path, module in modules: - _append_module_factory_nodes( - bucket=bucket, - module=module, - file_path=file_path, - project_dir=project_dir, - provider_by_name=provider_by_name, - direct_functions=direct_functions, - ) - finally: - _restore_python_root_modules(saved_modules=saved_modules) - return bucket - - -def _python_root_module_names() -> tuple[str, ...]: - return tuple( - module_name - for module_name in sys.modules - if module_name == PYTHON_NODE_ROOT or module_name.startswith(f"{PYTHON_NODE_ROOT}.") - ) - - -def _install_python_root_package(*, node_root: Path, project_dir: Path) -> dict[str, ModuleType]: - """Bind python to this project's python/ folder and return displaced unrelated modules.""" - - existing: ModuleType | None = sys.modules.get(PYTHON_NODE_ROOT) - owned: bool = existing is None or bool(getattr(existing, _PROJECT_PYTHON_PACKAGE_MARKER, False)) - saved_modules: dict[str, ModuleType] = {} - module_name: str - for module_name in _python_root_module_names(): - module: ModuleType | None = sys.modules.pop(module_name, None) - if not owned and module is not None: - saved_modules[module_name] = module - importlib.invalidate_caches() - init_file: Path = node_root / f"{PYTHON_INIT_MODULE_STEM}.py" - spec: ModuleSpec | None = ( - importlib.util.spec_from_file_location( - PYTHON_NODE_ROOT, init_file, submodule_search_locations=[str(node_root)] - ) - if init_file.is_file() - else ModuleSpec(PYTHON_NODE_ROOT, None, is_package=True) - ) - if spec is None: - raise PythonNodeDiscoveryError(f"Could not load Python node package {node_root}") - spec.submodule_search_locations = [str(node_root)] - package: ModuleType = importlib.util.module_from_spec(spec) - setattr(package, _PROJECT_PYTHON_PACKAGE_MARKER, True) - sys.modules[PYTHON_NODE_ROOT] = package - if spec.loader is None: - return saved_modules - old_path: list[str] = list(sys.path) - sys.path.insert(0, str(project_dir)) - try: - spec.loader.exec_module(package) - except Exception as error: - _restore_python_root_modules(saved_modules=saved_modules) - raise PythonNodeDiscoveryError( - f"Failed to import Python node file {init_file.relative_to(project_dir)}: {error}" - ) from error - finally: - sys.path = old_path - return saved_modules - - -def _restore_python_root_modules(*, saved_modules: dict[str, ModuleType]) -> None: - """Put back an unrelated python package that discovery displaced.""" - - if not saved_modules: - return - module_name: str - for module_name in _python_root_module_names(): - sys.modules.pop(module_name, None) - sys.modules.update(saved_modules) - - -def _import_python_root_module(*, file_path: Path, project_dir: Path) -> ModuleType: - """Import a python/ module under its package name, reusing it within a discovery pass.""" - - relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) - module_name: str = ".".join(relative_path.with_suffix("").parts) - old_path: list[str] = list(sys.path) - sys.path.insert(0, str(project_dir)) - try: - return importlib.import_module(module_name) - except Exception as error: - raise PythonNodeDiscoveryError( - f"Failed to import Python node file {relative_path}: {error}" - ) from error - finally: - sys.path = old_path - - -def _append_module_direct_nodes( - *, - bucket: _PythonNodeDiscoveryBucket, - module: ModuleType, - file_path: Path, - project_dir: Path, - provider_by_name: dict[str, DiscoveredProvider], -) -> tuple[Callable[..., object], ...]: - _append_module_audit_factories( - bucket=bucket, - module=module, - file_path=file_path, - project_dir=project_dir, - ) - registered: list[Callable[..., object]] = [] - for _, value in inspect.getmembers(module, inspect.isfunction): - if value.__module__ != module.__name__: - continue - if _append_python_node_function( - bucket=bucket, - function=value, - file_path=file_path, - project_dir=project_dir, - provider_by_name=provider_by_name, - ): - registered.append(value) - return tuple(registered) - - -def _append_module_factory_nodes( - *, - bucket: _PythonNodeDiscoveryBucket, - module: ModuleType, - file_path: Path, - project_dir: Path, - provider_by_name: dict[str, DiscoveredProvider], - direct_functions: dict[int, Callable[..., object]], -) -> None: - for _, value in inspect.getmembers(module, inspect.isfunction): - if value.__module__ != module.__name__: - continue - factory_definition: FactoryDefinition | None = read_factory_definition(value) - if factory_definition is None: - continue - generated_functions: tuple[Callable[..., object], ...] = _call_factory( - factory=value, - factory_definition=factory_definition, - file_path=file_path, - project_dir=project_dir, - ) - index: int - generated_function: Callable[..., object] - for index, generated_function in enumerate(generated_functions): - if direct_functions.get(id(generated_function)) is generated_function: - continue - if not _append_python_node_function( - bucket=bucket, - function=generated_function, - file_path=file_path, - project_dir=project_dir, - provider_by_name=provider_by_name, - ): - raise PythonNodeDiscoveryError( - f"Factory '{factory_definition.name}' in " - f"{file_path.relative_to(project_dir)} returned item {index} that is not " - "a SQLBuild task, asset, loader, or check" - ) - - -def _append_module_audit_factories( - *, - bucket: _PythonNodeDiscoveryBucket, - module: ModuleType, - file_path: Path, - project_dir: Path, -) -> None: - """Collect audit factories without routing them through Python-node discovery.""" - - for _, value in inspect.getmembers(module, inspect.isfunction): - if value.__module__ != module.__name__: - continue - definition: AuditFactoryDefinition | None = read_audit_factory_definition(value) - if definition is None: - continue - relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) - if _python_node_definition_names(value): - kinds: str = ", ".join(_python_node_definition_names(value)) - raise PythonNodeDiscoveryError( - f"Audit factory '{definition.name}' in {relative_path} cannot also be decorated " - f"as a Python-node kind ({kinds})" - ) - existing: DiscoveredAuditFactory | None = next( - (item for item in bucket.audit_factories if item.name == definition.name), None - ) - if existing is not None: - raise PythonNodeDiscoveryError( - f"Duplicate audit factory name '{definition.name}' found in " - f"{existing.relative_path} and {relative_path}" - ) - cases: tuple[AuditCase, ...] = _call_audit_factory( - factory=value, - definition=definition, - file_path=file_path, - project_dir=project_dir, - ) - bucket.add_audit_factory( - DiscoveredAuditFactory( - name=definition.name, - function=value, - file_path=file_path, - relative_path=relative_path, - line=inspect.getsourcelines(value)[1], - cases=cases, - ) - ) - - -def _python_node_definition_names(function: Callable[..., object]) -> tuple[str, ...]: - definitions: tuple[tuple[str, object | None], ...] = ( - ("factory", read_factory_definition(function)), - ("loader", read_loader_definition(function)), - ("task", read_task_definition(function)), - ("asset", read_asset_definition(function)), - ("check", read_check_definition(function)), - ) - return tuple(name for name, definition in definitions if definition is not None) - - -def _call_audit_factory( - *, - factory: Callable[..., object], - definition: AuditFactoryDefinition, - file_path: Path, - project_dir: Path, -) -> tuple[AuditCase, ...]: - relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) - if inspect.signature(factory).parameters: - raise PythonNodeDiscoveryError( - f"Audit factory '{definition.name}' in {relative_path} must not require arguments" - ) - try: - result: object = factory() - except Exception as error: - raise PythonNodeDiscoveryError( - f"Audit factory '{definition.name}' in {relative_path} failed during discovery: {error}" - ) from error - if not isinstance(result, list | tuple): - raise PythonNodeDiscoveryError( - f"Audit factory '{definition.name}' in {relative_path} must return a list or tuple " - "of AuditCase instances" - ) - cases: list[AuditCase] = [] - for index, item in enumerate(result): - if not isinstance(item, AuditCase): - raise PythonNodeDiscoveryError( - f"Audit factory '{definition.name}' in {relative_path} returned item {index} " - "that is not an AuditCase" - ) - cases.append(item) - return tuple(cases) - - -def _append_python_node_function( - *, - bucket: _PythonNodeDiscoveryBucket, - function: Callable[..., object], - file_path: Path, - project_dir: Path, - provider_by_name: dict[str, DiscoveredProvider] | None = None, -) -> bool: - resolved_provider_by_name: dict[str, DiscoveredProvider] = provider_by_name or {} - loader_definition: LoaderDefinition | None = read_loader_definition(function) - if loader_definition is not None: - bucket.add_loader( - DiscoveredLoaderFunction( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - name=loader_definition.name, - function=function, - depends_on=loader_definition.depends_on, - destination=loader_definition.destination, - write_strategy=loader_definition.write_strategy, - cursor_column=loader_definition.cursor_column, - unique_key=loader_definition.unique_key, - columns=loader_definition.columns, - contract=loader_definition.contract, - description=loader_definition.description, - provider_usages=_provider_usages( - function=function, - provider_by_name=resolved_provider_by_name, - ), - ) - ) - return True - task_definition: TaskDefinition | None = read_task_definition(function) - if task_definition is not None: - bucket.add_task( - DiscoveredTaskFunction( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - name=task_definition.name, - function=function, - depends_on=task_definition.depends_on, - tags=task_definition.tags, - group=task_definition.group, - description=task_definition.description, - meta=task_definition.meta, - retry=task_definition.retry, - provider_usages=_provider_usages( - function=function, - provider_by_name=resolved_provider_by_name, - ), - ) - ) - return True - asset_definition: AssetDefinition | None = read_asset_definition(function) - if asset_definition is not None: - bucket.add_asset( - DiscoveredAssetFunction( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - name=asset_definition.name, - function=function, - depends_on=asset_definition.depends_on, - tags=asset_definition.tags, - group=asset_definition.group, - description=asset_definition.description, - meta=asset_definition.meta, - columns=asset_definition.columns, - column_lineage=asset_definition.column_lineage, - retry=asset_definition.retry, - provider_usages=_provider_usages( - function=function, - provider_by_name=resolved_provider_by_name, - ), - ) - ) - return True - check_definition: CheckDefinition | None = read_check_definition(function) - if check_definition is not None: - bucket.add_check( - DiscoveredCheckFunction( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - name=check_definition.name, - function=function, - depends_on=check_definition.depends_on, - severity=check_definition.severity, - tags=check_definition.tags, - group=check_definition.group, - description=check_definition.description, - meta=check_definition.meta, - provider_usages=_provider_usages( - function=function, - provider_by_name=resolved_provider_by_name, - ), - ) - ) - return True - return False - - -def _call_factory( - *, - factory: Callable[..., object], - factory_definition: FactoryDefinition, - file_path: Path, - project_dir: Path, -) -> tuple[Callable[..., object], ...]: - parameters: tuple[inspect.Parameter, ...] = tuple( - inspect.signature(factory).parameters.values() - ) - if parameters: - raise PythonNodeDiscoveryError( - f"Factory '{factory_definition.name}' in {file_path.relative_to(project_dir)} " - "must not require arguments" - ) - try: - result: object = factory() - except Exception as error: - raise PythonNodeDiscoveryError( - f"Factory '{factory_definition.name}' in {file_path.relative_to(project_dir)} " - f"failed during discovery: {error}" - ) from error - return _normalize_factory_result( - result=result, - factory_definition=factory_definition, - file_path=file_path, - project_dir=project_dir, - ) - - -def _normalize_factory_result( - *, - result: object, - factory_definition: FactoryDefinition, - file_path: Path, - project_dir: Path, -) -> tuple[Callable[..., object], ...]: - if callable(result): - return (result,) - if isinstance(result, str | bytes | dict) or not isinstance(result, list | tuple | set): - raise PythonNodeDiscoveryError( - f"Factory '{factory_definition.name}' in {file_path.relative_to(project_dir)} " - "must return a SQLBuild node function or a list, tuple, or set of node functions" - ) - functions: list[Callable[..., object]] = [] - index: int - item: object - for index, item in enumerate(result): - if not callable(item): - raise PythonNodeDiscoveryError( - f"Factory '{factory_definition.name}' in {file_path.relative_to(project_dir)} " - f"returned item {index} that is not a SQLBuild task, asset, loader, or check" - ) - functions.append(item) - return tuple(functions) - - -def _load_python_node_module(*, file_path: Path, project_dir: Path) -> ModuleType: - return _exec_project_module( - module_name="sqlbuild_project_python_node_" - + "_".join( - project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts - ), - file_path=file_path, - project_dir=project_dir, - file_label="Python node", - error_type=PythonNodeDiscoveryError, - ) - - -def _load_provider_module(*, file_path: Path, project_dir: Path) -> ModuleType: - return _load_project_package_module( - file_path=file_path, - project_dir=project_dir, - file_label="provider", - error_type=ProviderDiscoveryError, - ) - - -def _load_sink_module(*, file_path: Path, project_dir: Path) -> ModuleType: - return _load_project_package_module( - file_path=file_path, - project_dir=project_dir, - file_label="sink", - error_type=EventExporterDiscoveryError, - ) - - -def _load_project_package_module( - *, - file_path: Path, - project_dir: Path, - file_label: str, - error_type: type[Exception], -) -> ModuleType: - module_name: str = ".".join( - project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts - ) - _evict_stale_project_package_modules( - root_module=module_name.split(".", maxsplit=1)[0], - project_dir=project_dir, - ) - existing_module: ModuleType | None = sys.modules.get(module_name) - if existing_module is not None: - existing_file: object = getattr(existing_module, "__file__", None) - if isinstance(existing_file, str) and Path(existing_file).resolve() == file_path.resolve(): - return existing_module - sys.modules.pop(module_name, None) - return _exec_project_module( - module_name=module_name, - file_path=file_path, - project_dir=project_dir, - file_label=file_label, - error_type=error_type, - ) - - -def _exec_project_module( - *, - module_name: str, - file_path: Path, - project_dir: Path, - file_label: str, - error_type: type[Exception], -) -> ModuleType: - spec: ModuleSpec | None = importlib.util.spec_from_file_location(module_name, file_path) - if spec is None or spec.loader is None: - raise error_type(f"Could not load {file_label} file {file_path}") - module: ModuleType = importlib.util.module_from_spec(spec) - old_path: list[str] = list(sys.path) - sys.path.insert(0, str(project_dir)) - try: - sys.modules[module_name] = module - spec.loader.exec_module(module) - except Exception as error: - raise error_type( - f"Failed to import {file_label} file {file_path.relative_to(project_dir)}: {error}" - ) from error - finally: - sys.path = old_path - return module - - -def _evict_stale_project_package_modules(*, root_module: str, project_dir: Path) -> None: - root_path: Path = (project_dir / root_module).resolve() - module_name: str - module: ModuleType - for module_name, module in tuple(sys.modules.items()): - if module_name != root_module and not module_name.startswith(f"{root_module}."): - continue - module_file: object = getattr(module, "__file__", None) - if isinstance(module_file, str): - try: - Path(module_file).resolve().relative_to(project_dir.resolve()) - continue - except ValueError: - sys.modules.pop(module_name, None) - continue - module_paths: object = getattr(module, "__path__", None) - if module_paths is None: - sys.modules.pop(module_name, None) - continue - if any(Path(path).resolve() == root_path for path in module_paths): - continue - sys.modules.pop(module_name, None) - - -def _load_materialization_module(*, file_path: Path, project_dir: Path) -> ModuleType: - removed_hook_name: str = "prepare_version" - removed_hook_message: str = ( - f"materialization '{file_path.stem}' at {file_path} defines " - "'prepare_version', which was removed with virtual environments; " - "projects run in direct mode" - ) - try: - module_ast: ast.Module = ast.parse(file_path.read_text(encoding="utf-8")) - except SyntaxError as error: - raise PythonNodeDiscoveryError( - f"Failed to import materialization file {file_path.relative_to(project_dir)}: {error}" - ) from error - if any( - isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == removed_hook_name - for node in module_ast.body - ): - raise PythonNodeDiscoveryError(removed_hook_message) - module: ModuleType = _exec_project_module( - module_name=".".join( - project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts - ), - file_path=file_path, - project_dir=project_dir, - file_label="materialization", - error_type=PythonNodeDiscoveryError, - ) - if hasattr(module, removed_hook_name): - raise PythonNodeDiscoveryError(removed_hook_message) - return module diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py new file mode 100644 index 0000000000..4134e3df5b --- /dev/null +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py @@ -0,0 +1,396 @@ +"""Import project Python modules and read live definitions for native discovery.""" + +from __future__ import annotations + +import ast +import importlib.util +import inspect +import sys +from collections.abc import Callable +from importlib.machinery import ModuleSpec +from pathlib import Path +from types import ModuleType +from typing import get_type_hints + +from sqlbuild._native import ( + EventExporterDiscoveryError, + ProviderDiscoveryError, + PythonNodeDiscoveryError, + project_relative_path, +) +from sqlbuild.compiler.discovery.constants import ( + PYTHON_INIT_MODULE_STEM, + PYTHON_NODE_ROOT, +) +from sqlbuild.compiler.discovery.models import DiscoveredProvider, DiscoveredProviderUsage +from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORTER_EVENT_PARAMETER_NAME +from sqlbuild.runtime.observability.models import LifecycleEvent + +_PROJECT_PYTHON_PACKAGE_MARKER: str = "__sqlbuild_project_python__" + + +def module_functions(module: ModuleType) -> tuple[Callable[..., object], ...]: + """Functions defined in one loaded module, in `inspect.getmembers` order.""" + + return tuple( + value + for _, value in inspect.getmembers(module, inspect.isfunction) + if value.__module__ == module.__name__ + ) + + +def module_provider_classes(module: ModuleType) -> tuple[type, ...]: + """Concrete provider classes defined in one loaded module.""" + + from sqlbuild.provider.classes.provider import Provider + + return tuple( + value + for _, value in inspect.getmembers(module, inspect.isclass) + if value.__module__ == module.__name__ + and value is not Provider + and issubclass(value, Provider) + and not inspect.isabstract(value) + ) + + +def requires_arguments(function: Callable[..., object]) -> bool: + """Whether a loaded user function declares any parameter.""" + + return bool(inspect.signature(function).parameters) + + +def source_line(function: Callable[..., object]) -> int: + """The first source line of a loaded user function.""" + + return inspect.getsourcelines(function)[1] + + +def parameter_annotations(function: Callable[..., object]) -> tuple[tuple[str, object], ...]: + """Each parameter name of a loaded user function with its resolved annotation.""" + + parameters: tuple[inspect.Parameter, ...] = tuple( + inspect.signature(function).parameters.values() + ) + type_hints: dict[str, object] = get_type_hints(function) + return tuple( + (parameter.name, type_hints.get(parameter.name, parameter.annotation)) + for parameter in parameters + ) + + +def validate_event_exporter_signature( + *, function: Callable[..., object], exporter_name: str, relative_path: Path +) -> None: + """Reject lifecycle sink signatures the runtime cannot call.""" + + if inspect.iscoroutinefunction(function): + raise EventExporterDiscoveryError( + f"Event exporter '{exporter_name}' in {relative_path} must be synchronous" + ) + parameters: tuple[inspect.Parameter, ...] = tuple( + inspect.signature(function).parameters.values() + ) + if not parameters or parameters[0].name != EVENT_EXPORTER_EVENT_PARAMETER_NAME: + raise EventExporterDiscoveryError( + f"Event exporter '{exporter_name}' in {relative_path} must declare event first" + ) + if any( + parameter.kind + in { + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.VAR_POSITIONAL, + inspect.Parameter.VAR_KEYWORD, + } + for parameter in parameters + ): + raise EventExporterDiscoveryError( + f"Event exporter '{exporter_name}' in {relative_path} must use named parameters" + ) + if any(parameter.default is not inspect.Parameter.empty for parameter in parameters): + raise EventExporterDiscoveryError( + f"Event exporter '{exporter_name}' in {relative_path} parameters must not have defaults" + ) + try: + type_hints: dict[str, object] = get_type_hints(function) + except (NameError, TypeError) as error: + raise EventExporterDiscoveryError( + f"Event exporter '{exporter_name}' in {relative_path} has invalid annotations" + ) from error + event_annotation: object = type_hints.get("event", parameters[0].annotation) + if event_annotation not in {inspect.Parameter.empty, LifecycleEvent}: + raise EventExporterDiscoveryError( + f"Event exporter '{exporter_name}' in {relative_path} event must be LifecycleEvent" + ) + return_annotation: object = type_hints.get( + "return", inspect.signature(function).return_annotation + ) + if return_annotation not in {inspect.Signature.empty, None, type(None)}: + raise EventExporterDiscoveryError( + f"Event exporter '{exporter_name}' in {relative_path} return annotation must be None" + ) + + +def install_python_root_package(*, node_root: Path, project_dir: Path) -> dict[str, ModuleType]: + """Bind python to this project's python/ folder and return displaced unrelated modules.""" + + existing: ModuleType | None = sys.modules.get(PYTHON_NODE_ROOT) + owned: bool = existing is None or bool(getattr(existing, _PROJECT_PYTHON_PACKAGE_MARKER, False)) + saved_modules: dict[str, ModuleType] = {} + module_name: str + for module_name in _python_root_module_names(): + module: ModuleType | None = sys.modules.pop(module_name, None) + if not owned and module is not None: + saved_modules[module_name] = module + importlib.invalidate_caches() + init_file: Path = node_root / f"{PYTHON_INIT_MODULE_STEM}.py" + spec: ModuleSpec | None = ( + importlib.util.spec_from_file_location( + PYTHON_NODE_ROOT, init_file, submodule_search_locations=[str(node_root)] + ) + if init_file.is_file() + else ModuleSpec(PYTHON_NODE_ROOT, None, is_package=True) + ) + if spec is None: + raise PythonNodeDiscoveryError(f"Could not load Python node package {node_root}") + spec.submodule_search_locations = [str(node_root)] + package: ModuleType = importlib.util.module_from_spec(spec) + setattr(package, _PROJECT_PYTHON_PACKAGE_MARKER, True) + sys.modules[PYTHON_NODE_ROOT] = package + if spec.loader is None: + return saved_modules + old_path: list[str] = list(sys.path) + sys.path.insert(0, str(project_dir)) + try: + spec.loader.exec_module(package) + except Exception as error: + restore_python_root_modules(saved_modules=saved_modules) + raise PythonNodeDiscoveryError( + f"Failed to import Python node file {init_file.relative_to(project_dir)}: {error}" + ) from error + finally: + sys.path = old_path + return saved_modules + + +def restore_python_root_modules(*, saved_modules: dict[str, ModuleType]) -> None: + """Put back an unrelated python package that discovery displaced.""" + + if not saved_modules: + return + module_name: str + for module_name in _python_root_module_names(): + sys.modules.pop(module_name, None) + sys.modules.update(saved_modules) + + +def import_python_root_module(*, file_path: Path, project_dir: Path) -> ModuleType: + """Import a python/ module under its package name, reusing it within a discovery pass.""" + + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) + module_name: str = ".".join(relative_path.with_suffix("").parts) + old_path: list[str] = list(sys.path) + sys.path.insert(0, str(project_dir)) + try: + return importlib.import_module(module_name) + except Exception as error: + raise PythonNodeDiscoveryError( + f"Failed to import Python node file {relative_path}: {error}" + ) from error + finally: + sys.path = old_path + + +def load_python_node_module(*, file_path: Path, project_dir: Path) -> ModuleType: + """Execute one hook module under a private module name.""" + + return _exec_project_module( + module_name="sqlbuild_project_python_node_" + + "_".join( + project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts + ), + file_path=file_path, + project_dir=project_dir, + file_label="Python node", + error_type=PythonNodeDiscoveryError, + ) + + +def load_provider_module(*, file_path: Path, project_dir: Path) -> ModuleType: + """Import one providers/ module under its package name.""" + + return _load_project_package_module( + file_path=file_path, + project_dir=project_dir, + file_label="provider", + error_type=ProviderDiscoveryError, + ) + + +def load_sink_module(*, file_path: Path, project_dir: Path) -> ModuleType: + """Import one sinks/ module under its package name.""" + + return _load_project_package_module( + file_path=file_path, + project_dir=project_dir, + file_label="sink", + error_type=EventExporterDiscoveryError, + ) + + +def load_materialize_function( + *, file_path: Path, project_dir: Path +) -> Callable[..., object] | None: + """Execute one materializations/ module and return its callable `materialize`, if any.""" + + removed_hook_name: str = "prepare_version" + removed_hook_message: str = ( + f"materialization '{file_path.stem}' at {file_path} defines " + "'prepare_version', which was removed with virtual environments; " + "projects run in direct mode" + ) + try: + module_ast: ast.Module = ast.parse(file_path.read_text(encoding="utf-8")) + except SyntaxError as error: + raise PythonNodeDiscoveryError( + f"Failed to import materialization file {file_path.relative_to(project_dir)}: {error}" + ) from error + if any( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == removed_hook_name + for node in module_ast.body + ): + raise PythonNodeDiscoveryError(removed_hook_message) + module: ModuleType = _exec_project_module( + module_name=".".join( + project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts + ), + file_path=file_path, + project_dir=project_dir, + file_label="materialization", + error_type=PythonNodeDiscoveryError, + ) + if hasattr(module, removed_hook_name): + raise PythonNodeDiscoveryError(removed_hook_message) + materialize_fn: object = getattr(module, "materialize", None) + return materialize_fn if callable(materialize_fn) else None + + +def _python_root_module_names() -> tuple[str, ...]: + return tuple( + module_name + for module_name in sys.modules + if module_name == PYTHON_NODE_ROOT or module_name.startswith(f"{PYTHON_NODE_ROOT}.") + ) + + +def _load_project_package_module( + *, + file_path: Path, + project_dir: Path, + file_label: str, + error_type: type[Exception], +) -> ModuleType: + module_name: str = ".".join( + project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts + ) + _evict_stale_project_package_modules( + root_module=module_name.split(".", maxsplit=1)[0], + project_dir=project_dir, + ) + existing_module: ModuleType | None = sys.modules.get(module_name) + if existing_module is not None: + existing_file: object = getattr(existing_module, "__file__", None) + if isinstance(existing_file, str) and Path(existing_file).resolve() == file_path.resolve(): + return existing_module + sys.modules.pop(module_name, None) + return _exec_project_module( + module_name=module_name, + file_path=file_path, + project_dir=project_dir, + file_label=file_label, + error_type=error_type, + ) + + +def _exec_project_module( + *, + module_name: str, + file_path: Path, + project_dir: Path, + file_label: str, + error_type: type[Exception], +) -> ModuleType: + spec: ModuleSpec | None = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None or spec.loader is None: + raise error_type(f"Could not load {file_label} file {file_path}") + module: ModuleType = importlib.util.module_from_spec(spec) + old_path: list[str] = list(sys.path) + sys.path.insert(0, str(project_dir)) + try: + sys.modules[module_name] = module + spec.loader.exec_module(module) + except Exception as error: + raise error_type( + f"Failed to import {file_label} file {file_path.relative_to(project_dir)}: {error}" + ) from error + finally: + sys.path = old_path + return module + + +def _evict_stale_project_package_modules(*, root_module: str, project_dir: Path) -> None: + root_path: Path = (project_dir / root_module).resolve() + module_name: str + module: ModuleType + for module_name, module in tuple(sys.modules.items()): + if module_name != root_module and not module_name.startswith(f"{root_module}."): + continue + module_file: object = getattr(module, "__file__", None) + if isinstance(module_file, str): + try: + Path(module_file).resolve().relative_to(project_dir.resolve()) + continue + except ValueError: + sys.modules.pop(module_name, None) + continue + module_paths: object = getattr(module, "__path__", None) + if module_paths is None: + sys.modules.pop(module_name, None) + continue + if any(Path(path).resolve() == root_path for path in module_paths): + continue + sys.modules.pop(module_name, None) + + +def provider_usages( + *, function: Callable[..., object], provider_by_name: dict[str, DiscoveredProvider] +) -> tuple[DiscoveredProviderUsage, ...]: + """Provider parameters of a loaded user function; `TypeError` hints fall back to raw ones.""" + + usages: list[DiscoveredProviderUsage] = [] + try: + type_hints: dict[str, object] = get_type_hints(function) + except TypeError: + type_hints = {} + parameter: inspect.Parameter + for parameter in inspect.signature(function).parameters.values(): + discovered_provider: DiscoveredProvider | None = provider_by_name.get(parameter.name) + if discovered_provider is None: + continue + from sqlbuild.provider.classes.provider import Provider + + annotation: object = type_hints.get(parameter.name, parameter.annotation) + annotation_class_name: str | None = None + annotation_module: str | None = None + if isinstance(annotation, type) and issubclass(annotation, Provider): + annotation_class_name = annotation.__name__ + annotation_module = annotation.__module__ + usages.append( + DiscoveredProviderUsage( + provider_name=discovered_provider.name, + parameter_name=parameter.name, + annotation_class_name=annotation_class_name, + annotation_module=annotation_module, + ) + ) + return tuple(usages) diff --git a/src/sqlbuild/compiler/discovery/exceptions.py b/src/sqlbuild/compiler/discovery/exceptions.py deleted file mode 100644 index 12ffcbe54f..0000000000 --- a/src/sqlbuild/compiler/discovery/exceptions.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Expected discovery-stage exception types.""" - -from __future__ import annotations - - -class DiscoveryError(ValueError): - """Base error for expected discovery failures.""" - - code: str = "D000" - - def __init__(self, message: str, *, code: str | None = None, help: str | None = None) -> None: - super().__init__(message) - self.message = message - self.code = code if code is not None else self.code - self.help = help - - -class ProjectConfigError(DiscoveryError): - """Raised when shared or local project config files are invalid.""" - - code: str = "D001" - - -class ModelSqlParseError(DiscoveryError): - """Raised when an authored SQL model file has an invalid shape.""" - - code: str = "D002" - - -class SqlTestParseError(DiscoveryError): - """Raised when an authored SQL-native test file has an invalid shape.""" - - code: str = "D003" - - -class SqlAuditParseError(DiscoveryError): - """Raised when an authored SQL audit file has an invalid shape.""" - - code: str = "D004" - - -class SchemaParseError(DiscoveryError): - """Raised when a schema.yml file has an invalid shape.""" - - code: str = "D005" - - -class SourceParseError(DiscoveryError): - """Raised when a sources/*.yml file has an invalid shape.""" - - code: str = "D006" - - -class DiscoveryConflictError(DiscoveryError): - """Raised when discovered project inputs conflict across files.""" - - code: str = "D007" - - -class SeedDiscoveryError(DiscoveryError): - """Raised when declared seed metadata does not match local seed files.""" - - code: str = "D008" - - -class SqlScenarioParseError(DiscoveryError): - """Raised when an authored SQL scenario file has an invalid shape.""" - - code: str = "D009" - - -class PythonNodeDiscoveryError(DiscoveryError): - """Raised when project Python nodes cannot be discovered.""" - - code: str = "D011" - - -class ProviderDiscoveryError(DiscoveryError): - """Raised when project providers cannot be discovered.""" - - code: str = "D012" - - -class DeclarationParseError(DiscoveryError): - """Raised when an authored public SQL declaration is invalid.""" - - code: str = "D013" - - -class SqlHookParseError(DiscoveryError): - """Raised when an authored SQL hook resource has an invalid shape.""" - - code: str = "D014" - - -class EventExporterDiscoveryError(DiscoveryError): - """Raised when project lifecycle event exporters are invalid.""" - - code: str = "D015" - - -class ProjectPythonPathError(DiscoveryError): - """Raised when project-owned Python lives outside a supported extension root.""" - - code: str = "D016" - - -class ProjectPathError(DiscoveryError): - """Raised when an authored project path cannot be read as valid UTF-8.""" - - code: str = "D016" diff --git a/src/sqlbuild/compiler/discovery/main/runtime_extensions.py b/src/sqlbuild/compiler/discovery/main/runtime_extensions.py index 6af1f392ea..32b378c44a 100644 --- a/src/sqlbuild/compiler/discovery/main/runtime_extensions.py +++ b/src/sqlbuild/compiler/discovery/main/runtime_extensions.py @@ -4,16 +4,16 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( - bind_command_output_sink_declarations, - discover_command_output_sink_declarations, -) -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( +from sqlbuild._native import ( + ProjectConfigError, bind_event_exporter_declarations, discover_event_exporter_declarations, discover_provider_classes, ) -from sqlbuild.compiler.discovery.exceptions import ProjectConfigError +from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( + bind_command_output_sink_declarations, + discover_command_output_sink_declarations, +) from sqlbuild.compiler.discovery.models import ( DiscoveredCommandOutputSink, DiscoveredCommandOutputSinkDeclaration, @@ -22,7 +22,6 @@ DiscoveredProvider, DiscoveredRuntimeExtensions, ) -from sqlbuild.runtime.event_exporting.main.stricter_severity import stricter_severity from sqlbuild.spec.contracts.models import ( LifecycleEventSinkFilterConfig, LifecycleEventSinksConfig, @@ -89,5 +88,7 @@ def _apply_runtime_filter( for runtime_filter in filters: if runtime_filter.event_kinds is not None: kinds = kinds & runtime_filter.event_kinds - severity = stricter_severity(first=severity, second=runtime_filter.min_severity) + severity = _native.stricter_event_export_severity( + first=severity, second=runtime_filter.min_severity + ) return replace(exporter, event_kinds=kinds, min_severity=severity) diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index 17ee06a0c9..9958fad246 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -6,11 +6,23 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild._native import DiscoveredSqlModelFile, NativeDiscoverySession -from sqlbuild.compiler.auditing.models import MeasurementContract -from sqlbuild.compiler.auditing.types import AuditEvaluationMode -from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE -from sqlbuild.compiler.compile.types import SqlTestMode +from sqlbuild._native import ( + CollectionRendering, + DiscoveredAdapterFile, + DiscoveredAuditFile, + DiscoveredConstantFile, + DiscoveredEnumFile, + DiscoveredModelSchemaFile, + DiscoveredPythonFunctionFile, + DiscoveredSchemaFile, + DiscoveredSeedFile, + DiscoveredSqlFunctionFile, + DiscoveredSqlHookFile, + DiscoveredSqlModelFile, + DiscoveredSqlTestBlock, + EnumMember, + NativeDiscoverySession, +) from sqlbuild.compiler.discovery.constants import ( SQL_SCENARIOS_OWNERSHIP_ROOT, SQL_TESTS_OWNERSHIP_ROOT, @@ -19,7 +31,7 @@ LoaderConnectionMode, ProjectProvider, ) -from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind +from sqlbuild.compiler.scopes.types import ScopeKind from sqlbuild.python_nodes.models import AuditCase, ColumnLineageRef, RetryPolicy, SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORT_KINDS @@ -27,15 +39,12 @@ from sqlbuild.spec.contracts.models import ( LocalConfig, ProjectConfig, - SchemaColumn, - SchemaModelEntry, - SchemaSeedEntry, SourceColumnEntry, SourceEntry, ) from sqlbuild.spec.contracts.types import EventExportSeverity, SourceWriteStrategy from sqlbuild.sql_values.models import SqlLogicalType, SqlValue -from sqlbuild.sql_values.types import CollectionRendering, SqlValueKind +from sqlbuild.sql_values.types import SqlValueKind @dataclass(frozen=True) @@ -51,14 +60,6 @@ class SqlHookEntry: reads: tuple[SqlResourceRef, ...] = () -@dataclass(frozen=True) -class NamedSqlHookEntry: - """An unresolved invocation of a discovered SQL hook resource.""" - - name: str - kwargs: dict[str, object] - - @dataclass(frozen=True) class PythonHookEntry: """A model lifecycle hook that invokes a discovered Python hook.""" @@ -67,42 +68,6 @@ class PythonHookEntry: kwargs: dict[str, object] -@dataclass(frozen=True) -class DiscoveredSqlFunctionFile: - """A discovered SQL function file and its raw contents.""" - - file_path: Path - relative_path: Path - contents: str - header_values: dict[str, object] - body_sql: str - - -@dataclass(frozen=True) -class DiscoveredSqlHookFile: - """A discovered named SQL lifecycle hook resource.""" - - file_path: Path - relative_path: Path - contents: str - header_values: dict[str, object] - sql_body: str - name: str - description: str | None = None - scope_kind: ScopeKind = ScopeKind.GLOBAL - ownership_root: Path | None = None - owning_path: Path | None = None - declaration_root: Path | None = None - - -@dataclass(frozen=True) -class EnumMember: - """One named member in a SQLBuild enum declaration.""" - - name: str - value: str | int - - @dataclass(frozen=True) class EnumDeclaration: """One validated public or model-local enum declaration.""" @@ -145,82 +110,6 @@ def scalar_type(self) -> str: return names.get(self.value.kind, self.value.logical_type.display_name.upper()) -@dataclass(frozen=True) -class DiscoveredEnumFile: - """A public SQL file containing one or more enum declarations.""" - - file_path: Path - relative_path: Path - contents: str - declarations: tuple[EnumDeclaration, ...] - scope_kind: ScopeKind = ScopeKind.GLOBAL - ownership_root: Path | None = None - owning_path: Path | None = None - declaration_root: Path | None = None - - -@dataclass(frozen=True) -class DiscoveredConstantFile: - """A public SQL file containing one or more constant declarations.""" - - file_path: Path - relative_path: Path - contents: str - declarations: tuple[ConstantDeclaration, ...] - scope_kind: ScopeKind = ScopeKind.GLOBAL - ownership_root: Path | None = None - owning_path: Path | None = None - declaration_root: Path | None = None - - -@dataclass(frozen=True) -class ModelSchemaDeclaration: - """One public reusable model column schema declaration.""" - - name: str - columns: tuple[SchemaColumn, ...] - relative_path: Path - extends: str | None = None - description: str | None = None - - -@dataclass(frozen=True) -class DiscoveredModelSchemaFile: - """A public SQL file containing reusable model schemas.""" - - file_path: Path - relative_path: Path - contents: str - declarations: tuple[ModelSchemaDeclaration, ...] - scope_kind: ScopeKind = ScopeKind.GLOBAL - ownership_root: Path | None = None - owning_path: Path | None = None - declaration_root: Path | None = None - - -@dataclass(frozen=True) -class DiscoveredPythonFunctionFile: - """A discovered Python function file and its parsed UDF metadata.""" - - file_path: Path - relative_path: Path - contents: str - header_values: dict[str, object] - entry_point: str - body_python: str - - -@dataclass(frozen=True) -class DiscoveredSchemaFile: - """A discovered schema.yml file and its raw contents.""" - - file_path: Path - relative_path: Path - contents: str - model_entries: tuple[SchemaModelEntry, ...] - seed_entries: tuple[SchemaSeedEntry, ...] - - @dataclass(frozen=True) class DiscoveredSourceFile: """A discovered source declaration file and its raw contents.""" @@ -231,14 +120,6 @@ class DiscoveredSourceFile: source_entries: tuple[SourceEntry, ...] -@dataclass(frozen=True) -class DiscoveredSeedFile: - """A discovered seed file.""" - - file_path: Path - relative_path: Path - - @dataclass(frozen=True) class DiscoveredSqlTestFile: """A discovered SQL-native test file and its raw contents.""" @@ -259,31 +140,6 @@ class SqlTestParameterDeclaration: nullable: bool = False -@dataclass(frozen=True) -class DiscoveredSqlTestCase: - """One ordered named case belonging to a SQL-native test template.""" - - name: str - values: tuple[tuple[str, SqlValue], ...] - case_index: int - - -@dataclass(frozen=True) -class DiscoveredSqlTestBlock: - """One raw TEST(...) block discovered from a SQL-native test file.""" - - test_index: int - header_values: dict[str, object] - sql_body: str - name: str | None = None - mode: SqlTestMode = DEFAULT_SQL_TEST_MODE - parameters: tuple[SqlTestParameterDeclaration, ...] = field(default_factory=tuple) - cases: tuple[DiscoveredSqlTestCase, ...] = field(default_factory=tuple) - cursor_start: str | None = None - cursor_end: str | None = None - sql_body_span: tuple[int, int] | None = None - - @dataclass(frozen=True) class DiscoveredSqlScenarioFile: """A discovered SQL-native scenario file and its raw contents.""" @@ -298,35 +154,6 @@ class DiscoveredSqlScenarioFile: sql_body_span: tuple[int, int] | None = None -@dataclass(frozen=True) -class DiscoveredAuditFile: - """A discovered audit SQL file and its raw contents.""" - - file_path: Path - relative_path: Path - contents: str - blocks: tuple[DiscoveredAuditBlock, ...] - declaration_kind: DeclarationKind = DeclarationKind.AUDIT - scope_kind: ScopeKind = ScopeKind.GLOBAL - ownership_root: Path | None = None - owning_path: Path | None = None - declaration_root: Path | None = None - - -@dataclass(frozen=True) -class DiscoveredAuditBlock: - """One raw AUDIT(...) block discovered from a SQL audit file.""" - - audit_index: int - header_values: dict[str, object] - sql_body: str - name: str | None = None - evaluation_mode: AuditEvaluationMode = AuditEvaluationMode.VIOLATIONS - measurement_contract: MeasurementContract | None = None - measure_sql: str | None = None - evidence_sql: str | None = None - - @dataclass(frozen=True) class DiscoveredMacroFile: """A discovered project macro file and its raw contents.""" @@ -340,14 +167,6 @@ class DiscoveredMacroFile: declaration_root: Path | None = None -@dataclass(frozen=True) -class DiscoveredAdapterFile: - """A detected project adapter Python file.""" - - file_path: Path - relative_path: Path - - @dataclass(frozen=True) class DiscoveredMaterializationFile: """A discovered custom materialization Python file.""" @@ -588,22 +407,3 @@ class DiscoveredRuntimeExtensions: event_exporters: tuple[DiscoveredEventExporter, ...] = field(default_factory=tuple) command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = field(default_factory=tuple) lifecycle_shutdown_timeout_seconds: int | None = None - - -@dataclass(frozen=True) -class DiscoveryFileFault: - """One project-relative authored file fault captured during tolerant discovery.""" - - path: Path | None - message: str - - -@dataclass(frozen=True) -class TolerantScopeDiscovery: - """Parsed scope inputs and independently categorized tolerant discovery faults.""" - - discovered_inputs: DiscoveredProjectInputs - resource_faults: tuple[DiscoveryFileFault, ...] = field(default_factory=tuple) - declaration_faults: tuple[DiscoveryFileFault, ...] = field(default_factory=tuple) - relationship_faults: tuple[DiscoveryFileFault, ...] = field(default_factory=tuple) - config_faults: tuple[DiscoveryFileFault, ...] = field(default_factory=tuple) diff --git a/src/sqlbuild/compiler/fingerprints/exceptions.py b/src/sqlbuild/compiler/fingerprints/exceptions.py deleted file mode 100644 index e715da351c..0000000000 --- a/src/sqlbuild/compiler/fingerprints/exceptions.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Expected fingerprint storage exception types.""" - -from __future__ import annotations - - -class FingerprintInputError(ValueError): - """Raised when fingerprint storage inputs or rows are invalid.""" - - code: str = "F000" - - def __init__(self, message: str, *, code: str | None = None, help: str | None = None) -> None: - super().__init__(message) - self.message = message - self.code = code if code is not None else self.code - self.help = help - - -class QueryFingerprintError(ValueError): - """Raised when SQL cannot be tokenized into a change fingerprint.""" diff --git a/src/sqlbuild/compiler/frontier/exceptions.py b/src/sqlbuild/compiler/frontier/exceptions.py deleted file mode 100644 index e423066531..0000000000 --- a/src/sqlbuild/compiler/frontier/exceptions.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Compiler frontier errors.""" - - -class NativeStageMismatchError(RuntimeError): - """Raised when native state the compiler replays is inconsistent with its inputs.""" diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py deleted file mode 100644 index 302627d586..0000000000 --- a/src/sqlbuild/compiler/frontier/types.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Frontier stage identities.""" - -from enum import StrEnum - - -class CompilerStage(StrEnum): - """Frontier objects handed from one compiler stage to the next, in pipeline order.""" - - DISCOVERED_PROJECT_INPUTS = "discovered_project_inputs" - COMPILE_PROJECT_INPUTS = "compile_project_inputs" - COMPILED_PROJECT = "compiled_project" diff --git a/src/sqlbuild/compiler/graph/models.py b/src/sqlbuild/compiler/graph/models.py deleted file mode 100644 index 9dd0c15426..0000000000 --- a/src/sqlbuild/compiler/graph/models.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Structured graph facts shared by compile validation and execution planning.""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.graph.types import HookReadType - - -@dataclass(frozen=True) -class LineageGraphViews: - """A native project graph's lineage edges and selector indexes as Python dicts.""" - - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] - tag_index: dict[str, frozenset[CompiledObjectKey]] - path_index: dict[CompiledObjectKey, str] - all_keys: dict[str, CompiledObjectKey] - - -@dataclass(frozen=True) -class AttachedAuditGateEdge: - """One ordering edge: the gated node waits for a resource its attached audit reads.""" - - audit_name: str - target: CompiledObjectKey - gated: CompiledObjectKey - read: CompiledObjectKey - - -@dataclass(frozen=True) -class HookReadEdge: - """One ordering edge: a model waits for a resource one of its hooks reads.""" - - hook_name: str - gated: CompiledObjectKey - read: CompiledObjectKey - hook_type: HookReadType = HookReadType.PYTHON - hook_path: Path | None = None - - @property - def label(self) -> str: - """Return how diagnostics name the hook, such as ``SQL hook 'grant_access'``.""" - - if self.hook_type is HookReadType.INLINE_SQL: - return f"inline SQL hook {self.hook_name}" - return f"{self.hook_type.value} hook '{self.hook_name}'" diff --git a/src/sqlbuild/compiler/graph/types.py b/src/sqlbuild/compiler/graph/types.py deleted file mode 100644 index 2924bb2f75..0000000000 --- a/src/sqlbuild/compiler/graph/types.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Enumerations and aliases shared by graph facts.""" - -from __future__ import annotations - -from enum import StrEnum - - -class HookReadType(StrEnum): - """Kind of model hook that declares or writes a read.""" - - PYTHON = "Python" - SQL = "SQL" - INLINE_SQL = "inline SQL" diff --git a/src/sqlbuild/compiler/lineage/exceptions.py b/src/sqlbuild/compiler/lineage/exceptions.py deleted file mode 100644 index 5913f3a8a7..0000000000 --- a/src/sqlbuild/compiler/lineage/exceptions.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Column-lineage failures.""" - - -class UnknownLineageDialectError(ValueError): - """Raised, as Python's parser raised `ValueError`, for a dialect Polyglot does not know.""" diff --git a/src/sqlbuild/compiler/lineage/models.py b/src/sqlbuild/compiler/lineage/models.py deleted file mode 100644 index 34d5e52baf..0000000000 --- a/src/sqlbuild/compiler/lineage/models.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Column lineage result models.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.lineage.types import ( - ColumnLineageConfidence, - ColumnTransformKind, - InferredNullability, -) - - -@dataclass(frozen=True) -class QualifiedLineageColumn: - """A resource-qualified column in the collapsed SQLBuild lineage graph.""" - - resource_type: CompiledResourceType | str - resource_name: str - column_name: str - - def __post_init__(self) -> None: - object.__setattr__(self, "resource_type", CompiledResourceType(self.resource_type)) - - -@dataclass(frozen=True) -class ColumnLineageSource: - """One upstream column dependency for an output column.""" - - resource_type: CompiledResourceType | str - resource_name: str - column_name: str - - def __post_init__(self) -> None: - object.__setattr__(self, "resource_type", CompiledResourceType(self.resource_type)) - - def as_qualified_column(self) -> QualifiedLineageColumn: - return QualifiedLineageColumn( - resource_type=self.resource_type, - resource_name=self.resource_name, - column_name=self.column_name, - ) - - -@dataclass(frozen=True) -class ColumnLineageEdge: - """One collapsed SQLBuild lineage graph edge.""" - - source: QualifiedLineageColumn - target: QualifiedLineageColumn - transform_kind: ColumnTransformKind = ColumnTransformKind.UNKNOWN - confidence: ColumnLineageConfidence = ColumnLineageConfidence.UNKNOWN - - -@dataclass(frozen=True) -class DirectSemanticColumnUse: - """One direct non-projection use of a resolved resource column.""" - - consumer_model: str - context: str - expression_sql: str - source: QualifiedLineageColumn - confidence: ColumnLineageConfidence = ColumnLineageConfidence.UNKNOWN - line: int | None = None - column: int | None = None - - -@dataclass(frozen=True) -class ColumnLineage: - """Lineage for one output column in one model.""" - - output_column: str - transform_kind: ColumnTransformKind - expression_sql: str | None - upstream_columns: tuple[ColumnLineageSource, ...] - nullability: InferredNullability = InferredNullability.UNKNOWN - confidence: ColumnLineageConfidence = ColumnLineageConfidence.UNKNOWN - - -@dataclass(frozen=True) -class ModelColumnLineage: - """Column lineage extracted for one compiled model.""" - - model_name: str - columns: tuple[ColumnLineage, ...] - has_star: bool = False diff --git a/src/sqlbuild/compiler/lineage/types.py b/src/sqlbuild/compiler/lineage/types.py index 1a7ba6da30..5c1151ce5a 100644 --- a/src/sqlbuild/compiler/lineage/types.py +++ b/src/sqlbuild/compiler/lineage/types.py @@ -5,76 +5,9 @@ from enum import StrEnum -class ColumnTransformKind(StrEnum): - """High-level transform classification for one output column.""" - - DIRECT = "direct" - CAST = "cast" - EXPRESSION = "expression" - AGGREGATION = "aggregation" - STAR = "star" - CONSTANT = "constant" - UNKNOWN = "unknown" - - class InferredNullability(StrEnum): """Conservative nullability state inferred for an output column.""" NON_NULL = "non_null" NULLABLE = "nullable" UNKNOWN = "unknown" - - -class ColumnLineageConfidence(StrEnum): - """Coarse confidence that a lineage edge is fully understood.""" - - HIGH = "high" - MEDIUM = "medium" - LOW = "low" - UNKNOWN = "unknown" - - -class ColumnLineageMode(StrEnum): - """Column lineage analyzer mode.""" - - RICH = "rich" - FAST = "fast" - - -class PolyglotAnalysisDialect(StrEnum): - """Dialect names accepted by Polyglot analyze_query.""" - - GENERIC = "generic" - POSTGRESQL = "postgresql" - MYSQL = "mysql" - BIGQUERY = "bigquery" - SNOWFLAKE = "snowflake" - DUCKDB = "duckdb" - SQLITE = "sqlite" - HIVE = "hive" - SPARK = "spark" - TRINO = "trino" - PRESTO = "presto" - REDSHIFT = "redshift" - TSQL = "tsql" - ORACLE = "oracle" - CLICKHOUSE = "clickhouse" - DATABRICKS = "databricks" - ATHENA = "athena" - TERADATA = "teradata" - DORIS = "doris" - STARROCKS = "starrocks" - MATERIALIZE = "materialize" - RISINGWAVE = "risingwave" - SINGLESTORE = "singlestore" - COCKROACHDB = "cockroachdb" - TIDB = "tidb" - DRUID = "druid" - SOLR = "solr" - TABLEAU = "tableau" - DUNE = "dune" - FABRIC = "fabric" - DRILL = "drill" - DREMIO = "dremio" - EXASOL = "exasol" - DATAFUSION = "datafusion" diff --git a/src/sqlbuild/compiler/migrations/main/__init__.py b/src/sqlbuild/compiler/migrations/main/__init__.py deleted file mode 100644 index c0e9893620..0000000000 --- a/src/sqlbuild/compiler/migrations/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Model migration state entry modules.""" diff --git a/src/sqlbuild/compiler/migrations/models.py b/src/sqlbuild/compiler/migrations/models.py index 3ad791bf8e..115abbdb10 100644 --- a/src/sqlbuild/compiler/migrations/models.py +++ b/src/sqlbuild/compiler/migrations/models.py @@ -5,10 +5,10 @@ from dataclasses import dataclass from datetime import datetime +from sqlbuild._native import MigrationDiscovery from sqlbuild.compiler.migrations.types import ( ColumnMigrationDecision, MigrationDecision, - MigrationDiscovery, OldNameViewDropReason, OldNameViewEventType, OldNameViewStatus, diff --git a/src/sqlbuild/compiler/migrations/types.py b/src/sqlbuild/compiler/migrations/types.py index 56eba6ffd6..ff24918059 100644 --- a/src/sqlbuild/compiler/migrations/types.py +++ b/src/sqlbuild/compiler/migrations/types.py @@ -55,13 +55,6 @@ def label(self) -> str: ) -class MigrationDiscovery(StrEnum): - """How a migration was requested.""" - - MANUAL = "manual" - AUTOMATIC = "automatic" - - class MigrationPromotion(StrEnum): """How a completed migration stage replaces or creates the destination.""" @@ -76,14 +69,6 @@ def label(self) -> str: return self.value.replace("_", " ") -class MigrationCompatibility(StrEnum): - """Outcome of checking the origin relation against the destination model.""" - - COMPATIBLE = "compatible" - INCOMPATIBLE = "incompatible" - NOT_CHECKED = "not_checked" - - class ColumnMigrationDecision(StrEnum): """Per-run decision for one declared or detected column rename.""" @@ -147,15 +132,6 @@ class OldNameViewStatus(StrEnum): DROPPED = "dropped" -class OldNameViewAction(StrEnum): - """What a build does at a migrated model's old name.""" - - ARCHIVE_AND_VIEW = "archive_and_view" - VIEW_ONLY = "view_only" - LIVE = "live" - NONE = "none" - - _RECORDING_COLUMN_DECISIONS: frozenset[ColumnMigrationDecision] = frozenset( {ColumnMigrationDecision.RENAME, ColumnMigrationDecision.RECORD} ) diff --git a/src/sqlbuild/compiler/pipeline/models.py b/src/sqlbuild/compiler/pipeline/models.py index 2eeaaddabc..db16309d72 100644 --- a/src/sqlbuild/compiler/pipeline/models.py +++ b/src/sqlbuild/compiler/pipeline/models.py @@ -8,28 +8,16 @@ from typing import Any import sqlbuild._native as _native -from sqlbuild._native import lineage_graph_views +from sqlbuild._native import ( + LineageGraphViews, + PythonPlanEntry, + lineage_graph_views, +) from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProviderUsage -from sqlbuild.compiler.graph.models import LineageGraphViews from sqlbuild.compiler.planner.models import ( - CloneSourcePlanEntry, - CursorOverrides, - FunctionPlanEntry, - ModelPlanEntry, - PlannerRelationsContext, - PlannerScope, PlanOutput, - SeedPlanEntry, - SqlTestSelection, -) -from sqlbuild.compiler.python_nodes.types import ( - PythonIdentityStatus, - PythonNodeKind, - PythonRunPhase, ) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver @dataclass(frozen=True) @@ -76,58 +64,6 @@ def all_keys(self) -> dict[str, CompiledObjectKey]: return self.lineage_views.all_keys -@dataclass(frozen=True) -class CompilePipelineOptions: - """Selection, deferral, and planning options for one compile pipeline run.""" - - selected_target: str | None = None - no_sql_validation: bool = False - defer_to: str | None = None - defer_sources_to: str | None = None - source_deferral_enabled: bool = True - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - cursor_overrides: CursorOverrides | None = None - full_refresh: bool = False - auto_load_sources: bool = False - reload_sources: bool = False - connection_config: dict[str, object] | None = None - cli_vars: dict[str, object] | None = None - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None - resolve_python_run_selectors: bool = False - no_cache: bool = False - max_microbatches: int | None = None - selection_diagnostics: bool = False - plan_sql_tests: bool = True - record_migration_fingerprints: bool = True - accepts_unit_test_selectors: bool = False - sql_test_selection: SqlTestSelection = field(default_factory=SqlTestSelection) - - -@dataclass(frozen=True) -class PythonPlanEntry: - """Display-ready Python node entry for plan output.""" - - name: str - kind: PythonNodeKind - phase: PythonRunPhase - identity_status: PythonIdentityStatus = PythonIdentityStatus.UNKNOWN - current_definition_json: str | None = None - previous_definition_json: str | None = None - current_metadata_json: str | None = None - previous_metadata_json: str | None = None - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class PythonRunPlanOutputs: - """Plan output and Python entries after python-aware run selection.""" - - plan_output: PlanOutput - python_plan_entries: tuple[PythonPlanEntry, ...] - selected_python_node_names: frozenset[str] - - @dataclass(frozen=True) class CompilePipelineResult: """Complete output from the compile-and-plan pipeline.""" @@ -139,58 +75,3 @@ class CompilePipelineResult: python_plan_entries: tuple[PythonPlanEntry, ...] = field(default_factory=tuple) compile_seconds: float | None = None planning_seconds: float | None = None - - -@dataclass(frozen=True) -class CompiledProjectPhaseResult: - """Canonical project compilation output before command-specific planning.""" - - project: CompiledProjectView - connection_config: dict[str, object] - compile_seconds: float - - -@dataclass(frozen=True) -class StaticCommandContext: - """Shared compile, selection, and relation phases for focused commands.""" - - project: CompiledProjectView - scope: PlannerScope - relations: PlannerRelationsContext - connection_config: dict[str, object] - compile_seconds: float - - -@dataclass(frozen=True) -class ClonePipelineOptions: - """Compilation and selection options for one clone pipeline.""" - - no_sql_validation: bool = False - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - cli_vars: dict[str, object] | None = None - no_cache: bool = False - - -@dataclass(frozen=True) -class ClonePipelineConnection: - """Resolved destination connection used throughout clone compilation and planning.""" - - config: dict[str, object] - handle: Any - - -@dataclass(frozen=True) -class ClonePipelineResult: - """Prepared clone inputs for origin and destination target environments.""" - - origin_project: CompiledProjectView - destination_project: CompiledProjectView - clone_plan: PlanOutput - destination_source_entries: tuple[CloneSourcePlanEntry, ...] = field(default_factory=tuple) - destination_model_entries: tuple[ModelPlanEntry, ...] = field(default_factory=tuple) - destination_seed_entries: tuple[SeedPlanEntry, ...] = field(default_factory=tuple) - destination_function_entries: tuple[FunctionPlanEntry, ...] = field(default_factory=tuple) - origin_model_entries: tuple[ModelPlanEntry, ...] = field(default_factory=tuple) - origin_seed_entries: tuple[SeedPlanEntry, ...] = field(default_factory=tuple) - origin_source_entries: tuple[CloneSourcePlanEntry, ...] = field(default_factory=tuple) diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 5fafa8a24c..eedaf65977 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -12,7 +12,12 @@ BackfillResult, ColumnRenameHint, CursorWatermarkMode, + DirectSourceFreshnessPlanningResult, + FunctionReturnColumn, GraphResourceKind, + MigrationCompatibility, + MigrationDiscovery, + OldNameViewAction, PlanProviderUsage, PlanWarning, ScenarioArtifactName, @@ -20,9 +25,10 @@ SchemaAction, SchemaChangeKind, SchemaColumnSource, + SqlTestAssertionStep, ) from sqlbuild._native import cursor_parse as parse -from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RetentionRequest +from sqlbuild.adapter.contract.models import ColumnInfo, RetentionRequest from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.compiler.auditing.models import MeasurementThresholds from sqlbuild.compiler.auditing.types import ( @@ -31,25 +37,17 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, - FunctionReturnColumn, -) +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, - CompiledModelView, FunctionLanguage, ) from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, SqlTestParameterDeclaration from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.migrations.types import ( ColumnMigrationDecision, - MigrationCompatibility, MigrationDecision, - MigrationDiscovery, MigrationPromotion, - OldNameViewAction, ) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.types import ( @@ -63,10 +61,6 @@ RetentionDirection, RetentionPlanPhase, ScenarioArtifactKind, - SelectorKind, -) -from sqlbuild.compiler.source_freshness.models import ( - DirectSourceFreshnessPlanningResult, ) from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError from sqlbuild.cursor_algebra.models import ( @@ -195,26 +189,6 @@ def _parse_integer(*, field_name: str, value: str | CursorScalar | None) -> Curs raise PlannerInputError(f"{field_name} value '{value}' is not a valid integer") -@dataclass(frozen=True) -class ParsedSelector: - """One parsed selector token before graph resolution.""" - - kind: SelectorKind - value: str - upstream: bool = False - downstream: bool = False - - -@dataclass(frozen=True) -class PathSelector: - """A directed path selector between two model names with optional endpoint expansion.""" - - start_name: str - end_name: str - upstream: bool = False - downstream: bool = False - - @dataclass(frozen=True) class EmptyCursorInputDecision: """Cursor inputs whose lack of rows leaves no window, and whether other inputs have rows.""" @@ -563,45 +537,6 @@ class SchemaFinding: actual_type: str | None = None -@dataclass(frozen=True) -class SqlTestSelection: - """Unit tests selected or excluded by name, beside the tests of the selected resources.""" - - names: frozenset[str] = frozenset() - excluded_names: frozenset[str] = frozenset() - models_selected: bool = True - - -@dataclass(frozen=True) -class PlannerScope: - """Resolved graph scope for one planner invocation.""" - - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] - all_keys: dict[str, CompiledObjectKey] - models_by_name: dict[str, CompiledModelView] - selected_keys: frozenset[CompiledObjectKey] - execution_order: tuple[CompiledObjectKey, ...] - user_selected_keys: frozenset[CompiledObjectKey] = frozenset() - python_read_source_names: frozenset[str] = frozenset() - sql_test_selection: SqlTestSelection = field(default_factory=SqlTestSelection) - - -@dataclass(frozen=True) -class PlannerRelationsContext: - """Resolved relation and source inputs for plan entry construction.""" - - model_locations: dict[str, CompiledRelationLocation] - seed_locations: dict[str, CompiledRelationLocation] - function_locations: dict[str, CompiledRelationLocation] - source_map: dict[str, SourceEntry] - source_read_map: dict[str, SourceEntry] - python_source_read_map: dict[str, SourceEntry] - source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]] - star_exclude_keyword: str - listed_source_names: frozenset[str] = frozenset() - - @dataclass(frozen=True) class RetentionPlanEntry: """Warehouse retention work independent of model identity actions.""" @@ -970,16 +905,6 @@ class ChainStep: expected_lifted_ctes: tuple[tuple[str, str], ...] = field(default_factory=tuple) -@dataclass(frozen=True) -class SqlTestAssertionStep: - """One zero-row assertion step in a SQL-native unit test.""" - - name: str - resolved_sql: str - lifted_ctes: tuple[tuple[str, str], ...] = field(default_factory=tuple) - comparison_body_sql: str | None = None - - @dataclass(frozen=True) class SqlTestPlanEntry: """Per-test execution plan entry with chained resolution.""" @@ -1124,46 +1049,3 @@ def python_source_entries(self) -> dict[str, SourceEntry]: """Return the source entries Python reads resolve, as a selected SQL read sees them.""" return self.python_source_read_map or self.source_read_map or self.source_map - - -@dataclass(frozen=True) -class PlannerSelection: - """User-facing selection inputs for one planner invocation.""" - - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - selected_keys: frozenset[CompiledObjectKey] | None = None - python_read_source_names: frozenset[str] = frozenset() - sql_test_selection: SqlTestSelection = field(default_factory=SqlTestSelection) - - -@dataclass(frozen=True) -class PlannerOverrides: - """Explicit user overrides that force or bound planner decisions.""" - - cursor_overrides: CursorOverrides | None = None - full_refresh: bool = False - reload_sources: bool = False - forced_stale_model_names: tuple[str, ...] = () - external_blocked_model_names: tuple[str, ...] = () - max_microbatches: int | None = None - - -@dataclass(frozen=True) -class DeferralInputs: - """Deferred relation and source-deferral inputs for planning.""" - - deferred_locations: dict[str, CompiledRelationLocation] | None = None - deferred_relations: dict[str, RelationInfo] | None = None - defer_sources_to: str | None = None - source_deferral_enabled: bool = True - - -@dataclass(frozen=True) -class PlannerPolicies: - """Behavior policies selected by the caller for one planner invocation.""" - - auto_load_sources: bool = False - selection_diagnostics: bool = False - plan_sql_tests: bool = True - record_migration_fingerprints: bool = True diff --git a/src/sqlbuild/compiler/planner/types.py b/src/sqlbuild/compiler/planner/types.py index c0a9ec7c69..75c869829c 100644 --- a/src/sqlbuild/compiler/planner/types.py +++ b/src/sqlbuild/compiler/planner/types.py @@ -5,19 +5,6 @@ from enum import StrEnum -class SelectorKind(StrEnum): - NAME = "name" - SEED = "seed" - SOURCE = "source" - TASK = "task" - ASSET = "asset" - LOADER = "loader" - CHECK = "check" - TAG = "tag" - PATH = "path" - TEST = "test" - - class BackfillAction(StrEnum): FULL = "full" BOUNDED = "bounded" diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py b/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py index c16204c3a8..03f3fd1e3c 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py @@ -1,16 +1,12 @@ -"""Python-node facts read from loaded user functions: identities, providers and dependencies.""" +"""Python-node facts read from loaded user functions: identities and dependencies.""" from __future__ import annotations -import inspect from collections.abc import Callable from pathlib import Path -from typing import get_type_hints -from sqlbuild.compiler.discovery.models import DiscoveredProvider, DiscoveredProviderUsage from sqlbuild.compiler.python_nodes._helpers.identity import build_python_identity from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity -from sqlbuild.provider.classes.provider import Provider from sqlbuild.python_nodes.main.read_asset_definition import read_asset_definition from sqlbuild.python_nodes.main.read_check_definition import read_check_definition from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition @@ -23,58 +19,25 @@ ) -def python_node_user_facts( +def python_node_identity( *, kind: str, name: str, function: Callable[..., object], file_path: Path, - provider_by_name: dict[str, DiscoveredProvider], decorator_config: dict[str, object], -) -> tuple[tuple[DiscoveredProviderUsage, ...], PythonNodeIdentity]: - """Read provider parameters and the identity of one loaded user node function.""" +) -> PythonNodeIdentity: + """Read the identity of one loaded user node function.""" - return ( - provider_usages(function=function, provider_by_name=provider_by_name), - build_python_identity( - node_type=kind, - node_name=name, - function=function, - project_dir=file_path.parent, - decorator_config=decorator_config, - ), + return build_python_identity( + node_type=kind, + node_name=name, + function=function, + project_dir=file_path.parent, + decorator_config=decorator_config, ) -def provider_usages( - *, function: Callable[..., object], provider_by_name: dict[str, DiscoveredProvider] -) -> tuple[DiscoveredProviderUsage, ...]: - """Provider parameters of one loaded user function and their annotation classes.""" - - usages: list[DiscoveredProviderUsage] = [] - type_hints: dict[str, object] = get_type_hints(function) - parameter: inspect.Parameter - for parameter in inspect.signature(function).parameters.values(): - discovered_provider: DiscoveredProvider | None = provider_by_name.get(parameter.name) - if discovered_provider is None: - continue - annotation: object = type_hints.get(parameter.name, parameter.annotation) - annotation_class_name: str | None = None - annotation_module: str | None = None - if isinstance(annotation, type) and issubclass(annotation, Provider): - annotation_class_name = annotation.__name__ - annotation_module = annotation.__module__ - usages.append( - DiscoveredProviderUsage( - provider_name=discovered_provider.name, - parameter_name=parameter.name, - annotation_class_name=annotation_class_name, - annotation_module=annotation_module, - ) - ) - return tuple(usages) - - def python_node_dependency_key(dependency: object) -> object | tuple[str, str]: """The node name a decorated dependency declares, or the dependency itself.""" diff --git a/src/sqlbuild/compiler/python_nodes/models.py b/src/sqlbuild/compiler/python_nodes/models.py index 6ec5d49a17..601582f2dd 100644 --- a/src/sqlbuild/compiler/python_nodes/models.py +++ b/src/sqlbuild/compiler/python_nodes/models.py @@ -6,50 +6,16 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import ( + DiscoveredPythonAssetMetadata, + DiscoveredPythonCheckMetadata, + DiscoveredPythonLoaderMetadata, + DiscoveredPythonTaskMetadata, + PythonNodeDependencyEdge, +) from sqlbuild.compiler.discovery.models import DiscoveredProviderUsage -from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.python_nodes.types import PythonNodeKind -from sqlbuild.python_nodes.models import ColumnLineageRef, RetryPolicy, SqlResourceRef -from sqlbuild.python_nodes.types import PythonCheckSeverity -from sqlbuild.spec.contracts.models import SourceColumnEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy - - -@dataclass(frozen=True) -class DiscoveredPythonLoaderMetadata: - """Loader-specific metadata carried by an internal Python loader node.""" - - destination: str | None = None - write_strategy: SourceWriteStrategy | None = None - cursor_column: str | None = None - unique_key: tuple[str, ...] = field(default_factory=tuple) - columns: tuple[SourceColumnEntry, ...] = field(default_factory=tuple) - contract: str | None = None - connection_mode: LoaderConnectionMode = LoaderConnectionMode.SQLBUILD - - -@dataclass(frozen=True) -class DiscoveredPythonTaskMetadata: - """Task-specific metadata carried by an internal Python task node.""" - - retry: RetryPolicy | None = None - - -@dataclass(frozen=True) -class DiscoveredPythonAssetMetadata: - """Asset-specific metadata carried by an internal Python asset node.""" - - columns: tuple[SourceColumnEntry, ...] = field(default_factory=tuple) - column_lineage: dict[str, tuple[ColumnLineageRef, ...]] | None = None - retry: RetryPolicy | None = None - - -@dataclass(frozen=True) -class DiscoveredPythonCheckMetadata: - """Check-specific metadata carried by an internal Python check node.""" - - severity: PythonCheckSeverity = PythonCheckSeverity.ERROR +from sqlbuild.python_nodes.models import SqlResourceRef @dataclass(frozen=True) @@ -75,14 +41,6 @@ class DiscoveredPythonNode: identity: PythonNodeIdentity | None = None -@dataclass(frozen=True) -class PythonNodeDependencyEdge: - """Internal dependency edge between two discovered Python nodes.""" - - upstream_name: str - downstream_name: str - - @dataclass(frozen=True) class PythonNodeGraph: """Internal graph inventory for discovered executable Python nodes.""" @@ -114,14 +72,6 @@ def selected_sql_refs(self, *, selected_names: frozenset[str]) -> frozenset[SqlR return frozenset(refs) -@dataclass(frozen=True) -class PythonSqlRunSelection: - """Unified run-command selection excluding Python checks.""" - - sql_keys: frozenset[CompiledObjectKey] - python_node_names: frozenset[str] - - @dataclass(frozen=True) class PythonSqlRunLifecyclePlan: """Lifecycle-aware run classification for SQL resources and Python nodes.""" diff --git a/src/sqlbuild/compiler/python_nodes/types.py b/src/sqlbuild/compiler/python_nodes/types.py index 21b5428992..f697d9a70a 100644 --- a/src/sqlbuild/compiler/python_nodes/types.py +++ b/src/sqlbuild/compiler/python_nodes/types.py @@ -35,19 +35,3 @@ class PythonNodeFanInAction(StrEnum): RUN = "run" SKIP = "skip" BLOCK = "block" - - -class PythonRunPhase(StrEnum): - """Lifecycle-aware Python execution phase.""" - - PRE_SQL_INGRESS = "pre_sql_ingress" - READ_SIDE = "read_side" - - -class PythonIdentityStatus(StrEnum): - """Read-only identity diff status for Python plan display.""" - - NEW = "new" - CHANGED = "changed" - UNCHANGED = "unchanged" - UNKNOWN = "unknown" diff --git a/src/sqlbuild/compiler/refactoring/exceptions.py b/src/sqlbuild/compiler/refactoring/exceptions.py deleted file mode 100644 index 87a4e53647..0000000000 --- a/src/sqlbuild/compiler/refactoring/exceptions.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Expected refactoring errors.""" - -from __future__ import annotations - - -class RefactorValueError(ValueError): - """An unhandled value failure the Python planner also leaves to the caller.""" - - -class RefactorIOError(OSError): - """A file read or write failure the Python planner also leaves to the caller.""" diff --git a/src/sqlbuild/compiler/refactoring/models.py b/src/sqlbuild/compiler/refactoring/models.py deleted file mode 100644 index 577f489a22..0000000000 --- a/src/sqlbuild/compiler/refactoring/models.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Immutable models for planned and applied project refactorings.""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ProjectGraph - - -@dataclass(frozen=True) -class RefactorProject: - """Compiler facts of one project, as the refactoring planner reads them.""" - - project_dir: Path - graph: ProjectGraph - discovered: DiscoveredProjectInputs - - -@dataclass(frozen=True) -class ManualLocation: - """A location a refactoring cannot rewrite safely.""" - - path: str - line: int | None - column: int | None - reason: str diff --git a/src/sqlbuild/compiler/references/models.py b/src/sqlbuild/compiler/references/models.py index 957eaa2491..510208584c 100644 --- a/src/sqlbuild/compiler/references/models.py +++ b/src/sqlbuild/compiler/references/models.py @@ -3,9 +3,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from pathlib import Path -from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind from sqlbuild.python_nodes.models import SqlResourceRef @@ -32,15 +30,3 @@ class ProjectRelationIndex: """Every relation name that counts as a project relation for hard-coded name checks.""" relations: tuple[ProjectRelation, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class LiteralSqlRelation: - """A relation named in literal SQL in Python code that matched no project relation.""" - - owner_label: str - owner_kind: HardCodedRelationOwnerKind - relative_path: Path - line: int - method: str - relation: RelationName diff --git a/src/sqlbuild/compiler/resource_names/exceptions.py b/src/sqlbuild/compiler/resource_names/exceptions.py deleted file mode 100644 index 48b5e48e34..0000000000 --- a/src/sqlbuild/compiler/resource_names/exceptions.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Resource name validation exceptions.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery.exceptions import DiscoveryError - - -class ResourceIdentityError(DiscoveryError): - """Raised when an authored resource identity is not canonical snake_case.""" - - code: str = "D016" diff --git a/src/sqlbuild/compiler/scopes/exceptions.py b/src/sqlbuild/compiler/scopes/exceptions.py deleted file mode 100644 index 739a0f3b82..0000000000 --- a/src/sqlbuild/compiler/scopes/exceptions.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Expected scope-domain exception types.""" - -from __future__ import annotations - -from sqlbuild.compiler.scopes.models import ScopeDiagnostic - - -class ScopeError(ValueError): - """Base error for invalid scope-domain input.""" - - code: str = "S000" - - def __init__(self, message: str, *, code: str | None = None) -> None: - super().__init__(message) - self.message = message - self.code = code if code is not None else self.code - - -class InvalidScopePathError(ScopeError): - """Raised when a project-relative scope path is invalid.""" - - code: str = "S011" - - -class InvalidQualifiedIdentityError(ScopeError): - """Raised when a qualified scope identity cannot be parsed.""" - - code: str = "S002" - - -class UnindexableScopeError(ScopeError): - """Raised when the native scope index cannot place a discovered path or resource kind.""" - - def __init__(self) -> None: - super().__init__( - "Native scope indexing could not place a discovered path or resource kind. Discovery " - "always yields normalized project-relative paths, so this is a SQLBuild bug; please " - "file an issue with the project layout" - ) - - -class ScopeValidationError(ScopeError): - """Aggregate error raised only when strict scope validation is requested.""" - - def __init__(self, diagnostics: tuple[ScopeDiagnostic, ...]) -> None: - self.diagnostics = diagnostics - message: str = "Invalid declaration scope index:\n" + "\n".join( - f"[{diagnostic.code.value}] {diagnostic.message}" for diagnostic in diagnostics - ) - super().__init__(message) diff --git a/src/sqlbuild/compiler/source_freshness/models.py b/src/sqlbuild/compiler/source_freshness/models.py index 7cb94a2d42..8b20d3d929 100644 --- a/src/sqlbuild/compiler/source_freshness/models.py +++ b/src/sqlbuild/compiler/source_freshness/models.py @@ -2,14 +2,11 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass from datetime import datetime from sqlbuild._native import ( - DirectSourceFreshnessPropagationResult, - SourceFreshnessAgeStatus, SourceFreshnessIdentity, - SourceFreshnessUnknown, require_aware_utc_datetime, ) @@ -44,19 +41,3 @@ def identity(self) -> SourceFreshnessIdentity: target_schema=self.target_schema, target_name=self.target_name, ) - - -@dataclass(frozen=True) -class DirectSourceFreshnessPlanningResult: - """Direct planning-time source freshness observations and comparisons.""" - - observed_records: tuple[SourceFreshnessRecord, ...] = () - previous_records: tuple[SourceFreshnessRecord, ...] = () - changed_identities: frozenset[SourceFreshnessIdentity] = frozenset() - unchanged_identities: frozenset[SourceFreshnessIdentity] = frozenset() - unknown_source_names: tuple[str, ...] = () - unknown_sources: dict[str, SourceFreshnessUnknown] = field(default_factory=dict) - age_statuses: dict[SourceFreshnessIdentity, SourceFreshnessAgeStatus] = field( - default_factory=dict - ) - propagation: DirectSourceFreshnessPropagationResult | None = None diff --git a/src/sqlbuild/compiler/sql_analysis/exceptions.py b/src/sqlbuild/compiler/sql_analysis/exceptions.py deleted file mode 100644 index fac70583d0..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/exceptions.py +++ /dev/null @@ -1,5 +0,0 @@ -"""SQL analysis boundary failures.""" - - -class UnsupportedLexicalSyntaxError(ValueError): - """Raised when an adapter declares lexical rules the native SQL scanners do not support.""" diff --git a/src/sqlbuild/compiler/sql_analysis/models.py b/src/sqlbuild/compiler/sql_analysis/models.py index 65c926503b..47f779ab2b 100644 --- a/src/sqlbuild/compiler/sql_analysis/models.py +++ b/src/sqlbuild/compiler/sql_analysis/models.py @@ -2,10 +2,9 @@ from __future__ import annotations -from collections.abc import Mapping -from dataclasses import dataclass, field -from typing import Any +from dataclasses import dataclass +from sqlbuild._native import UnsupportedLexicalSyntaxError from sqlbuild.compiler.sql_analysis.constants import ( SQL_BLOCK_COMMENT_OPEN, SQL_ESCAPABLE_QUOTE_CHARACTERS, @@ -14,40 +13,6 @@ SQL_TRIPLE_QUOTE_LENGTH, SUPPORTED_SQL_LINE_COMMENT_PREFIXES, ) -from sqlbuild.compiler.sql_analysis.exceptions import UnsupportedLexicalSyntaxError - - -@dataclass(frozen=True) -class SqlBindingDiagnostic: - """One native binding error or non-blocking runtime-conversion warning.""" - - code: str - message: str - line: int | None = None - column: int | None = None - start: int | None = None - end: int | None = None - severity: str = "error" - - -@dataclass(frozen=True) -class SqlBindingResult: - """Stable result of schema-aware semantic SQL validation.""" - - diagnostics: tuple[SqlBindingDiagnostic, ...] = () - - -@dataclass(frozen=True) -class SqlSchemaValidationRequest: - """One expanded SQL query and its complete relation schemas.""" - - sql: str - dialect: str | None - schema: Mapping[str, Mapping[str, str]] - known_functions: tuple[str, ...] = () - known_types: tuple[str, ...] = () - quoted_identifiers_ignore_case: bool = False - catalog: Any | None = field(default=None, repr=False, compare=False) @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/sql_test_glue/models.py b/src/sqlbuild/compiler/sql_test_glue/models.py deleted file mode 100644 index ebb8d1fce7..0000000000 --- a/src/sqlbuild/compiler/sql_test_glue/models.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Requests and results of the native SQL-test glue, read from compiled objects.""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Literal - -from sqlbuild.compiler.compile.models import ( - CompiledSqlTest, - CompilerDiagnostic, -) -from sqlbuild.compiler.compile.types import CompiledModelView - - -@dataclass(frozen=True) -class NativeSqlTestPlanningRequest: - """Compiled models and tests to plan, with the adapter facts native planning cannot derive.""" - - models: tuple[CompiledModelView, ...] - rendered_model_sql: dict[int, str] - functions: tuple[tuple[str, str, str, str, str], ...] - tests: tuple[CompiledSqlTest, ...] - rendered_test_overrides: dict[int, dict[str, str]] - sql_analysis_enabled: bool - sql_analysis_dialect: str | None - set_difference_operator: str - requires_derived_table_aliases: bool - lexical_syntax: dict[str, object] - render_sql: bool - include_plan: bool - - -@dataclass(frozen=True) -class NativeSqlTestChainRequest: - """Compiled models and the tests whose unmocked model chains to resolve.""" - - models: tuple[CompiledModelView, ...] - tests: tuple[CompiledSqlTest, ...] - lexical_syntax: dict[str, object] - - -@dataclass(frozen=True) -class NativeSqlTestFailure: - """The error one test's assembly raises: its kind and message.""" - - kind: Literal["input", "internal", "decimal_overflow"] - message: str - - -@dataclass(frozen=True) -class NativeSqlTestAssembly: - """One natively assembled test, its keyed diagnostics, and the error its assembly raises.""" - - test: CompiledSqlTest | None - diagnostics: tuple[tuple[tuple[str, ...], CompilerDiagnostic], ...] - failure: NativeSqlTestFailure | None = None diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_names.py b/src/sqlbuild/executor/janitor/_helpers/archive_names.py deleted file mode 100644 index 31885e515f..0000000000 --- a/src/sqlbuild/executor/janitor/_helpers/archive_names.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Strict janitor archive relation name grammar.""" - -from __future__ import annotations - -import re -from datetime import UTC, datetime - -from sqlbuild._native import fit_artifact_logical_name -from sqlbuild.executor.janitor.constants import ( - ARCHIVE_ARTIFACT_LABEL, - ARCHIVE_LOOKALIKE_PREFIX, - ARCHIVE_NAME_PREFIX, - ARCHIVE_NAME_SEPARATOR, - ARCHIVE_TIMESTAMP_FORMAT, -) -from sqlbuild.executor.janitor.models import JanitorParsedArchiveName - -_ARCHIVE_NAME_RE: re.Pattern[str] = re.compile( - rf"^{re.escape(ARCHIVE_NAME_PREFIX)}" - r"(?P[0-9]{8}T[0-9]{6}Z)" - rf"{re.escape(ARCHIVE_NAME_SEPARATOR)}" - r"(?P.+)$", - re.IGNORECASE, -) - - -def archive_timestamp(value: datetime) -> datetime: - """Normalize a timestamp to the UTC second resolution embedded in archive names.""" - - aware: datetime = value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC) - return aware.replace(microsecond=0) - - -def build_archive_name( - *, original_name: str, archived_at: datetime, identifier_limit: int, kind: str | None = None -) -> str: - """Build the fitted, case-folded archive name, keeping prefix, timestamp, and kind intact.""" - - kind_segment: str = "" if kind is None else f"{kind}{ARCHIVE_NAME_SEPARATOR}" - fixed_prefix: str = ( - f"{ARCHIVE_NAME_PREFIX}" - f"{archive_timestamp(archived_at).strftime(ARCHIVE_TIMESTAMP_FORMAT)}" - f"{ARCHIVE_NAME_SEPARATOR}" - f"{kind_segment}" - ) - logical_part: str = fit_artifact_logical_name( - logical_name=original_name, - fixed_prefix=fixed_prefix, - identifier_limit=identifier_limit, - artifact_label=ARCHIVE_ARTIFACT_LABEL, - ) - return f"{fixed_prefix}{logical_part}".lower() - - -def parse_archive_name(name: str) -> JanitorParsedArchiveName | None: - """Parse a strict archive name, returning None for anything else.""" - - match: re.Match[str] | None = _ARCHIVE_NAME_RE.fullmatch(name) - if match is None: - return None - try: - archived_at: datetime = datetime.strptime( - match.group("timestamp").upper(), ARCHIVE_TIMESTAMP_FORMAT - ).replace(tzinfo=UTC) - except ValueError: - return None - return JanitorParsedArchiveName( - archived_at=archived_at, - logical_name=match.group("logical_name"), - ) - - -def is_archive_lookalike_name(name: str) -> bool: - """Return whether a relation name is reserved for janitor archive handling.""" - - return name.lower().startswith(ARCHIVE_LOOKALIKE_PREFIX) diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_planning.py b/src/sqlbuild/executor/janitor/_helpers/archive_planning.py deleted file mode 100644 index 074659d08c..0000000000 --- a/src/sqlbuild/executor/janitor/_helpers/archive_planning.py +++ /dev/null @@ -1,208 +0,0 @@ -"""Direct-mode janitor archive and archive-expiry planning.""" - -from __future__ import annotations - -from dataclasses import replace -from datetime import datetime, timedelta - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.executor.janitor._helpers.archive_names import ( - archive_timestamp, - build_archive_name, - is_archive_lookalike_name, - parse_archive_name, -) -from sqlbuild.executor.janitor._helpers.classification import matching_exclude_pattern -from sqlbuild.executor.janitor._helpers.plan import relation_key as build_relation_key -from sqlbuild.executor.janitor._helpers.relation_addressing import ( - case_colliding_names, - unaddressable_relation_reason, -) -from sqlbuild.executor.janitor.constants import BUILT_IN_EXCLUDE_PATTERNS, MALFORMED_ARCHIVE_REASON -from sqlbuild.executor.janitor.models import ( - JanitorArchiveCandidate, - JanitorArchivedRelation, - JanitorArchivePlanning, - JanitorBlockedSchema, - JanitorDeleteCandidate, - JanitorParsedArchiveName, - JanitorRelationKey, - JanitorSchemaClassification, - JanitorSkippedRelation, - JanitorWarehouseFacts, -) - - -def plan_janitor_archives( - *, - direct_mode: bool, - adapter: BaseAdapter, - schemas: JanitorSchemaClassification, - facts: JanitorWarehouseFacts, - managed_target_schemas: set[tuple[str | None, str | None]], - exclude_patterns: tuple[str, ...], - archive_retention_days: int, - now: datetime, -) -> JanitorArchivePlanning: - """Plan archives and archive expiry.""" - - if not direct_mode: - return JanitorArchivePlanning( - archive_candidates=(), - archive_deletion_candidates=(), - retained_archives=(), - skipped_relations=(), - blocked_schemas=schemas.blocked_schemas, - ) - return _plan_direct_archives( - candidates=schemas.candidates, - facts=facts, - managed_target_schemas=managed_target_schemas, - blocked_schemas=schemas.blocked_schemas, - exclude_patterns=exclude_patterns, - archive_retention_days=archive_retention_days, - identifier_limit=adapter.maximum_identifier_length(), - now=now, - ) - - -def _plan_direct_archives( - *, - candidates: tuple[JanitorDeleteCandidate, ...], - facts: JanitorWarehouseFacts, - managed_target_schemas: set[tuple[str | None, str | None]], - blocked_schemas: tuple[JanitorBlockedSchema, ...], - exclude_patterns: tuple[str, ...], - archive_retention_days: int, - identifier_limit: int, - now: datetime, -) -> JanitorArchivePlanning: - """Plan archive renames and name-based expiry of existing archives.""" - - archived_at: datetime = archive_timestamp(now) - retention: timedelta = timedelta(days=archive_retention_days) - archive_candidates: tuple[JanitorArchiveCandidate, ...] = tuple( - _archive_candidate( - candidate=candidate, - archived_at=archived_at, - retention=retention, - identifier_limit=identifier_limit, - ) - for candidate in candidates - ) - blocked_schema_keys: frozenset[tuple[str | None, str | None]] = frozenset( - (blocked.database, blocked.schema) for blocked in blocked_schemas - ) - deletions: list[JanitorArchivedRelation] = [] - retained: list[JanitorArchivedRelation] = [] - skipped: list[JanitorSkippedRelation] = [] - suppressed_by_schema: dict[tuple[str | None, str | None], list[JanitorArchivedRelation]] = {} - effective_patterns: tuple[str, ...] = BUILT_IN_EXCLUDE_PATTERNS + exclude_patterns - schema_key: tuple[str | None, str | None] - for schema_key in sorted(managed_target_schemas, key=lambda key: (key[0] or "", key[1] or "")): - schema_relations: tuple[RelationInfo, ...] = facts.relations_by_schema.get(schema_key, ()) - colliding_names: frozenset[str] = case_colliding_names( - relation.name for relation in schema_relations - ) - relation: RelationInfo - for relation in schema_relations: - key: JanitorRelationKey = build_relation_key(relation) - if key in facts.desired_keys or not is_archive_lookalike_name(key.name): - continue - parsed: JanitorParsedArchiveName | None = parse_archive_name(key.name) - if parsed is None: - skipped.append( - JanitorSkippedRelation( - key=key, relation=relation, reason=MALFORMED_ARCHIVE_REASON - ) - ) - continue - skip_reason: str | None = _archive_skip_reason( - key=key, colliding_names=colliding_names, effective_patterns=effective_patterns - ) - if skip_reason is not None: - skipped.append( - JanitorSkippedRelation(key=key, relation=relation, reason=skip_reason) - ) - continue - archived: JanitorArchivedRelation = JanitorArchivedRelation( - key=key, - relation_type=relation.relation_type, - archived_at=parsed.archived_at, - expires_at=parsed.archived_at + retention, - ) - if archived.expires_at > now: - retained.append(archived) - elif schema_key in blocked_schema_keys: - suppressed_by_schema.setdefault(schema_key, []).append(archived) - else: - deletions.append(archived) - same_run_deletions: tuple[JanitorArchivedRelation, ...] = tuple( - JanitorArchivedRelation( - key=candidate.archive_key, - relation_type=candidate.relation.relation_type, - archived_at=candidate.archived_at, - expires_at=candidate.expires_at, - original_key=candidate.key, - ) - for candidate in archive_candidates - if candidate.expires_at <= archived_at - ) - return JanitorArchivePlanning( - archive_candidates=archive_candidates, - archive_deletion_candidates=(*deletions, *same_run_deletions), - retained_archives=tuple(retained), - skipped_relations=tuple(skipped), - blocked_schemas=tuple( - replace( - blocked, - suppressed_archive_deletions=tuple( - suppressed_by_schema.get((blocked.database, blocked.schema), ()) - ), - ) - for blocked in blocked_schemas - ), - ) - - -def _archive_skip_reason( - *, - key: JanitorRelationKey, - colliding_names: frozenset[str], - effective_patterns: tuple[str, ...], -) -> str | None: - addressing_reason: str | None = unaddressable_relation_reason( - name=key.name, colliding_names=colliding_names - ) - if addressing_reason is not None: - return addressing_reason - exclude_pattern: str | None = matching_exclude_pattern(key=key, patterns=effective_patterns) - if exclude_pattern is not None: - return f"relation matches exclude pattern {exclude_pattern!r}" - return None - - -def _archive_candidate( - *, - candidate: JanitorDeleteCandidate, - archived_at: datetime, - retention: timedelta, - identifier_limit: int, -) -> JanitorArchiveCandidate: - return JanitorArchiveCandidate( - key=candidate.key, - relation=candidate.relation, - age_timestamp=candidate.age_timestamp, - archive_key=JanitorRelationKey( - database=candidate.key.database, - schema=candidate.key.schema, - name=build_archive_name( - original_name=candidate.key.name, - archived_at=archived_at, - identifier_limit=identifier_limit, - ), - ), - archived_at=archived_at, - expires_at=archived_at + retention, - ) diff --git a/src/sqlbuild/executor/janitor/_helpers/classification.py b/src/sqlbuild/executor/janitor/_helpers/classification.py deleted file mode 100644 index 63bc113985..0000000000 --- a/src/sqlbuild/executor/janitor/_helpers/classification.py +++ /dev/null @@ -1,354 +0,0 @@ -"""Janitor relation fact gathering and delete-candidate classification.""" - -from __future__ import annotations - -from datetime import datetime, timedelta -from fnmatch import fnmatchcase -from typing import Any - -from sqlbuild._native import is_scenario_artifact_physical_name -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup -from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME -from sqlbuild.executor.diff.classes.query_artifact_lifecycle import QueryDiffArtifactLifecycle -from sqlbuild.executor.diff.models import QueryDiffArtifactInspection -from sqlbuild.executor.janitor._helpers.archive_names import is_archive_lookalike_name -from sqlbuild.executor.janitor._helpers.plan import ( - collect_desired_keys, - collect_source_schemas, - list_target_schema_relations, - relation_age_timestamp, -) -from sqlbuild.executor.janitor._helpers.plan import ( - relation_key as build_relation_key, -) -from sqlbuild.executor.janitor._helpers.relation_addressing import ( - case_colliding_names, - case_collision_reason, - unaddressable_relation_reason, -) -from sqlbuild.executor.janitor._helpers.tracking import collect_tracked_relation_keys -from sqlbuild.executor.janitor.classes.relation_age_reader import JanitorRelationAgeReader -from sqlbuild.executor.janitor.models import ( - JanitorDeleteCandidate, - JanitorDirectStatePruneCandidate, - JanitorQueryDiffArtifactCandidate, - JanitorRelationClassification, - JanitorRelationKey, - JanitorSkippedRelation, - JanitorWarehouseFacts, -) - - -def collect_query_diff_artifact_candidates( - *, - adapter: BaseAdapter, - connection: Any, - target_schemas: set[tuple[str | None, str]], - now: datetime, -) -> tuple[tuple[JanitorQueryDiffArtifactCandidate, ...], tuple[JanitorSkippedRelation, ...]]: - """Collect expired owned query artifacts and report untracked reserved names.""" - - candidates: list[JanitorQueryDiffArtifactCandidate] = [] - skipped: list[JanitorSkippedRelation] = [] - for database, schema in sorted(target_schemas, key=lambda value: (value[0] or "", value[1])): - inspection: QueryDiffArtifactInspection = QueryDiffArtifactLifecycle.inspect( - adapter=adapter, - connection=connection, - database=database, - schema=schema, - now=now, - ) - for artifact in inspection.expired: - candidates.append( - JanitorQueryDiffArtifactCandidate( - key=JanitorRelationKey( - database=artifact.database, - schema=artifact.schema, - name=artifact.name, - ) - ) - ) - for relation in inspection.untracked: - key: JanitorRelationKey = build_relation_key(relation) - skipped.append( - JanitorSkippedRelation( - key=key, - relation=relation, - reason="reserved query-diff name lacks matching ownership evidence", - ) - ) - return tuple(candidates), tuple(skipped) - - -def gather_janitor_warehouse_facts( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - target_schemas: set[tuple[str | None, str | None]], - delete_tracked_only: bool, -) -> JanitorWarehouseFacts: - """Collect desired, discovered, and tracked relation facts for planning.""" - - return JanitorWarehouseFacts( - desired_keys=frozenset(collect_desired_keys(project)), - source_schema_names=collect_source_schemas( - project=project, - default_database=adapter.default_database(), - default_schema=adapter.default_schema(), - ), - relations_by_schema=list_target_schema_relations( - adapter=adapter, - connection=connection, - target_schemas=target_schemas, - ), - tracked_relation_keys=frozenset( - collect_tracked_relation_keys( - adapter=adapter, - connection=connection, - target_schemas=target_schemas, - ) - if delete_tracked_only - else set() - ), - ) - - -def collect_direct_state_prune_candidates( - *, - adapter: BaseAdapter, - connection: Any, - target_schemas: set[tuple[str | None, str | None]], - direct_state_history_versions: int, -) -> tuple[JanitorDirectStatePruneCandidate, ...]: - """Collect fingerprint/freshness history prune candidates for target schemas.""" - - if direct_state_history_versions <= 0: - return () - state_table_locations: list[tuple[str | None, str | None, str]] = [] - state_table_database: str | None - state_table_schema: str | None - for state_table_database, state_table_schema in target_schemas: - if state_table_schema is None: - continue - state_table_locations.append( - (state_table_database, state_table_schema, FINGERPRINT_TABLE_NAME) - ) - state_table_locations.append( - (state_table_database, state_table_schema, SOURCE_FRESHNESS_TABLE_NAME) - ) - state_table_lookup: RelationLookup = build_relation_lookup( - adapter=adapter, - connection=connection, - locations=tuple(state_table_locations), - ) - candidates: list[JanitorDirectStatePruneCandidate] = [] - schema_key: tuple[str | None, str | None] - for schema_key in sorted(target_schemas, key=lambda key: (key[0] or "", key[1] or "")): - if schema_key[1] is None: - continue - candidates.extend( - _direct_state_prune_candidates( - adapter=adapter, - database=schema_key[0], - schema=schema_key[1], - retain_versions=direct_state_history_versions, - state_table_lookup=state_table_lookup, - ) - ) - return tuple(candidates) - - -def classify_janitor_relations( - *, - schema_relations: tuple[RelationInfo, ...], - facts: JanitorWarehouseFacts, - protected_relation_keys: frozenset[JanitorRelationKey], - protection_reasons: dict[JanitorRelationKey, str], - effective_exclude_patterns: tuple[str, ...], - delete_tracked_only: bool, - retention_days: int, - age_reader: JanitorRelationAgeReader, - now: datetime, - direct_mode: bool = True, -) -> JanitorRelationClassification: - """Split one schema's relations into delete candidates and skipped relations.""" - - candidates: list[JanitorDeleteCandidate] = [] - skipped_relations: list[JanitorSkippedRelation] = [] - eligible_relations: list[RelationInfo] = [] - colliding_names: frozenset[str] = case_colliding_names( - relation.name for relation in schema_relations - ) - relation: RelationInfo - for relation in schema_relations: - relation_key: JanitorRelationKey = build_relation_key(relation) - if relation_key in facts.desired_keys: - continue - if direct_mode and is_archive_lookalike_name(relation_key.name): - continue - skip_reason: str | None = _relation_skip_reason( - relation_key=relation_key, - facts=facts, - protected_relation_keys=protected_relation_keys, - protection_reasons=protection_reasons, - effective_exclude_patterns=effective_exclude_patterns, - delete_tracked_only=delete_tracked_only, - ) - if skip_reason is None and direct_mode: - skip_reason = unaddressable_relation_reason( - name=relation_key.name, colliding_names=colliding_names - ) - elif skip_reason is None: - skip_reason = case_collision_reason( - name=relation_key.name, colliding_names=colliding_names - ) - if skip_reason is not None: - skipped_relations.append( - JanitorSkippedRelation(key=relation_key, relation=relation, reason=skip_reason) - ) - continue - eligible_relations.append(relation) - age_supported: bool = age_reader.supported() - aged_relation: RelationInfo - eligible: tuple[RelationInfo, ...] = tuple(eligible_relations) - for aged_relation in age_reader.read(eligible) if retention_days > 0 else eligible: - relation_key = build_relation_key(aged_relation) - age_timestamp: datetime | None = relation_age_timestamp(aged_relation) - if retention_days > 0: - retention_skip_reason: str | None = _retention_skip_reason( - age_timestamp=age_timestamp, - age_supported=age_supported, - retention_days=retention_days, - now=now, - ) - if retention_skip_reason is not None: - skipped_relations.append( - JanitorSkippedRelation( - key=relation_key, - relation=aged_relation, - reason=retention_skip_reason, - ) - ) - continue - candidates.append( - JanitorDeleteCandidate( - key=relation_key, - relation=aged_relation, - age_timestamp=age_timestamp, - ) - ) - return JanitorRelationClassification( - candidates=tuple(candidates), - skipped_relations=tuple(skipped_relations), - ) - - -def _relation_skip_reason( - *, - relation_key: JanitorRelationKey, - facts: JanitorWarehouseFacts, - protected_relation_keys: frozenset[JanitorRelationKey], - protection_reasons: dict[JanitorRelationKey, str], - effective_exclude_patterns: tuple[str, ...], - delete_tracked_only: bool, -) -> str | None: - if QueryDiffArtifactLifecycle.is_artifact_name(relation_key.name): - return "query-diff artifacts use fingerprint expiry cleanup" - if relation_key in protected_relation_keys: - return protection_reasons.get( - relation_key, - "relation is protected from cleanup", - ) - exclude_pattern: str | None = matching_exclude_pattern( - key=relation_key, - patterns=effective_exclude_patterns, - ) - if exclude_pattern is not None: - return f"relation matches exclude pattern {exclude_pattern!r}" - if ( - delete_tracked_only - and relation_key not in facts.tracked_relation_keys - and not is_scenario_artifact_physical_name(relation_key.name) - ): - return "relation is not tracked by SQLBuild" - return None - - -def _retention_skip_reason( - *, - age_timestamp: datetime | None, - age_supported: bool, - retention_days: int, - now: datetime, -) -> str | None: - if not age_supported: - return "adapter does not expose relation age metadata" - if age_timestamp is None: - return "relation age is unavailable" - if age_timestamp > now - timedelta(days=retention_days): - return f"relation is newer than {retention_days} days" - return None - - -def matching_exclude_pattern( - *, - key: JanitorRelationKey, - patterns: tuple[str, ...], -) -> str | None: - """Return the first exclude pattern matching a relation or qualified name, ignoring case.""" - - name: str = key.name.lower() - display_name: str = key.display_name().lower() - pattern: str - for pattern in patterns: - folded_pattern: str = pattern.lower() - if fnmatchcase(name, folded_pattern) or fnmatchcase(display_name, folded_pattern): - return pattern - return None - - -def _direct_state_prune_candidates( - *, - adapter: BaseAdapter, - database: str | None, - schema: str, - retain_versions: int, - state_table_lookup: RelationLookup, -) -> tuple[JanitorDirectStatePruneCandidate, ...]: - candidates: list[JanitorDirectStatePruneCandidate] = [] - if state_table_lookup.exists(database=database, schema=schema, name=FINGERPRINT_TABLE_NAME): - candidates.append( - JanitorDirectStatePruneCandidate( - database=database, - schema=schema, - table_name=FINGERPRINT_TABLE_NAME, - retain_versions=retain_versions, - prune_sql=adapter.render_prune_fingerprint_history_sql( - database=database, - schema=schema, - retain_versions=retain_versions, - ), - ) - ) - if state_table_lookup.exists( - database=database, schema=schema, name=SOURCE_FRESHNESS_TABLE_NAME - ): - candidates.append( - JanitorDirectStatePruneCandidate( - database=database, - schema=schema, - table_name=SOURCE_FRESHNESS_TABLE_NAME, - retain_versions=retain_versions, - prune_sql=adapter.render_prune_source_freshness_history_sql( - database=database, - schema=schema, - retain_versions=retain_versions, - ), - ) - ) - return tuple(candidates) diff --git a/src/sqlbuild/executor/janitor/_helpers/plan.py b/src/sqlbuild/executor/janitor/_helpers/plan.py deleted file mode 100644 index 8ed9ca6362..0000000000 --- a/src/sqlbuild/executor/janitor/_helpers/plan.py +++ /dev/null @@ -1,245 +0,0 @@ -"""Janitor planning helpers.""" - -from __future__ import annotations - -from collections import defaultdict -from dataclasses import replace -from datetime import UTC, datetime - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup -from sqlbuild.compiler.compile.constants import MIGRATE_FROM_CONFIG_KEY -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledSeedView, - CompiledSourceView, -) -from sqlbuild.executor.janitor.constants import QUALIFIED_ORIGIN_PARTS -from sqlbuild.executor.janitor.models import JanitorRelationKey - - -def collect_desired_keys(project: CompiledProjectView) -> set[JanitorRelationKey]: - """Collect relation keys that belong to the compiled project.""" - - keys: set[JanitorRelationKey] = set() - model: CompiledModelView - for model in project.models: - keys.add( - JanitorRelationKey( - database=model.destination.database, - schema=model.destination.schema, - name=model.destination.name, - ) - ) - seed: CompiledSeedView - for seed in project.seeds: - keys.add( - JanitorRelationKey( - database=seed.destination.database, - schema=seed.destination.schema, - name=seed.destination.name, - ) - ) - return keys - - -def pending_migration_origins( - *, - project: CompiledProjectView, - relations_by_schema: dict[tuple[str | None, str | None], tuple[RelationInfo, ...]], -) -> dict[JanitorRelationKey, str]: - """Return listed origins of migrations whose destination relation does not exist yet.""" - - listed: dict[tuple[str | None, str | None, str], JanitorRelationKey] = {} - relations: tuple[RelationInfo, ...] - for relations in relations_by_schema.values(): - listed.update({_folded(relation_key(item)): relation_key(item) for item in relations}) - schemas: set[tuple[str | None, str | None]] = collect_target_schemas(project) - pending: dict[JanitorRelationKey, str] = {} - model: CompiledModelView - for model in project.models: - raw: object | None = model.config.values.get(MIGRATE_FROM_CONFIG_KEY) - destination: JanitorRelationKey = JanitorRelationKey( - database=model.destination.database, - schema=model.destination.schema, - name=model.destination.name, - ) - if not isinstance(raw, str) or _folded(destination) in listed: - continue - origin: JanitorRelationKey - for origin in _origin_keys(raw=raw, destination=destination, schemas=schemas): - found: JanitorRelationKey | None = listed.get(_folded(origin)) - if found is not None: - pending.setdefault(found, f"pending migration origin for {model.name}") - return pending - - -def _origin_keys( - *, raw: str, destination: JanitorRelationKey, schemas: set[tuple[str | None, str | None]] -) -> tuple[JanitorRelationKey, ...]: - parts: list[str] = [part.strip().strip('"`') for part in raw.split(".")] - if len(parts) == 1: - return tuple( - JanitorRelationKey(database=database, schema=schema, name=parts[0]) - for database, schema in sorted(schemas, key=_schema_sort_key) - ) - return ( - JanitorRelationKey( - database=parts[0] if len(parts) == QUALIFIED_ORIGIN_PARTS else destination.database, - schema=parts[-2], - name=parts[-1], - ), - ) - - -def _folded(key: JanitorRelationKey) -> tuple[str | None, str | None, str]: - return RelationLookup.key(database=key.database, schema=key.schema, name=key.name) - - -def collect_target_schemas(project: CompiledProjectView) -> set[tuple[str | None, str | None]]: - """Collect schemas where the compiled project writes relations, one spelling per schema.""" - - schemas: list[tuple[str | None, str | None]] = [] - model: CompiledModelView - for model in project.models: - schemas.append((model.destination.database, model.destination.schema)) - seed: CompiledSeedView - for seed in project.seeds: - schemas.append((seed.destination.database, seed.destination.schema)) - return _dedupe_schema_keys(schemas) - - -def collect_scan_schemas( - *, - managed_target_schemas: set[tuple[str | None, str | None]], - relation_keys: frozenset[JanitorRelationKey], -) -> set[tuple[str | None, str | None]]: - """Add schemas of scanned or protected relations without duplicating folded spellings.""" - - return _dedupe_schema_keys( - [ - *sorted(managed_target_schemas, key=_schema_sort_key), - *sorted(((key.database, key.schema) for key in relation_keys), key=_schema_sort_key), - ] - ) - - -def _dedupe_schema_keys( - schema_keys: list[tuple[str | None, str | None]], -) -> set[tuple[str | None, str | None]]: - kept: dict[tuple[str | None, str | None], tuple[str | None, str | None]] = {} - schema_key: tuple[str | None, str | None] - for schema_key in schema_keys: - kept.setdefault(normalized_schema_key(schema_key), schema_key) - return set(kept.values()) - - -def _schema_sort_key(schema_key: tuple[str | None, str | None]) -> tuple[str, str]: - return (schema_key[0] or "", schema_key[1] or "") - - -def collect_source_schemas( - *, - project: CompiledProjectView, - default_database: str | None, - default_schema: str | None, -) -> dict[tuple[str | None, str | None], set[str]]: - """Collect schemas containing active configured sources.""" - - source_schemas: dict[tuple[str | None, str | None], set[str]] = defaultdict(set) - source: CompiledSourceView - for source in project.sources: - if source.source_entry.expression is not None: - continue - database: str | None = ( - source.source_entry.database - if source.source_entry.database is not None - else default_database - ) - schema: str | None = ( - source.source_entry.schema if source.source_entry.schema is not None else default_schema - ) - source_schemas[(database, schema)].add(source.name) - return source_schemas - - -def list_target_schema_relations( - *, - adapter: BaseAdapter, - connection: object, - target_schemas: set[tuple[str | None, str | None]], -) -> dict[tuple[str | None, str | None], tuple[RelationInfo, ...]]: - """List relations matched case-insensitively and respelled as their target schema.""" - - target_by_normalized: dict[tuple[str | None, str | None], tuple[str | None, str | None]] = { - normalized_schema_key(schema_key): schema_key for schema_key in target_schemas - } - by_database: dict[str | None, set[str | None]] = defaultdict(set) - schema_key: tuple[str | None, str | None] - for schema_key in target_schemas: - by_database[schema_key[0]].add(schema_key[1]) - - result: dict[tuple[str | None, str | None], list[RelationInfo]] = defaultdict(list) - database: str | None - schemas: set[str | None] - for database, schemas in by_database.items(): - concrete_schemas: tuple[str, ...] | None = tuple( - sorted(schema for schema in schemas if schema is not None) - ) - if not concrete_schemas: - concrete_schemas = None - relation: RelationInfo - for relation in adapter.list_relations( - connection=connection, - database=database, - schemas=concrete_schemas, - ): - target_key: tuple[str | None, str | None] | None = target_by_normalized.get( - normalized_schema_key((relation.database, relation.schema)) - ) - if target_key is None: - continue - result[target_key].append( - replace(relation, database=target_key[0], schema=target_key[1]) - ) - - return {key: tuple(value) for key, value in result.items()} - - -def normalized_schema_key( - schema_key: tuple[str | None, str | None], -) -> tuple[str | None, str | None]: - """Return the case-insensitive comparison form of one database and schema pair.""" - - database, schema, _ = RelationLookup.key(database=schema_key[0], schema=schema_key[1], name="") - return (database, schema) - - -def relation_key(relation: RelationInfo) -> JanitorRelationKey: - """Build a janitor relation key from adapter relation metadata.""" - - return JanitorRelationKey( - database=relation.database, - schema=relation.schema, - name=relation.name, - ) - - -def relation_age_timestamp(relation: RelationInfo) -> datetime | None: - """Return the latest known relation timestamp.""" - - known_times: list[datetime] = [] - if relation.created_at is not None: - known_times.append(_ensure_aware(relation.created_at)) - if relation.last_altered_at is not None: - known_times.append(_ensure_aware(relation.last_altered_at)) - if not known_times: - return None - return max(known_times) - - -def _ensure_aware(value: datetime) -> datetime: - if value.tzinfo is None: - return value.replace(tzinfo=UTC) - return value.astimezone(UTC) diff --git a/src/sqlbuild/executor/janitor/_helpers/relation_addressing.py b/src/sqlbuild/executor/janitor/_helpers/relation_addressing.py deleted file mode 100644 index d59933ae78..0000000000 --- a/src/sqlbuild/executor/janitor/_helpers/relation_addressing.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Janitor guards for relation names that are safe to address unquoted.""" - -from __future__ import annotations - -import re -from collections import Counter -from collections.abc import Iterable - -from sqlbuild.executor.janitor.constants import ( - CASE_COLLISION_REASON, - UNQUOTED_ADDRESSING_REASON, -) - -_PLAIN_LOWERCASE_IDENTIFIER_RE: re.Pattern[str] = re.compile(r"^[a-z_][a-z0-9_$]*$") - - -def case_colliding_names(names: Iterable[str]) -> frozenset[str]: - """Return case-folded names shared by more than one relation in a listing.""" - - counts: Counter[str] = Counter(name.lower() for name in names) - return frozenset(name for name, count in counts.items() if count > 1) - - -def unaddressable_relation_reason(*, name: str, colliding_names: frozenset[str]) -> str | None: - """Return why the janitor must not rename or drop a relation by its catalog name.""" - - if _PLAIN_LOWERCASE_IDENTIFIER_RE.fullmatch(name) is None: - return UNQUOTED_ADDRESSING_REASON - return case_collision_reason(name=name, colliding_names=colliding_names) - - -def case_collision_reason(*, name: str, colliding_names: frozenset[str]) -> str | None: - """Return why a relation whose folded name is shared in its listing must not be dropped.""" - - if name.lower() in colliding_names: - return CASE_COLLISION_REASON - return None diff --git a/src/sqlbuild/executor/janitor/_helpers/schema_planning.py b/src/sqlbuild/executor/janitor/_helpers/schema_planning.py deleted file mode 100644 index 6e72f870b4..0000000000 --- a/src/sqlbuild/executor/janitor/_helpers/schema_planning.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Classify janitor candidates across target schemas.""" - -from __future__ import annotations - -from datetime import datetime - -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.executor.janitor._helpers.classification import classify_janitor_relations -from sqlbuild.executor.janitor._helpers.source_safety import ( - blocking_source_names, - source_names_for_schema, -) -from sqlbuild.executor.janitor.classes.relation_age_reader import JanitorRelationAgeReader -from sqlbuild.executor.janitor.constants import BUILT_IN_EXCLUDE_PATTERNS -from sqlbuild.executor.janitor.models import ( - JanitorBlockedSchema, - JanitorDeleteCandidate, - JanitorRelationClassification, - JanitorRelationScope, - JanitorSchemaClassification, - JanitorSkippedRelation, - JanitorSkippedSchema, - JanitorWarehouseFacts, -) - - -def classify_target_schemas( - *, - target_schemas: set[tuple[str | None, str | None]], - managed_target_schemas: set[tuple[str | None, str | None]], - facts: JanitorWarehouseFacts, - scope: JanitorRelationScope, - exclude_patterns: tuple[str, ...], - delete_tracked_only: bool, - retention_days: int, - age_reader: JanitorRelationAgeReader, - now: datetime, - direct_mode: bool, -) -> JanitorSchemaClassification: - """Classify all target schemas while preserving source safety behavior.""" - - candidates: list[JanitorDeleteCandidate] = [] - skipped_relations: list[JanitorSkippedRelation] = [] - skipped_schemas: list[JanitorSkippedSchema] = [] - blocked_schemas: list[JanitorBlockedSchema] = [] - for schema_key in sorted(target_schemas, key=lambda key: (key[0] or "", key[1] or "")): - schema_relations: tuple[RelationInfo, ...] = facts.relations_by_schema.get(schema_key, ()) - source_names: set[str] = source_names_for_schema( - schema_key=schema_key, source_schema_names=facts.source_schema_names - ) - blocking_sources: tuple[str, ...] = blocking_source_names( - schema_key=schema_key, - managed_schema_keys=managed_target_schemas, - source_schema_names=facts.source_schema_names, - ) - if source_names or blocking_sources: - if direct_mode and blocking_sources: - suppressed: JanitorRelationClassification = _classify_schema( - schema_relations=schema_relations, - facts=facts, - scope=scope, - exclude_patterns=exclude_patterns, - delete_tracked_only=delete_tracked_only, - retention_days=retention_days, - age_reader=age_reader, - now=now, - direct_mode=direct_mode, - ) - blocked_schemas.append( - JanitorBlockedSchema( - database=schema_key[0], - schema=schema_key[1], - source_names=blocking_sources, - suppressed_candidates=suppressed.candidates, - ) - ) - continue - skipped_schemas.append( - JanitorSkippedSchema( - database=schema_key[0], - schema=schema_key[1], - source_names=tuple(sorted(source_names or set())), - skipped_relations=schema_relations, - ) - ) - continue - classification: JanitorRelationClassification = _classify_schema( - schema_relations=schema_relations, - facts=facts, - scope=scope, - exclude_patterns=exclude_patterns, - delete_tracked_only=delete_tracked_only, - retention_days=retention_days, - age_reader=age_reader, - now=now, - direct_mode=direct_mode, - ) - candidates.extend(classification.candidates) - skipped_relations.extend(classification.skipped_relations) - return JanitorSchemaClassification( - candidates=tuple(candidates), - skipped_relations=tuple(skipped_relations), - skipped_schemas=tuple(skipped_schemas), - blocked_schemas=tuple(blocked_schemas), - ) - - -def _classify_schema( - *, - schema_relations: tuple[RelationInfo, ...], - facts: JanitorWarehouseFacts, - scope: JanitorRelationScope, - exclude_patterns: tuple[str, ...], - delete_tracked_only: bool, - retention_days: int, - age_reader: JanitorRelationAgeReader, - now: datetime, - direct_mode: bool, -) -> JanitorRelationClassification: - return classify_janitor_relations( - schema_relations=schema_relations, - facts=facts, - protected_relation_keys=scope.protected_relation_keys, - protection_reasons=scope.protected_relation_reasons or {}, - effective_exclude_patterns=BUILT_IN_EXCLUDE_PATTERNS + exclude_patterns, - delete_tracked_only=delete_tracked_only, - retention_days=retention_days, - age_reader=age_reader, - now=now, - direct_mode=direct_mode, - ) diff --git a/src/sqlbuild/executor/janitor/_helpers/source_safety.py b/src/sqlbuild/executor/janitor/_helpers/source_safety.py deleted file mode 100644 index 695061f74c..0000000000 --- a/src/sqlbuild/executor/janitor/_helpers/source_safety.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Source and managed-target schema overlap helpers.""" - -from __future__ import annotations - -from sqlbuild.executor.janitor._helpers.plan import normalized_schema_key - - -def blocking_source_names( - *, - schema_key: tuple[str | None, str | None], - managed_schema_keys: set[tuple[str | None, str | None]], - source_schema_names: dict[tuple[str | None, str | None], set[str]], -) -> tuple[str, ...]: - """Return active sources when one managed schema has mixed ownership.""" - - normalized: tuple[str | None, str | None] = normalized_schema_key(schema_key) - if not any(normalized_schema_key(candidate) == normalized for candidate in managed_schema_keys): - return () - return tuple( - sorted( - source_names_for_schema(schema_key=schema_key, source_schema_names=source_schema_names) - ) - ) - - -def source_names_for_schema( - *, - schema_key: tuple[str | None, str | None], - source_schema_names: dict[tuple[str | None, str | None], set[str]], -) -> set[str]: - """Return active source names configured in one schema, compared case-insensitively.""" - - normalized: tuple[str | None, str | None] = normalized_schema_key(schema_key) - names: set[str] = set() - for candidate, source_names in source_schema_names.items(): - if normalized_schema_key(candidate) == normalized: - names.update(source_names) - return names diff --git a/src/sqlbuild/executor/janitor/_helpers/tracking.py b/src/sqlbuild/executor/janitor/_helpers/tracking.py deleted file mode 100644 index f477104677..0000000000 --- a/src/sqlbuild/executor/janitor/_helpers/tracking.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Janitor tracking-state helpers.""" - -from __future__ import annotations - -import logging -from typing import Any - -from sqlbuild._native import read_latest_fingerprints -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationLookup -from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet -from sqlbuild.diagnostics.main.log_debug_event import log_debug_event -from sqlbuild.executor.janitor.models import JanitorRelationKey - -_DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") - - -def collect_tracked_relation_keys( - *, - adapter: BaseAdapter, - connection: Any, - target_schemas: set[tuple[str | None, str | None]], -) -> set[JanitorRelationKey]: - """Collect exact physical relation keys from fingerprint metadata.""" - - tracked_keys: set[JanitorRelationKey] = set() - fingerprint_table_lookup: RelationLookup = build_relation_lookup( - adapter=adapter, - connection=connection, - locations=tuple( - (database, schema, FINGERPRINT_TABLE_NAME) - for database, schema in target_schemas - if schema is not None - ), - ) - schema_key: tuple[str | None, str | None] - for schema_key in target_schemas: - database: str | None = schema_key[0] - schema: str | None = schema_key[1] - if schema is None: - continue - try: - fingerprint_set: FingerprintSet = read_latest_fingerprints( - connection=connection, - execute=adapter.execute, - table_exists=fingerprint_table_lookup.exists( - database=database, - schema=schema, - name=FINGERPRINT_TABLE_NAME, - ), - database=database, - schema=schema, - render_qualified_name=adapter.render_qualified_name, - render_read_latest_sql=adapter.render_read_latest_fingerprints_sql, - ) - except Exception as error: - log_debug_event( - logger=_DEBUG_LOGGER, - message="janitor fingerprint tracking read failed; skipping schema", - database=database, - schema=schema, - sqlbuild_error=str(error), - ) - continue - fingerprint: Fingerprint - for fingerprint in fingerprint_set.fingerprints.values(): - if fingerprint.target_schema is None or fingerprint.target_name is None: - continue - tracked_keys.add( - JanitorRelationKey( - database=fingerprint.target_database, - schema=fingerprint.target_schema, - name=fingerprint.target_name, - ) - ) - return tracked_keys diff --git a/src/sqlbuild/executor/janitor/classes/relation_age_reader.py b/src/sqlbuild/executor/janitor/classes/relation_age_reader.py deleted file mode 100644 index 250082de9f..0000000000 --- a/src/sqlbuild/executor/janitor/classes/relation_age_reader.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Candidate-scoped relation age metadata reader for janitor planning.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationInfo - - -class JanitorRelationAgeReader: - """Adapter connection used to read age metadata for janitor candidates only.""" - - def __init__(self, *, adapter: BaseAdapter, connection: Any) -> None: - self.adapter: BaseAdapter = adapter - self.connection: Any = connection - - def supported(self) -> bool: - """Return whether the adapter reports reliable relation age metadata.""" - - return self.adapter.supports_relation_age_metadata() - - def read(self, relations: tuple[RelationInfo, ...]) -> tuple[RelationInfo, ...]: - """Return candidate relations with age timestamps filled when supported.""" - - if not relations or not self.supported(): - return relations - return self.adapter.with_relation_age_metadata( - connection=self.connection, relations=relations - ) diff --git a/src/sqlbuild/executor/janitor/constants.py b/src/sqlbuild/executor/janitor/constants.py deleted file mode 100644 index ce81109fec..0000000000 --- a/src/sqlbuild/executor/janitor/constants.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Janitor constants.""" - -from __future__ import annotations - -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.migrations.constants import ( - COLUMN_MIGRATION_TABLE_NAME, - MIGRATION_TABLE_NAME, - OLD_NAME_VIEW_TABLE_NAME, -) -from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME -from sqlbuild.executor.audit_results.constants import AUDIT_RESULTS_TABLE_NAME -from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME -from sqlbuild.executor.node_results.constants import NODE_RESULTS_TABLE_NAME -from sqlbuild.microbatches.constants import MICROBATCH_TABLE_NAME - -BUILT_IN_EXCLUDE_PATTERNS: tuple[str, ...] = ( - FINGERPRINT_TABLE_NAME, - MICROBATCH_TABLE_NAME, - SOURCE_FRESHNESS_TABLE_NAME, - NODE_RESULTS_TABLE_NAME, - AUDIT_RESULTS_TABLE_NAME, - JANITOR_EVENTS_TABLE_NAME, - MIGRATION_TABLE_NAME, - COLUMN_MIGRATION_TABLE_NAME, - OLD_NAME_VIEW_TABLE_NAME, -) - -DIRECT_DEFAULT_RETENTION_DAYS: int = 14 - -ARCHIVE_NAME_PREFIX: str = "_SQB_ARCHIVE__" -ARCHIVE_NAME_SEPARATOR: str = "__" -ARCHIVE_TIMESTAMP_FORMAT: str = "%Y%m%dT%H%M%SZ" -ARCHIVE_LOOKALIKE_PREFIX: str = "_sqb_archive" -ARCHIVE_ARTIFACT_LABEL: str = "Janitor archive" -UNQUOTED_ADDRESSING_REASON: str = ( - "relation name is not a plain lowercase identifier and may require quoting; " - "janitor does not act on it" -) -CASE_COLLISION_REASON: str = ( - "relation name collides case-insensitively with another relation; janitor does not act on it" -) -MALFORMED_ARCHIVE_REASON: str = ( - "name resembles a janitor archive but does not match the strict archive grammar" -) -QUALIFIED_ORIGIN_PARTS: int = 3 diff --git a/src/sqlbuild/executor/janitor/main/_build_archive_name.py b/src/sqlbuild/executor/janitor/main/_build_archive_name.py deleted file mode 100644 index e7f55b8153..0000000000 --- a/src/sqlbuild/executor/janitor/main/_build_archive_name.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public janitor archive naming for relations archived outside the janitor.""" - -from __future__ import annotations - -from datetime import datetime - -from sqlbuild.executor.janitor._helpers.archive_names import build_archive_name - - -def build_janitor_archive_name( - *, original_name: str, archived_at: datetime, identifier_limit: int, kind: str -) -> str: - """Build a strict archive name that janitor expires by its embedded timestamp.""" - - return build_archive_name( - original_name=original_name, - archived_at=archived_at, - identifier_limit=identifier_limit, - kind=kind, - ) diff --git a/src/sqlbuild/executor/janitor/main/plan.py b/src/sqlbuild/executor/janitor/main/plan.py deleted file mode 100644 index e5e7645008..0000000000 --- a/src/sqlbuild/executor/janitor/main/plan.py +++ /dev/null @@ -1,233 +0,0 @@ -"""Plan janitor cleanup.""" - -from __future__ import annotations - -from dataclasses import replace -from datetime import UTC, datetime -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.executor.diff.classes.query_artifact_lifecycle import QueryDiffArtifactLifecycle -from sqlbuild.executor.janitor._helpers.archive_planning import plan_janitor_archives -from sqlbuild.executor.janitor._helpers.classification import ( - collect_direct_state_prune_candidates, - collect_query_diff_artifact_candidates, - gather_janitor_warehouse_facts, -) -from sqlbuild.executor.janitor._helpers.plan import ( - collect_scan_schemas, - collect_target_schemas, - pending_migration_origins, -) -from sqlbuild.executor.janitor._helpers.schema_planning import classify_target_schemas -from sqlbuild.executor.janitor.classes.relation_age_reader import JanitorRelationAgeReader -from sqlbuild.executor.janitor.models import ( - JanitorArchivePlanning, - JanitorDirectModeSettings, - JanitorDirectStatePruneCandidate, - JanitorOldNameViewPlanning, - JanitorPlan, - JanitorRelationKey, - JanitorRelationScope, - JanitorSchemaClassification, - JanitorWarehouseFacts, -) -from sqlbuild.executor.old_name_views.main._plan_old_name_view_cleanup import ( - plan_old_name_view_cleanup, -) -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle - - -def build_janitor_plan( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - retention_days: int, - delete_tracked_only: bool = True, - exclude_patterns: tuple[str, ...] = (), - relation_scope: JanitorRelationScope | None = None, - direct_settings: JanitorDirectModeSettings | None = None, - early_old_name_view_drops: tuple[str, ...] = (), -) -> JanitorPlan: - """Build a desired-vs-warehouse cleanup plan for target schemas.""" - - scope: JanitorRelationScope = ( - relation_scope if relation_scope is not None else JanitorRelationScope() - ) - direct: JanitorDirectModeSettings = direct_settings or JanitorDirectModeSettings() - managed_target_schemas: set[tuple[str | None, str | None]] = collect_target_schemas(project) - target_schemas: set[tuple[str | None, str | None]] = collect_scan_schemas( - managed_target_schemas=managed_target_schemas, - relation_keys=scope.protected_relation_keys | scope.scan_relation_keys, - ) - query_artifact_schemas: set[tuple[str | None, str]] = { - (database, schema) for database, schema in target_schemas if schema is not None - } - if project.effective_target_schema is not None: - query_artifact_schemas.add( - (project.effective_target_database, project.effective_target_schema) - ) - now: datetime = datetime.now(UTC) - ( - query_diff_artifact_candidates, - query_diff_artifact_skipped, - ) = collect_query_diff_artifact_candidates( - adapter=adapter, - connection=connection, - target_schemas=query_artifact_schemas, - now=now, - ) - if not target_schemas: - return JanitorPlan( - target_name=project.effective_target_name, - retention_days=retention_days, - direct_mode=direct.enabled, - archive_retention_days=direct.archive_retention_days, - query_diff_artifact_candidates=query_diff_artifact_candidates, - direct_state_prune_candidates=(), - skipped_relations=query_diff_artifact_skipped, - scanned_schema_count=len(query_artifact_schemas), - age_metadata_supported=adapter.supports_relation_age_metadata(), - planned_at=now, - ) - - with OperationLifecycle( - operation_kind="janitor", operation_name="janitor_warehouse_inspection" - ) as inspection: - facts: JanitorWarehouseFacts = gather_janitor_warehouse_facts( - project=project, - adapter=adapter, - connection=connection, - target_schemas=target_schemas, - delete_tracked_only=delete_tracked_only, - ) - direct_state_prune_candidates: tuple[JanitorDirectStatePruneCandidate, ...] = ( - collect_direct_state_prune_candidates( - adapter=adapter, - connection=connection, - target_schemas=target_schemas, - direct_state_history_versions=direct.state_history_versions, - ) - ) - inspection.completed(metadata={"item_count": len(target_schemas)}) - old_names: JanitorOldNameViewPlanning - old_names, scope = _plan_protected_names( - adapter=adapter, - connection=connection, - managed_target_schemas=managed_target_schemas, - facts=facts, - early_drops=early_old_name_view_drops, - project=project, - scope=scope, - now=now, - ) - age_reader: JanitorRelationAgeReader = JanitorRelationAgeReader( - adapter=adapter, connection=connection - ) - age_supported: bool = adapter.supports_relation_age_metadata() - schemas: JanitorSchemaClassification = classify_target_schemas( - age_reader=age_reader, - target_schemas=target_schemas, - managed_target_schemas=managed_target_schemas, - facts=facts, - scope=scope, - exclude_patterns=exclude_patterns, - delete_tracked_only=delete_tracked_only, - retention_days=retention_days, - now=now, - direct_mode=direct.enabled, - ) - archives: JanitorArchivePlanning = plan_janitor_archives( - direct_mode=direct.enabled, - adapter=adapter, - schemas=schemas, - facts=facts, - managed_target_schemas=managed_target_schemas, - exclude_patterns=exclude_patterns, - archive_retention_days=direct.archive_retention_days, - now=now, - ) - - return JanitorPlan( - target_name=project.effective_target_name, - retention_days=retention_days, - direct_mode=direct.enabled, - archive_retention_days=direct.archive_retention_days, - candidates=schemas.candidates, - archive_candidates=archives.archive_candidates, - archive_deletion_candidates=archives.archive_deletion_candidates, - retained_archives=archives.retained_archives, - query_diff_artifact_candidates=query_diff_artifact_candidates, - direct_state_prune_candidates=direct_state_prune_candidates, - skipped_relations=( - *( - skipped - for skipped in schemas.skipped_relations - if not QueryDiffArtifactLifecycle.is_artifact_name(skipped.key.name) - and skipped.key not in old_names.keys - ), - *archives.skipped_relations, - *query_diff_artifact_skipped, - ), - skipped_schemas=schemas.skipped_schemas, - blocked_schemas=archives.blocked_schemas, - old_name_views=old_names, - scanned_schema_count=len(target_schemas | set(query_artifact_schemas)), - age_metadata_supported=age_supported, - planned_at=now, - ) - - -def _plan_protected_names( - *, - adapter: BaseAdapter, - connection: Any, - managed_target_schemas: set[tuple[str | None, str | None]], - facts: JanitorWarehouseFacts, - early_drops: tuple[str, ...], - project: CompiledProjectView, - scope: JanitorRelationScope, - now: datetime, -) -> tuple[JanitorOldNameViewPlanning, JanitorRelationScope]: - """Plan old-name views, then keep general cleanup away from pending migration origins.""" - - old_names: JanitorOldNameViewPlanning - old_names, scope = plan_old_name_view_cleanup( - adapter=adapter, - connection=connection, - managed_target_schemas=managed_target_schemas, - relations_by_schema=facts.relations_by_schema, - target_name=project.effective_target_name, - early_drops=early_drops, - project=project, - scope=scope, - now=now, - ) - return old_names, _protect_pending_origins( - project=project, facts=facts, scope=scope, old_names=old_names - ) - - -def _protect_pending_origins( - *, - project: CompiledProjectView, - facts: JanitorWarehouseFacts, - scope: JanitorRelationScope, - old_names: JanitorOldNameViewPlanning, -) -> JanitorRelationScope: - pending: dict[JanitorRelationKey, str] = { - key: reason - for key, reason in pending_migration_origins( - project=project, relations_by_schema=facts.relations_by_schema - ).items() - if key not in old_names.keys - } - if not pending: - return scope - return replace( - scope, - protected_relation_keys=scope.protected_relation_keys | frozenset(pending), - protected_relation_reasons={**(scope.protected_relation_reasons or {}), **pending}, - ) diff --git a/src/sqlbuild/executor/janitor/main/resolve_retention_days.py b/src/sqlbuild/executor/janitor/main/resolve_retention_days.py deleted file mode 100644 index 71a342c723..0000000000 --- a/src/sqlbuild/executor/janitor/main/resolve_retention_days.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Janitor relation retention resolution entrypoint.""" - -from __future__ import annotations - -from sqlbuild.executor.janitor.constants import ( - DIRECT_DEFAULT_RETENTION_DAYS, -) - - -def resolve_janitor_retention_days(*, override: int | None, configured: int | None) -> int: - """Resolve CLI override, explicit config, then the mode-specific default.""" - - if override is not None: - return override - if configured is not None: - return configured - return DIRECT_DEFAULT_RETENTION_DAYS diff --git a/src/sqlbuild/executor/janitor/models.py b/src/sqlbuild/executor/janitor/models.py index 4655857c9c..965584d612 100644 --- a/src/sqlbuild/executor/janitor/models.py +++ b/src/sqlbuild/executor/janitor/models.py @@ -52,14 +52,6 @@ class JanitorDeleteCandidate: age_timestamp: datetime | None -@dataclass(frozen=True) -class JanitorParsedArchiveName: - """Structured parts of one strict janitor archive relation name.""" - - archived_at: datetime - logical_name: str - - @dataclass(frozen=True) class JanitorArchiveCandidate: """One stale direct-mode relation that will be renamed to an archive name.""" @@ -196,63 +188,6 @@ def display_name(self) -> str: return "" -@dataclass(frozen=True) -class JanitorWarehouseFacts: - """Desired, discovered, and tracked relation facts for planning.""" - - desired_keys: frozenset[JanitorRelationKey] - source_schema_names: dict[tuple[str | None, str | None], set[str]] - relations_by_schema: dict[tuple[str | None, str | None], tuple[RelationInfo, ...]] - tracked_relation_keys: frozenset[JanitorRelationKey] - - -@dataclass(frozen=True) -class JanitorRelationClassification: - """Delete candidates and skipped relations for one target schema.""" - - candidates: tuple[JanitorDeleteCandidate, ...] - skipped_relations: tuple[JanitorSkippedRelation, ...] - - -@dataclass(frozen=True) -class JanitorSchemaClassification: - """Combined relation decisions across target schemas.""" - - candidates: tuple[JanitorDeleteCandidate, ...] - skipped_relations: tuple[JanitorSkippedRelation, ...] - skipped_schemas: tuple[JanitorSkippedSchema, ...] - blocked_schemas: tuple[JanitorBlockedSchema, ...] - - -@dataclass(frozen=True) -class JanitorArchivePlanning: - """Direct-mode archive and archive-expiry decisions for one janitor plan.""" - - archive_candidates: tuple[JanitorArchiveCandidate, ...] - archive_deletion_candidates: tuple[JanitorArchivedRelation, ...] - retained_archives: tuple[JanitorArchivedRelation, ...] - skipped_relations: tuple[JanitorSkippedRelation, ...] - blocked_schemas: tuple[JanitorBlockedSchema, ...] - - -@dataclass(frozen=True) -class JanitorRelationScope: - """Scan scope and protection rules for one janitor plan.""" - - scan_relation_keys: frozenset[JanitorRelationKey] = frozenset() - protected_relation_keys: frozenset[JanitorRelationKey] = frozenset() - protected_relation_reasons: dict[JanitorRelationKey, str] | None = None - - -@dataclass(frozen=True) -class JanitorDirectModeSettings: - """Direct-mode state-history settings.""" - - enabled: bool = True - state_history_versions: int = 20 - archive_retention_days: int = 14 - - @dataclass(frozen=True) class JanitorPlan: """Complete janitor preview and execution plan.""" diff --git a/src/sqlbuild/executor/migrations/_helpers/columns.py b/src/sqlbuild/executor/migrations/_helpers/columns.py index 195558d9c2..8c62013c05 100644 --- a/src/sqlbuild/executor/migrations/_helpers/columns.py +++ b/src/sqlbuild/executor/migrations/_helpers/columns.py @@ -10,10 +10,12 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation from sqlbuild.compiler.migrations.types import ColumnMigrationDecision from sqlbuild.compiler.planner.models import ColumnMigrationPlanEntry +from sqlbuild.executor.migrations._helpers.relation_for_location import ( + migration_relation_for_location, +) def apply_model_column_renames( diff --git a/src/sqlbuild/executor/migrations/_helpers/naming.py b/src/sqlbuild/executor/migrations/_helpers/naming.py index 7c4a2ae6c0..03d29b933c 100644 --- a/src/sqlbuild/executor/migrations/_helpers/naming.py +++ b/src/sqlbuild/executor/migrations/_helpers/naming.py @@ -6,6 +6,7 @@ from datetime import datetime, timedelta from typing import Any +from sqlbuild._native import build_janitor_archive_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( @@ -13,7 +14,6 @@ ) from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.executor.janitor.main._build_archive_name import build_janitor_archive_name from sqlbuild.executor.migrations.constants import ( MIGRATION_NAME_ATTEMPTS, MIGRATION_PREVIOUS_ARCHIVE_KIND, diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py b/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py index 1ad02e77bf..b05e84d7d1 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py @@ -13,10 +13,12 @@ ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location from sqlbuild.compiler.migrations.models import MigrationRelation, OldNameViewEvent from sqlbuild.compiler.migrations.types import OldNameViewEventType from sqlbuild.compiler.planner.models import OldNameViewPlanEntry +from sqlbuild.executor.migrations._helpers.relation_for_location import ( + migration_relation_for_location, +) def old_name_fact( diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py index e8495b0dd5..c83c502598 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py @@ -8,7 +8,11 @@ from datetime import UTC, datetime, timedelta from typing import Any -from sqlbuild._native import deterministic_migration_event_id, read_recorded_column_renames +from sqlbuild._native import ( + build_janitor_archive_name, + deterministic_migration_event_id, + read_recorded_column_renames, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.main.grants_for_columns import grants_for_columns @@ -20,7 +24,6 @@ from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.migrations.constants import COLUMN_MIGRATION_TABLE_NAME -from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location from sqlbuild.compiler.migrations.models import ( ColumnMigrationEvent, MigrationRelation, @@ -30,12 +33,14 @@ from sqlbuild.compiler.planner.models import OldNameView, OldNameViewPlanEntry, PlanOutput from sqlbuild.cursor_algebra.models import Duration from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.executor.janitor.main._build_archive_name import build_janitor_archive_name from sqlbuild.executor.migrations._helpers.old_name_facts import ( old_name_fact, record_old_name_fact, recorded_old_name_facts, ) +from sqlbuild.executor.migrations._helpers.relation_for_location import ( + migration_relation_for_location, +) from sqlbuild.executor.migrations.constants import ( MIGRATION_NAME_ATTEMPTS, MIGRATION_ORIGIN_ARCHIVE_KIND, diff --git a/src/sqlbuild/executor/migrations/_helpers/promotion.py b/src/sqlbuild/executor/migrations/_helpers/promotion.py index 42d2f1be39..9e3dac3889 100644 --- a/src/sqlbuild/executor/migrations/_helpers/promotion.py +++ b/src/sqlbuild/executor/migrations/_helpers/promotion.py @@ -9,7 +9,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation, OldNameViewEvent from sqlbuild.compiler.migrations.types import MigrationDecision, OldNameViewEventType from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, OldNameViewPlanEntry @@ -18,6 +17,9 @@ old_name_state_table_sql, record_old_name_fact, ) +from sqlbuild.executor.migrations._helpers.relation_for_location import ( + migration_relation_for_location, +) from sqlbuild.executor.migrations.models import MigrationArtifactNames from sqlbuild.executor.run.main.promote_staged_relation import promote_staged_relation diff --git a/src/sqlbuild/compiler/migrations/main/relation_for_location.py b/src/sqlbuild/executor/migrations/_helpers/relation_for_location.py similarity index 100% rename from src/sqlbuild/compiler/migrations/main/relation_for_location.py rename to src/sqlbuild/executor/migrations/_helpers/relation_for_location.py diff --git a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py index 3abcd272a3..299d04eb72 100644 --- a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py +++ b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py @@ -2,14 +2,11 @@ from __future__ import annotations -from dataclasses import replace from datetime import UTC, datetime from typing import Any from sqlbuild._native import ( deterministic_old_name_view_event_id, - read_old_name_view_histories, - stored_old_name_view_columns, write_migration_state_event, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -20,106 +17,19 @@ resolve_qualified_name_parts, ) from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.migrations.constants import ( - MIGRATION_TABLE_NAME, - OLD_NAME_VIEW_TABLE_NAME, -) -from sqlbuild.compiler.migrations.models import OldNameViewEvent, OldNameViewHistory +from sqlbuild.compiler.migrations.models import OldNameViewEvent from sqlbuild.compiler.migrations.types import ( OldNameViewDropReason, OldNameViewEventType, - OldNameViewStatus, ) from sqlbuild.executor.janitor.models import ( JanitorOldNameView, JanitorOldNameViewPlanning, JanitorRelationKey, - JanitorRelationScope, ) from sqlbuild.executor.migrations.main._compatibility_view_sql import compatibility_view_sql -def plan_old_name_views( - *, - adapter: BaseAdapter, - connection: Any, - managed_target_schemas: set[tuple[str | None, str | None]], - relations_by_schema: dict[tuple[str | None, str | None], tuple[RelationInfo, ...]], - target_name: str | None, - early_drops: tuple[str, ...], - project_destinations: dict[JanitorRelationKey, str], - now: datetime, -) -> JanitorOldNameViewPlanning: - """Split recorded views into live, droppable, and gone; another relation's name is gone.""" - - histories: tuple[OldNameViewHistory, ...] = tuple( - history - for history in _read_histories( - adapter=adapter, - connection=connection, - managed_target_schemas=managed_target_schemas, - relations_by_schema=relations_by_schema, - ) - if history.move.target_name in (None, target_name) - and history.status(now=now) in (OldNameViewStatus.LIVE, OldNameViewStatus.EXPIRED) - ) - if not histories and not early_drops: - return JanitorOldNameViewPlanning() - physical: dict[JanitorRelationKey, RelationInfo] = _physical_old_names( - adapter=adapter, - connection=connection, - histories=histories, - relations_by_schema=relations_by_schema, - ) - requested: dict[str, JanitorOldNameView | None] = dict.fromkeys(early_drops) - live: list[JanitorOldNameView] = [] - drops: list[JanitorOldNameView] = [] - missing: list[JanitorOldNameView] = [] - history: OldNameViewHistory - for history in histories: - key: JanitorRelationKey = _old_key(history) - view: JanitorOldNameView = _view(history) - matched: tuple[str, ...] = tuple(name for name in requested if _names(name=name, key=key)) - requested.update(dict.fromkeys(matched, view)) - relation: RelationInfo | None = physical.get(key) - if relation is None: - missing.append(_with_reason(view=view, reason=OldNameViewDropReason.MISSING)) - elif not is_compatibility_view( - adapter=adapter, connection=connection, view=view, relation=relation - ): - missing.append(_occupied(view=view, project_destinations=project_destinations)) - elif matched: - drops.append(_with_reason(view=view, reason=OldNameViewDropReason.EARLY)) - elif history.status(now=now) == OldNameViewStatus.EXPIRED: - drops.append(_with_reason(view=view, reason=OldNameViewDropReason.EXPIRED)) - else: - live.append(view) - return JanitorOldNameViewPlanning( - live=tuple(live), - drops=tuple(drops), - missing=tuple(missing), - unknown_requests=tuple(name for name, view in requested.items() if view is None), - project_destinations=project_destinations, - ) - - -def protect_old_names( - *, scope: JanitorRelationScope, old_names: JanitorOldNameViewPlanning -) -> JanitorRelationScope: - """Keep general cleanup away from every old name this planning handles itself.""" - - if not old_names.keys: - return scope - reasons: dict[JanitorRelationKey, str] = dict(scope.protected_relation_reasons or {}) - reasons.update({key: "compatibility view at an old model name" for key in old_names.keys}) - return replace( - scope, - protected_relation_keys=scope.protected_relation_keys | old_names.keys, - protected_relation_reasons=reasons, - ) - - def apply_old_name_view_drops( *, plan: JanitorOldNameViewPlanning, @@ -189,139 +99,6 @@ def _record_drop( ) -def _read_histories( - *, - adapter: BaseAdapter, - connection: Any, - managed_target_schemas: set[tuple[str | None, str | None]], - relations_by_schema: dict[tuple[str | None, str | None], tuple[RelationInfo, ...]], -) -> tuple[OldNameViewHistory, ...]: - histories: list[OldNameViewHistory] = [] - database: str | None - schema: str | None - for database, schema in sorted( - managed_target_schemas, key=lambda key: (key[0] or "", key[1] or "") - ): - names: frozenset[str] = frozenset( - relation.name.lower() for relation in relations_by_schema.get((database, schema), ()) - ) - if schema is None or not {OLD_NAME_VIEW_TABLE_NAME, MIGRATION_TABLE_NAME} <= names: - continue - histories.extend( - read_old_name_view_histories( - connection=connection, - execute=adapter.execute, - database=database, - schema=schema, - stored_columns=stored_old_name_view_columns( - adapter=adapter, connection=connection, database=database, schema=schema - ), - render_qualified_name=adapter.render_qualified_name, - ) - ) - return tuple(histories) - - -def _physical_old_names( - *, - adapter: BaseAdapter, - connection: Any, - histories: tuple[OldNameViewHistory, ...], - relations_by_schema: dict[tuple[str | None, str | None], tuple[RelationInfo, ...]], -) -> dict[JanitorRelationKey, RelationInfo]: - """Look up each old name, listing only schemas the janitor did not already scan.""" - - known: dict[str, tuple[RelationInfo, ...]] = { - (schema or "").lower(): relations for (_, schema), relations in relations_by_schema.items() - } - unscanned: tuple[OldNameViewHistory, ...] = tuple( - history for history in histories if (history.old.schema or "").lower() not in known - ) - listed: tuple[RelationInfo, ...] = ( - adapter.list_relations( - connection=connection, - database=unscanned[0].old.database, - schemas=tuple(sorted({history.old.schema or "" for history in unscanned})), - names=tuple(sorted({history.old.name for history in unscanned})), - ) - if unscanned - else () - ) - found: dict[JanitorRelationKey, RelationInfo] = {} - scanned: list[RelationInfo] = list(listed) - schema_relations: tuple[RelationInfo, ...] - for schema_relations in known.values(): - scanned.extend(schema_relations) - relation: RelationInfo - for relation in scanned: - _ = found.setdefault( - JanitorRelationKey( - database=relation.database, schema=relation.schema, name=relation.name - ), - relation, - ) - physical: dict[JanitorRelationKey, RelationInfo] = {} - history: OldNameViewHistory - for history in histories: - key: JanitorRelationKey = _old_key(history) - match: RelationInfo | None = _lookup(found=found, key=key) - if match is not None: - physical[key] = match - return physical - - -def _view(history: OldNameViewHistory) -> JanitorOldNameView: - return JanitorOldNameView( - key=_old_key(history), - history=history, - expires_at=None if history.created is None else history.created.expires_at, - ) - - -def project_destinations(project: CompiledProjectView) -> dict[JanitorRelationKey, str]: - """Map each project model's destination to the model's name.""" - - return { - JanitorRelationKey( - database=model.destination.database, - schema=model.destination.schema, - name=model.destination.name, - ): model.name - for model in project.models - } - - -def _old_key(history: OldNameViewHistory) -> JanitorRelationKey: - return JanitorRelationKey( - database=history.old.database, schema=history.old.schema, name=history.old.name - ) - - -def _lookup( - *, found: dict[JanitorRelationKey, RelationInfo], key: JanitorRelationKey -) -> RelationInfo | None: - exact: RelationInfo | None = found.get(key) - if exact is not None or key.database is not None: - return exact - return next( - ( - relation - for candidate, relation in found.items() - if (candidate.schema or "").lower() == (key.schema or "").lower() - and candidate.name.lower() == key.name.lower() - ), - None, - ) - - -def _names(*, name: str, key: JanitorRelationKey) -> bool: - parts: list[str] = [part.strip().strip('"').lower() for part in name.split(".")] - spelled: list[str] = [ - part.lower() for part in (key.database, key.schema, key.name) if part is not None - ] - return spelled[-len(parts) :] == parts - - def _with_reason(*, view: JanitorOldNameView, reason: OldNameViewDropReason) -> JanitorOldNameView: return JanitorOldNameView( key=view.key, history=view.history, expires_at=view.expires_at, drop_reason=reason @@ -375,37 +152,5 @@ def _listed_relation( ) -def _occupied( - *, view: JanitorOldNameView, project_destinations: dict[JanitorRelationKey, str] -) -> JanitorOldNameView: - return JanitorOldNameView( - key=view.key, - history=view.history, - expires_at=view.expires_at, - drop_reason=OldNameViewDropReason.MISSING, - occupied=True, - claimed_by=_claiming_model(key=view.key, project_destinations=project_destinations), - ) - - -def _claiming_model( - *, key: JanitorRelationKey, project_destinations: dict[JanitorRelationKey, str] -) -> str | None: - return next( - ( - model_name - for destination, model_name in project_destinations.items() - if (destination.schema or "").lower() == (key.schema or "").lower() - and destination.name.lower() == key.name.lower() - and ( - destination.database is None - or key.database is None - or destination.database.lower() == key.database.lower() - ) - ), - None, - ) - - def _is_view(relation: RelationInfo) -> bool: return normalize_relation_type(relation.relation_type) == RelationType.VIEW diff --git a/src/sqlbuild/executor/old_name_views/main/_plan_old_name_view_cleanup.py b/src/sqlbuild/executor/old_name_views/main/_plan_old_name_view_cleanup.py deleted file mode 100644 index 790cd5560b..0000000000 --- a/src/sqlbuild/executor/old_name_views/main/_plan_old_name_view_cleanup.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Plan janitor cleanup of compatibility views at old model names.""" - -from __future__ import annotations - -from datetime import datetime -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.executor.janitor.models import JanitorOldNameViewPlanning, JanitorRelationScope -from sqlbuild.executor.old_name_views._helpers.janitor import ( - plan_old_name_views, - project_destinations, - protect_old_names, -) - - -def plan_old_name_view_cleanup( - *, - adapter: BaseAdapter, - connection: Any, - managed_target_schemas: set[tuple[str | None, str | None]], - relations_by_schema: dict[tuple[str | None, str | None], tuple[RelationInfo, ...]], - target_name: str | None, - early_drops: tuple[str, ...], - project: CompiledProjectView, - scope: JanitorRelationScope, - now: datetime, -) -> tuple[JanitorOldNameViewPlanning, JanitorRelationScope]: - """Return old-name view actions and a scope that keeps general cleanup away from them.""" - - old_names: JanitorOldNameViewPlanning = plan_old_name_views( - adapter=adapter, - connection=connection, - managed_target_schemas=managed_target_schemas, - relations_by_schema=relations_by_schema, - target_name=target_name, - early_drops=early_drops, - project_destinations=project_destinations(project), - now=now, - ) - return old_names, protect_old_names(scope=scope, old_names=old_names) diff --git a/src/sqlbuild/integrations/dagster/_helpers/invocation.py b/src/sqlbuild/integrations/dagster/_helpers/invocation.py index 1a218f9ce1..bd8a689c70 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/invocation.py +++ b/src/sqlbuild/integrations/dagster/_helpers/invocation.py @@ -1,47 +1,19 @@ -"""SQLBuild CLI invocation helpers for Dagster.""" +"""Dagster glue around native SQLBuild CLI invocation planning.""" from __future__ import annotations -import json -import tempfile from collections.abc import Mapping from concurrent.futures import Future, ThreadPoolExecutor -from enum import Enum from pathlib import Path from typing import IO, TYPE_CHECKING, Any, TextIO -from sqlbuild._native import IntegrationAssetResult, IntegrationResultEnvelope -from sqlbuild.integrations.dagster._helpers.translation import ( - is_dagster_asset_enabled, - is_dagster_check_enabled, -) -from sqlbuild.integrations.dagster.constants import ( - ASSET_SELECTION_COMMANDS, - CHECK_COMMAND, - CHECK_METADATA_EXCLUDED_KEYS, - CHECK_NAME_SEPARATOR_CHARACTER, - CLONE_COMMAND, - COMPLETED_EXECUTION_STATUSES, - DAGSTER_CHECK_NAME_FIELD, - DEFAULT_SELECTABLE_NODE_KINDS, - EVENT_OUTPUT_FLAG, - EXPLICIT_SELECTION_FLAGS, - INSUFFICIENT_CHECK_STATUS, - JSON_OUTPUT_FLAG, - JSON_OUTPUT_FLAGS, - LIVE_EVENT_COMMANDS, - LOAD_COMMAND, - LOAD_SELECTABLE_NODE_KINDS, - LOADER_NODE_KIND, - MATERIALIZABLE_NODE_KINDS, - SCENARIO_CHECK_KIND, - SCENARIO_TEST_COMMAND, - SCENARIO_VALUE_FLAGS, - SELECT_FILE_FLAG, - SOURCE_NODE_KIND, - STANDALONE_CHECK_COMMANDS, - SUCCESS_EXECUTION_STATUS, - WARNING_CHECK_SEVERITY, +from sqlbuild._native import ( + IntegrationResultEnvelope, + dagster_execution_payload, + dagster_integration_results, + dagster_payload_results, + dagster_selected_asset_results, + dagster_selector_log_lines, ) if TYPE_CHECKING: @@ -49,6 +21,9 @@ SqlBuildCliInvocation, ) +_CHECK_RESULT: str = "check" +_ASSET_MATERIALIZATION: str = "materialization" + def _log_invocation(*, context: Any, invocation: SqlBuildCliInvocation) -> None: if context is None: @@ -63,227 +38,10 @@ def _log_invocation(*, context: Any, invocation: SqlBuildCliInvocation) -> None: logger.info("SQLBuild selector file:") logger.info(" %s", invocation.selector_file_path) logger.info("SQLBuild selected assets from Dagster (%s):", len(invocation.selection)) - for line in _wrap_selectors(selectors=invocation.selection): + for line in dagster_selector_log_lines(selectors=list(invocation.selection)): logger.info(" %s", line) -def _wrap_selectors(*, selectors: tuple[str, ...], width: int = 100) -> tuple[str, ...]: - lines: list[str] = [] - current: str = "" - for selector in selectors: - candidate: str = selector if not current else f"{current} {selector}" - if len(candidate) <= width: - current = candidate - continue - if current: - lines.append(current) - current = selector - if current: - lines.append(current) - return tuple(lines) - - -def _with_selected_asset_args( - *, args: tuple[str, ...], context: Any, dag: Mapping[str, Any] | None -) -> tuple[tuple[str, ...], tuple[str, ...], Path | None]: - if dag is None or context is None or not args: - return args, (), None - scenario_test_argument_count: int = 2 - if len(args) >= scenario_test_argument_count and args[:2] == SCENARIO_TEST_COMMAND: - return _with_selected_scenario_args(args=args, context=context, dag=dag) - if args[0] not in ASSET_SELECTION_COMMANDS: - return args, (), None - if not EXPLICIT_SELECTION_FLAGS.isdisjoint(args): - return args, (), None - selected_paths: set[tuple[str, ...]] = _selected_asset_paths(context=context) - selected_check_keys: set[tuple[tuple[str, ...], str]] = _selected_asset_check_keys( - context=context - ) - if args[0] == CHECK_COMMAND and selected_check_keys: - check_selectors: tuple[str, ...] = _selected_check_selectors( - dag=dag, selected_check_keys=selected_check_keys - ) - if check_selectors: - selector_file: Path = _write_selector_file(check_selectors) - return ( - (*args, SELECT_FILE_FLAG, str(selector_file)), - check_selectors, - selector_file, - ) - if selected_check_keys: - selected_paths.update( - _selected_check_asset_paths(dag=dag, selected_check_keys=selected_check_keys) - ) - if not selected_paths: - return args, (), None - selectors: list[str] = [] - for node in _sort_nodes_topologically(dag=dag): - if tuple(str(part) for part in node["asset_key"]) not in selected_paths: - continue - if str(node.get("kind")) not in _selectable_kinds_for_command(args[0]): - continue - selector: object = node.get("name") - if selector is not None: - selectors.append(str(selector)) - if not selectors: - return args, (), None - selector_file: Path = _write_selector_file(tuple(selectors)) - return (*args, SELECT_FILE_FLAG, str(selector_file)), tuple(selectors), selector_file - - -def _with_selected_scenario_args( - *, args: tuple[str, ...], context: Any, dag: Mapping[str, Any] -) -> tuple[tuple[str, ...], tuple[str, ...], Path | None]: - if _has_explicit_scenario_selector(args=args): - return args, (), None - selected_paths: set[tuple[str, ...]] = _selected_asset_paths(context=context) - selected_check_keys: set[tuple[tuple[str, ...], str]] = _selected_asset_check_keys( - context=context - ) - if not selected_paths: - selected_paths = {asset_key for asset_key, _check_name in selected_check_keys} - if not selected_paths and not selected_check_keys: - return args, (), None - nodes_by_id: dict[str, Mapping[str, Any]] = {} - selected_asset_ids: set[str] = set() - for node in dag.get("nodes", ()): - node_id: str = str(node.get("id")) - nodes_by_id[node_id] = node - asset_key: list[str] = [] - for part in node.get("asset_key", ()): - asset_key.append(str(part)) - if tuple(asset_key) in selected_paths: - selected_asset_ids.add(node_id) - selectors: list[str] = [] - check: Mapping[str, Any] - for check in dag.get("checks", ()): # type: ignore[assignment] - if not is_dagster_check_enabled(check): - continue - if str(check.get("kind")) != SCENARIO_CHECK_KIND: - continue - if not selected_asset_ids.intersection( - str(id_) for id_ in check.get("checked_asset_ids", ()) - ): - continue - check_name: str = _dagster_check_name(check) - if selected_check_keys and not _check_is_selected( - check=check, - check_name=check_name, - nodes_by_id=nodes_by_id, - selected_check_keys=selected_check_keys, - ): - continue - selector: object = check.get("name") - if selector is not None: - selectors.append(str(selector)) - if not selectors: - return args, (), None - return (*args, *selectors), tuple(selectors), None - - -def _has_explicit_scenario_selector(*, args: tuple[str, ...]) -> bool: - skip_next: bool = False - for arg in args[2:]: - if skip_next: - skip_next = False - continue - if arg in SCENARIO_VALUE_FLAGS: - skip_next = True - continue - if arg.startswith("-"): - continue - return True - return False - - -def _check_is_selected( - *, - check: Mapping[str, Any], - check_name: str, - nodes_by_id: Mapping[str, Mapping[str, Any]], - selected_check_keys: set[tuple[tuple[str, ...], str]], -) -> bool: - for asset_id in check.get("checked_asset_ids", ()): - node: Mapping[str, Any] | None = nodes_by_id.get(str(asset_id)) - if node is None: - continue - asset_path: tuple[str, ...] = tuple(str(part) for part in node.get("asset_key", ())) - if (asset_path, check_name) in selected_check_keys: - return True - return False - - -def _with_json_output_args( - *, args: tuple[str, ...], context: Any, dag: Mapping[str, Any] | None -) -> tuple[tuple[str, ...], Path | None]: - if dag is None or context is None or not args or _json_output_requested(args=args): - return args, None - if args[0] in ASSET_SELECTION_COMMANDS: - path: Path = _create_execution_json_path() - return (*args, JSON_OUTPUT_FLAG, str(path)), path - scenario_test_argument_count: int = 2 - if len(args) >= scenario_test_argument_count and args[:2] == SCENARIO_TEST_COMMAND: - path = _create_execution_json_path() - return (*args, JSON_OUTPUT_FLAG, str(path)), path - return args, None - - -def _json_output_requested(*, args: tuple[str, ...]) -> bool: - return any( - argument in JSON_OUTPUT_FLAGS or argument.startswith(f"{JSON_OUTPUT_FLAG}=") - for argument in args - ) - - -def _caller_json_output_path(*, args: tuple[str, ...]) -> Path | None: - for index, argument in enumerate(args): - if argument == JSON_OUTPUT_FLAG and index + 1 < len(args): - return Path(args[index + 1]) - prefix: str = f"{JSON_OUTPUT_FLAG}=" - if argument.startswith(prefix) and argument != prefix: - return Path(argument.removeprefix(prefix)) - return None - - -def _create_execution_json_path() -> Path: - handle: IO[str] = tempfile.NamedTemporaryFile( - mode="w", - encoding="utf-8", - prefix="sqlbuild-dagster-execution-", - suffix=".json", - delete=False, - ) - with handle: - return Path(handle.name) - - -def _with_event_output_args( - *, args: tuple[str, ...], context: Any, dag: Mapping[str, Any] | None -) -> tuple[tuple[str, ...], Path | None]: - if ( - dag is None - or context is None - or not args - or args[0] not in LIVE_EVENT_COMMANDS - or EVENT_OUTPUT_FLAG in args - ): - return args, None - path: Path = _create_execution_event_path() - return args, path - - -def _create_execution_event_path() -> Path: - handle: IO[str] = tempfile.NamedTemporaryFile( - mode="w", - encoding="utf-8", - prefix="sqlbuild-dagster-events-", - suffix=".jsonl", - delete=False, - ) - with handle: - return Path(handle.name) - - def _start_stream_future( *, executor: ThreadPoolExecutor, @@ -332,86 +90,84 @@ def _streams_share_file_descriptor(*, first: TextIO, second: TextIO) -> bool: return False -def _write_selector_file(selectors: tuple[str, ...]) -> Path: - handle: IO[str] = tempfile.NamedTemporaryFile( - mode="w", - encoding="utf-8", - prefix="sqlbuild-dagster-select-", - suffix=".txt", - delete=False, +def _selected_asset_paths(*, context: Any) -> list[tuple[str, ...]]: + selected_keys: object = ( + getattr(context, "selected_asset_keys", None) if context is not None else None ) - with handle: - for selector in selectors: - handle.write(f"{selector}\n") - return Path(handle.name) + if selected_keys is None: + return [] + paths: list[tuple[str, ...]] = [] + for key in selected_keys: + paths.append(tuple(str(part) for part in key.path)) + return paths -def _build_results_for_selected_assets( - *, dg: Any, dag: Mapping[str, Any], command: tuple[str, ...], context: Any -) -> tuple[Any, ...]: +def _selected_asset_check_keys(*, context: Any) -> list[tuple[tuple[str, ...], str]]: selected_keys: object = ( - getattr(context, "selected_asset_keys", None) if context is not None else None + getattr(context, "selected_asset_check_keys", None) if context is not None else None ) - selected_paths: set[tuple[str, ...]] = set() - if selected_keys is not None: - selected_paths = {tuple(key.path) for key in selected_keys} - nodes: list[Mapping[str, Any]] = _sort_nodes_topologically(dag=dag) - if selected_paths: - selected_nodes: list[Mapping[str, Any]] = [] - for node in nodes: - asset_path: list[str] = [] - for part in node["asset_key"]: - asset_path.append(str(part)) - if tuple(asset_path) in selected_paths: - selected_nodes.append(node) - nodes = selected_nodes - results: list[Any] = [] - for node in nodes: - if not is_dagster_asset_enabled(node): + if selected_keys is None: + return [] + check_keys: list[tuple[tuple[str, ...], str]] = [] + for key in selected_keys: + asset_key: object = getattr(key, "asset_key", None) + check_name: object = getattr(key, "name", None) + if asset_key is None or check_name is None: continue - if not _is_materializable_node_kind(str(node.get("kind"))): + path: object = getattr(asset_key, "path", None) + if path is None: continue - asset_path = [] - for part in node["asset_key"]: - asset_path.append(str(part)) - results.append( - dg.MaterializeResult( - asset_key=dg.AssetKey(asset_path), - metadata={"command": " ".join(command), "sqlbuild_id": node.get("id")}, - ) - ) - return tuple(results) + check_keys.append((tuple(str(part) for part in path), str(check_name))) + return check_keys -def _selectable_kinds_for_command(command: str) -> frozenset[str]: - if command == LOAD_COMMAND: - return LOAD_SELECTABLE_NODE_KINDS - return DEFAULT_SELECTABLE_NODE_KINDS +def _check_selection_is_explicit(*, context: Any) -> bool: + return context is not None and getattr(context, "selected_asset_check_keys", None) is not None -def _is_materializable_node_kind(kind: str) -> bool: - return kind in MATERIALIZABLE_NODE_KINDS +def _dagster_events(*, dg: Any, plans: list[tuple[Any, ...]]) -> tuple[Any, ...]: + events: list[Any] = [] + for kind, path, metadata, check_name, passed, warn in plans: + asset_key: Any = dg.AssetKey(path) + if kind == _CHECK_RESULT: + events.append( + dg.AssetCheckResult( + passed=passed, + asset_key=asset_key, + check_name=check_name, + metadata=metadata, + severity=dg.AssetCheckSeverity.WARN if warn else dg.AssetCheckSeverity.ERROR, + ) + ) + continue + event_type: Any = ( + dg.AssetMaterialization if kind == _ASSET_MATERIALIZATION else dg.MaterializeResult + ) + events.append(event_type(asset_key=asset_key, metadata=metadata)) + return tuple(events) + + +def _build_results_for_selected_assets( + *, dg: Any, dag: Mapping[str, Any], command: tuple[str, ...], context: Any +) -> tuple[Any, ...]: + return _dagster_events( + dg=dg, + plans=dagster_selected_asset_results( + dag=dag, + command=list(command), + selected_asset_paths=_selected_asset_paths(context=context), + ), + ) def _load_execution_payload(stdout: str) -> Mapping[str, Any] | None: - stripped: str = stdout.strip() - if not stripped: - return None - try: - payload: Any = json.loads(stripped) - except json.JSONDecodeError: - return None - if not isinstance(payload, Mapping): - return None - if payload.get("version") != 1: - return None - return payload + return dagster_execution_payload(text=stdout) def _load_execution_payload_from_path(path: Path | None) -> Mapping[str, Any] | None: if path is None or not path.exists(): return None - return _load_execution_payload(path.read_text(encoding="utf-8")) + return dagster_execution_payload(text=path.read_text(encoding="utf-8")) def _build_results_from_execution_payload( @@ -422,96 +178,17 @@ def _build_results_from_execution_payload( command: tuple[str, ...], context: Any, ) -> tuple[Any, ...]: - selected_paths: set[tuple[str, ...]] = _selected_asset_paths(context=context) - selected_check_keys: set[tuple[tuple[str, ...], str]] = _selected_asset_check_keys( - context=context - ) - is_clone: bool = str(payload.get("command")) == CLONE_COMMAND - if is_clone: - selected_paths = set() - nodes_by_name: dict[tuple[str, str], Mapping[str, Any]] = { - (str(node.get("kind")), str(node.get("name"))): node for node in dag.get("nodes", ()) - } - nodes_by_id: dict[str, Mapping[str, Any]] = { - str(node.get("id")): node for node in dag.get("nodes", ()) - } - asset_results_by_id: dict[str, Mapping[str, Any]] = {} - payload_asset: Mapping[str, Any] - for payload_asset in payload.get("assets", ()): # type: ignore[assignment] - execution_status: str = str(payload_asset.get("status")) - if execution_status not in COMPLETED_EXECUTION_STATUSES: - continue - if is_clone and execution_status != SUCCESS_EXECUTION_STATUS: - continue - payload_kind: str = str(payload_asset.get("kind")) - if payload_asset.get("loader") is not None: - payload_kind = SOURCE_NODE_KIND - node: Mapping[str, Any] | None = nodes_by_name.get( - (payload_kind, str(payload_asset.get("name"))) - ) - if node is None: - continue - asset_results_by_id[str(node.get("id"))] = payload_asset - if str(node.get("kind")) == SOURCE_NODE_KIND and payload_asset.get("loader") is not None: - loader_result: tuple[str, Mapping[str, Any]] | None = _loader_result_for_source_payload( - dag=dag, - source_node=node, - payload_asset=payload_asset, - loader_name=str(payload_asset["loader"]), - ) - if loader_result is not None: - loader_id, loader_payload = loader_result - asset_results_by_id[loader_id] = loader_payload - results: list[Any] = [] - for node in _sort_nodes_topologically(dag=dag): - node_id: str = str(node.get("id")) - execution_asset: Mapping[str, Any] | None = asset_results_by_id.get(node_id) - if execution_asset is None: - continue - if not is_dagster_asset_enabled(node): - continue - asset_key: Any = dg.AssetKey([str(part) for part in node["asset_key"]]) - if selected_paths and tuple(asset_key.path) not in selected_paths: - continue - metadata: Mapping[str, Any] = ( - _metadata_from_mapping(execution_asset) - if execution_asset is not None - else {"kind": "source", "name": node.get("name"), "status": "observed"} - ) - materialization_type: Any = dg.AssetMaterialization if is_clone else dg.MaterializeResult - results.append( - materialization_type( - asset_key=asset_key, - metadata={ - "command": " ".join(command), - "sqlbuild_id": node.get("id"), - **metadata, - }, - ) - ) - check: Mapping[str, Any] - seen_check_outputs: set[tuple[tuple[str, ...], str]] = set() - selected_check_keys: set[tuple[tuple[str, ...], str]] = _selected_asset_check_keys( - context=context - ) - check_selection_is_explicit: bool = _check_selection_is_explicit(context=context) - for check in payload.get("checks", ()): # type: ignore[assignment] - check_results, seen_check_outputs = _build_check_results_from_execution_check( - dg=dg, + return _dagster_events( + dg=dg, + plans=dagster_payload_results( dag=dag, - nodes_by_id=nodes_by_id, - nodes_by_name=nodes_by_name, - check=check, - selected_paths=selected_paths, - selected_check_keys=selected_check_keys, - emitted_asset_paths=None, - check_selection_is_explicit=check_selection_is_explicit, - seen_check_outputs=seen_check_outputs, - ) - results.extend(check_results) - if _asset_check_only_context(context=context): - return tuple(result for result in results if isinstance(result, dg.AssetCheckResult)) - return tuple(results) + payload=payload, + command=list(command), + selected_asset_paths=_selected_asset_paths(context=context), + selected_check_keys=_selected_asset_check_keys(context=context), + check_selection_is_explicit=_check_selection_is_explicit(context=context), + ), + ) def _build_results_from_integration_result( @@ -525,386 +202,15 @@ def _build_results_from_integration_result( ) -> tuple[Any, ...]: """Translate one canonical integration result directly into Dagster events.""" - selected_paths: set[tuple[str, ...]] = _selected_asset_paths(context=context) - selected_check_keys: set[tuple[tuple[str, ...], str]] = _selected_asset_check_keys( - context=context - ) - nodes_by_id: dict[str, Mapping[str, Any]] = { - str(node.get("id")): node for node in dag.get("nodes", ()) - } - nodes_by_name: dict[tuple[str, str], Mapping[str, Any]] = { - (str(node.get("kind")), str(node.get("name"))): node for node in dag.get("nodes", ()) - } - results: list[Any] = [] - asset: IntegrationAssetResult | None = envelope.asset - if asset is not None and asset.status in COMPLETED_EXECUTION_STATUSES: - node: Mapping[str, Any] | None = nodes_by_id.get(envelope.resource_id) - loader_result: tuple[str, Mapping[str, Any]] | None = None - if ( - node is not None - and str(node.get("kind")) == SOURCE_NODE_KIND - and asset.loader is not None - ): - loader_result = _loader_result_for_source_payload( - dag=dag, - source_node=node, - payload_asset=asset.as_dict(), - loader_name=asset.loader, - ) - asset_nodes: dict[str, tuple[Mapping[str, Any], Mapping[str, Any] | None]] = {} - if node is not None: - asset_nodes[str(node.get("id"))] = (node, asset.as_dict()) - if loader_result is not None: - loader_id, loader_payload = loader_result - loader_node: Mapping[str, Any] | None = nodes_by_id.get(loader_id) - if loader_node is not None: - asset_nodes[loader_id] = (loader_node, loader_payload) - for candidate in _sort_nodes_topologically(dag=dag): - projected: tuple[Mapping[str, Any], Mapping[str, Any] | None] | None = asset_nodes.get( - str(candidate.get("id")) - ) - if projected is None: - continue - projected_node, projected_asset = projected - if not is_dagster_asset_enabled(candidate): - continue - asset_key: Any = dg.AssetKey([str(part) for part in candidate["asset_key"]]) - if selected_paths and tuple(asset_key.path) not in selected_paths: - continue - materialization_type: Any = ( - dg.AssetMaterialization - if envelope.command == CLONE_COMMAND - else dg.MaterializeResult - ) - results.append( - materialization_type( - asset_key=asset_key, - metadata={ - "command": " ".join(command), - "sqlbuild_id": candidate.get("id"), - "event_id": envelope.event_id, - "resource_attempt_id": envelope.resource_attempt_id, - "resource_id": envelope.resource_id, - "event_sequence": envelope.event_sequence, - "duration_ms": envelope.duration_ms, - **( - _metadata_from_mapping(projected_asset) - if projected_asset is not None - else { - "kind": "source", - "name": projected_node.get("name"), - "status": "observed", - } - ), - }, - ) - ) - seen_check_outputs: set[tuple[tuple[str, ...], str]] = set() - for check in envelope.checks: - check_results, seen_check_outputs = _build_check_results_from_execution_check( - dg=dg, + return _dagster_events( + dg=dg, + plans=dagster_integration_results( dag=dag, - nodes_by_id=nodes_by_id, - nodes_by_name=nodes_by_name, - check={ - **check.as_dict(), - "event_id": envelope.event_id, - "resource_attempt_id": envelope.resource_attempt_id, - "resource_id": envelope.resource_id, - "event_sequence": envelope.event_sequence, - }, - selected_paths=selected_paths, - selected_check_keys=selected_check_keys, - emitted_asset_paths=( - None - if _asset_check_only_context(context=context) - or envelope.command in STANDALONE_CHECK_COMMANDS - else emitted_asset_paths - ), + envelope=envelope, + command=list(command), + selected_asset_paths=_selected_asset_paths(context=context), + selected_check_keys=_selected_asset_check_keys(context=context), check_selection_is_explicit=_check_selection_is_explicit(context=context), - seen_check_outputs=seen_check_outputs, - ) - results.extend(check_results) - if _asset_check_only_context(context=context): - return tuple(result for result in results if isinstance(result, dg.AssetCheckResult)) - return tuple(results) - - -def _build_check_results_from_execution_check( - *, - dg: Any, - dag: Mapping[str, Any], - nodes_by_id: Mapping[str, Mapping[str, Any]], - nodes_by_name: Mapping[tuple[str, str], Mapping[str, Any]], - check: Mapping[str, Any], - selected_paths: set[tuple[str, ...]], - selected_check_keys: set[tuple[tuple[str, ...], str]], - emitted_asset_paths: set[tuple[str, ...]] | None, - check_selection_is_explicit: bool, - seen_check_outputs: set[tuple[tuple[str, ...], str]], -) -> tuple[tuple[Any, ...], set[tuple[tuple[str, ...], str]]]: - dag_check: Mapping[str, Any] | None = _dag_check_for_execution_check(dag=dag, check=check) - if dag_check is not None and not is_dagster_check_enabled(dag_check): - return (), seen_check_outputs - asset_ids: tuple[str, ...] - check_name: str - if dag_check is not None: - asset_ids = tuple(str(asset_id) for asset_id in dag_check.get("checked_asset_ids", ())) - check_name = _dagster_check_name(dag_check) - else: - asset_ids = _asset_ids_for_execution_check(check=check, nodes_by_name=nodes_by_name) - check_name = _dagster_check_name(check) - results: list[Any] = [] - asset_id: str - for asset_id in asset_ids: - node: Mapping[str, Any] | None = nodes_by_id.get(asset_id) - if node is None: - continue - if not is_dagster_asset_enabled(node): - continue - asset_key: Any = dg.AssetKey([str(part) for part in node["asset_key"]]) - output_key: tuple[tuple[str, ...], str] = (tuple(asset_key.path), check_name) - if emitted_asset_paths is not None and tuple(asset_key.path) not in emitted_asset_paths: - continue - if check_selection_is_explicit and output_key not in selected_check_keys: - continue - if ( - not check_selection_is_explicit - and selected_paths - and output_key[0] not in selected_paths - ): - continue - if output_key in seen_check_outputs: - continue - seen_check_outputs.add(output_key) - results.append( - dg.AssetCheckResult( - passed=bool(check.get("passed")), - asset_key=asset_key, - check_name=check_name, - metadata=_metadata_from_mapping(check), - severity=_dagster_check_severity(dg=dg, check=check), - ) - ) - return tuple(results), seen_check_outputs - - -def _loader_result_for_source_payload( - *, - dag: Mapping[str, Any], - source_node: Mapping[str, Any], - payload_asset: Mapping[str, Any], - loader_name: str, -) -> tuple[str, Mapping[str, Any]] | None: - nodes_by_id: dict[str, Mapping[str, Any]] = { - str(node.get("id")): node for node in dag.get("nodes", ()) - } - source_id: str = str(source_node.get("id")) - for edge in dag.get("edges", ()): # type: ignore[assignment] - if str(edge.get("to_id")) != source_id: - continue - upstream_node: Mapping[str, Any] | None = nodes_by_id.get(str(edge.get("from_id"))) - if ( - upstream_node is None - or str(upstream_node.get("kind")) != LOADER_NODE_KIND - or str(upstream_node.get("name")) != loader_name - ): - continue - return str(upstream_node.get("id")), { - "kind": "loader", - "name": upstream_node.get("name"), - "resource_id": upstream_node.get("id"), - "source": source_node.get("name"), - "source_relation": payload_asset.get("target"), - "status": payload_asset.get("status"), - } - return None - - -def _selected_asset_paths(*, context: Any) -> set[tuple[str, ...]]: - selected_keys: object = ( - getattr(context, "selected_asset_keys", None) if context is not None else None - ) - if selected_keys is None: - return set() - return {tuple(key.path) for key in selected_keys} - - -def _selected_asset_check_keys(*, context: Any) -> set[tuple[tuple[str, ...], str]]: - selected_keys: object = ( - getattr(context, "selected_asset_check_keys", None) if context is not None else None + emitted_asset_paths=list(emitted_asset_paths), + ), ) - if selected_keys is None: - return set() - check_keys: set[tuple[tuple[str, ...], str]] = set() - for key in selected_keys: - asset_key: object = getattr(key, "asset_key", None) - check_name: object = getattr(key, "name", None) - if asset_key is None or check_name is None: - continue - path: object = getattr(asset_key, "path", None) - if path is None: - continue - check_keys.add((tuple(str(part) for part in path), str(check_name))) - return check_keys - - -def _check_selection_is_explicit(*, context: Any) -> bool: - return context is not None and getattr(context, "selected_asset_check_keys", None) is not None - - -def _selected_check_asset_paths( - *, - dag: Mapping[str, Any], - selected_check_keys: set[tuple[tuple[str, ...], str]], -) -> set[tuple[str, ...]]: - nodes_by_id: dict[str, Mapping[str, Any]] = { - str(node.get("id")): node for node in dag.get("nodes", ()) - } - paths: set[tuple[str, ...]] = set() - for check in dag.get("checks", ()): # type: ignore[assignment] - if not is_dagster_check_enabled(check): - continue - check_name: str = _dagster_check_name(check) - if not _check_is_selected( - check=check, - check_name=check_name, - nodes_by_id=nodes_by_id, - selected_check_keys=selected_check_keys, - ): - continue - for asset_id in check.get("checked_asset_ids", ()): - node: Mapping[str, Any] | None = nodes_by_id.get(str(asset_id)) - if node is None: - continue - asset_path: tuple[str, ...] = tuple(str(part) for part in node.get("asset_key", ())) - paths.add(asset_path) - return paths - - -def _selected_check_selectors( - *, dag: Mapping[str, Any], selected_check_keys: set[tuple[tuple[str, ...], str]] -) -> tuple[str, ...]: - nodes_by_id: dict[str, Mapping[str, Any]] = { - str(node.get("id")): node for node in dag.get("nodes", ()) - } - selectors: list[str] = [] - for check in dag.get("checks", ()): # type: ignore[assignment] - if not is_dagster_check_enabled(check): - continue - if _check_is_selected( - check=check, - check_name=_dagster_check_name(check), - nodes_by_id=nodes_by_id, - selected_check_keys=selected_check_keys, - ): - selectors.append(str(check.get("name"))) - return tuple(dict.fromkeys(selectors)) - - -def _asset_check_only_context(*, context: Any) -> bool: - return bool(_selected_asset_check_keys(context=context)) and not bool( - _selected_asset_paths(context=context) - ) - - -def _dag_check_for_execution_check( - *, dag: Mapping[str, Any], check: Mapping[str, Any] -) -> Mapping[str, Any] | None: - dag_check: Mapping[str, Any] - dag_check_id: object = check.get("dag_check_id") or check.get("check_id") - for dag_check in dag.get("checks", ()): # type: ignore[assignment] - if dag_check.get("id") == dag_check_id: - return dag_check - return None - - -def _asset_ids_for_execution_check( - *, check: Mapping[str, Any], nodes_by_name: Mapping[tuple[str, str], Mapping[str, Any]] -) -> tuple[str, ...]: - asset_name: object = check.get("asset_name") - if asset_name is None: - return () - for kind in ("model", "source", "seed", "udf", "table_fn"): - node: Mapping[str, Any] | None = nodes_by_name.get((kind, str(asset_name))) - if node is not None: - return (str(node.get("id")),) - return () - - -def _dagster_check_name(check: Mapping[str, Any]) -> str: - translated_name: object = check.get(DAGSTER_CHECK_NAME_FIELD) - if translated_name is not None: - return str(translated_name) - parts: list[str] = [str(check.get("kind", "check")), str(check.get("name", "check"))] - if check.get("attached_column_name") is not None: - parts.append(str(check["attached_column_name"])) - elif check.get("attached_target_name") is not None: - parts.append(str(check["attached_target_name"])) - return "__".join(_normalize_check_name(part) for part in parts if part) - - -def _normalize_check_name(value: str) -> str: - normalized: str = "".join( - char - if char.isalnum() or char == CHECK_NAME_SEPARATOR_CHARACTER - else CHECK_NAME_SEPARATOR_CHARACTER - for char in value - ) - return normalized.strip("_") or "check" - - -def _metadata_from_mapping(value: Mapping[str, Any]) -> Mapping[str, Any]: - return { - str(key): _metadata_value(item) - for key, item in value.items() - if item is not None and key not in CHECK_METADATA_EXCLUDED_KEYS - } - - -def _metadata_value(value: Any) -> Any: - if isinstance(value, Enum): - return value.value - if isinstance(value, Mapping): - return {str(key): _metadata_value(item) for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [_metadata_value(item) for item in value] - return value - - -def _dagster_check_severity(*, dg: Any, check: Mapping[str, Any]) -> Any: - if ( - str(check.get("status")) in {INSUFFICIENT_CHECK_STATUS, WARNING_CHECK_SEVERITY} - or str(check.get("severity")) == WARNING_CHECK_SEVERITY - ): - return dg.AssetCheckSeverity.WARN - return dg.AssetCheckSeverity.ERROR - - -def _sort_nodes_topologically(*, dag: Mapping[str, Any]) -> list[Mapping[str, Any]]: - nodes: list[Mapping[str, Any]] = list(dag.get("nodes", ())) - nodes_by_id: dict[str, Mapping[str, Any]] = {str(node.get("id")): node for node in nodes} - incoming_by_id: dict[str, set[str]] = {node_id: set() for node_id in nodes_by_id} - outgoing_by_id: dict[str, list[str]] = {node_id: [] for node_id in nodes_by_id} - for edge in dag.get("edges", ()): - from_id: str = str(edge.get("from_id")) - to_id: str = str(edge.get("to_id")) - if from_id not in nodes_by_id or to_id not in nodes_by_id: - continue - incoming_by_id[to_id].add(from_id) - if to_id not in outgoing_by_id[from_id]: - outgoing_by_id[from_id].append(to_id) - - ordered: list[Mapping[str, Any]] = [] - ready: list[str] = [node_id for node_id in nodes_by_id if not incoming_by_id[node_id]] - while ready: - node_id: str = ready.pop(0) - ordered.append(nodes_by_id[node_id]) - for downstream_id in tuple(outgoing_by_id[node_id]): - incoming_by_id[downstream_id].discard(node_id) - if not incoming_by_id[downstream_id]: - ready.append(downstream_id) - - if len(ordered) != len(nodes): - return nodes - return ordered diff --git a/src/sqlbuild/integrations/dagster/_helpers/invocation_factory.py b/src/sqlbuild/integrations/dagster/_helpers/invocation_factory.py index 1ad51cb9a7..8c35568cff 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/invocation_factory.py +++ b/src/sqlbuild/integrations/dagster/_helpers/invocation_factory.py @@ -9,12 +9,11 @@ from pathlib import Path from typing import Any +from sqlbuild._native import dagster_cli_arguments from sqlbuild.cli.output.constants import INTEGRATION_RESULT_PATH_ENV from sqlbuild.integrations.dagster._helpers.invocation import ( - _caller_json_output_path, - _with_event_output_args, - _with_json_output_args, - _with_selected_asset_args, + _selected_asset_check_keys, + _selected_asset_paths, ) from sqlbuild.integrations.dagster._helpers.invocation_context import ( dagster_invocation_context, @@ -34,30 +33,21 @@ def start_sqlbuild_cli_invocation( ) -> SqlBuildCliInvocation: """Start a SQLBuild CLI subprocess and return its invocation wrapper.""" - selection: tuple[str, ...] - selector_file: Path | None - resolved_args: tuple[str, ...] - selected_args: tuple[str, ...] - selected_args, selection, selector_file = _with_selected_asset_args( - args=tuple(args), - context=context, - dag=dag, - ) - execution_json_path: Path | None - resolved_args, execution_json_path = _with_json_output_args( - args=selected_args, - context=context, - dag=dag, - ) - execution_json_owned: bool = execution_json_path is not None - if execution_json_path is None: - execution_json_path = _caller_json_output_path(args=resolved_args) - event_args: tuple[str, ...] - event_args, event_jsonl_path = _with_event_output_args( - args=resolved_args, - context=context, + planned: dict[str, Any] = dagster_cli_arguments( + args=list(args), dag=dag, + has_context=context is not None, + read_selection=lambda: ( + _selected_asset_paths(context=context), + _selected_asset_check_keys(context=context), + ), ) + event_args: tuple[str, ...] = planned["args"] + selection: tuple[str, ...] = planned["selection"] + selector_file: Path | None = planned["selector_file"] + execution_json_path: Path | None = planned["execution_json_path"] + execution_json_owned: bool = planned["execution_json_owned"] + event_jsonl_path: Path | None = planned["event_jsonl_path"] command: tuple[str, ...] = (*tuple(sqb_command), *event_args) process_environment: dict[str, str] = dict(os.environ) process_environment["PYTHONUNBUFFERED"] = "1" diff --git a/src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_invocation.py b/src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_invocation.py index c7361ba805..76a4e878c8 100644 --- a/src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_invocation.py +++ b/src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_invocation.py @@ -5,14 +5,18 @@ import json import subprocess import sys -import textwrap import time from collections.abc import Iterator, Mapping from concurrent.futures import Future, ThreadPoolExecutor from pathlib import Path from typing import IO, Any -from sqlbuild._native import IntegrationAssetResult, IntegrationResultEnvelope +from sqlbuild._native import ( + IntegrationAssetResult, + IntegrationResultEnvelope, + dagster_failure, + dagster_live_failure, +) from sqlbuild.integrations.dagster._helpers.imports import load_dagster from sqlbuild.integrations.dagster._helpers.invocation import ( _build_results_for_selected_assets, @@ -23,43 +27,9 @@ _log_invocation, _start_stream_future, ) -from sqlbuild.integrations.dagster.constants import ( - COMPLETED_EXECUTION_STATUSES, - FAILED_EXECUTION_STATUS, -) +from sqlbuild.integrations.dagster.constants import FAILED_EXECUTION_STATUS from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError -_MAX_DIAGNOSTIC_CHARS: int = 4_000 - - -def _diagnostic_tail(value: str) -> str: - return value[-_MAX_DIAGNOSTIC_CHARS:] - - -def _format_asset_failure(asset: Mapping[str, Any]) -> str: - status: str = str(asset.get("status")) - lines: list[str] = [f"{asset.get('kind')}:{asset.get('name')} ({status})"] - failed_phase: object = asset.get("failed_phase") - if failed_phase is not None: - lines[0] += f" during {failed_phase}" - error_message: object = asset.get("error_message") - if error_message is not None: - error_code: object = asset.get("error_code") - prefix: str = f"[{error_code}] " if error_code is not None else "" - lines.append(f" {prefix}{error_message}") - staging_relation: object = asset.get("staging_relation") - if staging_relation is not None: - lines.append(f" staging relation: {staging_relation}") - failed_sql: object = asset.get("failed_sql") - if isinstance(failed_sql, str) and failed_sql: - lines.append(" failed SQL:") - lines.append(textwrap.indent(failed_sql, " ")) - last_recorded_sql: object = asset.get("last_recorded_sql") - if isinstance(last_recorded_sql, str) and last_recorded_sql: - lines.append(" last recorded SQL:") - lines.append(textwrap.indent(last_recorded_sql, " ")) - return "\n".join(lines) - class SqlBuildCliInvocation: """A running or completed SQLBuild CLI subprocess.""" @@ -155,45 +125,18 @@ def get_error(self) -> Exception | None: if self.returncode is None or self.returncode == 0: return None dg: Any = load_dagster() - metadata: dict[str, Any] = { - "command": " ".join(self.command), - "project_dir": str(self.project_dir), - "selection": " ".join(self.selection), - "selector_file": self.selector_file_path, - } - description: str = ( - "SQLBuild CLI command failed with exit code " - f"{self.returncode}: {' '.join(self.command)}" + description: str + metadata: dict[str, Any] + description, metadata = dagster_failure( + command=list(self.command), + project_dir=self.project_dir, + selection=list(self.selection), + selector_file=self.selector_file_path, + returncode=self.returncode, + execution_payload=self.execution_payload, + stdout=self.stdout, + stderr=self.stderr, ) - if self.execution_payload is not None: - incomplete_assets: list[str] = [] - failure_details: list[str] = [] - microbatch_limits: dict[str, object] = {} - for asset in self.execution_payload.get("assets", ()): - status: str = str(asset.get("status")) - if status in COMPLETED_EXECUTION_STATUSES: - continue - incomplete_assets.append(f"{asset.get('kind')}:{asset.get('name')} ({status})") - failure_details.append(_format_asset_failure(asset)) - microbatch: object | None = asset.get("microbatch") - if isinstance(microbatch, dict) and microbatch.get("limit") is not None: - microbatch_limits[str(asset.get("name"))] = microbatch - metadata["execution_status"] = str(self.execution_payload.get("status")) - metadata["incomplete_assets"] = ", ".join(incomplete_assets) - if microbatch_limits: - metadata["microbatch_limits"] = microbatch_limits - if failure_details: - description += "\n\nFailures:\n\n" + "\n\n".join(failure_details) - else: - stdout_tail: str = _diagnostic_tail(self.stdout) - stderr_tail: str = _diagnostic_tail(self.stderr) - if stdout_tail: - metadata["stdout_tail"] = stdout_tail - if stderr_tail: - metadata["stderr_tail"] = stderr_tail - diagnostic: str = stderr_tail.strip() or stdout_tail.strip() - if diagnostic: - description += f"\n\n{diagnostic}" return dg.Failure(description=description, metadata=metadata) def get_artifact(self, artifact: str) -> dict[str, Any]: @@ -430,33 +373,14 @@ def _log_live_asset_failure( logger: Any = getattr(self.context, "log", None) if self.context is not None else None if logger is None: return logged_failed_assets - asset_label: str = f"{asset_kind}:{asset_name}" - phase: str = str(asset.failed_phase or "unknown") - code: str = str(envelope.error_code or "unknown") - message: str = "SQLBuild resource attempt failed; see final execution diagnostics" - metadata: dict[str, Any] = { - "sqlbuild_asset": asset_label, - "sqlbuild_asset_kind": asset_kind, - "sqlbuild_asset_name": asset_name, - "sqlbuild_phase": phase, - "sqlbuild_error_code": code, - "sqlbuild_error_message": message, - "sqlbuild_command": " ".join(self.command), - } - values: Mapping[str, object] = { - "target": asset.target, - "staging_relation": asset.staging_relation, - "duration_ms": envelope.duration_ms, - } - for key, value in values.items(): - if value is not None: - metadata[f"sqlbuild_{key}"] = value + log_arguments: list[str] + metadata: dict[str, Any] + log_arguments, metadata = dagster_live_failure( + asset=asset, envelope=envelope, command=list(self.command) + ) logger.error( "SQLBuild asset failed: asset=%s phase=%s code=%s message=%s", - asset_label, - phase, - code, - message, + *log_arguments, extra=metadata, ) return logged_failed_assets diff --git a/src/sqlbuild/integrations/dbt/exceptions.py b/src/sqlbuild/integrations/dbt/exceptions.py index 2028fdea69..55c227ad92 100644 --- a/src/sqlbuild/integrations/dbt/exceptions.py +++ b/src/sqlbuild/integrations/dbt/exceptions.py @@ -2,35 +2,7 @@ from __future__ import annotations - -class DbtInteropError(ValueError): - """Base class for expected dbt interop errors.""" - - code: str = "C240" - - def __init__(self, message: str, *, code: str | None = None, help: str | None = None) -> None: - super().__init__(message) - self.message = message - self.code = code if code is not None else self.code - self.help = help - - -class DbtInteropConfigError(DbtInteropError): - """Raised when dbt project configuration is missing or invalid.""" - - code: str = "C240" - - -class DbtInteropRuntimeError(DbtInteropError): - """Raised when dbt command execution fails.""" - - code: str = "C241" - - -class DbtInteropArgumentError(DbtInteropError): - """Raised when `sqb dbt` arguments cannot be routed safely.""" - - code: str = "C230" +from sqlbuild._native import DbtInteropConfigError class DbtProfileError(DbtInteropConfigError): diff --git a/src/sqlbuild/integrations/dbt/models.py b/src/sqlbuild/integrations/dbt/models.py index 8818e7c014..477de1ed86 100644 --- a/src/sqlbuild/integrations/dbt/models.py +++ b/src/sqlbuild/integrations/dbt/models.py @@ -2,25 +2,9 @@ from __future__ import annotations -from collections.abc import Callable -from dataclasses import dataclass, field +from dataclasses import dataclass from pathlib import Path -from sqlbuild._native import DbtLsNode, DbtNodeMessage - - -@dataclass(frozen=True) -class DbtCliOptions: - """Resolved dbt CLI options shared by dbt commands.""" - - project_dir: Path | None = None - profiles_dir: Path | None = None - target: str | None = None - target_path: Path | None = None - vars: str | None = None - state: Path | None = None - defer: bool = False - @dataclass(frozen=True) class SelectedDbtProfileOutput: @@ -47,90 +31,3 @@ def adapter_type(self) -> str: value: object | None = self.output.get("type") return value if isinstance(value, str) else "" - - -@dataclass(frozen=True) -class NormalizedDbtProfileConnection: - """Rendered dbt profile output normalized for SQLBuild.""" - - adapter: str - connection: dict[str, object] - target_schema: str | None = None - target_database: str | None = None - warnings: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DbtInitProgressCallbacks: - """Optional progress hooks for `sqb dbt init` phases.""" - - start: Callable[[str], None] | None = None - complete: Callable[[str], None] | None = None - - -@dataclass(frozen=True) -class DbtInitRequest: - """Request to initialize a SQLBuild project from a dbt project.""" - - cwd: Path - dbt_project_dir: Path - profiles_dir: Path | None - profile_name: str | None - target_name: str | None - sqb_output_dir: Path | None - dry_run: bool = False - overwrite: bool = False - skip_dbt_debug: bool = False - progress_callbacks: DbtInitProgressCallbacks = field(default_factory=DbtInitProgressCallbacks) - - -@dataclass(frozen=True) -class DbtInitResult: - """Result from `sqb dbt init`.""" - - output_dir: Path - project_file: Path - project_name: str - adapter: str - target_name: str - profile_name: str - toml: str - warnings: tuple[str, ...] = field(default_factory=tuple) - dry_run: bool = False - - -@dataclass(frozen=True) -class DbtCommandResult: - """Completed dbt command output.""" - - argv: tuple[str, ...] - returncode: int - stdout: str = "" - stderr: str = "" - - -@dataclass(frozen=True) -class DbtNodeExecutionResult: - """One dbt node execution result parsed from JSON logs.""" - - unique_id: str - resource_type: str - node_name: str - status: str - index: int | None - total: int | None - execution_time: float | None - materialized: str | None = None - relation_name: str | None = None - database: str | None = None - schema: str | None = None - node_checksum: str | None = None - messages: tuple[DbtNodeMessage, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DbtLsResult: - """Parsed result from one dbt ls invocation.""" - - nodes: tuple[DbtLsNode, ...] - command: DbtCommandResult diff --git a/src/sqlbuild/integrations/dlt/main/_loaders.py b/src/sqlbuild/integrations/dlt/main/_loaders.py new file mode 100644 index 0000000000..df463458d3 --- /dev/null +++ b/src/sqlbuild/integrations/dlt/main/_loaders.py @@ -0,0 +1,25 @@ +"""Loader functions that run declarative dlt sources.""" + +from __future__ import annotations + +from collections.abc import Callable + +from sqlbuild.executor.load.models import LoaderContext +from sqlbuild.integrations.dlt._helpers.runner import run_dlt_source +from sqlbuild.integrations.dlt.models import DltSourceConfig +from sqlbuild.spec.contracts.models import SourceEntry + + +def dlt_loader_function(*, source_entry: SourceEntry, name: str) -> Callable[[LoaderContext], None]: + """The loader function that runs one declarative dlt source under `name`.""" + + def run(ctx: LoaderContext) -> None: + config: object = ( + source_entry.integration_loader.config if source_entry.integration_loader else None + ) + if not isinstance(config, DltSourceConfig): + return None + return run_dlt_source(config=config, ctx=ctx) + + run.__name__ = name + return run diff --git a/src/sqlbuild/integrations/dlt/main/loaders.py b/src/sqlbuild/integrations/dlt/main/loaders.py deleted file mode 100644 index bd94f93787..0000000000 --- a/src/sqlbuild/integrations/dlt/main/loaders.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Synthetic loader functions for declarative dlt sources.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction, DiscoveredSourceFile -from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.executor.load.models import LoaderContext -from sqlbuild.integrations.dlt._helpers.runner import run_dlt_source -from sqlbuild.integrations.dlt.constants import DLT_INTEGRATION_KIND -from sqlbuild.integrations.dlt.models import DltSourceConfig -from sqlbuild.spec.contracts.models import SourceEntry - - -def build_dlt_loader_functions( - source_files: tuple[DiscoveredSourceFile, ...], -) -> tuple[DiscoveredLoaderFunction, ...]: - """Build synthetic loader functions for declarative dlt sources.""" - - loaders: list[DiscoveredLoaderFunction] = [] - source_file: DiscoveredSourceFile - for source_file in source_files: - source_entry: SourceEntry - for source_entry in source_file.source_entries: - if ( - source_entry.integration_loader is None - or source_entry.integration_loader.kind != DLT_INTEGRATION_KIND - ): - continue - loader_name: str = source_entry.name - loaders.append( - DiscoveredLoaderFunction( - file_path=source_file.file_path, - relative_path=source_file.relative_path, - name=loader_name, - function=_build_dlt_loader(source_entry=source_entry), - connection_mode=LoaderConnectionMode.EXTERNAL, - ) - ) - return tuple(loaders) - - -def _build_dlt_loader(*, source_entry: SourceEntry) -> Any: - def run(ctx: LoaderContext) -> None: - config: object = ( - source_entry.integration_loader.config if source_entry.integration_loader else None - ) - if not isinstance(config, DltSourceConfig): - return None - return run_dlt_source(config=config, ctx=ctx) - - run.__name__ = source_entry.name - return run diff --git a/src/sqlbuild/integrations/ingestr/main/loaders.py b/src/sqlbuild/integrations/ingestr/main/_loaders.py similarity index 51% rename from src/sqlbuild/integrations/ingestr/main/loaders.py rename to src/sqlbuild/integrations/ingestr/main/_loaders.py index 5f64eef03d..916dc95fff 100644 --- a/src/sqlbuild/integrations/ingestr/main/loaders.py +++ b/src/sqlbuild/integrations/ingestr/main/_loaders.py @@ -1,53 +1,23 @@ -"""Synthetic loader functions for declarative ingestr sources.""" +"""Loader functions that run declarative ingestr sources.""" from __future__ import annotations -from typing import Any +from collections.abc import Callable -from sqlbuild.compiler.discovery.models import ( - DiscoveredLoaderFunction, - DiscoveredSourceFile, -) -from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.executor.load.models import LoaderContext from sqlbuild.integrations.ingestr._helpers.command import build_ingestr_command from sqlbuild.integrations.ingestr._helpers.output import record_ingestr_output from sqlbuild.integrations.ingestr._helpers.runner import run_ingestr_command -from sqlbuild.integrations.ingestr.constants import INGESTR_INTEGRATION_KIND from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError from sqlbuild.integrations.ingestr.models import IngestrCommandResult from sqlbuild.spec.contracts.models import SourceEntry -def build_ingestr_loader_functions( - source_files: tuple[DiscoveredSourceFile, ...], -) -> tuple[DiscoveredLoaderFunction, ...]: - """Build synthetic loader functions for declarative ingestr sources.""" +def ingestr_loader_function( + *, source_entry: SourceEntry, name: str +) -> Callable[[LoaderContext], None]: + """The loader function that runs one declarative ingestr source under `name`.""" - loaders: list[DiscoveredLoaderFunction] = [] - source_file: DiscoveredSourceFile - for source_file in source_files: - source_entry: SourceEntry - for source_entry in source_file.source_entries: - if ( - source_entry.integration_loader is None - or source_entry.integration_loader.kind != INGESTR_INTEGRATION_KIND - ): - continue - loader_name: str = f"ingestr__{source_entry.name}" - loaders.append( - DiscoveredLoaderFunction( - file_path=source_file.file_path, - relative_path=source_file.relative_path, - name=loader_name, - function=_build_ingestr_loader(source_entry=source_entry), - connection_mode=LoaderConnectionMode.EXTERNAL, - ) - ) - return tuple(loaders) - - -def _build_ingestr_loader(*, source_entry: SourceEntry) -> Any: def run(ctx: LoaderContext) -> None: command: tuple[str, ...] = build_ingestr_command( source_entry=source_entry, @@ -70,7 +40,7 @@ def run(ctx: LoaderContext) -> None: _ = record_ingestr_output(ctx=ctx, result=result) return None - run.__name__ = f"ingestr__{source_entry.name}" + run.__name__ = name return run diff --git a/src/sqlbuild/lint/constants.py b/src/sqlbuild/lint/constants.py deleted file mode 100644 index f9c86a5037..0000000000 --- a/src/sqlbuild/lint/constants.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Constants for the lint and format layer.""" - -from __future__ import annotations - -from sqlbuild.lint.types import LintSeverity - -VIOLATION_SEVERITY_FAULT: LintSeverity = LintSeverity.FAULT -VIOLATION_SEVERITY_WARNING: LintSeverity = LintSeverity.WARNING - -DEFAULT_MAX_LITERAL_LENGTH: int = 100 -DEFAULT_MAX_RANKING_ORDER_BY: int = 5 diff --git a/src/sqlbuild/lint/exceptions.py b/src/sqlbuild/lint/exceptions.py deleted file mode 100644 index 30c96576e2..0000000000 --- a/src/sqlbuild/lint/exceptions.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Exceptions raised by the lint and format layer.""" - -from __future__ import annotations - - -class LintError(Exception): - """A lint or format run could not proceed.""" - - code: str = "L001" - - def __init__(self, message: str, *, code: str | None = None, help: str | None = None) -> None: - super().__init__(message) - self.message = message - self.code = code if code is not None else self.code - self.help = help - - -class NativeLintError(LintError): - """The SQLBuild-owned native lint engine rejected a request or response.""" - - code: str = "L003" - - -class ProjectCompileError(LintError): - """The project could not be compiled, so its SQL cannot be linted.""" - - code: str = "L005" - - -class LintStoppedError(Exception): - """A speculative lint run was stopped because its result is no longer needed.""" diff --git a/src/sqlbuild/lint/models.py b/src/sqlbuild/lint/models.py deleted file mode 100644 index 53e54892c7..0000000000 --- a/src/sqlbuild/lint/models.py +++ /dev/null @@ -1,219 +0,0 @@ -"""Structured models for the lint and format layer.""" - -from __future__ import annotations - -from collections import Counter -from collections.abc import Mapping -from dataclasses import dataclass, field -from pathlib import Path - -from sqlbuild.compiler.compile.models import CompiledObjectKey, ExpansionSpan -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.scopes.types import DeclarationKind -from sqlbuild.lint.constants import ( - DEFAULT_MAX_LITERAL_LENGTH, - DEFAULT_MAX_RANKING_ORDER_BY, - VIOLATION_SEVERITY_FAULT, - VIOLATION_SEVERITY_WARNING, -) -from sqlbuild.lint.exceptions import ProjectCompileError -from sqlbuild.lint.types import ( - DiagnosticIdentity, - LintSeverity, - NativeLintPreparationRequest, - RuleFixStatus, -) - - -@dataclass(frozen=True) -class HeaderSpan: - """A located DSL header region inside one file.""" - - kind: str - start: int - end: int - - -@dataclass(frozen=True) -class LintFileRole: - """Path-derived classification of one lint input, derived once per file.""" - - declaration_kind: DeclarationKind | None = None - in_project: bool = False - in_hook_directory: bool = False - - -@dataclass(frozen=True) -class InterpolationSite: - """One sqlbuild interpolation occurrence replaced by a unique sentinel.""" - - sentinel: str - neutralized_start: int - neutralized_end: int - original_start: int - original_end: int - original_text: str - - -@dataclass(frozen=True) -class LintBody: - """One authored SQL body prepared for linting, with its expansion spans.""" - - file_path: Path - body_start: int - body_end: int - lint_text: str - passes: tuple[tuple[ExpansionSpan, ...], ...] - external_identifiers: tuple[str, ...] = () - dependency_identifiers: tuple[str, ...] = () - externally_referenced_ctes: tuple[str, ...] = () - allows_ceremonial_select: bool = False - allows_dynamic_output_star: bool = False - allows_empty_fixture_star: bool = False - dependency_relations: tuple[tuple[str, str], ...] = () - - -@dataclass(frozen=True) -class PendingLintBody: - """One expanded authored body awaiting its native lexical preparation.""" - - file_path: Path - body_start: int - body_end: int - expansion_input: str - expanded: str - expansion_passes: tuple[tuple[ExpansionSpan, ...], ...] - pre_expansion_sites: tuple[InterpolationSite, ...] - external_identifiers: tuple[str, ...] - allows_ceremonial_select: bool - allows_dynamic_output_star: bool - allows_empty_fixture_star: bool - request: NativeLintPreparationRequest - - -@dataclass(frozen=True) -class ExpandedLintFile: - """The expanded bodies of one file, up to the first body that could not be expanded.""" - - file_path: Path - bodies: tuple[PendingLintBody, ...] - failure: ProjectCompileError | None - - -@dataclass(frozen=True) -class PreparedLintFiles: - """Lint bodies of every expandable file, and why the skipped files could not expand.""" - - bodies: tuple[LintBody, ...] - unexpandable: dict[Path, str] - - -@dataclass(frozen=True) -class LintEdit: - """One deterministic authored-source edit proposed by a lint rule.""" - - file_path: Path - code: str - start: int - end: int - replacement: str - - -@dataclass(frozen=True) -class LintViolation: - """One lint diagnostic reported against an authored file.""" - - file_path: Path - line: int - column: int - code: str - message: str - severity: LintSeverity - engine: str - end_line: int | None = None - end_column: int | None = None - remediation: str | None = None - fix: LintEdit | None = None - fix_unavailable_reason: str | None = None - - -@dataclass(frozen=True) -class RuleFixResult: - """One applied, refused, or unavailable semantic Rule fix.""" - - file_path: Path - code: str - line: int - status: RuleFixStatus - reason: str - - -@dataclass(frozen=True) -class LintConfig: - """Resolved lint and format configuration for one run.""" - - native_enabled: bool = True - max_description_lines: int = 10 - line_width: int = 100 - dialect: str = "generic" - enabled_native_rules: tuple[str, ...] | None = None - ignored_native_rules: tuple[str, ...] = () - header_rules_enabled: bool = True - max_literal_length: int = DEFAULT_MAX_LITERAL_LENGTH - max_ranking_order_by: int = DEFAULT_MAX_RANKING_ORDER_BY - allow_suppressions: bool = True - relation_keys: Mapping[str, tuple[tuple[str, ...], ...]] = field( - default_factory=dict, compare=False - ) - - -@dataclass(frozen=True) -class LintRunResult: - """Aggregated outcome of one lint or format run.""" - - files_checked: int - violations: tuple[LintViolation, ...] - formatted_files: tuple[Path, ...] - source_texts: Mapping[Path, str] = field(default_factory=dict, repr=False, compare=False) - unexpandable: Mapping[Path, str] = field(default_factory=dict) - - @property - def faults(self) -> tuple[LintViolation, ...]: - """Return violations with fault severity.""" - - return tuple( - violation - for violation in self.violations - if violation.severity == VIOLATION_SEVERITY_FAULT - ) - - @property - def warnings(self) -> tuple[LintViolation, ...]: - """Return violations with warning severity.""" - - return tuple( - violation - for violation in self.violations - if violation.severity == VIOLATION_SEVERITY_WARNING - ) - - -@dataclass(frozen=True) -class CompileFacts: - """Per-model outputs and per-owner diagnostics of one compilation.""" - - models: dict[CompiledObjectKey, CompiledModelView] - model_paths: dict[CompiledObjectKey, Path] - diagnostics: dict[Path, Counter[DiagnosticIdentity]] - - -@dataclass(frozen=True) -class FixVerdict: - """Edited files that fail verification, and whether any change is unattributable.""" - - failing: dict[Path, str] - unattributed: bool - - @property - def verified(self) -> bool: - return not self.failing and not self.unattributed diff --git a/src/sqlbuild/lint/types.py b/src/sqlbuild/lint/types.py deleted file mode 100644 index 10a7fee765..0000000000 --- a/src/sqlbuild/lint/types.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Lint domain types.""" - -from __future__ import annotations - -from enum import StrEnum -from typing import TYPE_CHECKING, Literal - -if TYPE_CHECKING: - from sqlbuild._native import LintPreparationRequest - - -class LintSeverity(StrEnum): - """Severity assigned to one lint violation.""" - - FAULT = "fault" - WARNING = "warning" - - -type RuleFixStatus = Literal["applied", "refused", "unavailable"] - -type RelationKeyRow = tuple[ - str, - str, - str, - str, - str, - bool, - list[str] | None, - list[tuple[str, list[str], bool]], - list[tuple[str, str | None]], -] - -type DiagnosticIdentity = tuple[str, str, str, str] - -type NativeLintPreparationRequest = LintPreparationRequest diff --git a/src/sqlbuild/presentation/models.py b/src/sqlbuild/presentation/models.py deleted file mode 100644 index 9dc8455ead..0000000000 --- a/src/sqlbuild/presentation/models.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Presentation models.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.presentation.constants import DEFAULT_MAX_DISPLAY_ENTRIES - - -@dataclass(frozen=True) -class DisplayOptions: - """Options controlling bounded human output.""" - - max_entries_per_section: int | None = DEFAULT_MAX_DISPLAY_ENTRIES - overflow_flag: str = "--verbose" diff --git a/src/sqlbuild/provider/_helpers/session.py b/src/sqlbuild/provider/_helpers/session.py index 14285017d3..6776f19815 100644 --- a/src/sqlbuild/provider/_helpers/session.py +++ b/src/sqlbuild/provider/_helpers/session.py @@ -8,8 +8,8 @@ from sqlbuild.provider.classes.session import ProviderSession if TYPE_CHECKING: + from sqlbuild._native import EventExporterCommandScope from sqlbuild.provider.classes.provider import Provider - from sqlbuild.runtime.event_exporting.classes.command_scope import EventExporterCommandScope def build_provider_session( @@ -21,9 +21,7 @@ def build_provider_session( """Build a runtime provider session with fresh provider instances.""" if allow_shared: - from sqlbuild.runtime.event_exporting.main.current_event_exporter_command_scope import ( - current_event_exporter_command_scope, - ) + from sqlbuild._native import current_event_exporter_command_scope scope: EventExporterCommandScope | None = current_event_exporter_command_scope() if scope is not None and scope.provider_session is not None: diff --git a/src/sqlbuild/rule_engine/models.py b/src/sqlbuild/rule_engine/models.py index 42c1434bbf..362c0a90b3 100644 --- a/src/sqlbuild/rule_engine/models.py +++ b/src/sqlbuild/rule_engine/models.py @@ -3,15 +3,20 @@ from __future__ import annotations from collections.abc import Callable -from concurrent.futures import Future from dataclasses import dataclass, field from pathlib import Path from typing import Any -from sqlbuild.compiler.compile.models import DeclarationScopeBuild, SqlExpansionContext -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.lint.models import LintRunResult +from sqlbuild._native import ( + GraphEdgeExclusion, + RuleExemption, + RuleGuidance, + RuleIgnore, + RulesCacheConfig, + SelectStarAllow, + SqlTestRulesConfig, + ThresholdOverride, +) from sqlbuild.rule_engine.types import RuleCheck, RuleOptionValue, RuleSubject @@ -175,15 +180,6 @@ def integer_list( ) -@dataclass(frozen=True) -class RuleGuidance: - """Structured authoring guidance shared by diagnostics and generated documentation.""" - - good_example: str - anti_tautology: str - mutation_check: str - - @dataclass(frozen=True) class Rule: """Immutable metadata and implementation for one rule.""" @@ -225,53 +221,6 @@ class Finding: fixable: bool = False -@dataclass(frozen=True) -class RuleExemption: - rule: str - path: str - reason: str - - -@dataclass(frozen=True) -class GraphEdgeExclusion: - """One exact, temporary internal-layer dependency exception.""" - - consumer: str - dependency: str - reason: str - - -@dataclass(frozen=True) -class RuleIgnore: - rules: tuple[str, ...] - paths: tuple[str, ...] - reason: str - selectors: tuple[str, ...] = () - - -@dataclass(frozen=True) -class SelectStarAllow: - paths: tuple[str, ...] - reason: str - - -@dataclass(frozen=True) -class ThresholdOverride: - paths: tuple[str, ...] - thresholds: dict[str, int] - reason: str - - -@dataclass(frozen=True) -class RulesCacheConfig: - enabled: bool = True - - -@dataclass(frozen=True) -class SqlTestRulesConfig: - pipeline_directory: str = "pipelines" - - @dataclass(frozen=True) class LayoutConfig: levels: tuple[str, ...] = ( @@ -315,71 +264,6 @@ class RulesResult: custom_cpu_ms: int = 0 -@dataclass(frozen=True) -class PreparedSqlLintResult: - """Reusable expansion context and completed preliminary SQL checks.""" - - result: LintRunResult - context: SqlExpansionContext - - -@dataclass(frozen=True) -class SqlExpansionReuse: - """Compiler declaration scope for post-assembly SQL checks; assembly still runs its macros.""" - - discovered_inputs: DiscoveredProjectInputs - declaration_scope: DeclarationScopeBuild - - -@dataclass(frozen=True) -class PreparedSqlLint: - """Process-local SQL checks started after compiler expansion has completed.""" - - codes: tuple[str, ...] - future: Future[PreparedSqlLintResult] - project_dir: Path | None = None - - -@dataclass(frozen=True) -class SqlRulesEvaluation: - """SQL rule findings of one run, their cache accounting and elapsed time.""" - - findings: tuple[Finding, ...] - cache_hits: int - cache_misses: int - sql_ms: int = 0 - - -@dataclass(frozen=True) -class RuleRunPlan: - """The catalogue, selected rules, project slice and settings one rules run evaluates.""" - - effective_config: RulesConfig - project_dir: Path - catalogue: tuple[Rule, ...] - custom_payloads: list[dict[str, object]] - selected: tuple[Rule, ...] - skipped_type_proof_rules: tuple[str, ...] - model_paths: frozenset[str] | None - project: CompiledProjectView - config: RulesConfig - - -@dataclass(frozen=True) -class RulesRunResult: - """Unified findings and phase measurements for configured rules.""" - - findings: tuple[Finding, ...] - evaluated_models: int - built_in_ms: int - custom_ms: int - custom_cpu_ms: int = 0 - unevaluated_resources: int = 0 - cache_hits: int = 0 - cache_misses: int = 0 - skipped_type_proof_rules: tuple[str, ...] = () - - @dataclass(frozen=True) class RuleFile: path: str @@ -443,30 +327,6 @@ class CustomRuleRun: stateful_codes: frozenset[str] = frozenset() -@dataclass(frozen=True) -class NativeRulesEvaluation: - """The built-in rules response, decoded natively.""" - - findings: tuple[Finding, ...] - selected_codes: tuple[str, ...] - evaluated_models: int - cache_hits: int - cache_misses: int - built_in_ms: int - reused: bool - - -@dataclass(frozen=True) -class CustomRulesOutcome: - """Custom-rule findings in canonical order and their cache accounting.""" - - findings: tuple[Finding, ...] - cache_hits: int - cache_misses: int - custom_ms: int - custom_cpu_ms: int = 0 - - @dataclass(frozen=True) class CustomHostPartition: """How one host of a split run detects module state and learns the run was cancelled.""" diff --git a/src/sqlbuild/runtime/event_exporting/README.md b/src/sqlbuild/runtime/event_exporting/README.md index e405764971..db6a7a4aad 100644 --- a/src/sqlbuild/runtime/event_exporting/README.md +++ b/src/sqlbuild/runtime/event_exporting/README.md @@ -4,6 +4,9 @@ Project exporters receive only validated, immutable `LifecycleEvent` values. The exporter observer after canonical history and native projections, and diagnostics remain on the separate diagnostic channel. +The dispatcher, queue, accounting and command scope are native (`sqlbuild._native`); Python keeps +only the public sink declarations and the two calls into user sink functions. + Each command owns one bounded event queue and one daemon dispatcher. Exporters run in deterministic declaration order. An invocation runs on a daemon isolation thread so an exporter blocked in destination code cannot keep the command's non-daemon teardown alive. The dispatcher waits at most @@ -39,7 +42,8 @@ Project config can only narrow that opt-in. `[sinks.lifecycle]` supplies global kinds are the intersection of declaration, global, and named sets. Effective minimum severity is the strictest of those three values. Unknown names, kinds, and severities fail before execution. -The single authoritative lifecycle mapping is `LIFECYCLE_EXPORT_DIMENSIONS`: failed terminal event +The single authoritative lifecycle mapping is the native `lifecycle_export_policy` +(`crates/sqlbuild-core/src/event_export`), derived from the lifecycle event catalogue: failed terminal event types are `error`; completed and skipped terminals are `info`; `retry_scheduled` is `warning`; and starts plus `statement_submitted` are `debug`. Diagnostics are not lifecycle events and never enter this queue. diff --git a/src/sqlbuild/runtime/event_exporting/_helpers/__init__.py b/src/sqlbuild/runtime/event_exporting/_helpers/__init__.py deleted file mode 100644 index d5c4790884..0000000000 --- a/src/sqlbuild/runtime/event_exporting/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Internal event exporter helpers.""" diff --git a/src/sqlbuild/runtime/event_exporting/_helpers/policy.py b/src/sqlbuild/runtime/event_exporting/_helpers/policy.py deleted file mode 100644 index ad45ad9231..0000000000 --- a/src/sqlbuild/runtime/event_exporting/_helpers/policy.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Authoritative destination-neutral lifecycle export policy.""" - -from __future__ import annotations - -from collections.abc import Mapping -from types import MappingProxyType - -from sqlbuild.runtime.event_exporting.constants import ( - EVENT_EXPORT_SEVERITY_RANKS, - LIFECYCLE_EXPORT_DIMENSIONS, -) -from sqlbuild.runtime.event_exporting.models import LifecycleExportPolicy -from sqlbuild.runtime.observability.models import LifecycleEvent -from sqlbuild.spec.contracts.types import EventExportSeverity - -_LIFECYCLE_EXPORT_POLICY: Mapping[str, LifecycleExportPolicy] = MappingProxyType( - { - event_type: LifecycleExportPolicy(*dimensions) - for event_type, dimensions in LIFECYCLE_EXPORT_DIMENSIONS.items() - } -) - - -def lifecycle_export_policy(event: LifecycleEvent) -> LifecycleExportPolicy: - """Return destination-neutral dimensions for an already validated event.""" - - return _LIFECYCLE_EXPORT_POLICY[event.event_type] - - -def lifecycle_export_policy_catalog() -> Mapping[str, LifecycleExportPolicy]: - """Return the immutable policy catalog for validation and documentation tests.""" - - return _LIFECYCLE_EXPORT_POLICY - - -def severity_at_least(*, severity: EventExportSeverity, minimum: EventExportSeverity) -> bool: - """Return whether a catalogued severity meets a configured minimum.""" - - return EVENT_EXPORT_SEVERITY_RANKS[severity] >= EVENT_EXPORT_SEVERITY_RANKS[minimum] - - -def stricter_severity( - *, first: EventExportSeverity, second: EventExportSeverity | None -) -> EventExportSeverity: - """Return the stricter of a required and optional minimum severity.""" - - if second is None or EVENT_EXPORT_SEVERITY_RANKS[first] >= EVENT_EXPORT_SEVERITY_RANKS[second]: - return first - return second diff --git a/src/sqlbuild/runtime/event_exporting/_helpers/sanitization.py b/src/sqlbuild/runtime/event_exporting/_helpers/sanitization.py deleted file mode 100644 index 181acc0dbb..0000000000 --- a/src/sqlbuild/runtime/event_exporting/_helpers/sanitization.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Safe exporter health diagnostic fields.""" - -import re - -_MAX_EXCEPTION_TYPE_LENGTH: int = 100 - - -def sanitized_exception_type(error: BaseException) -> str: - """Return a bounded low-cardinality exception class name.""" - - sanitized: str = re.sub(r"[^A-Za-z0-9_]", "_", type(error).__name__) - return sanitized[:_MAX_EXCEPTION_TYPE_LENGTH] or "Exception" diff --git a/src/sqlbuild/runtime/event_exporting/_helpers/scope.py b/src/sqlbuild/runtime/event_exporting/_helpers/scope.py deleted file mode 100644 index 529a76ac37..0000000000 --- a/src/sqlbuild/runtime/event_exporting/_helpers/scope.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Current CLI event exporter command scope.""" - -from __future__ import annotations - -from collections.abc import Iterator -from contextlib import contextmanager -from contextvars import ContextVar, Token -from pathlib import Path - -from sqlbuild.compiler.discovery.models import ( - DiscoveredCommandOutputSink, - DiscoveredEventExporter, - DiscoveredProjectInputs, - DiscoveredProvider, -) -from sqlbuild.runtime.event_exporting.classes.command_scope import EventExporterCommandScope - -_CURRENT_SCOPE: ContextVar[EventExporterCommandScope | None] = ContextVar( - "sqlbuild_event_exporter_command_scope", default=None -) - - -@contextmanager -def event_exporter_command_scope( - scope: EventExporterCommandScope, -) -> Iterator[EventExporterCommandScope]: - token: Token[EventExporterCommandScope | None] = _CURRENT_SCOPE.set(scope) - try: - yield scope - finally: - _CURRENT_SCOPE.reset(token) - - -def current_event_exporter_command_scope() -> EventExporterCommandScope | None: - return _CURRENT_SCOPE.get() - - -def configure_discovered_event_exporters(discovered_inputs: DiscoveredProjectInputs) -> None: - scope: EventExporterCommandScope | None = current_event_exporter_command_scope() - if scope is not None: - scope.configure(discovered_inputs) - - -def cached_event_exporter_extensions( - *, project_dir: Path -) -> ( - tuple[ - tuple[DiscoveredProvider, ...], - tuple[DiscoveredEventExporter, ...], - tuple[DiscoveredCommandOutputSink, ...], - ] - | None -): - scope: EventExporterCommandScope | None = current_event_exporter_command_scope() - return None if scope is None else scope.cached_extensions(project_dir=project_dir) diff --git a/src/sqlbuild/runtime/event_exporting/classes/__init__.py b/src/sqlbuild/runtime/event_exporting/classes/__init__.py deleted file mode 100644 index 7bcfc91d90..0000000000 --- a/src/sqlbuild/runtime/event_exporting/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Event exporter runtime classes.""" diff --git a/src/sqlbuild/runtime/event_exporting/classes/command_scope.py b/src/sqlbuild/runtime/event_exporting/classes/command_scope.py deleted file mode 100644 index 6ce33dfd62..0000000000 --- a/src/sqlbuild/runtime/event_exporting/classes/command_scope.py +++ /dev/null @@ -1,149 +0,0 @@ -"""CLI command ownership for typed sink delivery and shared providers.""" - -from __future__ import annotations - -from pathlib import Path -from threading import Lock - -from sqlbuild.compiler.discovery.models import ( - DiscoveredCommandOutputSink, - DiscoveredEventExporter, - DiscoveredProjectInputs, - DiscoveredProvider, -) -from sqlbuild.provider.classes.session import ProviderSession -from sqlbuild.provider.main.session import build_provider_session -from sqlbuild.runtime.event_exporting.classes.dispatcher import EventExporterDispatcher -from sqlbuild.runtime.event_exporting.models import BoundEventExporter, EventExportSummary -from sqlbuild.runtime.output_capture.models import ( - BoundCommandOutputSink, - CommandOutputRecord, -) - - -class EventExporterCommandScope: - """Bind discovered typed sinks and own their command-scoped provider session.""" - - def __init__(self, *, dispatcher: EventExporterDispatcher) -> None: - self.dispatcher = dispatcher - self.provider_session: ProviderSession | None = None - self.project_dir: Path | None = None - self.providers: tuple[DiscoveredProvider, ...] = () - self.event_exporters: tuple[DiscoveredEventExporter, ...] = () - self.command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = () - self._configured = False - self._close_lock = Lock() - self._provider_close_registered = False - - def configure(self, discovered_inputs: DiscoveredProjectInputs) -> None: - """Bind once after normal project discovery has validated declarations.""" - - self.configure_extensions( - providers=discovered_inputs.providers, - event_exporters=discovered_inputs.event_exporters, - command_output_sinks=discovered_inputs.command_output_sinks, - ) - - def configure_extensions( - self, - *, - project_dir: Path | None = None, - providers: tuple[DiscoveredProvider, ...], - event_exporters: tuple[DiscoveredEventExporter, ...], - command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = (), - ) -> None: - """Bind declarations discovered at the outer command startup boundary.""" - - if self._configured: - return - self._configured = True - self.project_dir = project_dir.resolve() if project_dir is not None else None - self.providers = providers - self.event_exporters = event_exporters - self.command_output_sinks = command_output_sinks - if not event_exporters and not command_output_sinks: - self.dispatcher.bind(()) - self.dispatcher.bind_command_output_sinks(()) - return - session: ProviderSession = build_provider_session( - discovered_providers=providers, - allow_shared=False, - ) - session.hold_open() - try: - bindings: list[BoundEventExporter] = [] - for exporter in event_exporters: - provider_arguments: dict[str, object] = { - usage.parameter_name: session.get(usage.provider_name) - for usage in exporter.provider_usages - } - bindings.append( - BoundEventExporter( - name=exporter.name, - function=exporter.function, - provider_arguments=provider_arguments, - event_kinds=exporter.event_kinds, - min_severity=exporter.min_severity, - ) - ) - output_bindings: list[BoundCommandOutputSink] = [] - for sink in command_output_sinks: - output_provider_arguments: dict[str, object] = { - usage.parameter_name: session.get(usage.provider_name) - for usage in sink.provider_usages - } - output_bindings.append( - BoundCommandOutputSink( - name=sink.name, - function=sink.function, - provider_arguments=output_provider_arguments, - streams=sink.streams, - ) - ) - except BaseException: - session.release() - session.close(force=True) - raise - self.provider_session = session - self.dispatcher.bind(tuple(bindings)) - self.dispatcher.bind_command_output_sinks(tuple(output_bindings)) - - def cached_extensions( - self, *, project_dir: Path - ) -> ( - tuple[ - tuple[DiscoveredProvider, ...], - tuple[DiscoveredEventExporter, ...], - tuple[DiscoveredCommandOutputSink, ...], - ] - | None - ): - """Return startup declarations only for the same project root.""" - - if self.project_dir is None or project_dir.resolve() != self.project_dir: - return None - return self.providers, self.event_exporters, self.command_output_sinks - - def close(self) -> EventExportSummary: - """Flush sink delivery before reverse-order provider teardown.""" - - summary: EventExportSummary = self.dispatcher.shutdown() - session: ProviderSession | None = self.provider_session - with self._close_lock: - if session is None or self._provider_close_registered: - return summary - self._provider_close_registered = True - - def close_provider_session() -> None: - try: - session.release() - finally: - session.close(force=True) - - self.dispatcher.finalize_when_idle(close_provider_session) - return summary - - def export_output(self, records: tuple[CommandOutputRecord, ...]) -> None: - """Reuse bound sink functions and provider instances for output records.""" - - self.dispatcher.export_output(records) diff --git a/src/sqlbuild/runtime/event_exporting/classes/dispatcher.py b/src/sqlbuild/runtime/event_exporting/classes/dispatcher.py deleted file mode 100644 index 034bb04a19..0000000000 --- a/src/sqlbuild/runtime/event_exporting/classes/dispatcher.py +++ /dev/null @@ -1,549 +0,0 @@ -"""Bounded priority delivery of canonical lifecycle events.""" - -from __future__ import annotations - -import math -import queue -import threading -import time -from collections.abc import Callable - -from sqlbuild.runtime.event_exporting._helpers.sanitization import sanitized_exception_type -from sqlbuild.runtime.event_exporting.classes.finite_priority_event_queue import ( - FinitePriorityEventQueue, -) -from sqlbuild.runtime.event_exporting.classes.mutable_event_exporter_counts import ( - MutableEventExporterCounts, -) -from sqlbuild.runtime.event_exporting.constants import ( - DEFAULT_EVENT_EXPORT_HEALTH_INTERVAL_SECONDS, - DEFAULT_EVENT_EXPORT_INVOCATION_TIMEOUT_SECONDS, - DEFAULT_EVENT_EXPORT_NOTIFICATION_QUEUE_CAPACITY, - DEFAULT_EVENT_EXPORT_QUEUE_CAPACITY, - DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS, -) -from sqlbuild.runtime.event_exporting.exceptions import ( - EventExporterInputError, - EventExporterStateError, -) -from sqlbuild.runtime.event_exporting.main._lifecycle_export_policy import ( - lifecycle_export_policy, -) -from sqlbuild.runtime.event_exporting.main._severity_at_least import severity_at_least -from sqlbuild.runtime.event_exporting.models import ( - BoundEventExporter, - EventExporterAccounting, - EventExporterCounts, - EventExporterFailure, - EventExportSummary, - LifecycleExportPolicy, - QueuedLifecycleEvent, -) -from sqlbuild.runtime.observability.models import LifecycleEvent -from sqlbuild.runtime.output_capture.constants import COMMAND_OUTPUT_RECORD_TYPE -from sqlbuild.runtime.output_capture.models import ( - BoundCommandOutputSink, - CommandOutputRecord, -) - - -class EventExporterDispatcher: - """Deliver filtered lifecycle events off execution threads with bounded memory.""" - - def __init__( - self, - *, - exporters: tuple[BoundEventExporter, ...] | None = None, - queue_capacity: int = DEFAULT_EVENT_EXPORT_QUEUE_CAPACITY, - shutdown_timeout_seconds: float = DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS, - invocation_timeout_seconds: float = DEFAULT_EVENT_EXPORT_INVOCATION_TIMEOUT_SECONDS, - notification_queue_capacity: int = DEFAULT_EVENT_EXPORT_NOTIFICATION_QUEUE_CAPACITY, - health_interval_seconds: float = DEFAULT_EVENT_EXPORT_HEALTH_INTERVAL_SECONDS, - failure_callback: Callable[[EventExporterFailure], object] | None = None, - summary_callback: Callable[[EventExportSummary], object] | None = None, - ) -> None: - if notification_queue_capacity < 1: - raise EventExporterInputError("event exporter queue capacities must be at least 1") - if shutdown_timeout_seconds < 0 or invocation_timeout_seconds <= 0: - raise EventExporterInputError("event exporter timeouts must be positive") - if ( - isinstance(health_interval_seconds, bool) - or not isinstance(health_interval_seconds, int | float) - or not math.isfinite(health_interval_seconds) - or health_interval_seconds <= 0 - ): - raise EventExporterInputError( - "event exporter health_interval_seconds must be positive and finite" - ) - self._queue = FinitePriorityEventQueue(queue_capacity) - self._exporters: tuple[BoundEventExporter, ...] = exporters or () - self._command_output_sinks: tuple[BoundCommandOutputSink, ...] = () - self._counts: list[MutableEventExporterCounts] = [ - MutableEventExporterCounts() for _ in self._exporters - ] - self._bound = threading.Event() - if exporters is not None: - self._bound.set() - self._shutdown_timeout_seconds = shutdown_timeout_seconds - self._invocation_timeout_seconds = invocation_timeout_seconds - self._health_interval_seconds = float(health_interval_seconds) - self._failure_callback = failure_callback - self._summary_callback = summary_callback - self._notification_queue: queue.Queue[EventExporterFailure] | None = ( - queue.Queue(maxsize=notification_queue_capacity) - if failure_callback is not None or summary_callback is not None - else None - ) - self._lock = threading.Lock() - self._notification_stopping = threading.Event() - self._notification_wakeup = threading.Event() - self._summary_notification_lock = threading.Lock() - self._pending_periodic_summary: EventExportSummary | None = None - self._periodic_inflight_signature: tuple[int, ...] | None = None - self._final_notification_summary: EventExportSummary | None = None - self._final_notification_delivered = False - self._last_published_periodic_signature: tuple[int, ...] = ( - 0, - 0, - 0, - 0, - 0, - 0, - queue_capacity, - ) - self._notification_thread: threading.Thread | None = None - if self._notification_queue is not None: - self._notification_thread = threading.Thread( - target=self._run_notifications, - name="sqlbuild-event-exporter-notifier", - daemon=True, - ) - self._health_stopping = threading.Event() - self._health_thread: threading.Thread | None = None - if summary_callback is not None: - self._health_thread = threading.Thread( - target=self._run_periodic_health, - name="sqlbuild-event-exporter-health", - daemon=True, - ) - self._stopping = threading.Event() - self._force_stop = threading.Event() - self._accepting = True - self._deadline: float | None = None - self._blocked_exporters: set[int] = set() - self._live_invocations: set[threading.Thread] = set() - self._idle_finalizers: list[Callable[[], object]] = [] - self._dispatcher_running = True - self._shutdown_started = False - self._shutdown_complete = threading.Event() - self._final_summary: EventExportSummary | None = None - self._sequence = 0 - if self._notification_thread is not None: - self._notification_thread.start() - if self._health_thread is not None: - self._health_thread.start() - self._thread = threading.Thread( - target=self._run, name="sqlbuild-event-exporter-dispatcher", daemon=True - ) - self._thread.start() - - @property - def thread(self) -> threading.Thread: - return self._thread - - @property - def notification_thread(self) -> threading.Thread | None: - return self._notification_thread - - def enqueue(self, event: LifecycleEvent) -> None: - """Filter and enqueue eligible exporter attempts without waiting.""" - - policy: LifecycleExportPolicy = lifecycle_export_policy(event) - with self._lock: - if not self._bound.is_set(): - raise EventExporterStateError("event exporters must be bound before enqueue") - eligible: list[int] = [] - for index, exporter in enumerate(self._exporters): - if policy.kind not in exporter.event_kinds or not severity_at_least( - severity=policy.severity, minimum=exporter.min_severity - ): - self._counts[index].filtered += 1 - else: - self._counts[index].accepted += 1 - eligible.append(index) - if not eligible: - return - if not self._accepting: - self._drop_indices(tuple(eligible)) - return - item: QueuedLifecycleEvent = QueuedLifecycleEvent( - self._sequence, event, policy, tuple(eligible) - ) - self._sequence += 1 - inserted, displaced = self._queue.put_nowait(item) - if displaced is not None: - self._drop_indices(displaced.eligible_exporters) - if not inserted: - self._drop_indices(item.eligible_exporters) - - def bind(self, exporters: tuple[BoundEventExporter, ...]) -> None: - """Bind validated exporters once before lifecycle publication starts.""" - - with self._lock: - if self._bound.is_set(): - raise EventExporterStateError("event exporters are already bound") - self._exporters = exporters - self._counts = [MutableEventExporterCounts() for _ in exporters] - self._bound.set() - - def bind_command_output_sinks(self, sinks: tuple[BoundCommandOutputSink, ...]) -> None: - """Bind explicit command-output sinks before capture starts.""" - - with self._lock: - self._command_output_sinks = sinks - - def export_output(self, records: tuple[CommandOutputRecord, ...]) -> None: - """Deliver output records through the already-bound destination providers.""" - - invocation: threading.Thread = threading.current_thread() - failed: bool = False - with self._lock: - self._live_invocations.add(invocation) - try: - for record in records: - for exporter in self._command_output_sinks: - if ( - record.record_type == COMMAND_OUTPUT_RECORD_TYPE - and record.stream not in exporter.streams - ): - continue - try: - exporter.function(record=record, **exporter.provider_arguments) - except BaseException: - failed = True - finally: - self._invocation_finished(invocation) - if failed: - raise EventExporterStateError("one or more output export attempts failed") - - def shutdown(self) -> EventExportSummary: - """Stop acceptance and drain until the fixed shutdown deadline.""" - - owns_shutdown = False - with self._lock: - if not self._shutdown_started: - self._shutdown_started = True - owns_shutdown = True - self._accepting = False - self._deadline = time.monotonic() + self._shutdown_timeout_seconds - self._stopping.set() - self._health_stopping.set() - deadline: float = self._deadline or time.monotonic() - if not owns_shutdown: - self._shutdown_complete.wait() - with self._lock: - if self._final_summary is None: - raise EventExporterStateError( - "event exporter shutdown completed without summary" - ) - return self._final_summary - self._thread.join(timeout=max(0.0, deadline - time.monotonic()) + 0.05) - if self._thread.is_alive(): - self._force_stop.set() - self._thread.join() - with self._lock: - summary: EventExportSummary = self._snapshot(final=True) - self._final_summary = summary - self._shutdown_complete.set() - self._store_final_summary(summary) - self._notification_stopping.set() - self._notification_wakeup.set() - return summary - - def finalize_when_idle(self, finalizer: Callable[[], object]) -> None: - run_now = False - with self._lock: - if self._live_invocations or self._dispatcher_running: - self._idle_finalizers.append(finalizer) - else: - run_now = True - if run_now: - self._run_finalizer(finalizer) - - def summary(self) -> EventExportSummary: - """Return a thread-safe immutable point-in-time snapshot.""" - - with self._lock: - return self._snapshot(final=False) - - def _snapshot(self, *, final: bool) -> EventExportSummary: - per_exporter: tuple[EventExporterAccounting, ...] = tuple( - EventExporterAccounting(exporter.name, counts.freeze()) - for exporter, counts in zip(self._exporters, self._counts, strict=True) - ) - aggregate: EventExporterCounts = EventExporterCounts( - accepted=sum(item.counts.accepted for item in per_exporter), - filtered=sum(item.counts.filtered for item in per_exporter), - delivered=sum(item.counts.delivered for item in per_exporter), - dropped=sum(item.counts.dropped for item in per_exporter), - failed=sum(item.counts.failed for item in per_exporter), - ) - flush_complete: bool = ( - final - and not self._dispatcher_running - and not self._live_invocations - and len(self._queue) == 0 - and aggregate.accepted == aggregate.delivered + aggregate.failed + aggregate.dropped - ) - return EventExportSummary( - aggregate=aggregate, - per_exporter=per_exporter, - queue_depth=len(self._queue), - queue_capacity=self._queue.capacity, - flush_complete=flush_complete, - ) - - def _run(self) -> None: - try: - while not self._bound.wait(timeout=0.01): - if self._stopping.is_set() or self._force_stop.is_set(): - return - while True: - if self._past_deadline() or self._force_stop.is_set(): - self._drop_queued_events() - return - try: - item: QueuedLifecycleEvent = self._queue.get(timeout=0.01) - except queue.Empty: - if self._stopping.is_set(): - return - continue - self._deliver(item) - finally: - self._dispatcher_finished() - - def _deliver(self, item: QueuedLifecycleEvent) -> None: - for position, index in enumerate(item.eligible_exporters): - if self._past_deadline(): - with self._lock: - self._drop_indices(item.eligible_exporters[position:]) - return - if index in self._blocked_exporters: - with self._lock: - self._counts[index].dropped += 1 - continue - self._invoke(index=index, item=item) - - def _invoke(self, *, index: int, item: QueuedLifecycleEvent) -> None: - exporter: BoundEventExporter = self._exporters[index] - completed: threading.Event = threading.Event() - failure_types: queue.SimpleQueue[str] = queue.SimpleQueue() - - def invoke() -> None: - try: - exporter.function(event=item.event, **exporter.provider_arguments) - except BaseException as error: - failure_types.put(sanitized_exception_type(error)) - finally: - self._invocation_finished(threading.current_thread()) - completed.set() - - invocation: threading.Thread = threading.Thread( - target=invoke, name=f"sqlbuild-event-exporter-{exporter.name}", daemon=True - ) - with self._lock: - self._live_invocations.add(invocation) - try: - invocation.start() - except BaseException as error: - self._invocation_finished(invocation) - self._record_failure( - index=index, policy=item.policy, error_type=sanitized_exception_type(error) - ) - return - if not self._wait_for_invocation(completed): - self._blocked_exporters.add(index) - self._record_failure(index=index, policy=item.policy, error_type="TimeoutError") - elif not failure_types.empty(): - self._record_failure(index=index, policy=item.policy, error_type=failure_types.get()) - else: - with self._lock: - self._counts[index].delivered += 1 - - def _wait_for_invocation(self, completed: threading.Event) -> bool: - invocation_deadline: float = time.monotonic() + self._invocation_timeout_seconds - while True: - shutdown_deadline: float | None = self._deadline - effective: float = min(invocation_deadline, shutdown_deadline or invocation_deadline) - remaining: float = effective - time.monotonic() - if remaining <= 0 or self._force_stop.is_set(): - return completed.is_set() - if completed.wait(timeout=min(remaining, 0.01)): - return True - - def _record_failure( - self, *, index: int, policy: LifecycleExportPolicy, error_type: str - ) -> None: - with self._lock: - self._counts[index].failed += 1 - self._enqueue_failure( - EventExporterFailure( - self._exporters[index].name, error_type, policy.kind, policy.severity - ) - ) - - def _drop_indices(self, indices: tuple[int, ...]) -> None: - for index in indices: - self._counts[index].dropped += 1 - - def _drop_queued_events(self) -> None: - items: tuple[QueuedLifecycleEvent, ...] = self._queue.drain() - with self._lock: - for item in items: - self._drop_indices(item.eligible_exporters) - - def _enqueue_failure(self, notification: EventExporterFailure) -> None: - if self._notification_queue is None: - return - if self._failure_callback is None: - return - try: - self._notification_queue.put_nowait(notification) - except queue.Full: - return - self._notification_wakeup.set() - - def _run_periodic_health(self) -> None: - while not self._health_stopping.wait(timeout=self._health_interval_seconds): - summary: EventExportSummary = self.summary() - signature: tuple[int, ...] = self._health_signature(summary) - with self._summary_notification_lock: - if self._health_stopping.is_set() or self._final_notification_summary is not None: - return - if signature in { - self._last_published_periodic_signature, - self._periodic_inflight_signature, - }: - self._pending_periodic_summary = None - continue - self._pending_periodic_summary = summary - self._notification_wakeup.set() - - @staticmethod - def _health_signature(summary: EventExportSummary) -> tuple[int, ...]: - return ( - summary.accepted, - summary.filtered, - summary.delivered, - summary.dropped, - summary.failed, - summary.queue_depth, - summary.queue_capacity, - ) - - def _store_final_summary(self, summary: EventExportSummary) -> None: - if self._summary_callback is None: - return - with self._summary_notification_lock: - if self._final_notification_summary is None and not self._final_notification_delivered: - self._final_notification_summary = summary - self._notification_wakeup.set() - - def _take_periodic_summary(self) -> EventExportSummary | None: - with self._summary_notification_lock: - summary: EventExportSummary | None = self._pending_periodic_summary - self._pending_periodic_summary = None - self._periodic_inflight_signature = ( - None if summary is None else self._health_signature(summary) - ) - return summary - - def _take_final_summary(self) -> EventExportSummary | None: - with self._summary_notification_lock: - if self._final_notification_delivered: - return None - return self._final_notification_summary - - def _mark_final_delivered(self) -> None: - with self._summary_notification_lock: - self._final_notification_delivered = True - self._final_notification_summary = None - - def _mark_periodic_published(self, summary: EventExportSummary) -> None: - with self._summary_notification_lock: - self._last_published_periodic_signature = self._health_signature(summary) - self._periodic_inflight_signature = None - - def _run_notifications(self) -> None: - if self._notification_queue is None: - return - while True: - self._notification_wakeup.clear() - periodic_summary: EventExportSummary | None = self._take_periodic_summary() - if periodic_summary is not None: - try: - if self._summary_callback is not None: - self._summary_callback(periodic_summary) - except BaseException: - pass - finally: - self._mark_periodic_published(periodic_summary) - continue - final_summary: EventExportSummary | None = self._take_final_summary() - if final_summary is not None: - try: - if self._summary_callback is not None: - self._summary_callback(final_summary) - except BaseException: - pass - finally: - self._mark_final_delivered() - return - try: - notification: EventExporterFailure = self._notification_queue.get_nowait() - except queue.Empty: - if self._notification_stopping.is_set(): - return - self._notification_wakeup.wait(timeout=0.01) - continue - try: - if self._failure_callback is not None: - self._failure_callback(notification) - except BaseException: - pass - finally: - self._notification_queue.task_done() - - def _invocation_finished(self, invocation: threading.Thread) -> None: - finalizers: tuple[Callable[[], object], ...] = () - with self._lock: - self._live_invocations.discard(invocation) - if ( - not self._live_invocations - and not self._dispatcher_running - and self._idle_finalizers - ): - finalizers = tuple(self._idle_finalizers) - self._idle_finalizers.clear() - for finalizer in finalizers: - self._run_finalizer(finalizer) - - def _dispatcher_finished(self) -> None: - finalizers: tuple[Callable[[], object], ...] = () - with self._lock: - self._dispatcher_running = False - if not self._live_invocations and self._idle_finalizers: - finalizers = tuple(self._idle_finalizers) - self._idle_finalizers.clear() - for finalizer in finalizers: - self._run_finalizer(finalizer) - - @staticmethod - def _run_finalizer(finalizer: Callable[[], object]) -> None: - try: - finalizer() - except BaseException: - pass - - def _past_deadline(self) -> bool: - return self._deadline is not None and time.monotonic() >= self._deadline diff --git a/src/sqlbuild/runtime/event_exporting/classes/finite_priority_event_queue.py b/src/sqlbuild/runtime/event_exporting/classes/finite_priority_event_queue.py deleted file mode 100644 index f422c4392d..0000000000 --- a/src/sqlbuild/runtime/event_exporting/classes/finite_priority_event_queue.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Bounded finite-priority lifecycle event queue.""" - -import queue -import threading -import time - -from sqlbuild.runtime.event_exporting.constants import INVOCATION_TERMINAL_EVENT_TYPES -from sqlbuild.runtime.event_exporting.exceptions import EventExporterInputError -from sqlbuild.runtime.event_exporting.models import QueuedLifecycleEvent - - -class FinitePriorityEventQueue: - """Thread-safe queue, highest priority first and FIFO within priority.""" - - def __init__(self, capacity: int) -> None: - if capacity < 1: - raise EventExporterInputError("event exporter queue capacities must be at least 1") - self.capacity = capacity - self._condition = threading.Condition() - self._items: list[QueuedLifecycleEvent] = [] - - def put_nowait(self, item: QueuedLifecycleEvent) -> tuple[bool, QueuedLifecycleEvent | None]: - """Insert, displacing the oldest item at the lowest lower priority when full.""" - - with self._condition: - displaced: QueuedLifecycleEvent | None = None - if len(self._items) >= self.capacity: - lowest: int = min(queued.policy.priority for queued in self._items) - if lowest >= item.policy.priority: - return False, None - displaced = min( - (queued for queued in self._items if queued.policy.priority == lowest), - key=lambda queued: queued.sequence, - ) - self._items.remove(displaced) - self._items.append(item) - self._condition.notify() - return True, displaced - - def get(self, timeout: float) -> QueuedLifecycleEvent: - """Remove the oldest item at the highest available priority.""" - - deadline: float = time.monotonic() + timeout - with self._condition: - while not self._items: - remaining: float = deadline - time.monotonic() - if remaining <= 0: - raise queue.Empty - self._condition.wait(timeout=remaining) - selectable: list[QueuedLifecycleEvent] = [ - item for item in self._items if not _is_invocation_terminal(item) - ] - if not selectable: - selectable = self._items - highest: int = max(item.policy.priority for item in selectable) - selected: QueuedLifecycleEvent = min( - (item for item in selectable if item.policy.priority == highest), - key=lambda item: item.sequence, - ) - self._items.remove(selected) - return selected - - def drain(self) -> tuple[QueuedLifecycleEvent, ...]: - """Atomically remove all queued items.""" - - with self._condition: - items: tuple[QueuedLifecycleEvent, ...] = tuple(self._items) - self._items.clear() - return items - - def __len__(self) -> int: - with self._condition: - return len(self._items) - - -def _is_invocation_terminal(item: QueuedLifecycleEvent) -> bool: - return item.event.event_type in INVOCATION_TERMINAL_EVENT_TYPES diff --git a/src/sqlbuild/runtime/event_exporting/classes/mutable_event_exporter_counts.py b/src/sqlbuild/runtime/event_exporting/classes/mutable_event_exporter_counts.py deleted file mode 100644 index 3cf050007b..0000000000 --- a/src/sqlbuild/runtime/event_exporting/classes/mutable_event_exporter_counts.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Mutable dispatcher accounting state.""" - -from sqlbuild.runtime.event_exporting.models import EventExporterCounts - - -class MutableEventExporterCounts: - """Dispatcher-owned mutable counters guarded by its accounting lock.""" - - def __init__(self) -> None: - self.accepted = 0 - self.filtered = 0 - self.delivered = 0 - self.dropped = 0 - self.failed = 0 - - def freeze(self) -> EventExporterCounts: - return EventExporterCounts( - accepted=self.accepted, - filtered=self.filtered, - delivered=self.delivered, - dropped=self.dropped, - failed=self.failed, - ) diff --git a/src/sqlbuild/runtime/event_exporting/constants.py b/src/sqlbuild/runtime/event_exporting/constants.py index 07dfe4569f..227df104bd 100644 --- a/src/sqlbuild/runtime/event_exporting/constants.py +++ b/src/sqlbuild/runtime/event_exporting/constants.py @@ -1,49 +1,10 @@ -"""Destination-neutral event exporter delivery defaults.""" - -from collections.abc import Mapping -from types import MappingProxyType +"""Lifecycle-event sink declaration constants.""" from sqlbuild.runtime.event_exporting.types import LifecycleEventKind -from sqlbuild.runtime.observability.constants import LIFECYCLE_EVENT_CATALOG from sqlbuild.spec.contracts.types import EventExportSeverity -DEFAULT_EVENT_EXPORT_QUEUE_CAPACITY: int = 1024 -DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS: float = 2.0 -DEFAULT_EVENT_EXPORT_INVOCATION_TIMEOUT_SECONDS: float = 1.0 -DEFAULT_EVENT_EXPORT_NOTIFICATION_QUEUE_CAPACITY: int = 256 -DEFAULT_EVENT_EXPORT_HEALTH_INTERVAL_SECONDS: float = 30.0 EVENT_EXPORTER_EVENT_PARAMETER_NAME: str = "event" EVENT_EXPORT_SEVERITIES: tuple[str, ...] = tuple(item.value for item in EventExportSeverity) EVENT_EXPORT_KINDS: frozenset[str] = frozenset(kind.value for kind in LifecycleEventKind) COMMAND_OUTPUT_SINK_ATTRIBUTE: str = "__sqlbuild_command_output_sink__" LIFECYCLE_EVENT_SINK_ATTRIBUTE: str = "__sqlbuild_lifecycle_event_sink__" -INVOCATION_TERMINAL_EVENT_TYPES: frozenset[str] = frozenset( - {"invocation_completed", "invocation_failed"} -) -EVENT_EXPORT_SEVERITY_RANKS: Mapping[EventExportSeverity, int] = MappingProxyType( - {severity: index for index, severity in enumerate(EventExportSeverity)} -) -RETRY_EVENT_TYPE: str = "retry_scheduled" -RESOURCE_EVENT_PREFIX: str = "resource" -LIFECYCLE_EXPORT_DIMENSIONS: Mapping[str, tuple[str, EventExportSeverity, int]] = MappingProxyType( - { - event_type: ( - ("retry", EventExportSeverity.WARNING, 1) - if event_type == RETRY_EVENT_TYPE - else ( - ( - "resource" - if event_type.split("_", maxsplit=1)[0] == RESOURCE_EVENT_PREFIX - else event_type.split("_", maxsplit=1)[0] - ), - EventExportSeverity.ERROR - if event_type.endswith("_failed") - else EventExportSeverity.INFO - if definition.terminal - else EventExportSeverity.DEBUG, - 3 if event_type.endswith("_failed") else 2 if definition.terminal else 0, - ) - ) - for event_type, definition in LIFECYCLE_EVENT_CATALOG.items() - } -) diff --git a/src/sqlbuild/runtime/event_exporting/main/_configure_discovered_event_exporters.py b/src/sqlbuild/runtime/event_exporting/main/_configure_discovered_event_exporters.py deleted file mode 100644 index de7e1b3e6b..0000000000 --- a/src/sqlbuild/runtime/event_exporting/main/_configure_discovered_event_exporters.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Discovered event exporter configuration entrypoint.""" - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.runtime.event_exporting._helpers.scope import ( - configure_discovered_event_exporters as _configure_discovered_event_exporters, -) - - -def configure_discovered_event_exporters(discovered_inputs: DiscoveredProjectInputs) -> None: - """Configure the active exporter scope from validated discovery inputs.""" - - _ = _configure_discovered_event_exporters(discovered_inputs) diff --git a/src/sqlbuild/runtime/event_exporting/main/_event_exporter_command_scope.py b/src/sqlbuild/runtime/event_exporting/main/_event_exporter_command_scope.py deleted file mode 100644 index 90e3b0d391..0000000000 --- a/src/sqlbuild/runtime/event_exporting/main/_event_exporter_command_scope.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Event exporter command scope entrypoint.""" - -from collections.abc import Iterator -from contextlib import contextmanager - -from sqlbuild.runtime.event_exporting._helpers.scope import ( - event_exporter_command_scope as _event_exporter_command_scope, -) -from sqlbuild.runtime.event_exporting.classes.command_scope import EventExporterCommandScope - - -@contextmanager -def event_exporter_command_scope( - scope: EventExporterCommandScope, -) -> Iterator[EventExporterCommandScope]: - """Install one command-owned exporter scope.""" - - with _event_exporter_command_scope(scope) as installed: - yield installed diff --git a/src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.py b/src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.py new file mode 100644 index 0000000000..844e0e5382 --- /dev/null +++ b/src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.py @@ -0,0 +1,22 @@ +"""Call one lifecycle-event sink on its delivery thread.""" + +from collections.abc import Callable, Mapping + +from sqlbuild.runtime.observability.models import LifecycleEvent + + +def invoke_lifecycle_event_sink( + *, + function: Callable[..., object], + provider_arguments: Mapping[str, object], + event: LifecycleEvent, + completion: Callable[[BaseException | None], None], +) -> None: + """Call a user sink with one event, then report how the call ended.""" + + try: + _ = function(event=event, **provider_arguments) + except BaseException as error: + _ = completion(error) + return + _ = completion(None) diff --git a/src/sqlbuild/runtime/event_exporting/main/_lifecycle_export_policy.py b/src/sqlbuild/runtime/event_exporting/main/_lifecycle_export_policy.py deleted file mode 100644 index 5dcf763539..0000000000 --- a/src/sqlbuild/runtime/event_exporting/main/_lifecycle_export_policy.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Destination-neutral lifecycle export policy entrypoints.""" - -from sqlbuild.runtime.event_exporting._helpers.policy import ( - lifecycle_export_policy as _lifecycle_export_policy, -) -from sqlbuild.runtime.event_exporting.models import LifecycleExportPolicy -from sqlbuild.runtime.observability.models import LifecycleEvent - - -def lifecycle_export_policy(event: LifecycleEvent) -> LifecycleExportPolicy: - return _lifecycle_export_policy(event) diff --git a/src/sqlbuild/runtime/event_exporting/main/_lifecycle_export_policy_catalog.py b/src/sqlbuild/runtime/event_exporting/main/_lifecycle_export_policy_catalog.py deleted file mode 100644 index fcc1fdef4a..0000000000 --- a/src/sqlbuild/runtime/event_exporting/main/_lifecycle_export_policy_catalog.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Lifecycle export policy catalog entrypoint.""" - -from collections.abc import Mapping - -from sqlbuild.runtime.event_exporting._helpers.policy import ( - lifecycle_export_policy_catalog as _lifecycle_export_policy_catalog, -) -from sqlbuild.runtime.event_exporting.models import LifecycleExportPolicy - - -def lifecycle_export_policy_catalog() -> Mapping[str, LifecycleExportPolicy]: - return _lifecycle_export_policy_catalog() diff --git a/src/sqlbuild/runtime/event_exporting/main/_run_command_output_sinks.py b/src/sqlbuild/runtime/event_exporting/main/_run_command_output_sinks.py new file mode 100644 index 0000000000..4a8f547472 --- /dev/null +++ b/src/sqlbuild/runtime/event_exporting/main/_run_command_output_sinks.py @@ -0,0 +1,24 @@ +"""Call command-output sinks on their dispatcher's single delivery thread.""" + +from collections.abc import Callable, Mapping +from queue import SimpleQueue + +from sqlbuild.runtime.output_capture.models import CommandOutputRecord + + +def run_command_output_sinks( + *, + calls: SimpleQueue[ + tuple[Callable[..., object], Mapping[str, object], CommandOutputRecord, SimpleQueue[bool]] + | None + ], +) -> None: + """Call each queued user sink in order, reporting whether it raised, until `None` arrives.""" + + for function, provider_arguments, record, outcome in iter(calls.get, None): + try: + _ = function(record=record, **provider_arguments) + except BaseException: + outcome.put(True) + else: + outcome.put(False) diff --git a/src/sqlbuild/runtime/event_exporting/main/_severity_at_least.py b/src/sqlbuild/runtime/event_exporting/main/_severity_at_least.py deleted file mode 100644 index d131903811..0000000000 --- a/src/sqlbuild/runtime/event_exporting/main/_severity_at_least.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Lifecycle export severity comparison entrypoint.""" - -from sqlbuild.runtime.event_exporting._helpers.policy import ( - severity_at_least as _severity_at_least, -) -from sqlbuild.spec.contracts.types import EventExportSeverity - - -def severity_at_least( - *, severity: str | EventExportSeverity, minimum: str | EventExportSeverity -) -> bool: - """Compare severities while preserving the public string-accepting API.""" - - return _severity_at_least( - severity=EventExportSeverity(severity), minimum=EventExportSeverity(minimum) - ) diff --git a/src/sqlbuild/runtime/event_exporting/main/cached_event_exporter_extensions.py b/src/sqlbuild/runtime/event_exporting/main/cached_event_exporter_extensions.py deleted file mode 100644 index eca03da267..0000000000 --- a/src/sqlbuild/runtime/event_exporting/main/cached_event_exporter_extensions.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Cached event exporter extension entrypoint.""" - -from pathlib import Path - -from sqlbuild.compiler.discovery.models import ( - DiscoveredCommandOutputSink, - DiscoveredEventExporter, - DiscoveredProvider, -) -from sqlbuild.runtime.event_exporting._helpers.scope import ( - cached_event_exporter_extensions as _cached_event_exporter_extensions, -) - - -def cached_event_exporter_extensions( - *, project_dir: Path -) -> ( - tuple[ - tuple[DiscoveredProvider, ...], - tuple[DiscoveredEventExporter, ...], - tuple[DiscoveredCommandOutputSink, ...], - ] - | None -): - """Return startup-discovered extensions for the active project root.""" - - return _cached_event_exporter_extensions(project_dir=project_dir) diff --git a/src/sqlbuild/runtime/event_exporting/main/current_event_exporter_command_scope.py b/src/sqlbuild/runtime/event_exporting/main/current_event_exporter_command_scope.py deleted file mode 100644 index 3cd5b4ebd1..0000000000 --- a/src/sqlbuild/runtime/event_exporting/main/current_event_exporter_command_scope.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Current event exporter command scope entrypoint.""" - -from sqlbuild.runtime.event_exporting._helpers.scope import ( - current_event_exporter_command_scope as _current_event_exporter_command_scope, -) -from sqlbuild.runtime.event_exporting.classes.command_scope import EventExporterCommandScope - - -def current_event_exporter_command_scope() -> EventExporterCommandScope | None: - """Return the active command-owned exporter scope, if any.""" - - return _current_event_exporter_command_scope() diff --git a/src/sqlbuild/runtime/event_exporting/main/stricter_severity.py b/src/sqlbuild/runtime/event_exporting/main/stricter_severity.py deleted file mode 100644 index d79100169e..0000000000 --- a/src/sqlbuild/runtime/event_exporting/main/stricter_severity.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Lifecycle export severity narrowing entrypoint.""" - -from sqlbuild.runtime.event_exporting._helpers.policy import ( - stricter_severity as _stricter_severity, -) -from sqlbuild.spec.contracts.types import EventExportSeverity - - -def stricter_severity( - *, - first: str | EventExportSeverity, - second: str | EventExportSeverity | None, -) -> EventExportSeverity: - """Select a stricter severity while preserving string-accepting callers.""" - - return _stricter_severity( - first=EventExportSeverity(first), - second=EventExportSeverity(second) if second is not None else None, - ) diff --git a/src/sqlbuild/runtime/event_exporting/models.py b/src/sqlbuild/runtime/event_exporting/models.py index e6d4102d95..a75ce68508 100644 --- a/src/sqlbuild/runtime/event_exporting/models.py +++ b/src/sqlbuild/runtime/event_exporting/models.py @@ -1,12 +1,9 @@ -"""Typed event exporter runtime contracts.""" +"""Typed lifecycle-event sink declarations.""" from __future__ import annotations -from collections.abc import Callable, Mapping from dataclasses import dataclass -from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORT_KINDS -from sqlbuild.runtime.observability.models import LifecycleEvent from sqlbuild.spec.contracts.types import EventExportSeverity @@ -17,93 +14,3 @@ class LifecycleEventSinkDefinition: name: str event_kinds: frozenset[str] min_severity: EventExportSeverity - - -@dataclass(frozen=True) -class BoundEventExporter: - """One validated exporter bound to command-scoped provider instances.""" - - name: str - function: Callable[..., object] - provider_arguments: Mapping[str, object] - event_kinds: frozenset[str] = EVENT_EXPORT_KINDS - min_severity: EventExportSeverity = EventExportSeverity.DEBUG - - -@dataclass(frozen=True) -class EventExporterCounts: - """Frozen attempt counters for one exporter or their aggregate.""" - - accepted: int = 0 - filtered: int = 0 - delivered: int = 0 - failed: int = 0 - dropped: int = 0 - - -@dataclass(frozen=True) -class EventExporterAccounting: - """Frozen accounting snapshot for one named exporter.""" - - exporter_name: str - counts: EventExporterCounts - - -@dataclass(frozen=True) -class EventExportSummary: - """Final or point-in-time best-effort delivery accounting.""" - - aggregate: EventExporterCounts - per_exporter: tuple[EventExporterAccounting, ...] - queue_depth: int - queue_capacity: int - flush_complete: bool - - @property - def accepted(self) -> int: - return self.aggregate.accepted - - @property - def filtered(self) -> int: - return self.aggregate.filtered - - @property - def delivered(self) -> int: - return self.aggregate.delivered - - @property - def failed(self) -> int: - return self.aggregate.failed - - @property - def dropped(self) -> int: - return self.aggregate.dropped - - -@dataclass(frozen=True) -class EventExporterFailure: - """Sanitized exporter failure information safe for diagnostics.""" - - exporter_name: str - error_type: str - event_kind: str - event_severity: EventExportSeverity - - -@dataclass(frozen=True) -class LifecycleExportPolicy: - """Catalogued export dimensions for one validated lifecycle type.""" - - kind: str - severity: EventExportSeverity - priority: int - - -@dataclass(frozen=True) -class QueuedLifecycleEvent: - """One queued event and its exact eligible exporter attempts.""" - - sequence: int - event: LifecycleEvent - policy: LifecycleExportPolicy - eligible_exporters: tuple[int, ...] diff --git a/src/sqlbuild/spec/contracts/exceptions.py b/src/sqlbuild/spec/contracts/exceptions.py deleted file mode 100644 index af45cd8880..0000000000 --- a/src/sqlbuild/spec/contracts/exceptions.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Project specification exceptions.""" - -from __future__ import annotations - - -class SpecConfigError(RuntimeError): - """Raised when project specification configuration is invalid.""" - - -class ConfigValueTypeError(ValueError): - """Raised when an authored config value has the wrong runtime type.""" - - def __init__(self, *, key: str, expected: str, actual_type: type[object]) -> None: - self.key = key - self.expected = expected - self.actual_type = actual_type - super().__init__(f"config key '{key}' expected {expected}, got {actual_type.__name__}") diff --git a/src/sqlbuild/spec/contracts/models.py b/src/sqlbuild/spec/contracts/models.py index 4abc24f472..84b803ef0d 100644 --- a/src/sqlbuild/spec/contracts/models.py +++ b/src/sqlbuild/spec/contracts/models.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Literal +from sqlbuild._native import CollectionRendering from sqlbuild.compiler.auditing.models import MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditSeverity from sqlbuild.cost.constants import DEFAULT_USD_PER_CREDIT @@ -26,7 +27,6 @@ TimeTravelRetentionSource, WarehouseGroup, ) -from sqlbuild.sql_values.types import CollectionRendering @dataclass(frozen=True) diff --git a/src/sqlbuild/sql_values/types.py b/src/sqlbuild/sql_values/types.py index 8636af335a..2e6643497a 100644 --- a/src/sqlbuild/sql_values/types.py +++ b/src/sqlbuild/sql_values/types.py @@ -20,13 +20,6 @@ class SqlValueKind(StrEnum): OBJECT = "object" -class CollectionRendering(StrEnum): - """Physical rendering choices for homogeneous SQL collections.""" - - VALUE_LIST = "value_list" - ARRAY = "array" - - class StateSqlValueType(StrEnum): """Closed physical value kinds used by warehouse state columns.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index f27228d539..86149d8555 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -40,12 +40,11 @@ semantic_compile_fingerprint, ) from scripts.compiler_differential.constants import FAILURE_BASE_FILES -from sqlbuild._native import compiler_cache_directory +from sqlbuild._native import ContractValidationResult, compiler_cache_directory from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.contracts.models import ContractValidationResult from sqlbuild.compiler.frontier.constants import ( COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index f1870dff72..4986554e52 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -8,10 +8,14 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import NativeProjectResources, NativeSqlTestArtifact, ProjectColumnLineage -from sqlbuild.compiler.analysis_session.models import NativeModelAnalyses -from sqlbuild.compiler.contracts.models import ContractValidationResult -from sqlbuild.compiler.sql_test_glue.models import NativeSqlTestAssembly +from sqlbuild._native import ( + ContractValidationResult, + NativeModelAnalyses, + NativeProjectResources, + NativeSqlTestArtifact, + NativeSqlTestAssembly, + ProjectColumnLineage, +) from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( NativeAnalysisSeamTestCase, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py index e89596ebf7..009abe88ae 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py @@ -114,3 +114,13 @@ class ProviderPlanOutputE2ETestCase: expected_text_fragments: tuple[str, ...] expected_provider_name: str expected_used_by: tuple[tuple[str, str, str], ...] + + +@dataclass(frozen=True) +class FilteredSinkFailureE2ETestCase: + description: str + command: tuple[str, ...] + expected_exit_code: int + expected_runs_only: tuple[str, ...] + expected_errors_only: tuple[str, ...] + expected_warning: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py index cca5fb17fe..b676e69f0d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py @@ -12,6 +12,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.providers._test_types import ( CommandOutputE2ETestCase, EventExporterE2ETestCase, + FilteredSinkFailureE2ETestCase, NoExporterCommandE2ETestCase, ReadOnlyProviderE2ETestCase, ) @@ -317,3 +318,106 @@ def test_given_no_exporters_and_exploding_provider_when_non_provider_command_run assert not marker_path.exists() assert helper_marker_path.exists() assert not (project_dir / ".sqlbuild").exists() + + +@pytest.mark.parametrize( + "test_case", + ( + FilteredSinkFailureE2ETestCase( + description="failed build filters per sink and reports the failing sink", + command=("build",), + expected_exit_code=1, + expected_runs_only=(), + expected_errors_only=("invocation_failed",), + expected_warning=( + "Warning: lifecycle event export was incomplete: 0 of {accepted} event " + "deliveries dropped, 1 failed (sink 'failing_on_errors'). Increase " + "sinks.lifecycle.shutdown_timeout or check sink health." + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_filtered_and_failing_sinks_when_build_fails_then_delivery_and_warning_match_filters( + test_case: FilteredSinkFailureE2ETestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + output_dir: Path = tmp_path / "filtered" + output_dir.mkdir() + monkeypatch.setenv("FILTERED_EXPORT_DIR", str(output_dir)) + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="filtered_sink_project", + repo_files={ + "sqlbuild_project.toml": dedent( + """ + name = "filtered_sink_project" + adapter = "duckdb" + + [connection] + database = "filtered_sink_project.duckdb" + + [sinks.lifecycle.named.errors_only] + min_severity = "error" + """ + ).lstrip(), + "sinks/filtered.py": dedent( + """ + import os + from pathlib import Path + + from sqlbuild.sinks import LifecycleEvent, LifecycleEventKind, lifecycle_event_sink + + + def _record(sink: str, event: LifecycleEvent) -> None: + path = Path(os.environ["FILTERED_EXPORT_DIR"]) / f"{sink}.txt" + with path.open("a", encoding="utf-8") as stream: + stream.write(event.event_type + "\\n") + + + @lifecycle_event_sink(event_kinds={LifecycleEventKind.RUN}, min_severity="info") + def runs_only(event: LifecycleEvent) -> None: + _record("runs_only", event) + + + @lifecycle_event_sink(event_kinds={LifecycleEventKind.INVOCATION}) + def errors_only(event: LifecycleEvent) -> None: + _record("errors_only", event) + + + @lifecycle_event_sink(name="failing_on_errors", min_severity="error") + def failing(event: LifecycleEvent) -> None: + raise RuntimeError("destination detail must not be reported") + """ + ), + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized table);\n\n" + "SELECT 1 AS order_id\n" + ), + "models/broken_orders.sql": ( + "MODEL (description 'Test model broken orders.', materialized table);\n\n" + 'SELECT missing_column FROM __ref("orders")\n' + ), + }, + ) + (output_dir / "runs_only.txt").touch() + (output_dir / "errors_only.txt").touch() + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *test_case.command), + project_dir=project_dir, + ) + + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + runs_only: tuple[str, ...] = tuple( + (output_dir / "runs_only.txt").read_text(encoding="utf-8").splitlines() + ) + errors_only: tuple[str, ...] = tuple( + (output_dir / "errors_only.txt").read_text(encoding="utf-8").splitlines() + ) + accepted: int = len(runs_only) + len(errors_only) + 1 + assert runs_only == test_case.expected_runs_only + assert errors_only == test_case.expected_errors_only + assert test_case.expected_warning.format(accepted=accepted) in result.stderr + assert "destination detail" not in result.stdout + result.stderr diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 755aefccff..088cb42294 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1129,3 +1129,11 @@ class CustomRuleCacheSequenceTestCase: description: str steps: tuple[tuple[str, str], ...] expected_hits_and_misses: tuple[tuple[int, int], ...] + + +@dataclass(frozen=True) +class ProviderAnnotationToleranceTestCase: + description: str + annotation: str + expected_exit_code: int + expected_annotation_class_name: str | None diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index ffb3c98ed1..cfc9777725 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -17,7 +17,9 @@ import sqlbuild._native as native_module import sqlbuild._native as plan_artifacts_module from sqlbuild._native import ( + CompilerStage, CustomHostObserver, + NativeQueryColumns, NativeSqlTestArtifact, PlannedStaticSqlTests, PreparedCompileArtifacts, @@ -31,8 +33,6 @@ ) from sqlbuild.adapter.contract.types import RetentionChangePhase from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.analysis_session.models import NativeQueryColumns -from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.planner.exceptions import NativeSqlTestPlanningError from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( RepeatedJsonParseTestCase, @@ -958,3 +958,38 @@ def observe_hosts( ) monkeypatch.setattr(native_module, "custom_host_observer", observer) return observer + + +PROVIDER_SOURCE: str = """ +from sqlbuild.providers import Provider + + +class OrdersProvider(Provider): + 'Order service settings.' + + region: str = "eu" +""" + + +def write_provider_task_project(*, project_dir: Path, annotation: str) -> None: + """Write a DuckDB project whose task takes a provider parameter with `annotation`.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders_tasks"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "orders.duckdb"\n' + ), + "providers/orders.py": PROVIDER_SOURCE, + "python/orders.py": ( + "from providers.orders import OrdersProvider\n" + "from sqlbuild.tasks import task\n\n\n" + "@task\n" + f"def refresh_orders(ctx, orders_provider: {annotation}):\n" + ' """Refresh the order snapshot."""\n' + " return None\n" + ), + } + for relative_path, text in files.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_provider_annotation_tolerance.py b/tests/integration/src/sqlbuild/cli/commands/main/test_provider_annotation_tolerance.py new file mode 100644 index 0000000000..6a4d7ddab6 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_provider_annotation_tolerance.py @@ -0,0 +1,51 @@ +"""Provider parameters whose annotations cannot be resolved keep compiling.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import discover_provider_classes, discover_python_node_functions +from sqlbuild._native import run_cli as main +from sqlbuild.compiler.discovery.models import DiscoveredPythonNodeFunctions +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + ProviderAnnotationToleranceTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import write_provider_task_project + + +@pytest.mark.parametrize( + "test_case", + [ + ProviderAnnotationToleranceTestCase( + description="provider class annotation is recorded", + annotation="OrdersProvider", + expected_exit_code=0, + expected_annotation_class_name="OrdersProvider", + ), + ProviderAnnotationToleranceTestCase( + description="annotation that get_type_hints rejects with TypeError counts as unannotated", + annotation='"list[int](3)"', + expected_exit_code=0, + expected_annotation_class_name=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_provider_parameter_annotation_when_compiling_then_usage_is_recorded_without_failing( + test_case: ProviderAnnotationToleranceTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_provider_task_project(project_dir=tmp_path, annotation=test_case.annotation) + + discovered: DiscoveredPythonNodeFunctions = discover_python_node_functions( + project_dir=tmp_path, providers=discover_provider_classes(project_dir=tmp_path) + ) + exit_code: int = main(["--project-dir", str(tmp_path), "compile"]) + + assert tuple(usage.annotation_class_name for usage in discovered.tasks[0].provider_usages) == ( + test_case.expected_annotation_class_name, + ) + assert exit_code == test_case.expected_exit_code, capsys.readouterr() diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 3c4eceb705..bd995a0a75 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -14,12 +14,17 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.render.user_macros as macros_module -from sqlbuild._native import CustomHostObserver, active_lint_stop, rules_bulk_cache_path +from sqlbuild._native import ( + CustomHostObserver, + DeclarationScopeBuild, + LintRunResult, + active_lint_stop, + rules_bulk_cache_path, +) from sqlbuild._native import run_cli as main -from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro +from sqlbuild.compiler.compile.models import LoadedMacro from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, DiscoveredProjectInputs from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.lint.models import LintRunResult from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( CustomHostFailureIntegrationTestCase, CustomHostInterruptIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py index eaf40bf223..4fa6e4789d 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py @@ -3,8 +3,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING +from sqlbuild._native import ColumnLineageMode from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.lineage.types import ColumnLineageMode if TYPE_CHECKING: from sqlbuild._native import QueryColumnsMode diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index fef49b8785..77b3164b53 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -15,6 +15,15 @@ import sqlbuild._native as native_module from sqlbuild._native import ( + AnalysisCacheContext, + CollectionRendering, + ColumnLineageMode, + CompileAdapterContext, + DynamicColumnContractProof, + ModelSqlAnalysis, + NativeModelAnalyses, + NativeModelAnalysisRequest, + NativePivotTables, analyze_native_model_sql, build_analysis_cache_context, build_compile_inputs, @@ -30,35 +39,18 @@ from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main -from sqlbuild.compiler.analysis_session.models import ( - NativeModelAnalyses, - NativeModelAnalysisRequest, - NativePivotTables, -) from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - AnalysisCacheContext, - CompactLineageFacts, - CompileAdapterContext, - DynamicColumnContractProof, - InferredColumn, - ModelSqlAnalysis, -) -from sqlbuild.compiler.compile.types import ( - CompiledProjectView, - CompileInputsView, - ModelInput, -) -from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability +from sqlbuild.compiler.compile.models import CompactLineageFacts, InferredColumn +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily -from sqlbuild.sql_values.types import CollectionRendering from tests.integration.src.sqlbuild.compiler.analysis_session._test_types import ( AdapterRuleCallbackTestCase, ) from tests.integration.src.sqlbuild.compiler.helpers import GoldenEntry, golden_entry if TYPE_CHECKING: - from sqlbuild._native import ProofRow + from sqlbuild._native import CompileInputsView, ModelInput, ProofRow LINEAGE_NATIVE: str = "native" diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py index 77a6e7efc4..73d1870fc9 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py @@ -6,18 +6,22 @@ import random from itertools import product from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any import pytest import sqlbuild._native as native_module -from sqlbuild._native import analyze_native_model_sql, assemble_project +from sqlbuild._native import ( + ColumnLineageMode, + NativeModelAnalysisRequest, + analyze_native_model_sql, + assemble_project, +) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest -from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.lineage.types import InferredNullability from tests.integration.src.sqlbuild.compiler.analysis_session._test_types import ( AdapterRuleCallbackTestCase, CyclicAnalysisTestCase, @@ -51,6 +55,9 @@ read_golden, ) +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + _DIALECT_PROFILES: tuple[ExpressionInferenceProfile, ...] = tuple( ExpressionInferenceProfile(sql_analysis_dialect=dialect) for dialect in ("duckdb", "postgres", "snowflake", "bigquery") diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py index 2bf800b5c8..78bfa1f58a 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py @@ -12,9 +12,8 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import infer_native_query_columns +from sqlbuild._native import NativeColumnQuery, NativeModelAnalyses, infer_native_query_columns from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeModelAnalyses from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.integration.src.sqlbuild.compiler.analysis_session._test_types import ( diff --git a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py index c57f4e8979..173909af0d 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py @@ -8,11 +8,8 @@ from typing import Any import sqlbuild._native as _native -from sqlbuild._native import find_macro_call_names, scan_sql_reference_calls +from sqlbuild._native import SqlReferenceScan, find_macro_call_names, scan_sql_reference_calls from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - SqlReferenceScan, -) from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py index e28c5e327a..f732d103b2 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -10,11 +10,14 @@ from functools import partial from itertools import compress from pathlib import Path -from typing import cast +from typing import TYPE_CHECKING, cast import pytest from sqlbuild._native import ( + CompileAdapterContext, + CompilerStage, + MacroExpansionResult, build_compile_inputs, compile_frontier, discover_project_inputs, @@ -25,20 +28,16 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompileAdapterContext, - LoadedMacro, - MacroContext, - MacroExpansionResult, -) -from sqlbuild.compiler.compile.types import CompileInputsView +from sqlbuild.compiler.compile.models import LoadedMacro, MacroContext from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE from sqlbuild.compiler.scopes.models import ResourceIdentity from sqlbuild.compiler.scopes.types import ResourceKind +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + MACRO_CALL_LOG_ENV_VAR: str = "SQLBUILD_TEST_MACRO_CALL_LOG" _RUN_ID_PATTERN: re.Pattern[str] = re.compile(r'"run_id": "[^"]*"') _DIFFERENTIAL_FRAGMENTS: tuple[str, ...] = ( diff --git a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py index 8504effbe6..6b1afe12aa 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py @@ -6,14 +6,14 @@ from typing import cast import sqlbuild._native as _native -from sqlbuild._native import native_model_config_session -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompileModelConfig, - CompileSqlReference, ModelValidationRequest, NativeModelConfigInputs, NativeModelConfigSession, + native_model_config_session, ) +from sqlbuild.compiler.compile.models import CompileSqlReference from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import ProjectConfig diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 9fc9989506..d95f5c52d3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -9,12 +9,16 @@ from dataclasses import replace from pathlib import Path from types import ModuleType -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast import pytest import sqlbuild._native as _native from sqlbuild._native import ( + CollectionRendering, + CompileAdapterContext, + CompileModelConfig, + CompileProjectInputs, assemble_project, build_compile_inputs, build_compiled_project, @@ -25,27 +29,21 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds from sqlbuild.compiler.compile import models as compile_models -from sqlbuild.compiler.compile.models import ( - CompileAdapterContext, - CompileModelConfig, - CompileModelInput, - CompileProjectInputs, - CompilerDiagnostic, -) +from sqlbuild.compiler.compile.models import CompileModelInput, CompilerDiagnostic from sqlbuild.compiler.compile.types import ( CompiledAuditView, CompiledFunctionView, CompiledModelView, CompiledProjectView, - CompileInputsView, - ModelInput, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceColumnEntry -from sqlbuild.sql_values.types import CollectionRendering from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView, ModelInput + TYPE_PROOF_FILES: dict[str, str] = { "sqlbuild_project.toml": ( 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRCONTRACT105"]\n\n' @@ -89,7 +87,7 @@ def skip_model_retention(*, monkeypatch: pytest.MonkeyPatch) -> None: """ build_inputs: Callable[..., tuple[ModelInput, ...]] = cast( - Callable[..., tuple[ModelInput, ...]], _native.build_model_inputs + "Callable[..., tuple[ModelInput, ...]]", _native.build_model_inputs ) def discarded(**kwargs: Any) -> tuple[ModelInput, ...]: @@ -118,7 +116,7 @@ def count_legacy_constructions(*, monkeypatch: pytest.MonkeyPatch) -> Counter[st counts: Counter[str] = Counter() for name in LEGACY_PROJECT_TYPES: - model_type: type = getattr(compile_models, name) + model_type: type = getattr(compile_models, name, None) or getattr(_native, name) monkeypatch.setattr( model_type, "__init__", _counted_init(counts=counts, name=name, model_type=model_type) ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_audits.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_audits.py index 212d0679cf..d10fcc3fb3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_audits.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_audits.py @@ -1,11 +1,14 @@ """Audits attached by the native model loop and assembled into the native compiled project.""" +from __future__ import annotations + from pathlib import Path +from typing import TYPE_CHECKING import pytest from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import AuditInput, CompiledAuditView +from sqlbuild.compiler.compile.types import CompiledAuditView from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( NativeAuditAttachmentTestCase, NativeAuditErrorTestCase, @@ -18,6 +21,9 @@ measurement_policy, ) +if TYPE_CHECKING: + from sqlbuild.compiler.compile.types import AuditInput + @pytest.mark.parametrize( "test_case", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py index e45dc2ab34..e20cb23d88 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py @@ -1,20 +1,25 @@ """The rules request reads model facts the compile-input stage retained natively.""" +from __future__ import annotations + import json import pickle from collections import Counter from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any import pytest -from sqlbuild._native import build_compiled_project, discover_project_inputs +from sqlbuild._native import ( + InvalidQualifiedIdentityError, + build_compiled_project, + discover_project_inputs, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import InferredColumn -from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.scopes.exceptions import InvalidQualifiedIdentityError +from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( ImmediateSyntaxErrorTestCase, ModelLoopErrorTestCase, @@ -47,6 +52,9 @@ ) from tests.integration.src.sqlbuild.compiler.graph.helpers import ORDERS_PROJECT_FILES, write_files +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + @pytest.mark.parametrize( "test_case", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py index 8ed82f5416..bd2c76ed63 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py @@ -1,11 +1,14 @@ """Functions attached by the native model loop and assembled into the native compiled project.""" +from __future__ import annotations + from pathlib import Path +from typing import TYPE_CHECKING import pytest from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import CompiledFunctionView, FunctionInput +from sqlbuild.compiler.compile.types import CompiledFunctionView from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( NativeFunctionAttachmentTestCase, NativeFunctionErrorTestCase, @@ -16,6 +19,9 @@ function_summary, ) +if TYPE_CHECKING: + from sqlbuild._native import FunctionInput + @pytest.mark.parametrize( "test_case", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py index f1b4bb5061..6ecd0837a2 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py @@ -1,26 +1,33 @@ """Adapter literals and quoted hook arguments cross real native attachment boundaries.""" +from __future__ import annotations + import json from pathlib import Path -from typing import cast +from typing import TYPE_CHECKING, cast import pytest -from sqlbuild._native import build_compile_inputs, discover_project_inputs +from sqlbuild._native import ( + CollectionRendering, + CompileAdapterContext, + build_compile_inputs, + discover_project_inputs, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.models import CompileAdapterContext -from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.sql_values.types import CollectionRendering from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( AdapterParameterBoundaryTestCase, QuotedHookBoundaryTestCase, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + @pytest.mark.parametrize( "test_case", diff --git a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py index 55ee2c2be6..f5d3da81f7 100644 --- a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py @@ -13,6 +13,9 @@ import sqlbuild._native as native_module from sqlbuild._native import ( + ContractValidationResult, + DynamicColumnContractProof, + DynamicColumnFamilyProof, build_project_graph, compiled_model_record, compiled_project_record, @@ -20,14 +23,8 @@ ) from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompilerDiagnostic, - DynamicColumnContractProof, - DynamicColumnFamilyProof, - InferredColumn, -) +from sqlbuild.compiler.compile.models import CompilerDiagnostic, InferredColumn from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.contracts.models import ContractValidationResult from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.planner.types import ContractPolicy, IncrementalMode, MaterializationType from sqlbuild.spec.contracts.models import ( diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index 7a011212c2..195e1d6238 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -23,6 +23,9 @@ from sqlbuild._native import ( AuthoredSqlSet, AuthoredSqlValueCall, + DiscoveryFileFault, + NamedSqlHookEntry, + TolerantScopeDiscovery, discover_audit_files, discover_constant_files, discover_enum_files, @@ -38,11 +41,8 @@ DiscoveredProjectInputs, DiscoveredSourceFile, DiscoveredSqlModelFile, - DiscoveryFileFault, - NamedSqlHookEntry, PythonHookEntry, SqlHookEntry, - TolerantScopeDiscovery, ) from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( GeneratedDeclarationFileTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_discovery_failure_text.py b/tests/integration/src/sqlbuild/compiler/discovery/test_discovery_failure_text.py index 679b4b4c0e..a4f0f132d8 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_discovery_failure_text.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_discovery_failure_text.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.discovery.models import DiscoveryFileFault +from sqlbuild._native import DiscoveryFileFault from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( FailureTextTestCase, TolerantFailureTextTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py index 5e1eef9078..8e5e73f284 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py @@ -8,8 +8,8 @@ import pytest from sqlbuild import _native -from sqlbuild._native import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryFileFault +from sqlbuild._native import DiscoveryFileFault, discover_project_inputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( DeclarationFilesParityTestCase, DeepDeclarationHeaderTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py index 784ea0cd32..d9011ef856 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py @@ -8,8 +8,7 @@ import pytest -from sqlbuild._native import discover_project_inputs -from sqlbuild.compiler.discovery.exceptions import ProjectPathError +from sqlbuild._native import ProjectPathError, discover_project_inputs from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( UndecodableDeclarationNameTestCase, UndecodablePathToleranceTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py index fe1c968921..1c26a35c58 100644 --- a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py @@ -7,15 +7,16 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( + CollectionRendering, CompileAdapterContext, - CompilerDiagnostic, - LiteralSqlRelation, + assemble_project, + build_compile_inputs, + discover_project_inputs, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.models import CompilerDiagnostic, LiteralSqlRelation from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.sql_values.types import CollectionRendering _ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( value_renderer=DuckDbAdapter(), diff --git a/tests/integration/src/sqlbuild/compiler/lineage/helpers.py b/tests/integration/src/sqlbuild/compiler/lineage/helpers.py index 924b2d468b..a7a0aedc77 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/helpers.py @@ -11,6 +11,7 @@ import sqlbuild._native as native_module from sqlbuild._native import ( + ColumnLineageMode, ProjectColumnLineage, build_project_graph, compiled_model_record, @@ -19,7 +20,6 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.lineage.types import ColumnLineageMode _PROJECT_TOML: str = ( 'name = "orders_lineage"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index 11c98422f9..c3001f2298 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -7,12 +7,13 @@ from collections.abc import Callable, Iterator from dataclasses import dataclass, replace from pathlib import Path -from typing import TypeGuard, cast +from typing import TYPE_CHECKING, TypeGuard, cast import pytest import sqlbuild._native as _native from sqlbuild._native import ( + CompileAdapterContext, build_compile_inputs, discover_project_inputs, parse_audit_instances, @@ -20,11 +21,12 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileAdapterContext -from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.spec.contracts.models import SchemaModelEntry, SourceLocation +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + _TEXTS: tuple[object, ...] = ( "INTEGER", "Order amount", diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index f713823e15..18dbd2ce24 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -4,12 +4,16 @@ import json from collections.abc import Callable from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any import pytest import sqlbuild._native as native_module from sqlbuild._native import ( + CollectionRendering, + CompileAdapterContext, + CompiledLineageColumnFact, + CompilePipelineOptions, build_compile_inputs, build_compiled_project, build_manifest, @@ -21,15 +25,13 @@ from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter -from sqlbuild.compiler.compile.models import ( - CompileAdapterContext, - CompiledLineageColumnFact, -) -from sqlbuild.compiler.compile.types import AuditInput, CompiledModelView, CompiledProjectView +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult +from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.sql_values.types import CollectionRendering + +if TYPE_CHECKING: + from sqlbuild.compiler.compile.types import AuditInput _REPOSITORY_ROOT: Path = Path(__file__).resolve().parents[6] _ANALYSIS_CACHE_METRICS: tuple[str, ...] = ( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py index 37d65b5af1..9ec41a52a7 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py @@ -4,10 +4,10 @@ from collections.abc import Callable from pathlib import Path +from typing import TYPE_CHECKING import pytest -from sqlbuild.compiler.compile.types import AuditInput from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( AuditFactoryCompileIntegrationTestCase, ) @@ -16,6 +16,9 @@ compile_audits_for_project, ) +if TYPE_CHECKING: + from sqlbuild.compiler.compile.types import AuditInput + @pytest.mark.parametrize( "test_case", diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py index 8e58afe6ad..fec1e83cdd 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py @@ -6,7 +6,7 @@ import pytest -import sqlbuild.compiler.sql_analysis.models as sql_analysis_models +import sqlbuild._native as sql_analysis_models from sqlbuild._native import ( build_compiled_project, discover_project_inputs, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_reshaped_star_lineage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_reshaped_star_lineage.py index 2ce082132d..43743b6c37 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_reshaped_star_lineage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_reshaped_star_lineage.py @@ -9,7 +9,7 @@ import duckdb import pytest -from sqlbuild.compiler.compile.models import CompiledLineageColumnFact +from sqlbuild._native import CompiledLineageColumnFact from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( ReshapedStarIntegrationTestCase, ReshapedStarLineageIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py index 8ef5fcab46..b55df74441 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py @@ -6,11 +6,10 @@ import pytest -from sqlbuild._native import build_compiled_project, discover_project_inputs +from sqlbuild._native import ColumnLineageMode, build_compiled_project, discover_project_inputs from sqlbuild._native import run_cli as main from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.lineage.types import ColumnLineageMode from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( SnowflakeOutputInferenceCase, SnowflakeSemanticReleaseCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py index a932dd2486..517118e98c 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py @@ -9,11 +9,12 @@ import pytest from duckdb import sqltypes -from sqlbuild._native import build_compiled_project, discover_project_inputs -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledLineageColumnFact, + build_compiled_project, + discover_project_inputs, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( SetOperationLineageIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py index d55fbccb50..b4f86a36ea 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py @@ -5,12 +5,13 @@ import pytest -from sqlbuild._native import build_compiled_project, discover_project_inputs -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledDirectLogicSqlTestPayload, CompiledModelSqlTestPayload, + build_compiled_project, + discover_project_inputs, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ExpectedCountTestCase diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py index ddaaa19ad2..607dc40b84 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -10,6 +10,11 @@ from typing import Any, ClassVar from sqlbuild._native import ( + CompiledSeed, + CompiledSource, + CompileModelConfig, + DiscoveredSchemaFile, + DiscoveredSeedFile, WarehouseSnapshot, compiled_model_record, gather_warehouse_snapshot, @@ -23,9 +28,6 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompiledSeed, - CompiledSource, - CompileModelConfig, CompileSqlReference, ) from sqlbuild.compiler.compile.types import ( @@ -35,11 +37,7 @@ CompiledSeedView, CompiledSourceView, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredSchemaFile, - DiscoveredSeedFile, - DiscoveredSourceFile, -) +from sqlbuild.compiler.discovery.models import DiscoveredSourceFile from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import ( SchemaColumn, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index da3017754f..ed8184d490 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -8,6 +8,9 @@ from sqlbuild._native import ( BackfillResult, + CollectionRendering, + CompileAdapterContext, + CompileModelConfig, CursorOverridePair, ModelPlanContext, WarehouseSnapshot, @@ -19,11 +22,9 @@ from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( - CompileAdapterContext, CompiledModel, CompiledObjectKey, CompiledRelationLocation, - CompileModelConfig, CompileSqlReference, ) from sqlbuild.compiler.compile.types import ( @@ -37,7 +38,6 @@ from sqlbuild.compiler.planner.types import BackfillAction from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.sql_values.types import CollectionRendering class OrchestrationOverrideAdapter(DuckDbAdapter): diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 577cc71dea..3b7cdc52ba 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -4,25 +4,25 @@ from pathlib import Path +from sqlbuild._native import ( + CompiledModelSqlTestPayload, + CompileModelConfig, + CompileSqlTestCte, + DiscoveredSqlTestBlock, +) from sqlbuild.compiler.compile.models import ( CompiledModel, - CompiledModelSqlTestPayload, CompiledObjectKey, CompiledProject, CompiledRelationLocation, CompiledSqlTest, - CompileModelConfig, - CompileSqlTestCte, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredSqlTestBlock, - DiscoveredSqlTestFile, -) +from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile from sqlbuild.compiler.references.types import SqlReferenceKind from tests.integration.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( ExecuteChainTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py index a008d35f29..b185ed8256 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py @@ -6,17 +6,17 @@ import pytest -from sqlbuild._native import build_compiled_project, discover_project_inputs -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledDirectLogicSqlTestPayload, - CompiledObjectKey, - CompiledProject, - CompiledSqlTest, CompileSqlTestCte, + DiscoveredSqlTestBlock, + build_compiled_project, + discover_project_inputs, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile +from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from sqlbuild.executor.testing.main._execute import execute_sql_test from sqlbuild.executor.testing.models import SqlTestExecutionResult diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index f1b2b4c26d..07b92d16ff 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild._native import plan_audit +from sqlbuild._native import DiscoveredAuditBlock, DiscoveredAuditFile, plan_audit from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledAudit, @@ -13,10 +13,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledResourceType -from sqlbuild.compiler.discovery.models import ( - DiscoveredAuditBlock, - DiscoveredAuditFile, -) from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.spec.contracts.models import SourceEntry from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index e719bfb69a..8f26166f98 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( RelationFixtureCompletion, RelationFixturePlanningContext, + SqlTestSelection, build_compiled_project, build_relation_fixture_completion, build_relation_fixture_context, @@ -17,10 +18,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.models import ( - SqlTestPlanEntry, - SqlTestSelection, -) +from sqlbuild.compiler.planner.models import SqlTestPlanEntry from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( FixturePlanningIntegrationTestCase, FixtureSchemaIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 6df338e9b3..008c274148 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -13,6 +13,18 @@ from typing import Any, cast from sqlbuild._native import ( + CompiledFunction, + CompiledSeed, + CompiledSource, + CompileModelConfig, + CompilePipelineOptions, + DeferralInputs, + DiscoveredSchemaFile, + DiscoveredSeedFile, + FunctionArgument, + PlannerOverrides, + PlannerPolicies, + PlannerSelection, build_compiled_function_fingerprint_sql, build_execution_plan, compiled_model_record, @@ -25,16 +37,11 @@ from sqlbuild.adapter.contract.models import RelationInfo, RelationReadProbe from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( - CompiledFunction, CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompiledSeed, - CompiledSource, - CompileModelConfig, CompileSqlReference, - FunctionArgument, ) from sqlbuild.compiler.compile.types import ( CompiledFunctionView, @@ -45,21 +52,10 @@ CompiledSourceView, FunctionLanguage, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, - DiscoveredSchemaFile, - DiscoveredSeedFile, - DiscoveredSourceFile, -) +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSourceFile from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult -from sqlbuild.compiler.planner.models import ( - DeferralInputs, - PlannerOverrides, - PlannerPolicies, - PlannerSelection, - PlanOutput, -) +from sqlbuild.compiler.pipeline.models import CompilePipelineResult +from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.spec.contracts.models import ( diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py index 8b1e1cb324..a31b196ccb 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -8,12 +8,14 @@ from collections import Counter from collections.abc import Callable from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any import pytest import sqlbuild._native as _native from sqlbuild._native import ( + CollectionRendering, + CompileAdapterContext, assemble_project, build_compile_inputs, discover_project_inputs, @@ -21,11 +23,10 @@ record_compile_input_reads, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompileAdapterContext, -) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.sql_values.types import CollectionRendering +from sqlbuild.compiler.compile.types import CompiledProjectView + +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView _PYTHON_RESOURCE_FUNCTIONS: tuple[str, ...] = ("validate_sql_syntax", "validate_hook_sql_syntax") _ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py index 652a09570b..e3bc480ef9 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py @@ -6,6 +6,7 @@ import random from collections import Counter from pathlib import Path +from typing import TYPE_CHECKING import pytest @@ -13,7 +14,7 @@ from sqlbuild._native import assemble_project from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView +from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.helpers import ( GoldenEntry, golden_differences, @@ -42,6 +43,9 @@ windows_environ, ) +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + _PROJECT_TOML: str = ( 'name = "assembly"\nadapter = "duckdb"\n\n[connection]\ndatabase = ":memory:"\n\n' ) diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index 3c32f6e416..097ddd2d38 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -14,6 +14,9 @@ import sqlbuild._native as _native from sqlbuild._native import ( + DeclarationScopeBuild, + DeclarationScopeResolver, + DiscoveryError, build_declaration_scope, discover_project_inputs, resolve_declaration_context, @@ -23,13 +26,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - DeclarationResolutionContext, - DeclarationScopeBuild, - DeclarationScopeResolver, - LoadedMacro, -) -from sqlbuild.compiler.discovery.exceptions import DiscoveryError +from sqlbuild.compiler.compile.models import DeclarationResolutionContext, LoadedMacro from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, DiscoveredProjectInputs, diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py index 0332ad8888..ee3e91c7ba 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py @@ -8,20 +8,27 @@ from dataclasses import dataclass from itertools import chain, compress from pathlib import Path -from typing import cast +from typing import TYPE_CHECKING, cast import pytest import sqlbuild._native as native_module -from sqlbuild._native import build_project_graph, discover_project_inputs +from sqlbuild._native import ( + NativeSqlTestAssembly, + NativeSqlTestFailure, + build_project_graph, + discover_project_inputs, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledSqlTest, ) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.sql_test_glue.models import NativeSqlTestAssembly, NativeSqlTestFailure +from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.helpers import GoldenEntry, golden_entry +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + @dataclass(frozen=True) class PlanningCallOutcome: diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py index 8c27d98cef..b4a63cf3c8 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py @@ -7,11 +7,11 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import discover_project_inputs, run_compile_pipeline +from sqlbuild._native import CompilePipelineOptions, discover_project_inputs, run_compile_pipeline from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult +from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py index 51185b7896..1748ca2566 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py @@ -9,11 +9,11 @@ import pytest -from sqlbuild._native import discover_project_inputs, run_compile_pipeline +from sqlbuild._native import CompilePipelineOptions, discover_project_inputs, run_compile_pipeline from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult +from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( diff --git a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py index abc6853e1d..3b622651e7 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py @@ -425,10 +425,14 @@ def run_scheduler_build( ) -> tuple[Any, Any]: """Run a full build through the scheduler with custom materializations.""" - from sqlbuild._native import discover_project_inputs, run_compile_pipeline + from sqlbuild._native import ( + CompilePipelineOptions, + discover_project_inputs, + run_compile_pipeline, + ) from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs - from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult + from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( diff --git a/tests/integration/src/sqlbuild/executor/build/helpers.py b/tests/integration/src/sqlbuild/executor/build/helpers.py index cbe6dbdff1..3d0f0bc1d9 100644 --- a/tests/integration/src/sqlbuild/executor/build/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/helpers.py @@ -10,12 +10,12 @@ import duckdb -from sqlbuild._native import discover_project_inputs, run_compile_pipeline +from sqlbuild._native import CompilePipelineOptions, discover_project_inputs, run_compile_pipeline from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult +from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput, TableTypePlanEntry from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py index 059b3a7488..2f89e19bdc 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py @@ -13,10 +13,10 @@ from pathlib import Path from typing import Any -from sqlbuild._native import discover_project_inputs, run_compile_pipeline +from sqlbuild._native import CompilePipelineOptions, discover_project_inputs, run_compile_pipeline from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult +from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams from sqlbuild.executor.pipeline.main.run import run_build_pipeline from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py index 0a032be40f..ba35552546 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py @@ -8,10 +8,10 @@ import pytest -from sqlbuild._native import discover_project_inputs, run_compile_pipeline +from sqlbuild._native import CompilePipelineOptions, discover_project_inputs, run_compile_pipeline from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult +from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus diff --git a/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py b/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py index 249996f765..707a8d34e2 100644 --- a/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py +++ b/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py @@ -9,7 +9,12 @@ import duckdb import pytest -from sqlbuild._native import DiffCommandRequest, QueryDiffPreparation, execute_query_diff +from sqlbuild._native import ( + DiffCommandRequest, + QueryDiffPreparation, + build_janitor_plan, + execute_query_diff, +) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -19,9 +24,7 @@ from sqlbuild.executor.diff.classes.query_artifact_lifecycle import QueryDiffArtifactLifecycle from sqlbuild.executor.diff.models import QueryDiffArtifact from sqlbuild.executor.janitor.main.execute import execute_janitor_plan -from sqlbuild.executor.janitor.main.plan import build_janitor_plan from sqlbuild.executor.janitor.models import ( - JanitorDirectModeSettings, JanitorExecutionResult, JanitorPlan, ) @@ -97,7 +100,7 @@ def test_given_expired_owned_and_untracked_artifacts_when_janitor_runs_then_only adapter=adapter, connection=connection, retention_days=30, - direct_settings=JanitorDirectModeSettings(enabled=True), + direct_mode=True, ) assert tuple(candidate.key.name for candidate in plan.query_diff_artifact_candidates) == ( diff --git a/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py b/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py index 88b2e1653e..c588ec16b1 100644 --- a/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py @@ -2,15 +2,12 @@ from __future__ import annotations +from sqlbuild._native import MigrationCompatibility, MigrationDiscovery from sqlbuild.adapter.contract.models import MigrationStagePlan from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.migrations.types import ( - MigrationCompatibility, - MigrationDecision, - MigrationDiscovery, -) +from sqlbuild.compiler.migrations.types import MigrationDecision from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry from sqlbuild.executor.migrations.models import MigrationArtifactNames diff --git a/tests/integration/src/sqlbuild/integrations/dbt/helpers.py b/tests/integration/src/sqlbuild/integrations/dbt/helpers.py index 5bed9a503e..97893a3edc 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/helpers.py @@ -2,11 +2,13 @@ from pathlib import Path -from sqlbuild._native import build_compile_reference_resolver +from sqlbuild._native import ( + CollectionRendering, + CompileAdapterContext, + build_compile_reference_resolver, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompileAdapterContext from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.sql_values.types import CollectionRendering DUCKDB_COMPILE_ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( value_renderer=DuckDbAdapter(), diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py index 8a4d126f02..4284a9bbf6 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py @@ -2,11 +2,14 @@ import json from pathlib import Path +from typing import TYPE_CHECKING import pytest from sqlbuild._native import ( + DbtCliOptions, DbtCombinedGraph, + DbtCommandResult, DbtManifestIndex, DbtRunner, assemble_project, @@ -15,9 +18,8 @@ build_dbt_manifest_index, discover_project_inputs, ) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtCommandResult from tests.integration.src.sqlbuild.integrations.dbt._test_types import ( RealDbtCombinedGraphTestCase, ) @@ -28,6 +30,9 @@ ) from tests.unit.src.sqlbuild.integrations.dbt.helpers import downstream_stable_ids +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + pytestmark: pytest.MarkDecorator = pytest.mark.dbt diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py index 6b64d7125a..1f527821db 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py @@ -1,13 +1,18 @@ from __future__ import annotations from pathlib import Path +from typing import TYPE_CHECKING import pytest -from sqlbuild._native import DbtRunner, build_compile_inputs, discover_project_inputs -from sqlbuild.compiler.compile.types import CompileInputsView +from sqlbuild._native import ( + DbtCliOptions, + DbtCommandResult, + DbtRunner, + build_compile_inputs, + discover_project_inputs, +) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtCommandResult from tests.integration.src.sqlbuild.integrations.dbt._test_types import ( RealDbtManifestCompileTestCase, ) @@ -16,6 +21,9 @@ build_external_sql_reference_resolver, ) +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + pytestmark: pytest.MarkDecorator = pytest.mark.dbt diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_runner.py b/tests/integration/src/sqlbuild/integrations/dbt/test_runner.py index f98446c2d8..61da86f713 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_runner.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_runner.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import DbtRunner -from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtCommandResult, DbtLsResult +from sqlbuild._native import DbtCliOptions, DbtCommandResult, DbtLsResult, DbtRunner from tests.integration.src.sqlbuild.integrations.dbt._test_types import RealDbtRunnerTestCase pytestmark: pytest.MarkDecorator = pytest.mark.dbt diff --git a/tests/integration/src/sqlbuild/lint/main/test_fix_verdict.py b/tests/integration/src/sqlbuild/lint/main/test_fix_verdict.py index 9fe90a5ec7..abc54e7244 100644 --- a/tests/integration/src/sqlbuild/lint/main/test_fix_verdict.py +++ b/tests/integration/src/sqlbuild/lint/main/test_fix_verdict.py @@ -6,9 +6,8 @@ import pytest -from sqlbuild._native import compile_facts, fix_verdict +from sqlbuild._native import FixVerdict, compile_facts, fix_verdict from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.lint.models import FixVerdict from tests.integration.src.sqlbuild.lint.main._test_types import FixVerdictTestCase from tests.integration.src.sqlbuild.lint.main.helpers import compile_models, without_lineage diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py index 78fda746e9..f464671cd6 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py @@ -10,7 +10,8 @@ import pytest import sqlbuild._native as native -from sqlbuild.rule_engine.models import Finding, SqlRulesEvaluation +from sqlbuild._native import SqlRulesEvaluation +from sqlbuild.rule_engine.models import Finding from tests.integration.src.sqlbuild.rule_engine.native_rows._test_types import ( SqlRulePublicationTestCase, ) diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py index 70b36fb2ee..e7ee8d4540 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py @@ -3,10 +3,13 @@ from __future__ import annotations from dataclasses import dataclass +from typing import TYPE_CHECKING -from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + @dataclass(frozen=True) class DiscoveryKindsTestCase: diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py index aa8be654ca..6798ba8333 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py @@ -6,8 +6,8 @@ from itertools import filterfalse from typing import cast -from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES -from sqlbuild.compiler.compile.models import CompiledProject, CompileProjectInputs +from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES, CompileProjectInputs +from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs _VALUE_KIND: str = "sqlbuild.sql_values.types:SqlValueKind" diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py index daca70f1c0..4b51f34890 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py @@ -15,8 +15,7 @@ required_render_kinds, ) from scripts.compiler_differential.constants import RENDER_NON_COLLECTION_FIELDS -from sqlbuild._native import render_stage_capture -from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild._native import CompileProjectInputs, render_stage_capture from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py index 82059d5a6d..936a3d35a1 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -117,6 +117,7 @@ def test_given_native_source_when_checking_module_paths_then_reports_errors( "sqlbuild.compile", "sqlbuild.compile.reuse", "sqlbuild.cost", + "sqlbuild.execution", "sqlbuild.executor.runtime", "sqlbuild.fingerprints", "sqlbuild.lineage", diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_base_adapter.py b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_base_adapter.py index 8b5106839c..1cf9cd3ef8 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_base_adapter.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_base_adapter.py @@ -4,11 +4,11 @@ import pytest +from sqlbuild._native import FunctionArgument from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import ExpressionInferenceProfile, RetentionRequest from sqlbuild.adapter.contract.types import RetentionScope -from sqlbuild.compiler.compile.models import FunctionArgument from sqlbuild.compiler.compile.types import FunctionLanguage from tests.unit.src.sqlbuild.adapter.contract.classes.base_adapter._test_types import ( BaseAdapterDurableCloneTestCase, diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py b/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py index 4b6e1fac53..32f5e1245e 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py @@ -4,6 +4,7 @@ import pytest +from sqlbuild._native import FunctionArgument, FunctionReturnColumn from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import ( ColumnInfo, @@ -26,10 +27,6 @@ from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.bigquery.classes.bigquery_connection import _BigQueryConnection from sqlbuild.adapters.bigquery.classes.bigquery_cursor import _BigQueryCursor -from sqlbuild.compiler.compile.models import ( - FunctionArgument, - FunctionReturnColumn, -) from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.observability import ( diff --git a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py index 2b624a4b33..9618419132 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py @@ -5,6 +5,7 @@ import pytest +from sqlbuild._native import FunctionArgument, FunctionReturnColumn from sqlbuild.adapter.contract.models import ( ExpressionInferenceProfile, RelationInfo, @@ -13,10 +14,6 @@ ) from sqlbuild.adapter.contract.types import FunctionNullabilityRule, LoaderLogicalType from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter -from sqlbuild.compiler.compile.models import ( - FunctionArgument, - FunctionReturnColumn, -) from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.compiler.lineage.types import InferredNullability from tests.unit.src.sqlbuild.adapters.databricks._test_types import ( diff --git a/tests/unit/src/sqlbuild/adapters/duckdb/test_client.py b/tests/unit/src/sqlbuild/adapters/duckdb/test_client.py index ac373d6e7c..e3b4bb573e 100644 --- a/tests/unit/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/duckdb/test_client.py @@ -2,13 +2,10 @@ import pytest +from sqlbuild._native import FunctionArgument, FunctionReturnColumn from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapter.contract.types import CursorKind, FunctionNullabilityRule from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - FunctionArgument, - FunctionReturnColumn, -) from sqlbuild.compiler.lineage.types import InferredNullability from tests.unit.src.sqlbuild.adapter.contract.classes.base_adapter.helpers import ( RecordingConnection, diff --git a/tests/unit/src/sqlbuild/adapters/postgres/test_client.py b/tests/unit/src/sqlbuild/adapters/postgres/test_client.py index 8250f7e852..a8c2395705 100644 --- a/tests/unit/src/sqlbuild/adapters/postgres/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/postgres/test_client.py @@ -4,10 +4,10 @@ import pytest +from sqlbuild._native import FunctionArgument from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, SchemaDiffResult from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter -from sqlbuild.compiler.compile.models import FunctionArgument from tests.unit.src.sqlbuild.adapters.postgres._test_types import ( PostgresAdapterDefaultsTestCase, PostgresDescribeRelationTestCase, diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py index 25fb504975..4daf453e63 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild._native import FunctionArgument, FunctionReturnColumn from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import ( @@ -23,10 +24,6 @@ TableFreshnessStatus, ) from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile.models import ( - FunctionArgument, - FunctionReturnColumn, -) from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.compiler.lineage.types import InferredNullability from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( diff --git a/tests/unit/src/sqlbuild/adapters/sqlserver/test_client.py b/tests/unit/src/sqlbuild/adapters/sqlserver/test_client.py index 8636343e1b..51f5cd77bf 100644 --- a/tests/unit/src/sqlbuild/adapters/sqlserver/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/sqlserver/test_client.py @@ -5,6 +5,7 @@ import pytest +from sqlbuild._native import FunctionArgument from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( ColumnInfo, @@ -14,7 +15,6 @@ ) from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.models import FunctionArgument from tests.unit.src.sqlbuild.adapters.sqlserver._test_types import ( SqlServerAdapterDefaultsTestCase, SqlServerCursorBoundLiteralTestCase, diff --git a/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py b/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py index f9fda0f342..b580c0c654 100644 --- a/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py +++ b/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py @@ -6,10 +6,14 @@ import pytest -from sqlbuild._native import read_migration_state_events, write_migration_state_event +from sqlbuild._native import ( + MigrationDiscovery, + read_migration_state_events, + write_migration_state_event, +) from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation -from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery +from sqlbuild.compiler.migrations.types import MigrationDecision from tests.unit.src.sqlbuild.adapters.sqlserver._test_types import ( SqlServerMigrationEventReadTestCase, SqlServerMigrationEventWriteTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py index 1f9a593afd..9566ba298c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py @@ -8,12 +8,12 @@ import pytest from sqlbuild._native import ( + ColumnLineageMode, CompileLineageMode, ParsedCliInvocation, PlaygroundTemplate, parse_cli_invocation, ) -from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.compiler.scopes.types import DeclarationKind from tests.unit.src.sqlbuild.cli.commands._helpers.entry._test_types import ( AuditConcurrencyParsingTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py index 5fd1b6e710..4f2d58fa6b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py @@ -6,15 +6,15 @@ from sqlbuild import _native as native from sqlbuild._native import ( + NormalizedDbtProfileConnection, + ProjectConfigError, resolve_connection_config, resolve_project_connection_config, resolve_target_connection_config, ) from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.discovery.exceptions import ProjectConfigError from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt.models import NormalizedDbtProfileConnection from sqlbuild.spec.contracts.models import ( LocalConfig, LocalTargetConfig, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py index ab85e3c358..44a8306b08 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py @@ -6,13 +6,16 @@ import pytest -from sqlbuild._native import CostContext, DirectSourceFreshnessPropagationResult +from sqlbuild._native import ( + CostContext, + DirectSourceFreshnessPlanningResult, + DirectSourceFreshnessPropagationResult, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.freshness.source_freshness import ( append_eligible_direct_source_freshness_records, ) from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus diff --git a/tests/unit/src/sqlbuild/cli/commands/classes/test_cli_namespace.py b/tests/unit/src/sqlbuild/cli/commands/classes/test_cli_namespace.py index 9332c5a01e..104e04c36f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/classes/test_cli_namespace.py +++ b/tests/unit/src/sqlbuild/cli/commands/classes/test_cli_namespace.py @@ -2,8 +2,7 @@ import pytest -from sqlbuild._native import CliNamespace, CompileLineageMode -from sqlbuild.compiler.lineage.types import ColumnLineageMode +from sqlbuild._native import CliNamespace, ColumnLineageMode, CompileLineageMode from tests.unit.src.sqlbuild.cli.commands.classes._test_types import ( CliNamespaceAssignmentTestCase, CliNamespaceDefaultTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py b/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py index 546fdddd4f..1ef2d91a94 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py @@ -4,15 +4,13 @@ import pytest -from sqlbuild._native import resolve_defer_clone_selection +from sqlbuild._native import CompileModelConfig, PlannerScope, resolve_defer_clone_selection from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, CompiledRelationLocation, - CompileModelConfig, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.planner.models import PlannerScope from tests.unit.src.sqlbuild.cli.commands.main.build._test_types import ( DeferCloneBoundaryTestCase, DeferCloneModelCountTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index d9e5b92ba6..72c54c82d1 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -6,6 +6,9 @@ from typing import Any from sqlbuild._native import ( + CompiledFunction, + CompileModelConfig, + FunctionArgument, PreparedCompileArtifacts, WrittenTarget, project_with, @@ -19,13 +22,10 @@ AuditSeverity, ) from sqlbuild.compiler.compile.models import ( - CompiledFunction, CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompileModelConfig, - FunctionArgument, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 38d398bb1d..5ec96f130e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -12,9 +12,13 @@ import sqlbuild._native as discovery_entry import sqlbuild._native as plan_artifacts_module -from sqlbuild._native import CompileCommandRequest, CompileLineageMode, run_compile +from sqlbuild._native import ( + ColumnLineageMode, + CompileCommandRequest, + CompileLineageMode, + run_compile, +) from sqlbuild.adapter.discovery.main import resolve_adapter as runtime_adapters -from sqlbuild.compiler.lineage.types import ColumnLineageMode from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileColumnContractModeTestCase, CompileCommandTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py index d75ae8e2a6..52ece4d88d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( CompileLineageMode, + CompileModelConfig, ProjectColumnLineage, WrittenTarget, compiled_model_record, @@ -15,7 +16,7 @@ format_compile_text, project_with, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompileModelConfig +from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.compiler.pipeline.models import ProjectGraph diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 80ddcd32f0..87d117101e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -11,6 +11,7 @@ import sqlbuild._native as model_chain_module import sqlbuild._native as plan_artifacts_module from sqlbuild._native import ( + DiagnosticPhase, WrittenTarget, build_manifest, compiled_model_record, @@ -25,7 +26,6 @@ CompiledModelView, CompiledProjectView, CompiledResourceType, - DiagnosticPhase, ) from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py index a7a6672920..5cbfd27540 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py @@ -8,12 +8,13 @@ from _pytest.capture import CaptureResult from sqlbuild._native import ( + DbtInitRequest, + DbtInitResult, DbtInteropCommand, DbtInteropExecutionRequest, DbtInteropPlan, run_dbt_command, ) -from sqlbuild.integrations.dbt.models import DbtInitRequest, DbtInitResult from tests.unit.src.sqlbuild.cli.commands.main.dbt._test_types import ( DbtAutoInitTestCase, DbtDebugWrapperTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py index df806bc7b3..b232b21abc 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py @@ -7,9 +7,8 @@ import sqlbuild._native as dbt_init_module import sqlbuild._native as dbt_profile_init -from sqlbuild._native import DbtInitCommandRequest +from sqlbuild._native import DbtInitCommandRequest, DbtInitRequest, DbtInitResult from sqlbuild.integrations.dbt.exceptions import DbtProfileError -from sqlbuild.integrations.dbt.models import DbtInitRequest, DbtInitResult from tests.unit.src.sqlbuild.cli.commands.main.dbt._test_types import ( DbtInitOutputTestCase, DbtInitProjectDirDefaultTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py index 84f2efce91..334d3df385 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py @@ -2,11 +2,14 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild._native import CliUserError, CompileLineageMode +from sqlbuild._native import ( + CliUserError, + ColumnLineageMode, + CompileLineageMode, + ProjectConfigError, + SpecConfigError, +) from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.discovery.exceptions import ProjectConfigError -from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.spec.contracts.exceptions import SpecConfigError @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py index a22e51e0ac..6821a4d2f6 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py @@ -8,9 +8,9 @@ from sqlbuild._native import ( CliEntrypointHandlers, + SpecConfigError, loader_destination_parts, ) -from sqlbuild.spec.contracts.exceptions import SpecConfigError class JsonOutputRequest(Protocol): diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 0e86e4037c..59eb70930a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -14,6 +14,7 @@ BuildCommandRequest, CliUserError, CloneCommandRequest, + ColumnLineageMode, CompileCommandRequest, CompileLineageMode, CostCommandRequest, @@ -25,21 +26,20 @@ LoadCommandRequest, PlanCommandRequest, PlaygroundCommandRequest, + ProjectConfigError, RulesCommandRequest, ScenarioCaptureCommandRequest, ScenarioTestCommandRequest, + SpecConfigError, ) from sqlbuild._native import run_cli as _main_with_dependencies from sqlbuild._native import run_cli as main from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.discovery.exceptions import ProjectConfigError -from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.compiler.planner.models import CursorOverrides from sqlbuild.executor.pipeline.exceptions import AuditExecutionError from sqlbuild.executor.pipeline.models import AuditExecutionFailure from sqlbuild.observability import EventDispatcher, ExecutionIdentity, current_execution_identity from sqlbuild.runtime.observability.models import LifecycleEvent -from sqlbuild.spec.contracts.exceptions import SpecConfigError from tests.unit.src.sqlbuild.cli.commands.main.entry._test_types import ( AuditAggregateRenderingTestCase, CompileJsonErrorRenderingTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entrypoint/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/entrypoint/helpers.py index 6b40550aee..7447f03a7b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entrypoint/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entrypoint/helpers.py @@ -2,8 +2,7 @@ from __future__ import annotations -from sqlbuild.runtime.event_exporting.classes.command_scope import EventExporterCommandScope -from sqlbuild.runtime.event_exporting.classes.dispatcher import EventExporterDispatcher +from sqlbuild._native import EventExporterCommandScope, EventExporterDispatcher from sqlbuild.runtime.output_capture.models import ( BoundCommandOutputSink, CommandOutputRecord, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entrypoint/test_dispatch_with_output_capture.py b/tests/unit/src/sqlbuild/cli/commands/main/entrypoint/test_dispatch_with_output_capture.py index 6e8fe08857..78ed6ae108 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entrypoint/test_dispatch_with_output_capture.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entrypoint/test_dispatch_with_output_capture.py @@ -7,8 +7,8 @@ import pytest +from sqlbuild._native import EventExporterCommandScope from sqlbuild.observability import ExecutionIdentity -from sqlbuild.runtime.event_exporting.classes.command_scope import EventExporterCommandScope from sqlbuild.runtime.output_capture.constants import INVOCATION_CONTEXT_ENV from sqlbuild.runtime.output_capture.main.configured_output_capture_scope import ( configured_output_capture_scope, diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py index 39e0dbcb10..3efb0d4095 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py @@ -5,6 +5,7 @@ import re from pathlib import Path +from sqlbuild._native import SqlTestAssertionStep from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditOutcome, @@ -13,13 +14,7 @@ ) from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ( - ChainStep, - ModelPlanEntry, - PlanOutput, - SqlTestAssertionStep, - SqlTestPlanEntry, -) +from sqlbuild.compiler.planner.models import ChainStep, ModelPlanEntry, PlanOutput, SqlTestPlanEntry from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py index fbe30bb55e..24853fe94b 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py @@ -14,7 +14,7 @@ ) from tests.unit.src.sqlbuild.cli.compile_reuse.main.helpers import compile_in_fresh_process -_FULL_COMPILE_MODULES: tuple[str, ...] = ("sqlbuild.compiler.analysis_session.models",) +_FULL_COMPILE_MODULES: tuple[str, ...] = ("sqlbuild.compiler.compile.models",) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/_test_types.py b/tests/unit/src/sqlbuild/cli/output/main/plan/_test_types.py index c6400b32f1..a60630e85a 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/_test_types.py @@ -4,12 +4,11 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.pipeline.models import PythonPlanEntry +from sqlbuild._native import DisplayOptions, PythonPlanEntry from sqlbuild.compiler.planner.models import ( CursorInputRelation, PlanOutput, ) -from sqlbuild.presentation.models import DisplayOptions @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py index 088c9ba527..21cf8aca85 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py @@ -8,13 +8,16 @@ from sqlbuild._native import ( ColumnRenameHint, + DisplayOptions, GraphResourceKind, + PythonIdentityStatus, + PythonPlanEntry, + PythonRunPhase, SchemaChangeKind, WarningSeverity, format_plan, ) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.pipeline.models import PythonPlanEntry from sqlbuild.compiler.planner.models import ( CursorBounds, CursorInputRelation, @@ -26,12 +29,7 @@ PlanAction, PlanReason, ) -from sqlbuild.compiler.python_nodes.types import ( - PythonIdentityStatus, - PythonNodeKind, - PythonRunPhase, -) -from sqlbuild.presentation.models import DisplayOptions +from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.spec.contracts.types import SourceWriteStrategy from tests.unit.src.sqlbuild.cli.output.main.plan._test_types import ( FormatPlanColorTestCase, diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan_json/_test_types.py b/tests/unit/src/sqlbuild/cli/output/main/plan_json/_test_types.py index f820246ba0..22ea29272f 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan_json/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan_json/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.pipeline.models import PythonPlanEntry +from sqlbuild._native import PythonPlanEntry from sqlbuild.compiler.planner.models import PlanOutput diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py index d60b7efda3..b714af570a 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py @@ -8,10 +8,12 @@ import pytest from sqlbuild._native import ( + PythonIdentityStatus, + PythonPlanEntry, + PythonRunPhase, WarningSeverity, format_plan_json, ) -from sqlbuild.compiler.pipeline.models import PythonPlanEntry from sqlbuild.compiler.planner.models import ( CursorBounds, CursorInputEvidence, @@ -26,11 +28,7 @@ PlanAction, PlanReason, ) -from sqlbuild.compiler.python_nodes.types import ( - PythonIdentityStatus, - PythonNodeKind, - PythonRunPhase, -) +from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.spec.contracts.types import FutureCursorAction, MicrobatchLimitAction from tests.unit.src.sqlbuild.cli.output.main.plan.helpers import ( build_discovered_provider_usage, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index b81b5a40bf..459cc67b16 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -1,16 +1,14 @@ +from __future__ import annotations + from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path +from typing import TYPE_CHECKING +from sqlbuild._native import CompiledLineageColumnFact, ExpansionSpan from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import ( - CompiledLineageColumnFact, - CompiledObjectKey, - CompileSqlReference, - ExpansionSpan, - InferredColumn, -) -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompileInputsView +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompileSqlReference, InferredColumn +from sqlbuild.compiler.compile.types import AttachedAuditTargetKind from sqlbuild.compiler.lineage.types import ( InferredNullability, ) @@ -20,6 +18,9 @@ from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention, SourceLocation from sqlbuild.sql_values.models import SqlValue +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + @dataclass(frozen=True) class SqlReferenceExtractionTestCase: diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py index 06c6fd07b7..547af16fd6 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py @@ -2,10 +2,12 @@ import pytest +from sqlbuild._native import ( + DynamicColumnContractProof, + NativePivotTables, + prove_native_dynamic_contracts, +) from sqlbuild._native import dynamic_contract_proof as contract_proof -from sqlbuild._native import prove_native_dynamic_contracts -from sqlbuild.compiler.analysis_session.models import NativePivotTables -from sqlbuild.compiler.compile.models import DynamicColumnContractProof from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from tests.unit.src.sqlbuild.compiler.compile._helpers.analysis._test_types import ( DynamicPivotAnalysisTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py index 2c34d0dfde..0412e70802 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py @@ -1,21 +1,20 @@ """Helpers for model configuration reuse and native validation tests.""" +from __future__ import annotations + from pathlib import Path -from typing import cast +from typing import TYPE_CHECKING, cast import sqlbuild._native as _native from sqlbuild._native import ( + CompileModelConfig, + NativeModelConfigInputs, + NativeModelConfigSession, build_compile_inputs, discover_project_inputs, native_model_config_session, record_template_reads, ) -from sqlbuild.compiler.compile.models import ( - CompileModelConfig, - NativeModelConfigInputs, - NativeModelConfigSession, -) -from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.spec.contracts.models import ( DefaultsConfig, MaterializationDefaultsConfig, @@ -30,6 +29,9 @@ DUCKDB_COMPILE_ADAPTER_CONTEXT, ) +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + _NO_MATERIALIZATION_DEFAULTS: MaterializationDefaultsConfig = MaterializationDefaultsConfig() diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_model_config_reuse.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_model_config_reuse.py index 7c90fbbe9f..ca5b50acae 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_model_config_reuse.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_model_config_reuse.py @@ -2,10 +2,10 @@ from collections.abc import Callable from pathlib import Path +from typing import TYPE_CHECKING import pytest -from sqlbuild.compiler.compile.types import CompileInputsView from tests.unit.src.sqlbuild.compiler.compile._helpers.config._test_types import ( AttachedColumnsTestCase, ) @@ -14,6 +14,9 @@ compile_with_native_config, ) +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + @pytest.mark.parametrize( "test_case", diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py index 79653333a1..dec868dd0b 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py @@ -2,8 +2,8 @@ import pytest +from sqlbuild._native import CompileModelConfig from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileModelConfig from sqlbuild.spec.contracts.models import ( DefaultsConfig, MaterializationDefaultsConfig, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 2c312977cd..fc68224946 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -1,11 +1,25 @@ +from __future__ import annotations + import itertools import json from collections.abc import Callable from pathlib import Path from types import MappingProxyType -from typing import cast +from typing import TYPE_CHECKING, cast from sqlbuild._native import ( + CollectionRendering, + ColumnLineageConfidence, + ColumnTransformKind, + CompileAdapterContext, + CompiledDirectLogicSqlTestPayload, + CompiledLineageColumnFact, + CompiledModelSqlTestPayload, + DeclarationExpansionContext, + DeclarationRuntimeProjection, + DeclarationScopeResolver, + NativeColumnQuery, + NativeQueryColumns, assemble_project, build_compile_inputs, build_execution_upstream_deps, @@ -26,23 +40,15 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.models import ( - CompileAdapterContext, - CompiledDirectLogicSqlTestPayload, - CompiledLineageColumnFact, CompiledLineageSourceFact, - CompiledModelSqlTestPayload, CompiledObjectKey, CompiledRelationLocation, CompiledSqlTest, CompilerDiagnostic, CompileSqlReference, - DeclarationExpansionContext, DeclarationResolutionContext, - DeclarationRuntimeProjection, - DeclarationScopeResolver, LoadedMacro, MacroContext, ) @@ -50,18 +56,12 @@ CompiledModelView, CompiledProjectView, CompiledResourceType, - CompileInputsView, - ModelInput, ) from sqlbuild.compiler.discovery.models import ( DiscoveredMacroFile, DiscoveredProjectInputs, ) -from sqlbuild.compiler.lineage.types import ( - ColumnLineageConfidence, - ColumnTransformKind, - InferredNullability, -) +from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, @@ -73,7 +73,9 @@ ScopeLookup, ) from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind, ScopeKind -from sqlbuild.sql_values.types import CollectionRendering + +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView, ModelInput DUCKDB_COMPILE_ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( value_renderer=DuckDbAdapter(), diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py index 6fd7f9f166..24af81b145 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py @@ -3,13 +3,13 @@ import pytest from sqlbuild._native import ( + NativeColumnQuery, cursor_intrinsics_analysis_sql, infer_native_query_columns, resolve_cursor_intrinsics, validated_model_cursor_intrinsics, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.lineage.types import InferredNullability diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py index 6f0a067c5e..1c5b701ff9 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py @@ -2,7 +2,7 @@ from collections.abc import Callable from pathlib import Path -from typing import cast +from typing import TYPE_CHECKING, cast import pytest @@ -16,7 +16,6 @@ CompiledProjectView, CompiledResourceType, CompiledSeedView, - CompileInputsView, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, SqlHookEntry from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( @@ -35,6 +34,9 @@ base_repo_files, ) +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + @pytest.mark.parametrize( "test_case", diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py index 4facad6a36..3404d677c3 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py @@ -4,12 +4,12 @@ from collections.abc import Callable from pathlib import Path +from typing import TYPE_CHECKING import pytest from sqlbuild._native import build_compile_inputs, discover_project_inputs from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.spec.contracts.models import SchemaModelEntry, SourceLocation from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( AuditFactoryAttachmentTestCase, @@ -18,6 +18,9 @@ DUCKDB_COMPILE_ADAPTER_CONTEXT, ) +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + _PROJECT_FILE: str = 'name = "demo"\nadapter = "duckdb"\n' _AUDIT_FILE: str = 'AUDIT (); SELECT * FROM __ref("@model") WHERE NOT (@expression)' diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py index 541ed2451e..10d1a9b0e6 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py @@ -5,7 +5,7 @@ from dataclasses import replace from itertools import chain from pathlib import Path -from typing import cast +from typing import TYPE_CHECKING, cast import pytest @@ -16,7 +16,7 @@ scope_metadata_projection, ) from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView, ModelInput +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, DiscoveredSourceFile, @@ -49,6 +49,9 @@ usage_consumers, ) +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView, ModelInput + _PROJECT_FILE: str = """ name = "demo" adapter = "duckdb" diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py index 53f1b2665e..9a08befc21 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py @@ -3,11 +3,13 @@ import json from collections.abc import Callable from pathlib import Path -from typing import cast +from typing import TYPE_CHECKING, cast import pytest from sqlbuild._native import ( + DeclarationParseError, + ResourceIdentityError, assemble_project, build_compile_inputs, build_model_version_identity_metadata_json, @@ -15,15 +17,8 @@ discover_project_inputs, ) from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompileInputsView, - ModelInput, -) -from sqlbuild.compiler.discovery.exceptions import DeclarationParseError +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, SqlHookEntry -from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( CompileDeclarationsErrorTestCase, CompileDeclarationsTestCase, @@ -36,6 +31,9 @@ compile_first_model, ) +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView, ModelInput + _PROJECT_FILE: str = """ name = "demo" adapter = "duckdb" diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py index 88f9f0f96c..33ee290dfc 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py @@ -6,8 +6,13 @@ import pytest -from sqlbuild._native import expand_authored_sql_with_spans, map_expanded_offset -from sqlbuild.compiler.compile.models import ExpansionSpan, LoadedMacro, MacroContext, MappedOffset +from sqlbuild._native import ( + ExpansionSpan, + MappedOffset, + expand_authored_sql_with_spans, + map_expanded_offset, +) +from sqlbuild.compiler.compile.models import LoadedMacro, MacroContext from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ExpandWithSpansTestCase from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py index 8e08025218..7e93361a45 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py @@ -5,12 +5,12 @@ import pytest from sqlbuild._native import ( + CompileModelConfig, compute_cursor_bounds, resolve_effective_timestamp_grain, watermark_type_is_compatible, ) from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileModelConfig from sqlbuild.compiler.planner.models import CursorBounds, CursorInputRelation, ModelCursorSnapshot from sqlbuild.cursor_algebra.models import DateValue, TimestampValue from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py index a2dedcf3c7..c0948d8dfe 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py @@ -6,9 +6,8 @@ import pytest +from sqlbuild._native import CompileModelConfig, ConfigValueTypeError from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileModelConfig -from sqlbuild.spec.contracts.exceptions import ConfigValueTypeError from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention from sqlbuild.spec.contracts.types import TimeTravelRetentionSource from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py index f167adfc1f..7440b70bba 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py @@ -3,7 +3,7 @@ import json from collections.abc import Callable from pathlib import Path -from typing import cast +from typing import TYPE_CHECKING, cast import pytest @@ -19,11 +19,7 @@ from sqlbuild.compiler.compile.models import ( CompilerDiagnostic, ) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompileInputsView, -) +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.models import SchemaAuditInstance, SchemaModelEntry, SourceLocation from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( @@ -41,6 +37,9 @@ manifest_model_node, ) +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + _PROJECT_FILE: str = """ name = "demo" adapter = "duckdb" diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py index 6272cc9b6d..1a3b2a3110 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py @@ -3,6 +3,7 @@ from collections.abc import Callable from decimal import Decimal from pathlib import Path +from typing import TYPE_CHECKING import pytest @@ -17,7 +18,6 @@ from sqlbuild.compiler.compile.types import ( CompiledProjectView, CompiledResourceType, - CompileInputsView, TypedSqlValueRenderer, ) from sqlbuild.sql_values.models import SqlLogicalType, SqlValue @@ -31,6 +31,9 @@ from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import compile_project_inputs from tests.unit.src.sqlbuild.compiler.compile._test_helpers import base_repo_files +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + @pytest.mark.parametrize( "test_case", diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py index 5fdf06bc65..28a9954852 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py @@ -8,12 +8,13 @@ import pytest from sqlbuild._native import ( + DeclarationScopeBuild, build_declaration_scope, discover_project_inputs, rebind_declaration_scope, ) from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros -from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro +from sqlbuild.compiler.compile.models import LoadedMacro from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.scopes.models import DeclarationIdentity from sqlbuild.compiler.scopes.types import DeclarationKind diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_spans.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_spans.py index 359f824121..1389094096 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_spans.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_spans.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import map_expanded_offset -from sqlbuild.compiler.compile.models import ExpansionSpan, MappedOffset +from sqlbuild._native import ExpansionSpan, MappedOffset, map_expanded_offset from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( MapOutputOffsetTestCase, MapThroughPassesTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py index 07f3824c8f..0f73e79bee 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py @@ -2,21 +2,23 @@ import pytest -from sqlbuild._native import infer_native_query_columns, substitute_placeholder_defaults +from sqlbuild._native import ( + ColumnLineageConfidence, + ColumnTransformKind, + CompiledLineageColumnFact, + NativeColumnQuery, + NativeQueryColumns, + infer_native_query_columns, + substitute_placeholder_defaults, +) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns from sqlbuild.compiler.compile.models import ( - CompiledLineageColumnFact, CompiledLineageSourceFact, CompileSqlReference, InferredColumn, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.lineage.types import ( - ColumnLineageConfidence, - ColumnTransformKind, - InferredNullability, -) +from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( InferColumnsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_test_helpers.py index f4dcc0d762..20a9a56759 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_test_helpers.py @@ -3,11 +3,11 @@ from types import MappingProxyType from typing import cast -from sqlbuild._native import build_compile_reference_resolver -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompileDirectLogicSqlTestInputPayload, CompileModelSqlTestInputPayload, CompileSqlTestInput, + build_compile_reference_resolver, ) from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py index 8235239f3c..00c1a2ab70 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py @@ -2,11 +2,11 @@ from collections.abc import Callable from pathlib import Path +from typing import TYPE_CHECKING import pytest from sqlbuild._native import build_compile_inputs, discover_project_inputs -from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, @@ -17,6 +17,9 @@ CursorStartCompileInputsTestCase, ) +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + @pytest.mark.parametrize( "test_case", diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_main.py b/tests/unit/src/sqlbuild/compiler/compile/test_main.py index 61850194b7..8729c83215 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_main.py @@ -4,21 +4,23 @@ from collections.abc import Callable from dataclasses import replace from pathlib import Path +from typing import TYPE_CHECKING import pytest -from sqlbuild._native import build_compile_inputs, discover_project_inputs, resolve_target_config -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompileModelConfig, + ResourceIdentityError, + SpecConfigError, + build_compile_inputs, + discover_project_inputs, + resolve_target_config, ) -from sqlbuild.compiler.compile.types import CompileInputsView, FunctionInput from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, PythonHookEntry, SqlHookEntry, ) -from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError -from sqlbuild.spec.contracts.exceptions import SpecConfigError from sqlbuild.spec.contracts.models import ( AuthoredTimeTravelRetention, ClonePolicy, @@ -60,6 +62,9 @@ SeedRefRegressionTestCase, ) +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView, FunctionInput + @pytest.mark.parametrize( "test_case", diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py index 33024ae81d..a5c718b2d3 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py @@ -2,11 +2,11 @@ from collections.abc import Callable from pathlib import Path +from typing import TYPE_CHECKING import pytest from sqlbuild._native import build_compile_inputs, discover_project_inputs -from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, @@ -15,6 +15,9 @@ ResolvedConnectionCompileInputsTestCase, ) +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + _PROJECT_TOML: str = ( 'name = "demo"\nadapter = "duckdb"\n\n[connection]\ndatabase = "warehouse.duckdb"\n' ) diff --git a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py index 18c5bf41a2..6360d6504a 100644 --- a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py +++ b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py @@ -1,10 +1,9 @@ import pytest -from sqlbuild._native import compare_contract_evidence, edit_contract_model +from sqlbuild._native import ContractEvidence, compare_contract_evidence, edit_contract_model from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.contract_adoption.models import ContractEvidence from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from tests.unit.src.sqlbuild.compiler.contract_adoption._helpers._test_types import ( DynamicContractAdoptionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/contracts/helpers.py b/tests/unit/src/sqlbuild/compiler/contracts/helpers.py index 3df66a97ac..e8f754aaa2 100644 --- a/tests/unit/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/contracts/helpers.py @@ -3,13 +3,12 @@ from pathlib import Path from typing import cast +from sqlbuild._native import CompileModelConfig, DynamicColumnContractProof from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompileModelConfig, - DynamicColumnContractProof, InferredColumn, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType diff --git a/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py b/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py index 029785883e..47fb4428f7 100644 --- a/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py +++ b/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py @@ -4,13 +4,13 @@ import pytest -from sqlbuild._native import evaluate_model_contracts -from sqlbuild.adapter.contract.types import TypeDialect -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( + ContractValidationResult, DynamicColumnContractProof, DynamicColumnFamilyProof, + evaluate_model_contracts, ) -from sqlbuild.compiler.contracts.models import ContractValidationResult +from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily, SourceLocation from tests.unit.src.sqlbuild.compiler.contracts._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py index bcad829375..49834cbe00 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py @@ -2,20 +2,27 @@ from pathlib import Path +from sqlbuild._native import ( + CompiledFunction, + CompiledSeed, + CompiledSource, + CompileModelConfig, + DiscoveredAuditBlock, + DiscoveredAuditFile, + DiscoveredSchemaFile, + DiscoveredSeedFile, + DiscoveredSqlTestBlock, + FunctionArgument, +) from sqlbuild.compiler.auditing.types import AuditSeverity from sqlbuild.compiler.compile.models import ( CompiledAudit, - CompiledFunction, CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompiledSeed, - CompiledSource, CompiledSqlScenario, CompiledSqlTest, - CompileModelConfig, - FunctionArgument, ) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, @@ -24,14 +31,9 @@ FunctionLanguage, ) from sqlbuild.compiler.discovery.models import ( - DiscoveredAuditBlock, - DiscoveredAuditFile, DiscoveredLoaderFunction, - DiscoveredSchemaFile, - DiscoveredSeedFile, DiscoveredSourceFile, DiscoveredSqlScenarioFile, - DiscoveredSqlTestBlock, DiscoveredSqlTestFile, ) from sqlbuild.compiler.pipeline.models import ProjectGraph diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py index 826b3b8531..ba96505a1a 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py @@ -4,7 +4,7 @@ from collections.abc import Callable from dataclasses import replace from pathlib import Path -from typing import cast +from typing import TYPE_CHECKING, cast import pytest @@ -19,7 +19,7 @@ from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.dag.types import NodeKind from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph @@ -37,6 +37,9 @@ from tests.unit.src.sqlbuild.compiler.dag.main.helpers import build_dag_artifact_test_graph from tests.unit.src.sqlbuild.compiler.graph.helpers import project_graph_from_indexes +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + @pytest.mark.parametrize( "test_case", diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 8b11448b1c..12997eba18 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -2,9 +2,9 @@ from decimal import Decimal from pathlib import Path +from sqlbuild._native import CollectionRendering from sqlbuild.spec.contracts.models import AuthoredTimeTravelRetention, TargetWarehousesConfig from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy, RetentionDecreasePolicy -from sqlbuild.sql_values.types import CollectionRendering @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py index 8388c04f1b..0d0d63f921 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py @@ -4,6 +4,11 @@ from sqlbuild import _native from sqlbuild._native import ( + DiscoveredConstantFile, + DiscoveredEnumFile, + DiscoveredPythonFunctionFile, + DiscoveredSqlTestBlock, + DiscoveredSqlTestCase, discover_audit_files, discover_constant_files, discover_enum_files, @@ -13,15 +18,7 @@ discover_sql_function_files, discover_sql_hook_files, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredConstantFile, - DiscoveredEnumFile, - DiscoveredMacroFile, - DiscoveredPythonFunctionFile, - DiscoveredSqlTestBlock, - DiscoveredSqlTestCase, - SqlTestParameterDeclaration, -) +from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, SqlTestParameterDeclaration from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( LoadProjectConfigTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py index 5de55e3831..18b7540228 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py @@ -7,9 +7,10 @@ import pytest -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_python_node_functions -from sqlbuild.compiler.discovery.exceptions import PythonNodeDiscoveryError -from sqlbuild.compiler.discovery.models import DiscoveredPythonNodeFunctions +from sqlbuild._native import PythonNodeDiscoveryError, discover_python_node_functions +from sqlbuild.compiler.discovery.models import ( + DiscoveredPythonNodeFunctions, +) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( AuditFactoryDiscoveryTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_command_output_sinks.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_command_output_sinks.py index 125c477316..48cc3d5cb7 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_command_output_sinks.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_command_output_sinks.py @@ -5,14 +5,17 @@ import pytest +from sqlbuild._native import ( + EventExporterDiscoveryError, + discover_provider_classes, +) from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( discover_command_output_sink_functions, ) -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( - discover_provider_classes, +from sqlbuild.compiler.discovery.models import ( + DiscoveredCommandOutputSink, + DiscoveredProvider, ) -from sqlbuild.compiler.discovery.exceptions import EventExporterDiscoveryError -from sqlbuild.compiler.discovery.models import DiscoveredCommandOutputSink, DiscoveredProvider from sqlbuild.sinks import CommandOutputStream from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( CommandOutputSinkDiscoveryTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_event_exporters.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_event_exporters.py index dbacee8d69..7244ce5182 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_event_exporters.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_event_exporters.py @@ -5,12 +5,15 @@ import pytest -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( +from sqlbuild._native import ( + EventExporterDiscoveryError, discover_event_exporter_functions, discover_provider_classes, ) -from sqlbuild.compiler.discovery.exceptions import EventExporterDiscoveryError -from sqlbuild.compiler.discovery.models import DiscoveredEventExporter, DiscoveredProvider +from sqlbuild.compiler.discovery.models import ( + DiscoveredEventExporter, + DiscoveredProvider, +) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( EventExporterDiscoveryTestCase, EventExporterSignatureErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py index e23e33b0d1..ce2714758c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py @@ -7,9 +7,10 @@ import pytest -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_python_node_functions -from sqlbuild.compiler.discovery.exceptions import PythonNodeDiscoveryError -from sqlbuild.compiler.discovery.models import DiscoveredPythonNodeFunctions +from sqlbuild._native import PythonNodeDiscoveryError, discover_python_node_functions +from sqlbuild.compiler.discovery.models import ( + DiscoveredPythonNodeFunctions, +) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverPythonNodeFactoriesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_file_faults.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_file_faults.py index c53b3b7445..fa5c970dd5 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_file_faults.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_file_faults.py @@ -8,9 +8,16 @@ import pytest from sqlbuild._native import ( + DiscoveryFileFault, + EventExporterDiscoveryError, + ProviderDiscoveryError, + PythonNodeDiscoveryError, discover_audit_files, discover_constant_files, discover_enum_files, + discover_event_exporter_declarations, + discover_materialization_files, + discover_provider_classes, discover_scenario_files, discover_source_files, discover_sql_function_files, @@ -19,17 +26,6 @@ from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( discover_command_output_sink_declarations, ) -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( - discover_event_exporter_declarations, - discover_materialization_files, - discover_provider_classes, -) -from sqlbuild.compiler.discovery.exceptions import ( - EventExporterDiscoveryError, - ProviderDiscoveryError, - PythonNodeDiscoveryError, -) -from sqlbuild.compiler.discovery.models import DiscoveryFileFault from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverFileFaultsTestCase, DiscoverModuleImportFailureTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_hooks.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_hooks.py index ba4e5aa90f..071c411e05 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_hooks.py @@ -5,11 +5,11 @@ import pytest -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( +from sqlbuild._native import ( + PythonNodeDiscoveryError, discover_hook_functions, discover_provider_classes, ) -from sqlbuild.compiler.discovery.exceptions import PythonNodeDiscoveryError from sqlbuild.compiler.discovery.models import ( DiscoveredHookFunction, DiscoveredProvider, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py index 560c3393e6..74c3d57a2f 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py @@ -6,9 +6,10 @@ import pytest -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_python_node_functions -from sqlbuild.compiler.discovery.exceptions import PythonNodeDiscoveryError -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction +from sqlbuild._native import PythonNodeDiscoveryError, discover_python_node_functions +from sqlbuild.compiler.discovery.models import ( + DiscoveredLoaderFunction, +) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverLoaderFunctionsTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_materializations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_materializations.py index a2615c4d43..6e031cbed8 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_materializations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_materializations.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( +from sqlbuild._native import ( discover_materialization_files, discover_provider_classes, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_providers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_providers.py index dc90e2a7ee..d599d561f7 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_providers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_providers.py @@ -5,10 +5,14 @@ import pytest -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_provider_classes -from sqlbuild.compiler.discovery.exceptions import ProviderDiscoveryError -from sqlbuild.compiler.discovery.models import DiscoveredProvider -from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError +from sqlbuild._native import ( + ProviderDiscoveryError, + ResourceIdentityError, + discover_provider_classes, +) +from sqlbuild.compiler.discovery.models import ( + DiscoveredProvider, +) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverProviderCacheIsolationTestCase, DiscoverProviderClassesErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py index 8b032893aa..34dfb34ca0 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py @@ -10,10 +10,10 @@ import pytest -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( +from sqlbuild._native import ( + PythonNodeDiscoveryError, discover_python_node_functions, ) -from sqlbuild.compiler.discovery.exceptions import PythonNodeDiscoveryError from sqlbuild.compiler.discovery.models import ( DiscoveredAssetFunction, DiscoveredCheckFunction, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py index 4741d3441b..86778b6f9f 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py @@ -9,7 +9,11 @@ import pytest from sqlbuild._native import ( + DeclarationParseError, + DiscoveredConstantFile, + DiscoveredEnumFile, discover_audit_files, + discover_hook_functions, discover_macro_files, discover_project_inputs, discover_python_function_files, @@ -19,11 +23,7 @@ discover_sql_hook_files, discover_test_files, ) -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_hook_functions -from sqlbuild.compiler.discovery.exceptions import DeclarationParseError from sqlbuild.compiler.discovery.models import ( - DiscoveredConstantFile, - DiscoveredEnumFile, DiscoveredMacroFile, DiscoveredProjectInputs, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py index 029a48cdb5..c2f96bd1c5 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py @@ -7,10 +7,10 @@ import pytest -from sqlbuild.compiler.discovery.exceptions import ModelSqlParseError -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( DiscoveredPythonFunctionFile, DiscoveredSqlFunctionFile, + ModelSqlParseError, ) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( FunctionHeaderKeysTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py index 24bc174f74..9bb5032e8f 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py @@ -6,8 +6,11 @@ import pytest -from sqlbuild._native import discover_project_inputs, validate_resource_identity -from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError +from sqlbuild._native import ( + ResourceIdentityError, + discover_project_inputs, + validate_resource_identity, +) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( CanonicalResourceIdentityTestCase, DiscoveredResourceIdentityErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_audits.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_audits.py index 07084c8e4a..f99e1569f6 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_audits.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_audits.py @@ -5,9 +5,8 @@ import pytest +from sqlbuild._native import DiscoveredAuditBlock, DiscoveredAuditFile, SqlAuditParseError from sqlbuild.compiler.auditing.types import AuditEvaluationMode -from sqlbuild.compiler.discovery.exceptions import SqlAuditParseError -from sqlbuild.compiler.discovery.models import DiscoveredAuditBlock, DiscoveredAuditFile from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ParseMeasurementAuditTestCase, ParseSqlAuditFileErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py index 418dbfd95a..2c8db4c589 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py @@ -7,17 +7,16 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import AuthoredSqlValueCall -from sqlbuild.compiler.discovery.exceptions import DeclarationParseError -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, +from sqlbuild._native import ( + AuthoredSqlValueCall, + DeclarationParseError, DiscoveredConstantFile, DiscoveredEnumFile, DiscoveredModelSchemaFile, - EnumDeclaration, ModelSchemaDeclaration, + ResourceIdentityError, ) -from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError +from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration from sqlbuild.sql_values.models import SqlValue from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DeclarationResourceIdentityErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_hooks.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_hooks.py index 3ee4442bd7..bce356e0c0 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_hooks.py @@ -7,10 +7,10 @@ import pytest from sqlbuild._native import ( + DiscoveredSqlHookFile, + SqlHookParseError, discover_sql_hook_files, ) -from sqlbuild.compiler.discovery.exceptions import SqlHookParseError -from sqlbuild.compiler.discovery.models import DiscoveredSqlHookFile from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverSqlHooksTestCase, ParseSqlHookErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py index b97195f39b..35921e30df 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py @@ -8,16 +8,12 @@ from sqlbuild._native import ( AuthoredSqlSet, AuthoredSqlValueCall, - discover_model_files, - model_output_column_locations, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredSqlModelFile, DiscoveryFileFault, NamedSqlHookEntry, - PythonHookEntry, - SqlHookEntry, + discover_model_files, + model_output_column_locations, ) +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile, PythonHookEntry, SqlHookEntry from sqlbuild.spec.contracts.models import SourceLocation from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DeferredModelOutputLocationTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_scenarios.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_scenarios.py index 886f0e293c..9431ff703c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_scenarios.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_scenarios.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.discovery.exceptions import SqlScenarioParseError +from sqlbuild._native import SqlScenarioParseError from sqlbuild.compiler.discovery.models import DiscoveredSqlScenarioFile from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ParseSqlScenarioFileErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py index 30facdede3..35f64e200f 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py @@ -7,17 +7,14 @@ import pytest from sqlbuild._native import ( - discover_test_files, - omitted_ceremonial_select, -) -from sqlbuild.compiler.discovery.exceptions import SqlTestParseError -from sqlbuild.compiler.discovery.models import ( DiscoveredSqlTestBlock, DiscoveredSqlTestCase, - DiscoveredSqlTestFile, DiscoveryFileFault, - SqlTestParameterDeclaration, + SqlTestParseError, + discover_test_files, + omitted_ceremonial_select, ) +from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile, SqlTestParameterDeclaration from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ExpectedCountTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_statement_header_keys.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_statement_header_keys.py index 322f3db4bc..69a18d2866 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_statement_header_keys.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_statement_header_keys.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.discovery.exceptions import DiscoveryError, ProjectConfigError +from sqlbuild._native import DiscoveryError, ProjectConfigError from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( StatementHeaderKeysTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_diff.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_diff.py index b8ce19c597..3b3f6ef6be 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_diff.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_diff.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import resolve_target_config -from sqlbuild.compiler.discovery.exceptions import ProjectConfigError +from sqlbuild._native import ProjectConfigError, resolve_target_config from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( TargetDiffConfigErrorTestCase, TargetDiffConfigTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py index ae5088970b..1bfc803853 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import discover_project_inputs -from sqlbuild.compiler.discovery.exceptions import ProjectConfigError +from sqlbuild._native import ProjectConfigError, discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.models import TargetWarehousesConfig from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py index 4b21e81273..a70f2a3a52 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.discovery.exceptions import ProjectConfigError +from sqlbuild._native import CollectionRendering, ProjectConfigError from sqlbuild.runtime.event_exporting.types import LifecycleEventKind from sqlbuild.spec.contracts.models import ( AuthoredTimeTravelRetention, @@ -16,7 +16,6 @@ TargetConfig, ) from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy, RetentionDecreasePolicy -from sqlbuild.sql_values.types import CollectionRendering from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ColumnContractModeConfigErrorTestCase, ColumnContractModeConfigTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py index 938711839d..721e2e09c3 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild.compiler.discovery.exceptions import SourceParseError +from sqlbuild._native import SourceParseError from sqlbuild.integrations.dlt.models import DltSourceConfig from sqlbuild.integrations.ingestr.models import IngestrSourceConfig from sqlbuild.spec.contracts.models import SourceEntry diff --git a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py index fb6510bd32..40699cae19 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py @@ -1,11 +1,8 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild.compiler.discovery.models import ( - DiscoveredSqlModelFile, - DiscoveredSqlScenarioFile, - DiscoveredSqlTestBlock, -) +from sqlbuild._native import DiscoveredSqlTestBlock +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile, DiscoveredSqlScenarioFile from sqlbuild.spec.contracts.models import ( SchemaModelEntry, SchemaSeedEntry, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py index bf96a4ee0f..1ba30ebcf2 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py @@ -6,9 +6,13 @@ import pytest -from sqlbuild._native import discover_project_inputs -from sqlbuild.compiler.discovery.exceptions import ProjectConfigError, ProjectPythonPathError -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlTestBlock +from sqlbuild._native import ( + DiscoveredSqlTestBlock, + ProjectConfigError, + ProjectPythonPathError, + discover_project_inputs, +) +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.observability import EventDispatcher, LifecycleEvent, dispatcher_scope from tests.unit.src.sqlbuild.compiler.discovery._test_helpers import ( base_repo_files, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_runtime_event_exporter_config.py b/tests/unit/src/sqlbuild/compiler/discovery/test_runtime_event_exporter_config.py index ea76c13484..b1f5e23222 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/test_runtime_event_exporter_config.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/test_runtime_event_exporter_config.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.discovery.exceptions import ProjectConfigError +from sqlbuild._native import ProjectConfigError from sqlbuild.compiler.discovery.main.runtime_extensions import discover_runtime_extensions from sqlbuild.compiler.discovery.models import ( DiscoveredEventExporter, diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py index 8ed17b1b85..efc8b96a50 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_query_fingerprint.py @@ -2,8 +2,7 @@ import pytest -from sqlbuild._native import compute_query_hash, normalize_query_sql -from sqlbuild.compiler.fingerprints.exceptions import QueryFingerprintError +from sqlbuild._native import QueryFingerprintError, compute_query_hash, normalize_query_sql from tests.unit.src.sqlbuild.compiler.fingerprints.main._test_types import ( ComputeQueryHashStabilityTestCase, ComputeQueryHashTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_read.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_read.py index 67cd68449a..2461ffca0f 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_read.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_read.py @@ -5,9 +5,8 @@ import pytest -from sqlbuild._native import read_latest_fingerprints, write_fingerprint +from sqlbuild._native import FingerprintInputError, read_latest_fingerprints, write_fingerprint from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.exceptions import FingerprintInputError from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from tests.unit.src.sqlbuild.compiler.fingerprints.main._test_types import ( ReadLatestFingerprintsErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py b/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py index 3106cbbf5e..417789217d 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py @@ -5,7 +5,7 @@ from collections.abc import Callable from dataclasses import dataclass -from sqlbuild.compiler.frontier.types import CompilerStage +from sqlbuild._native import CompilerStage @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/helpers.py b/tests/unit/src/sqlbuild/compiler/frontier/helpers.py index e28c5cd385..192e943531 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/helpers.py @@ -7,8 +7,8 @@ from pathlib import PurePosixPath from typing import Any, cast -from sqlbuild._native import BindingCatalog -from sqlbuild.compiler.compile.models import CompactLineageFacts, DeclarationScopeBuild +from sqlbuild._native import BindingCatalog, DeclarationScopeBuild +from sqlbuild.compiler.compile.models import CompactLineageFacts HELPERS_MODULE: str = "tests.unit.src.sqlbuild.compiler.frontier.helpers" diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py index 9a50df5ea6..6eab3842bf 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py @@ -9,9 +9,8 @@ import pytest from scripts.compiler_differential.classes.capture_file import expand_capture_text -from sqlbuild._native import compile_frontier, render_stage_capture +from sqlbuild._native import CompilerStage, compile_frontier, render_stage_capture from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_DIR_ENV_VAR -from sqlbuild.compiler.frontier.types import CompilerStage from tests.unit.src.sqlbuild.compiler.frontier._test_types import ( FrontierCaptureTestCase, OmittedFieldTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py b/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py index b83ea020b9..29a9ea2709 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sqlbuild.compiler.lineage.types import ColumnTransformKind +from sqlbuild._native import ColumnTransformKind @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py index 086dbddaff..f0e2445b1d 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py @@ -3,14 +3,19 @@ from pathlib import Path import sqlbuild._native as native -from sqlbuild._native import ProjectColumnLineage +from sqlbuild._native import ( + ColumnLineageMode, + CompiledSeed, + CompileModelConfig, + DiscoveredSchemaFile, + DiscoveredSeedFile, + ProjectColumnLineage, +) from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompiledSeed, - CompileModelConfig, InferredColumn, ) from sqlbuild.compiler.compile.types import ( @@ -19,8 +24,6 @@ CompiledResourceType, CompiledSeedView, ) -from sqlbuild.compiler.discovery.models import DiscoveredSchemaFile, DiscoveredSeedFile -from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.spec.contracts.models import SchemaColumn, SchemaSeedEntry, SettingsConfig diff --git a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py index 9349c6c40f..2c0d24efa4 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py @@ -5,24 +5,21 @@ import pytest from sqlbuild._native import ( + ColumnLineage, + ColumnLineageConfidence, + ColumnLineageMode, + ColumnTransformKind, + CompiledLineageColumnFact, + ModelColumnLineage, ProjectColumnLineage, compiled_model_record, ) -from sqlbuild.compiler.compile.models import ( - CompiledLineageColumnFact, - CompiledLineageSourceFact, -) +from sqlbuild.compiler.compile.models import CompiledLineageSourceFact from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.lineage.models import ColumnLineage, ModelColumnLineage -from sqlbuild.compiler.lineage.types import ( - ColumnLineageConfidence, - ColumnLineageMode, - ColumnTransformKind, -) from tests.unit.src.sqlbuild.compiler.lineage._test_types import ( ColumnLineageAnalyzerTestCase, ExpectedCountTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py index 34ac7d46ae..f51152461b 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py @@ -5,8 +5,8 @@ from collections.abc import Callable from typing import cast +from sqlbuild._native import NativeStageMismatchError from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.frontier.exceptions import NativeStageMismatchError from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge diff --git a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py index 86c2bba61f..2d66988996 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py @@ -6,7 +6,15 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import build_manifest +from sqlbuild._native import ( + CompiledSeed, + CompiledSource, + CompileModelConfig, + DiscoveredSchemaFile, + DiscoveredSeedFile, + DiscoveredSqlHookFile, + build_manifest, +) from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditRunScope, @@ -17,9 +25,6 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompiledSeed, - CompiledSource, - CompileModelConfig, LoadedMacro, ) from sqlbuild.compiler.compile.types import ( @@ -29,12 +34,7 @@ CompiledSeedView, CompiledSourceView, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredSchemaFile, - DiscoveredSeedFile, - DiscoveredSourceFile, - DiscoveredSqlHookFile, -) +from sqlbuild.compiler.discovery.models import DiscoveredSourceFile from sqlbuild.compiler.planner.models import ( AuditPlanEntry, ChainStep, diff --git a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py index 9cf42aeebf..022f77edd1 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py @@ -9,6 +9,7 @@ import sqlbuild._native as _native from sqlbuild._native import ( + DiscoveredSqlHookFile, build_manifest, project_with, ) @@ -22,7 +23,7 @@ CompiledSeedView, CompiledSourceView, ) -from sqlbuild.compiler.discovery.models import DiscoveredSqlHookFile, SqlHookEntry +from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.compiler.planner.models import AuditPlanEntry, ChainStep, PlanOutput, SqlTestPlanEntry from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py index c796410eca..89b08c4e20 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py @@ -5,6 +5,7 @@ from datetime import UTC, datetime, timedelta from sqlbuild._native import ( + MigrationDiscovery, deterministic_column_migration_event_id, deterministic_migration_event_id, deterministic_old_name_view_event_id, @@ -18,7 +19,6 @@ from sqlbuild.compiler.migrations.types import ( ColumnMigrationDecision, MigrationDecision, - MigrationDiscovery, OldNameViewEventType, ) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/_test_types.py index f89f863b68..057679bf76 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/_test_types.py @@ -5,7 +5,7 @@ from collections.abc import Callable from dataclasses import dataclass -from sqlbuild.compiler.python_nodes.types import PythonIdentityStatus +from sqlbuild._native import PythonIdentityStatus @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py index 1083d400ef..b40947306c 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py @@ -5,12 +5,12 @@ from datetime import datetime from pathlib import Path +from sqlbuild._native import CompileModelConfig from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompileModelConfig, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py index 6bcaee331c..b52740a712 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py @@ -5,13 +5,13 @@ from pathlib import Path from typing import Any, ClassVar +from sqlbuild._native import CompileModelConfig from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompileModelConfig, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py index b83e6509d0..34b1eacb9d 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py @@ -2,14 +2,14 @@ import pytest -from sqlbuild._native import run_clone_pipeline_with_options -from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ( +from sqlbuild._native import ( ClonePipelineConnection, ClonePipelineOptions, ClonePipelineResult, + run_clone_pipeline_with_options, ) +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, TargetConfig from tests.unit.src.sqlbuild.compiler.pipeline._helpers._test_types import ( CloneConnectionInjectionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py index bc0e67322a..368bc2777b 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py @@ -8,18 +8,15 @@ import pytest from sqlbuild._native import ( + PythonIdentityStatus, + PythonPlanEntry, + PythonSqlRunSelection, build_python_plan_entries, build_python_sql_run_lifecycle, read_latest_fingerprints, ) from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet -from sqlbuild.compiler.pipeline.models import PythonPlanEntry -from sqlbuild.compiler.python_nodes.models import ( - PythonNodeGraph, - PythonNodeIdentity, - PythonSqlRunSelection, -) -from sqlbuild.compiler.python_nodes.types import PythonIdentityStatus +from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonNodeIdentity from tests.unit.src.sqlbuild.compiler.fingerprints.main.helpers import ( FakeFingerprintExecute, render_qualified_name, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py index 1fb2794b64..794f94504b 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from sqlbuild.compiler.pipeline.models import PythonPlanEntry +from sqlbuild._native import PythonPlanEntry from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.python_nodes.models import SqlResourceRef diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py index f3d84efc7b..73fef6bf95 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py @@ -3,12 +3,12 @@ from pathlib import Path from typing import Any, ClassVar +from sqlbuild._native import CompiledSource from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompiledSource, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredSourceFile diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index b17f2369ce..0da2b02e2e 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -9,13 +9,17 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import BuildPhaseTimingTracker, PartialBuildPhaseTimings +from sqlbuild._native import ( + BuildPhaseTimingTracker, + CompilePipelineOptions, + PartialBuildPhaseTimings, + RulesRunResult, +) from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.pipeline._helpers import materializations -from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult +from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.rule_engine.models import RulesRunResult from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import PipelinePhaseTimingTestCase diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_plan_work.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_plan_work.py index eca1fbe918..af5e7ec7e8 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_plan_work.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_plan_work.py @@ -2,14 +2,14 @@ import pytest -from sqlbuild._native import plan_has_executable_work -from sqlbuild.compiler.pipeline.models import PythonPlanEntry -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.python_nodes.types import ( +from sqlbuild._native import ( PythonIdentityStatus, - PythonNodeKind, + PythonPlanEntry, PythonRunPhase, + plan_has_executable_work, ) +from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild.compiler.python_nodes.types import PythonNodeKind from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import PlanWorkTestCase from tests.unit.src.sqlbuild.compiler.pipeline.main.helpers import build_plan_output_with_model diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index ebaad34eb9..65b24a5dce 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -5,6 +5,8 @@ from sqlbuild._native import ( ChangeKind, MissingUpstream, + ParsedSelector, + PathSelector, PlanWarning, ScenarioArtifactIdentity, ScenarioGraphPlan, @@ -24,8 +26,6 @@ from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import ( CursorBounds, - ParsedSelector, - PathSelector, ScenarioRelationMap, SchemaFinding, SourceLoadPlanEntry, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py index fb105642c1..4c8bf2aeff 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py @@ -9,6 +9,10 @@ from sqlbuild._native import ( ChangeKind, + CompiledFunction, + CompileModelConfig, + FunctionArgument, + PlannerScope, WarehouseFingerprints, WarehouseSnapshot, build_version_identity_metadata_json, @@ -16,13 +20,10 @@ ) from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.compiler.compile.models import ( - CompiledFunction, CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompileModelConfig, - FunctionArgument, ) from sqlbuild.compiler.compile.types import ( CompiledFunctionView, @@ -32,9 +33,6 @@ FunctionLanguage, ) from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import ( - PlannerScope, -) from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( DetectModelChangesTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 931a523c57..42e61cb32f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -10,6 +10,7 @@ ChangeDetectionResult, ChangeKind, PlannerChangeResults, + PlannerScope, PlanWarning, WarehouseFingerprints, WarehouseSnapshot, @@ -26,9 +27,6 @@ from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import ( - PlannerScope, -) from sqlbuild.compiler.planner.types import ( BackfillAction, PlanAction, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 47e15bb13f..9a281f15c5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -15,6 +15,20 @@ from sqlbuild._native import ( BackfillResult, ChangeDetectionResult, + CompiledDirectLogicSqlTestPayload, + CompiledFunction, + CompiledModelSqlTestPayload, + CompiledSeed, + CompiledSource, + CompileModelConfig, + CompileSqlScenarioCte, + CompileSqlTestCte, + DiscoveredAuditBlock, + DiscoveredAuditFile, + DiscoveredSchemaFile, + DiscoveredSeedFile, + DiscoveredSqlTestBlock, + FunctionArgument, PlannerChangeResults, ScenarioArtifactIdentity, WarehouseSnapshot, @@ -31,22 +45,13 @@ from sqlbuild.compiler.auditing.types import AuditSeverity from sqlbuild.compiler.compile.models import ( CompiledAudit, - CompiledDirectLogicSqlTestPayload, - CompiledFunction, CompiledModel, - CompiledModelSqlTestPayload, CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompiledSeed, - CompiledSource, CompiledSqlScenario, CompiledSqlTest, - CompileModelConfig, CompileSqlReference, - CompileSqlScenarioCte, - CompileSqlTestCte, - FunctionArgument, ) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, @@ -60,14 +65,9 @@ SqlTestMode, ) from sqlbuild.compiler.discovery.models import ( - DiscoveredAuditBlock, - DiscoveredAuditFile, DiscoveredLoaderFunction, - DiscoveredSchemaFile, - DiscoveredSeedFile, DiscoveredSourceFile, DiscoveredSqlScenarioFile, - DiscoveredSqlTestBlock, DiscoveredSqlTestFile, ) from sqlbuild.compiler.fingerprints.models import Fingerprint diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py index 709c37fed0..5a1712b79f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py @@ -7,12 +7,12 @@ from typing import TYPE_CHECKING from sqlbuild._native import ( + OldNameViewAction, QueryShape, build_migration_fingerprint, parse_query_shape, ) from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.migrations.types import OldNameViewAction from sqlbuild.compiler.planner.models import ( OldNameViewPlanEntry, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py index bcd1c93b32..0dea21e99b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py @@ -9,7 +9,11 @@ import pytest from sqlbuild._native import ( + CompileModelConfig, + DeferralInputs, + PlannerOverrides, PlannerRuntime, + PlannerScope, WarehouseFingerprints, WarehouseSnapshot, plan_column_migrations, @@ -20,16 +24,10 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompileModelConfig, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - DeferralInputs, - PlannerOverrides, - PlannerScope, -) from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( UnfingerprintableCandidateTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py index e8194607f9..16d6999afb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py @@ -5,7 +5,10 @@ from unittest.mock import Mock from sqlbuild._native import ( + CompileModelConfig, + PlannerRelationsContext, PlannerRuntime, + PlannerScope, PlannerWarehouseState, WarehouseSnapshot, ) @@ -16,17 +19,12 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompileModelConfig, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner.models import ( - PlannerRelationsContext, - PlannerScope, -) from sqlbuild.spec.contracts.models import ( LocalConfig, ProjectConfig, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py index 5d91b2ba62..31823edc7c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py @@ -11,13 +11,11 @@ DirectModelVersionIdentities, PlannerChangeResults, PlannerIdentityContext, + PlannerScope, PlannerScopeResolution, WarehouseSnapshot, detect_stale_warning_changes, ) -from sqlbuild.compiler.planner.models import ( - PlannerScope, -) from tests.unit.src.sqlbuild.compiler.planner._helpers.planning._test_types import ( ReusableExecutionModelNamesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py index 7935807286..7871c50b6d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( PlannerRuntime, + PlannerScope, PlannerWarehouseState, WarehouseFingerprints, plan_retention, @@ -26,11 +27,7 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - PlannerScope, - RetentionPlanEntry, - TableTypePlanEntry, -) +from sqlbuild.compiler.planner.models import RetentionPlanEntry, TableTypePlanEntry from sqlbuild.compiler.planner.types import ( MaterializationType, RetentionDirection, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py index 17cf987f13..c2047ede65 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py @@ -3,14 +3,12 @@ import pytest from sqlbuild._native import ( + PlannerPolicies, PlannerScopeResolution, + PlannerSelection, resolve_planner_scopes, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import ( - PlannerPolicies, - PlannerSelection, -) from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_test_project from tests.unit.src.sqlbuild.compiler.planner._helpers.planning._test_types import ( PlannerScopesTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index 75bdb881a9..82f2acdab3 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -5,6 +5,7 @@ from pathlib import Path from sqlbuild._native import ( + CompileModelConfig, ModelPlanContext, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -12,7 +13,6 @@ CompiledModel, CompiledObjectKey, CompiledRelationLocation, - CompileModelConfig, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/_test_types.py index 213756d6a9..1e0b74d81a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sqlbuild.compiler.planner.models import SqlTestSelection +from sqlbuild._native import SqlTestSelection @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py index 8ff43b33f1..bff9a49c97 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py @@ -5,14 +5,12 @@ import pytest from sqlbuild._native import ( + SqlTestSelection, UnitTestSelectorSplit, split_project_unit_test_selectors, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - SqlTestSelection, -) from tests.unit.src.sqlbuild.compiler.planner._helpers.selection._test_types import ( UnitTestSelectorErrorTestCase, UnitTestSelectorSplitTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 60fc1cb492..7d13ab9bc9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -8,6 +8,11 @@ from typing import Any from sqlbuild._native import ( + CompiledFunction, + CompiledModelSqlTestPayload, + CompileModelConfig, + CompileSqlTestCte, + DiscoveredSqlTestBlock, NativeSqlTestPlan, PlanWarning, SqlTestPlanResult, @@ -23,15 +28,11 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile.models import ( - CompiledFunction, CompiledModel, - CompiledModelSqlTestPayload, CompiledObjectKey, CompiledProject, CompiledRelationLocation, CompiledSqlTest, - CompileModelConfig, - CompileSqlTestCte, LoadedMacro, MacroContext, ) @@ -41,10 +42,7 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredSqlTestBlock, - DiscoveredSqlTestFile, -) +from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile from sqlbuild.compiler.planner.models import ( SqlTestPlanEntry, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index 6f47e2106d..8299d95df0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -6,7 +6,14 @@ import pytest from sqlbuild._native import ( + CompiledDirectLogicSqlTestPayload, + CompiledFunction, + CompiledModelSqlTestPayload, + CompileSqlTestCte, + DiscoveredSqlTestBlock, NativeSqlTestPlan, + PlannerScope, + PlannerSelection, PlanWarning, SqlTestPlanResult, WarningSeverity, @@ -23,25 +30,15 @@ from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError, SqlTestReferenceError from sqlbuild.compiler.compile.models import ( - CompiledDirectLogicSqlTestPayload, - CompiledFunction, - CompiledModelSqlTestPayload, CompiledObjectKey, CompiledProject, CompiledRelationLocation, CompiledSqlTest, - CompileSqlTestCte, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile +from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - ChainStep, - PlannerScope, - PlannerSelection, - PlanOutput, - SqlTestPlanEntry, -) +from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( AssertionChainCteErrorTestCase, NativePlanningDifferentialTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index fe2cd6aab1..a025b8a56d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -13,6 +13,7 @@ PlanWarning, RelationFixtureCompletion, RelationFixturePlanningContext, + SqlTestSelection, build_selected_test_entries, compiled_model_record, project_with, @@ -27,10 +28,7 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner.models import ( - SqlTestPlanEntry, - SqlTestSelection, -) +from sqlbuild.compiler.planner.models import SqlTestPlanEntry from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import SettingsConfig from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py index 2d2412f27a..3d3f1430d8 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( BackfillResult, + CompileModelConfig, WarehouseSnapshot, compute_microbatch_range, compute_plan_cursor_bounds, @@ -18,7 +19,6 @@ CompiledModel, CompiledObjectKey, CompiledRelationLocation, - CompileModelConfig, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py index a0639c392c..e15464501e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ( + CompileSqlScenarioCte, ScenarioGraphPlan, build_scenario_execution_plan, build_scenario_fixture_plans, @@ -20,10 +21,7 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.models import ( - CompiledSqlScenario, - CompileSqlScenarioCte, -) +from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.compiler.planner.exceptions import PlannerInputError diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index d8da9b9ed7..ed26cc5d80 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -9,8 +9,12 @@ ChangeDetectionResult, ChangeKind, DirectModelVersionIdentities, + DirectSourceFreshnessPlanningResult, PlannerChangeResults, + PlannerPolicies, + PlannerScope, PlannerScopeResolution, + PlannerSelection, PlanWarning, SourceFreshnessIdentity, WarehouseFingerprints, @@ -27,12 +31,6 @@ ) from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_SEED from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import ( - PlannerPolicies, - PlannerScope, - PlannerSelection, -) -from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CompleteModelChangesWorkTestCase, PlannerStaleWarningScopeTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py index 63ff653331..a190291669 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py @@ -3,6 +3,9 @@ import pytest from sqlbuild._native import ( + ParsedSelector, + PathSelector, + SelectorKind, build_downstream_deps, parse_project_selector, project_lineage_views, @@ -12,8 +15,6 @@ CompiledObjectKey, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector -from sqlbuild.compiler.planner.types import SelectorKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( BuildPathIndexTestCase, ParseSelectorErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py index 486e3bba2e..d3f480cabb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py @@ -7,6 +7,8 @@ import pytest from sqlbuild._native import ( + PlannerRelationsContext, + PlannerScope, build_cursor_input_relations, build_planner_relations_context, gather_source_columns, @@ -27,11 +29,7 @@ CompiledResourceType, ) from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - CursorInputRelation, - PlannerRelationsContext, - PlannerScope, -) +from sqlbuild.compiler.planner.models import CursorInputRelation from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py index e186a8891f..20cd18c124 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py @@ -4,11 +4,14 @@ import pytest -from sqlbuild._native import build_planner_source_freshness_result +from sqlbuild._native import ( + DirectSourceFreshnessPlanningResult, + PlannerRelationsContext, + PlannerScope, + build_planner_source_freshness_result, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope -from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py index 7a23ab1127..54bbd30a4f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py @@ -8,6 +8,7 @@ import pytest from sqlbuild._native import ( + CompileModelConfig, build_intermediate_source_map, build_source_load_entries, build_source_load_map, @@ -21,7 +22,6 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompileModelConfig, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py index 701313e83b..7cd8253437 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py @@ -6,13 +6,17 @@ from types import SimpleNamespace from typing import cast -from sqlbuild._native import check_selected_source_tables_exist +from sqlbuild._native import ( + CompiledSource, + PlannerRelationsContext, + PlannerScope, + check_selected_source_tables_exist, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompiledSource +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject from sqlbuild.compiler.compile.types import CompiledResourceType, CompiledSourceView from sqlbuild.compiler.discovery.models import DiscoveredSourceFile -from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope from sqlbuild.spec.contracts.models import SourceEntry diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/classes/_test_types.py index c45447080c..d494672a34 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/_test_types.py @@ -32,3 +32,11 @@ class FixtureColumnInferencesTestCase: description: str requests: tuple[tuple[str, str], ...] expected_inferences: int + + +@dataclass(frozen=True) +class PlanningExitTestCase: + description: str + script: str + expected_returncode: int + expected_output: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py index 9c6088189d..ca26c95c10 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py @@ -11,9 +11,12 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import build_migration_fingerprint, infer_fixture_column_facts +from sqlbuild._native import ( + FixtureColumnInference, + build_migration_fingerprint, + infer_fixture_column_facts, +) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import FixtureColumnInference ORDERS_SQL: str = 'SELECT o.order_id, o.amount_cents FROM __ref("stg_orders") AS o' CURSOR_METADATA: str = json.dumps( diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_fixture_column_inferences.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_fixture_column_inferences.py index 6f970978e9..8fa5c94b32 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_fixture_column_inferences.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_fixture_column_inferences.py @@ -5,10 +5,10 @@ import pytest from sqlbuild._native import ( + FixtureColumnInference, FixtureColumnInferences, infer_fixture_column_facts, ) -from sqlbuild.compiler.compile.models import FixtureColumnInference from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import ( FixtureColumnInferencesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py index f2c8c5e441..fbe40f6ebf 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py @@ -9,15 +9,13 @@ from sqlbuild._native import ( PlannedSqlTests, PlanWarning, + SqlTestSelection, WarningSeverity, start_sql_test_planning, ) from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - SqlTestSelection, -) from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import ( BackgroundSqlTestPlanningTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py new file mode 100644 index 0000000000..a1686d952e --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py @@ -0,0 +1,115 @@ +import subprocess +import sys +from pathlib import Path +from textwrap import dedent + +import pytest + +from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import PlanningExitTestCase + +_SLOW_TEST_PLANNING: str = dedent( + """ + import sys + import time + import types + + import sqlbuild._native as native + + + class SlowFinalizer: + def __del__(self): + print("finalizing", sys.is_finalizing(), file=sys.stderr, flush=True) + time.sleep(2.0) + + + holder = types.ModuleType("holder") + holder.finalizer = SlowFinalizer() + sys.modules["holder"] = holder + plan_tests = native.plan_selected_sql_tests + + + def slow_plan_selected_sql_tests(**arguments): + time.sleep(1.0) + return plan_tests(**arguments) + + + native.plan_selected_sql_tests = slow_plan_selected_sql_tests + """ +) +_FAILED_BUILD: str = dedent( + """ + from sqlbuild.cli.entry.main.entry import main + from sqlbuild.compiler.planner.exceptions import PlannerInputError + + + def failing_reconciliation(**arguments): + raise PlannerInputError("forced planning failure") + + + native.reconcile_execution_changes = failing_reconciliation + sys.exit(main(["--project-dir", sys.argv[1], "--no-color", "build"])) + """ +) +_UNJOINED_PLANNING: str = dedent( + """ + from sqlbuild._native import SqlTestSelection + + native.start_sql_test_planning( + project=None, + adapter=None, + selected_keys=frozenset(), + sql_test_selection=SqlTestSelection(), + enabled=True, + ) + print("planning started", file=sys.stderr, flush=True) + """ +) + + +@pytest.mark.parametrize( + "test_case", + ( + PlanningExitTestCase( + "plan fails while SQL tests are still planning", + _SLOW_TEST_PLANNING + _FAILED_BUILD, + 1, + "error[S000]: forced planning failure", + ), + PlanningExitTestCase( + "SQL test planning never joined", + _SLOW_TEST_PLANNING + _UNJOINED_PLANNING, + 0, + "planning started", + ), + ), + ids=lambda case: case.description, +) +def test_given_sql_test_planning_running_when_interpreter_finalizes_then_process_exits_cleanly( + test_case: PlanningExitTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = tmp_path / "orders_project" + (project_dir / "models").mkdir(parents=True) + (project_dir / "sqlbuild_project.toml").write_text( + 'name = "orders_project"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "orders_project.duckdb"\n', + encoding="utf-8", + ) + (project_dir / "models" / "orders.sql").write_text( + "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id\n", + encoding="utf-8", + ) + script: Path = tmp_path / "planning_exit.py" + script.write_text(test_case.script, encoding="utf-8") + + result: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, str(script), str(project_dir)], + capture_output=True, + text=True, + timeout=120, + check=False, + ) + + assert result.returncode == test_case.expected_returncode, result.stdout + result.stderr + assert test_case.expected_output in result.stdout + result.stderr + assert "finalizing True" in result.stderr diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py index 44a05fa22b..43420693fe 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py @@ -6,13 +6,12 @@ import pytest -from sqlbuild._native import build_display_only_sqlbuild_plan, format_plan +from sqlbuild._native import CompileModelConfig, build_display_only_sqlbuild_plan, format_plan from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompileModelConfig, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index a7cf81c185..7bd5efe351 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -6,6 +6,11 @@ from typing import Any from sqlbuild._native import ( + CompileModelConfig, + DeferralInputs, + PlannerOverrides, + PlannerPolicies, + PlannerSelection, SelectionStalenessGraph, SelectionStalenessNodeKey, build_execution_plan, @@ -19,7 +24,6 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompileModelConfig, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, @@ -27,13 +31,7 @@ CompiledResourceType, ) from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import ( - DeferralInputs, - PlannerOverrides, - PlannerPolicies, - PlannerSelection, - PlanOutput, -) +from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py index 2320037d58..0e14fac65d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py @@ -4,7 +4,12 @@ from types import SimpleNamespace from sqlbuild._native import ( + CompiledFunction, + CompileModelConfig, DirectModelVersionIdentities, + FunctionArgument, + FunctionReturnColumn, + PlannerScope, build_direct_model_version_identities, build_downstream_deps, build_model_version_identity_metadata_json, @@ -12,22 +17,15 @@ compiled_model_record, ) from sqlbuild.compiler.compile.models import ( - CompiledFunction, CompiledModel, CompiledObjectKey, CompiledRelationLocation, - CompileModelConfig, - FunctionArgument, - FunctionReturnColumn, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledResourceType, FunctionLanguage, ) -from sqlbuild.compiler.planner.models import ( - PlannerScope, -) from sqlbuild.spec.contracts.models import SchemaColumn, SeedCsvSettings from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py index 6e892c3296..065326faee 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py @@ -5,14 +5,12 @@ import pytest from sqlbuild._native import ( - DirectModelVersionIdentities, - build_function_local_hashes, -) -from sqlbuild.compiler.compile.models import ( CompiledFunction, - CompiledObjectKey, + DirectModelVersionIdentities, FunctionReturnColumn, + build_function_local_hashes, ) +from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function from tests.unit.src.sqlbuild.compiler.planner.main.identity._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/_test_types.py index 5604afa7d6..e4e1d8f073 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/_test_types.py @@ -1,11 +1,7 @@ from dataclasses import dataclass -from sqlbuild.compiler.migrations.types import ( - ColumnMigrationDecision, - MigrationCompatibility, - MigrationDecision, - MigrationDiscovery, -) +from sqlbuild._native import MigrationCompatibility, MigrationDiscovery +from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDecision @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py index 33ec9ff9f4..dcf6d63594 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py @@ -2,14 +2,17 @@ from __future__ import annotations -from sqlbuild._native import format_pre_build_work +from sqlbuild._native import ( + DisplayOptions, + MigrationCompatibility, + MigrationDiscovery, + format_pre_build_work, +) from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.migrations.types import ( ColumnMigrationDecision, - MigrationCompatibility, MigrationDecision, - MigrationDiscovery, MigrationPromotion, ) from sqlbuild.compiler.planner.models import ( @@ -17,7 +20,6 @@ ModelMigrationPlanEntry, PlanOutput, ) -from sqlbuild.presentation.models import DisplayOptions def migration_entry( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py index 04e9cc6423..45345c47cb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py @@ -4,15 +4,15 @@ import pytest -from sqlbuild._native import format_plan, format_pre_build_work -from sqlbuild.compiler.migrations.types import ( - ColumnMigrationDecision, +from sqlbuild._native import ( + DisplayOptions, MigrationCompatibility, - MigrationDecision, MigrationDiscovery, + format_plan, + format_pre_build_work, ) +from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDecision from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.presentation.models import DisplayOptions from tests.unit.src.sqlbuild.compiler.planner.main.pre_build._test_types import ( ColumnMigrationPlainTextTestCase, ColumnMigrationStyleTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py index 0398232fca..50a5f7cfac 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py @@ -10,6 +10,11 @@ import pytest from sqlbuild._native import ( + CompileModelConfig, + DiscoveredSeedFile, + PlannerRelationsContext, + PlannerScope, + PlannerSelection, build_audit_command_plan, build_seed_command_plan, build_test_command_plan, @@ -19,17 +24,9 @@ resolve_static_command_scope, resolve_static_relation_context, ) -from sqlbuild.compiler.compile.models import CompileModelConfig from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledSeedView -from sqlbuild.compiler.discovery.models import DiscoveredSeedFile from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import ( - PlannerRelationsContext, - PlannerScope, - PlannerSelection, - PlanOutput, - SeedPlanEntry, -) +from sqlbuild.compiler.planner.models import PlanOutput, SeedPlanEntry from sqlbuild.compiler.planner.types import PlanReason from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( PlannerTestAdapter, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/_test_types.py index 5221ff37ff..64cefbdae3 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/_test_types.py @@ -1,7 +1,6 @@ from dataclasses import dataclass -from sqlbuild.compiler.planner.models import ParsedSelector -from sqlbuild.compiler.python_nodes.models import PythonSqlRunSelection +from sqlbuild._native import ParsedSelector, PythonSqlRunSelection from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.python_nodes.models import SqlResourceRef diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py index ce7888d745..883331fe33 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py @@ -11,6 +11,9 @@ from typing import Any, cast from sqlbuild._native import ( + CompiledSource, + CompileModelConfig, + PythonSqlRunSelection, build_python_node_graph, compiled_graph_keys, resolve_python_sql_run_selection, @@ -20,8 +23,6 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompiledSource, - CompileModelConfig, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, @@ -39,7 +40,7 @@ ) from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunSelection +from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from sqlbuild.providers import Provider from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py index 560bcec84c..9af5d91594 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py @@ -4,14 +4,10 @@ import pytest -from sqlbuild._native import build_python_sql_run_lifecycle +from sqlbuild._native import PythonSqlRunSelection, build_python_sql_run_lifecycle from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.python_nodes.models import ( - PythonNodeGraph, - PythonSqlRunLifecyclePlan, - PythonSqlRunSelection, -) +from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunLifecyclePlan from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonSqlRunLifecycleTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py index a34cd53d4b..452fbbdd0e 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py @@ -4,8 +4,9 @@ import pytest +from sqlbuild._native import PythonSqlRunSelection from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunSelection +from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonSqlSelectorErrorTestCase, PythonSqlSelectorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py index 86b9b54830..5bdff1d5fb 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py @@ -4,9 +4,12 @@ import pytest -from sqlbuild._native import parse_project_selector, resolve_python_node_selection -from sqlbuild.compiler.planner.models import ParsedSelector -from sqlbuild.compiler.planner.types import SelectorKind +from sqlbuild._native import ( + ParsedSelector, + SelectorKind, + parse_project_selector, + resolve_python_node_selection, +) from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonNodeParseSelectorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py index 47297041ae..aef7c08cb9 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py @@ -4,10 +4,10 @@ import pytest -from sqlbuild._native import resolve_project_selectors +from sqlbuild._native import PythonSqlRunSelection, resolve_project_selectors from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunSelection +from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from sqlbuild.refs import seed from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonSqlSelectorErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py index 435c399591..9fbe192b5e 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py @@ -7,6 +7,15 @@ import pytest from sqlbuild._native import ( + DiscoveredAuditBlock, + DiscoveredAuditFile, + DiscoveredConstantFile, + DiscoveredEnumFile, + DiscoveredSqlFunctionFile, + DiscoveredSqlHookFile, + DiscoveredSqlTestBlock, + EnumMember, + ScopeValidationError, build_scope_index, build_scope_lookup, normalize_sql_value, @@ -17,21 +26,12 @@ from sqlbuild.compiler.compile.models import LoadedMacro from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, - DiscoveredAuditBlock, - DiscoveredAuditFile, - DiscoveredConstantFile, - DiscoveredEnumFile, DiscoveredMacroFile, DiscoveredSourceFile, - DiscoveredSqlFunctionFile, - DiscoveredSqlHookFile, DiscoveredSqlScenarioFile, - DiscoveredSqlTestBlock, DiscoveredSqlTestFile, EnumDeclaration, - EnumMember, ) -from sqlbuild.compiler.scopes.exceptions import ScopeValidationError from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py b/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py index 49ea1b4695..696d092b9c 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py @@ -4,8 +4,12 @@ import pytest -from sqlbuild._native import build_scope_lookup, explain_scope_declaration, query_scope_report -from sqlbuild.compiler.scopes.exceptions import InvalidQualifiedIdentityError +from sqlbuild._native import ( + InvalidQualifiedIdentityError, + build_scope_lookup, + explain_scope_declaration, + query_scope_report, +) from sqlbuild.compiler.scopes.models import ( DeclarationExplanation, DeclarationIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py b/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py index 2542649660..b51ef5ece4 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import normalize_scope_path -from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError +from sqlbuild._native import InvalidScopePathError, normalize_scope_path from tests.unit.src.sqlbuild.compiler.scopes._test_types import PathNormalizationCase diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py b/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py index 532f665da8..3d64124671 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py @@ -5,10 +5,10 @@ import pytest from sqlbuild._native import ( + ScopeValidationError, get_placement_validated_scope_index, validate_scope_index, ) -from sqlbuild.compiler.scopes.exceptions import ScopeValidationError from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py index dfb9b335ad..f9f2dd9bfc 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py @@ -7,12 +7,12 @@ from sqlbuild._native import ( AdapterSourceFreshnessBatch, + DirectSourceFreshnessPlanningResult, SourceFreshnessUnknownReason, build_direct_source_freshness_planning_result, observe_adapter_sources_freshness, ) from sqlbuild.adapter.contract.models import TableFreshnessMetadata -from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult from sqlbuild.observability import ( EventDispatcher, LifecycleEvent, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py index cc450556d5..d5b889758c 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ( + DirectSourceFreshnessPlanningResult, SourceFreshnessRenderers, build_direct_source_freshness_planning_result, source_freshness_data_version_hash, @@ -15,10 +16,7 @@ from sqlbuild.adapter.contract.models import TableFreshnessMetadata, TableFreshnessRequest from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.models import ( - DirectSourceFreshnessPlanningResult, - SourceFreshnessRecord, -) +from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from sqlbuild.spec.contracts.models import ( SourceEntry, SourceFreshnessAgePolicy, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_propagation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_propagation.py index 9a8cf4f190..0a1d900fe8 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_propagation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_propagation.py @@ -3,12 +3,12 @@ import pytest from sqlbuild._native import ( + DirectSourceFreshnessPlanningResult, DirectSourceFreshnessPropagationResult, + PlannerScope, SourceFreshnessAgeStatus, build_direct_source_freshness_propagation_result, ) -from sqlbuild.compiler.planner.models import PlannerScope -from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( DirectSourceFreshnessPropagationTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py index 8dce30ffd0..a831edc90e 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py @@ -6,8 +6,7 @@ from typing import TYPE_CHECKING, cast import sqlbuild._native as _native -from sqlbuild._native import BindingCatalog -from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic, SqlBindingResult +from sqlbuild._native import BindingCatalog, SqlBindingDiagnostic, SqlBindingResult if TYPE_CHECKING: from sqlbuild._native import ( diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_schema_validation_scopes.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_schema_validation_scopes.py index c0097fbf37..3c64529745 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_schema_validation_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_schema_validation_scopes.py @@ -2,8 +2,12 @@ import pytest -from sqlbuild._native import BindingCatalog, schema_validations -from sqlbuild.compiler.sql_analysis.models import SqlBindingResult, SqlSchemaValidationRequest +from sqlbuild._native import ( + BindingCatalog, + SqlBindingResult, + SqlSchemaValidationRequest, + schema_validations, +) from tests.unit.src.sqlbuild.compiler.sql_analysis.main._test_types import ( ExactColumnCatalogValidationTestCase, SchemaValidationScopeTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_semantic_diagnostics.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_semantic_diagnostics.py index 8a20c391f7..5920df37c4 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_semantic_diagnostics.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_semantic_diagnostics.py @@ -7,8 +7,7 @@ import pytest -from sqlbuild._native import BindingCatalog -from sqlbuild.compiler.sql_analysis.models import SqlBindingResult +from sqlbuild._native import BindingCatalog, SqlBindingResult from tests.unit.src.sqlbuild.compiler.sql_analysis.main._test_types import ( SemanticDiagnosticMappingCase, ) diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py index cbca5cf59e..985dd7a269 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py @@ -3,6 +3,7 @@ from pathlib import Path from typing import Any, ClassVar +from sqlbuild._native import CompileModelConfig from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.compile.models import ( @@ -10,7 +11,6 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompileModelConfig, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, diff --git a/tests/unit/src/sqlbuild/executor/diff/test_bounds.py b/tests/unit/src/sqlbuild/executor/diff/test_bounds.py index 3bb424e98e..fa93edb280 100644 --- a/tests/unit/src/sqlbuild/executor/diff/test_bounds.py +++ b/tests/unit/src/sqlbuild/executor/diff/test_bounds.py @@ -2,11 +2,11 @@ import pytest +from sqlbuild._native import ConfigValueTypeError from sqlbuild.adapter.contract.models import CursorValue from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.diff._helpers.bounds import resolve_bounded_cursors -from sqlbuild.spec.contracts.exceptions import ConfigValueTypeError from tests.unit.src.sqlbuild.executor.diff._test_types import ( ResolveBoundedCursorsErrorTestCase, ResolveBoundedCursorsTestCase, diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py index db317f90d9..4a3bc59c44 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py @@ -4,21 +4,6 @@ from datetime import datetime -@dataclass(frozen=True) -class ArchiveNameParseTestCase: - description: str - name: str - expected_archived_at: datetime - expected_logical_name: str - - -@dataclass(frozen=True) -class ArchiveNameRejectTestCase: - description: str - name: str - expected_lookalike: bool - - @dataclass(frozen=True) class ArchiveNameBuildTestCase: description: str @@ -29,11 +14,3 @@ class ArchiveNameBuildTestCase: expected_length: int expected_logical_name_is_original: bool kind: str | None = None - - -@dataclass(frozen=True) -class RelationAddressingTestCase: - description: str - name: str - listing: tuple[str, ...] - expected_reason: str | None diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py index a552c637d7..e035a8f4d0 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py @@ -1,93 +1,20 @@ from __future__ import annotations +import re from datetime import UTC, datetime import pytest -from sqlbuild.executor.janitor._helpers.archive_names import ( - build_archive_name, - is_archive_lookalike_name, - parse_archive_name, -) -from sqlbuild.executor.janitor.models import JanitorParsedArchiveName +from sqlbuild._native import build_janitor_archive_name from tests.unit.src.sqlbuild.executor.janitor._helpers._test_types import ( ArchiveNameBuildTestCase, - ArchiveNameParseTestCase, - ArchiveNameRejectTestCase, ) ARCHIVED_AT: datetime = datetime(2026, 9, 24, 10, 15, 0, tzinfo=UTC) LONG_NAME: str = "fulfillment_orders_" + "y" * 80 - - -@pytest.mark.parametrize( - "test_case", - [ - ArchiveNameParseTestCase( - description="canonical archive name parses", - name="_SQB_ARCHIVE__20260924T101500Z__orders", - expected_archived_at=ARCHIVED_AT, - expected_logical_name="orders", - ), - ArchiveNameParseTestCase( - description="lowercase folded archive name parses", - name="_sqb_archive__20260924t101500z__orders", - expected_archived_at=ARCHIVED_AT, - expected_logical_name="orders", - ), - ], - ids=lambda case: case.description, -) -def test_given_strict_archive_name_when_parsing_then_returns_timestamp_and_logical_name( - test_case: ArchiveNameParseTestCase, -) -> None: - parsed: JanitorParsedArchiveName | None = parse_archive_name(test_case.name) - - assert parsed == JanitorParsedArchiveName( - archived_at=test_case.expected_archived_at, - logical_name=test_case.expected_logical_name, - ) - assert is_archive_lookalike_name(test_case.name) is True - - -@pytest.mark.parametrize( - "test_case", - [ - ArchiveNameRejectTestCase( - description="impossible calendar timestamp is malformed", - name="_SQB_ARCHIVE__20261324T101500Z__orders", - expected_lookalike=True, - ), - ArchiveNameRejectTestCase( - description="missing timezone marker is malformed", - name="_SQB_ARCHIVE__20260924T101500__orders", - expected_lookalike=True, - ), - ArchiveNameRejectTestCase( - description="single separator after prefix is malformed", - name="_SQB_ARCHIVE_20260924T101500Z__orders", - expected_lookalike=True, - ), - ArchiveNameRejectTestCase( - description="empty original name is malformed", - name="_SQB_ARCHIVE__20260924T101500Z__", - expected_lookalike=True, - ), - ArchiveNameRejectTestCase( - description="ordinary relation is neither archive nor look-alike", - name="orders_archive", - expected_lookalike=False, - ), - ], - ids=lambda case: case.description, +ARCHIVE_NAME_PATTERN: re.Pattern[str] = re.compile( + r"^_sqb_archive__20260924t101500z__(?P.+)$" ) -def test_given_non_strict_name_when_parsing_archive_name_then_returns_none( - test_case: ArchiveNameRejectTestCase, -) -> None: - parsed: JanitorParsedArchiveName | None = parse_archive_name(test_case.name) - - assert parsed is None - assert is_archive_lookalike_name(test_case.name) is test_case.expected_lookalike @pytest.mark.parametrize( @@ -134,21 +61,20 @@ def test_given_non_strict_name_when_parsing_archive_name_then_returns_none( ], ids=lambda case: case.description, ) -def test_given_original_name_when_building_archive_name_then_round_trips_through_parser( +def test_given_original_name_when_building_archive_name_then_keeps_timestamp_and_fits( test_case: ArchiveNameBuildTestCase, ) -> None: - archive_name: str = build_archive_name( + archive_name: str = build_janitor_archive_name( original_name=test_case.original_name, archived_at=test_case.archived_at, identifier_limit=test_case.identifier_limit, kind=test_case.kind, ) - parsed: JanitorParsedArchiveName | None = parse_archive_name(archive_name) + parsed: re.Match[str] | None = ARCHIVE_NAME_PATTERN.fullmatch(archive_name) assert archive_name.startswith(test_case.expected_prefix) assert len(archive_name) == test_case.expected_length assert parsed is not None - assert parsed.archived_at == ARCHIVED_AT - assert (parsed.logical_name == test_case.original_name) is ( + assert (parsed.group("logical_name") == test_case.original_name) is ( test_case.expected_logical_name_is_original ) diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_relation_addressing.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_relation_addressing.py deleted file mode 100644 index f5bc3a9feb..0000000000 --- a/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_relation_addressing.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.executor.janitor._helpers.relation_addressing import ( - case_colliding_names, - unaddressable_relation_reason, -) -from sqlbuild.executor.janitor.constants import ( - CASE_COLLISION_REASON, - UNQUOTED_ADDRESSING_REASON, -) -from tests.unit.src.sqlbuild.executor.janitor._helpers._test_types import ( - RelationAddressingTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - RelationAddressingTestCase( - description="plain lowercase name is addressable", - name="old_orders", - listing=("old_orders", "customers"), - expected_reason=None, - ), - RelationAddressingTestCase( - description="lowercase name with digits dollar and leading underscore is addressable", - name="_orders$2026", - listing=("_orders$2026",), - expected_reason=None, - ), - RelationAddressingTestCase( - description="uppercase name is skipped", - name="OLD_ORDERS", - listing=("OLD_ORDERS",), - expected_reason=UNQUOTED_ADDRESSING_REASON, - ), - RelationAddressingTestCase( - description="mixed-case name is skipped", - name="OldOrders", - listing=("OldOrders",), - expected_reason=UNQUOTED_ADDRESSING_REASON, - ), - RelationAddressingTestCase( - description="name with punctuation is skipped", - name="old-orders;drop", - listing=("old-orders;drop",), - expected_reason=UNQUOTED_ADDRESSING_REASON, - ), - RelationAddressingTestCase( - description="name with a leading digit is skipped", - name="2026_orders", - listing=("2026_orders",), - expected_reason=UNQUOTED_ADDRESSING_REASON, - ), - RelationAddressingTestCase( - description="lowercase name colliding case-insensitively is skipped", - name="old_orders", - listing=("old_orders", "OLD_ORDERS"), - expected_reason=CASE_COLLISION_REASON, - ), - RelationAddressingTestCase( - description="uppercase side of a case collision is skipped", - name="OLD_ORDERS", - listing=("old_orders", "OLD_ORDERS"), - expected_reason=UNQUOTED_ADDRESSING_REASON, - ), - ], - ids=lambda case: case.description, -) -def test_given_catalog_name_when_checking_addressing_then_returns_expected_reason( - test_case: RelationAddressingTestCase, -) -> None: - reason: str | None = unaddressable_relation_reason( - name=test_case.name, colliding_names=case_colliding_names(test_case.listing) - ) - - assert reason == test_case.expected_reason diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py index 8561369fad..445c7dbf35 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py @@ -117,14 +117,6 @@ class JanitorAddressingPlanTestCase: expected_candidate_names: tuple[str, ...] = field(default_factory=tuple) -@dataclass(frozen=True) -class JanitorRetentionResolutionTestCase: - description: str - override: int | None - configured: int | None - expected_retention_days: int - - @dataclass(frozen=True) class JanitorFoldedIdentifierPlanTestCase: description: str diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index 05cb83d893..43417dd2fc 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Any, ClassVar, cast +from sqlbuild._native import CompiledSeed, CompiledSource, CompileModelConfig from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( @@ -25,9 +26,6 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompiledSeed, - CompiledSource, - CompileModelConfig, ) from sqlbuild.compiler.compile.types import ( CompiledProjectView, @@ -40,6 +38,17 @@ from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS from sqlbuild.spec.contracts.models import SchemaSeedEntry, SeedCsvSettings, SourceEntry +UNQUOTED_ADDRESSING_REASON: str = ( + "relation name is not a plain lowercase identifier and may require quoting; " + "janitor does not act on it" +) +CASE_COLLISION_REASON: str = ( + "relation name collides case-insensitively with another relation; janitor does not act on it" +) +MALFORMED_ARCHIVE_REASON: str = ( + "name resembles a janitor archive but does not match the strict archive grammar" +) + class FakeJanitorAdapter(BaseAdapter): adapter_name: ClassVar[str] = "janitor-test" diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py index 16c14497cc..5ad86f4cfa 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py @@ -6,10 +6,9 @@ import pytest +from sqlbuild._native import build_janitor_plan from sqlbuild.executor.janitor.main.execute import execute_janitor_plan -from sqlbuild.executor.janitor.main.plan import build_janitor_plan from sqlbuild.executor.janitor.models import ( - JanitorDirectModeSettings, JanitorExecutionResult, JanitorPlan, ) @@ -73,9 +72,8 @@ def test_given_direct_archive_plan_when_executing_then_renames_drops_and_writes_ connection=object(), retention_days=7, delete_tracked_only=False, - direct_settings=JanitorDirectModeSettings( - enabled=True, archive_retention_days=test_case.archive_retention_days - ), + direct_mode=True, + archive_retention_days=test_case.archive_retention_days, ) result: JanitorExecutionResult = execute_janitor_plan( @@ -150,7 +148,7 @@ def test_given_audit_write_failure_when_archiving_then_rename_persists_and_error connection=object(), retention_days=7, delete_tracked_only=False, - direct_settings=JanitorDirectModeSettings(enabled=True), + direct_mode=True, ) with pytest.raises(RuntimeError) as exc_info: diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py index cfbbbf145c..fbfd2166cc 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py @@ -5,18 +5,16 @@ import pytest from sqlbuild._native import ( + build_janitor_plan, build_scenario_artifact_name, compute_scenario_hash_prefix, ) from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.executor.janitor.main.execute import execute_janitor_plan -from sqlbuild.executor.janitor.main.plan import build_janitor_plan from sqlbuild.executor.janitor.models import ( - JanitorDirectModeSettings, JanitorExecutionResult, JanitorPlan, - JanitorRelationScope, ) from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( JanitorExecuteTestCase, @@ -235,13 +233,9 @@ def test_given_project_and_warehouse_when_building_janitor_plan_then_returns_exp retention_days=test_case.retention_days, delete_tracked_only=test_case.delete_tracked_only, exclude_patterns=test_case.exclude_patterns, - relation_scope=JanitorRelationScope( - protected_relation_keys=test_case.protected_relation_keys, - ), - direct_settings=JanitorDirectModeSettings( - enabled=test_case.direct_mode, - state_history_versions=test_case.direct_state_history_versions, - ), + protected_relation_keys=test_case.protected_relation_keys, + direct_mode=test_case.direct_mode, + state_history_versions=test_case.direct_state_history_versions, ) assert tuple(candidate.key.name for candidate in plan.candidates) == ( @@ -297,7 +291,7 @@ def test_given_janitor_plan_when_executing_then_drops_expected_relations( connection=object(), retention_days=7, delete_tracked_only=False, - direct_settings=JanitorDirectModeSettings(enabled=test_case.direct_mode), + direct_mode=test_case.direct_mode, ) result: JanitorExecutionResult = execute_janitor_plan( diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py index 547b933001..723dfe98ff 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py @@ -7,15 +7,9 @@ import pytest -from sqlbuild.executor.janitor.constants import ( - CASE_COLLISION_REASON, - MALFORMED_ARCHIVE_REASON, - UNQUOTED_ADDRESSING_REASON, -) -from sqlbuild.executor.janitor.main.plan import build_janitor_plan +from sqlbuild._native import build_janitor_plan from sqlbuild.executor.janitor.models import ( JanitorArchivedRelation, - JanitorDirectModeSettings, JanitorPlan, JanitorRelationKey, JanitorSkippedRelation, @@ -26,6 +20,9 @@ relation_info, ) from tests.unit.src.sqlbuild.executor.janitor.main.helpers import ( + CASE_COLLISION_REASON, + MALFORMED_ARCHIVE_REASON, + UNQUOTED_ADDRESSING_REASON, FakeJanitorAdapter, build_project, ) @@ -127,10 +124,8 @@ def test_given_direct_warehouse_when_planning_janitor_then_plans_archives_by_str retention_days=test_case.retention_days, delete_tracked_only=False, exclude_patterns=test_case.exclude_patterns, - direct_settings=JanitorDirectModeSettings( - enabled=test_case.direct_mode, - archive_retention_days=test_case.archive_retention_days, - ), + direct_mode=test_case.direct_mode, + archive_retention_days=test_case.archive_retention_days, ) assert ( @@ -202,7 +197,7 @@ def test_given_long_relation_name_when_planning_archive_then_fits_name_with_time connection=object(), retention_days=test_case.retention_days, delete_tracked_only=False, - direct_settings=JanitorDirectModeSettings(enabled=True), + direct_mode=True, ) assert ( @@ -262,7 +257,7 @@ def test_given_unaddressable_names_when_planning_direct_janitor_then_they_are_sk connection=object(), retention_days=7, delete_tracked_only=False, - direct_settings=JanitorDirectModeSettings(enabled=test_case.direct_mode), + direct_mode=test_case.direct_mode, ) assert ( diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_case_safety.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_case_safety.py index 885fbc3d2e..ed5bd1bcc4 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_case_safety.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_case_safety.py @@ -4,9 +4,9 @@ import pytest +from sqlbuild._native import build_janitor_plan from sqlbuild.executor.janitor.main.execute import execute_janitor_plan -from sqlbuild.executor.janitor.main.plan import build_janitor_plan -from sqlbuild.executor.janitor.models import JanitorDirectModeSettings, JanitorPlan +from sqlbuild.executor.janitor.models import JanitorPlan from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( JanitorCaseSafetyTestCase, relation_info, @@ -48,7 +48,7 @@ def test_given_case_variant_relations_when_running_janitor_then_excluded_tables_ connection=object(), retention_days=0, exclude_patterns=test_case.exclude_patterns, - direct_settings=JanitorDirectModeSettings(enabled=test_case.direct_mode), + direct_mode=test_case.direct_mode, ) execute_janitor_plan(plan=plan, adapter=adapter, connection=object()) diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_folded_identifiers.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_folded_identifiers.py index 01024ea183..c18a37970a 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_folded_identifiers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_folded_identifiers.py @@ -8,12 +8,10 @@ import pytest -from sqlbuild.executor.janitor.main.plan import build_janitor_plan +from sqlbuild._native import build_janitor_plan from sqlbuild.executor.janitor.models import ( JanitorDeleteCandidate, - JanitorDirectModeSettings, JanitorPlan, - JanitorRelationScope, ) from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( JanitorFoldedIdentifierPlanTestCase, @@ -128,10 +126,8 @@ def test_given_folded_catalog_identifiers_when_planning_janitor_then_keys_compar connection=object(), retention_days=7, delete_tracked_only=test_case.delete_tracked_only, - relation_scope=JanitorRelationScope( - protected_relation_keys=test_case.protected_relation_keys - ), - direct_settings=JanitorDirectModeSettings(enabled=test_case.direct_mode), + protected_relation_keys=test_case.protected_relation_keys, + direct_mode=test_case.direct_mode, ) assert ( diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py index 3c008eea98..6f31af948a 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py @@ -6,11 +6,9 @@ import pytest -from sqlbuild.executor.janitor.main.plan import build_janitor_plan +from sqlbuild._native import build_janitor_plan from sqlbuild.executor.janitor.models import ( - JanitorDirectModeSettings, JanitorPlan, - JanitorRelationScope, ) from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( JanitorAgeReadScopeTestCase, @@ -63,7 +61,7 @@ def test_given_adapter_reading_ages_separately_when_planning_janitor_then_uses_t connection=object(), retention_days=7, delete_tracked_only=False, - direct_settings=JanitorDirectModeSettings(enabled=True), + direct_mode=True, ) assert ( @@ -152,10 +150,8 @@ def test_given_filtered_relations_when_planning_janitor_then_ages_are_read_for_c retention_days=test_case.retention_days, delete_tracked_only=test_case.delete_tracked_only, exclude_patterns=test_case.exclude_patterns, - relation_scope=JanitorRelationScope( - protected_relation_keys=test_case.protected_relation_keys - ), - direct_settings=JanitorDirectModeSettings(enabled=test_case.direct_mode), + protected_relation_keys=test_case.protected_relation_keys, + direct_mode=test_case.direct_mode, ) assert tuple(adapter.age_metadata_requests) == test_case.expected_age_requests diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_resolve_retention_days.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_resolve_retention_days.py deleted file mode 100644 index 2afce5d559..0000000000 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_resolve_retention_days.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Tests for mode-specific janitor retention resolution.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.executor.janitor.main.resolve_retention_days import resolve_janitor_retention_days -from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( - JanitorRetentionResolutionTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - JanitorRetentionResolutionTestCase( - description="unset retention in direct mode defaults to 14", - override=None, - configured=None, - expected_retention_days=14, - ), - JanitorRetentionResolutionTestCase( - description="explicit retention applies in direct mode", - override=None, - configured=7, - expected_retention_days=7, - ), - JanitorRetentionResolutionTestCase( - description="cli override wins over explicit config", - override=3, - configured=7, - expected_retention_days=3, - ), - JanitorRetentionResolutionTestCase( - description="cli zero override wins over the mode default", - override=0, - configured=None, - expected_retention_days=0, - ), - ], - ids=lambda case: case.description, -) -def test_given_retention_inputs_when_resolving_janitor_retention_then_returns_expected_days( - test_case: JanitorRetentionResolutionTestCase, -) -> None: - resolved: int = resolve_janitor_retention_days( - override=test_case.override, - configured=test_case.configured, - ) - - assert resolved == test_case.expected_retention_days diff --git a/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py index 83fc645c18..8188fa6641 100644 --- a/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py @@ -8,16 +8,16 @@ import duckdb -from sqlbuild._native import deterministic_column_migration_event_id +from sqlbuild._native import ( + MigrationDiscovery, + OldNameViewAction, + deterministic_column_migration_event_id, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation -from sqlbuild.compiler.migrations.types import ( - ColumnMigrationDecision, - MigrationDiscovery, - OldNameViewAction, -) +from sqlbuild.compiler.migrations.types import ColumnMigrationDecision from sqlbuild.compiler.planner.models import ColumnMigrationPlanEntry, OldNameViewPlanEntry RELATION: str = "main.fct_orders" diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index 1d79f6747a..cbdb2539f1 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -11,7 +11,12 @@ from pathlib import Path from typing import Any, ClassVar, cast -from sqlbuild._native import CostContext, build_python_node_graph, build_python_sql_run_lifecycle +from sqlbuild._native import ( + CostContext, + PythonSqlRunSelection, + build_python_node_graph, + build_python_sql_run_lifecycle, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.assets import AssetContext, asset @@ -27,7 +32,6 @@ DiscoveredPythonNode, PythonNodeGraph, PythonSqlRunLifecyclePlan, - PythonSqlRunSelection, ) from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cost.models import CostResourceContext diff --git a/tests/unit/src/sqlbuild/executor/testing/main/helpers.py b/tests/unit/src/sqlbuild/executor/testing/main/helpers.py index daf30a3ff9..85e5494de1 100644 --- a/tests/unit/src/sqlbuild/executor/testing/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/testing/main/helpers.py @@ -1,3 +1,4 @@ +from sqlbuild._native import SqlTestAssertionStep from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -5,7 +6,7 @@ from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ChainStep, SqlTestAssertionStep, SqlTestPlanEntry +from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry def build_comparison_test_entry(*, sql_analysis_enabled: bool = True) -> SqlTestPlanEntry: diff --git a/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py b/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py index 0a49893e90..ff5d42a301 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py @@ -3,8 +3,7 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild._native import DbtInteropParsedArgs -from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtCommandResult +from sqlbuild._native import DbtCliOptions, DbtCommandResult, DbtInteropParsedArgs from sqlbuild.spec.contracts.models import DbtConfig, LocalDbtConfig diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index d11d0408dc..5fbed29ce9 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -10,8 +10,12 @@ from sqlbuild._native import ( BackfillResult, + CompileModelConfig, + CompileProjectInputs, DbtCliConfigOverrides, + DbtCliOptions, DbtCombinedGraph, + DbtCommandResult, DbtEventStream, DbtEventStreamUpdate, DbtRunner, @@ -25,9 +29,7 @@ from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, - CompileModelConfig, CompileModelInput, - CompileProjectInputs, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile @@ -39,7 +41,6 @@ PlanReason, ) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtCommandResult from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig _GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() diff --git a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_connection.py b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_connection.py index ba1f410299..cdffc0973a 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_connection.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_connection.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild._native import resolve_raw_dbt_profile_connection +from sqlbuild._native import NormalizedDbtProfileConnection, resolve_raw_dbt_profile_connection from sqlbuild.integrations.dbt.exceptions import DbtProfileError -from sqlbuild.integrations.dbt.models import NormalizedDbtProfileConnection from tests.unit.src.sqlbuild.integrations.dbt.profile._test_types import ( DbtProfileConnectionErrorTestCase, DbtProfileConnectionTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_init.py b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_init.py index 985c8bf50c..c640cbd226 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_init.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_init.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import build_dbt_init_project -from sqlbuild.integrations.dbt.models import DbtInitRequest, DbtInitResult +from sqlbuild._native import DbtInitRequest, DbtInitResult, build_dbt_init_project from tests.unit.src.sqlbuild.integrations.dbt.profile._test_types import ( DbtProfileInitDiscoveryTestCase, DbtProfileInitTomlTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py index dc0ec6ecd7..0cf38d3fce 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py @@ -4,12 +4,9 @@ import pytest -from sqlbuild._native import normalize_dbt_profile_connection +from sqlbuild._native import NormalizedDbtProfileConnection, normalize_dbt_profile_connection from sqlbuild.integrations.dbt.exceptions import DbtProfileError -from sqlbuild.integrations.dbt.models import ( - NormalizedDbtProfileConnection, - ResolvedDbtProfileOutput, -) +from sqlbuild.integrations.dbt.models import ResolvedDbtProfileOutput from tests.unit.src.sqlbuild.integrations.dbt.profile._test_types import ( DbtProfileNormalizeErrorTestCase, DbtProfileNormalizeTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_args.py b/tests/unit/src/sqlbuild/integrations/dbt/test_args.py index bc2ac4a2e9..1ffbb861cb 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_args.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_args.py @@ -3,13 +3,13 @@ import pytest from sqlbuild._native import ( + DbtInteropArgumentError, DbtInteropCommand, DbtInteropParsedArgs, DbtInteropRoutedArgs, parse_dbt_execution_args, route_dbt_interop_args, ) -from sqlbuild.integrations.dbt.exceptions import DbtInteropArgumentError from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtArgParseErrorTestCase, DbtArgParseTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_config.py b/tests/unit/src/sqlbuild/integrations/dbt/test_config.py index de65571383..d08b928f99 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_config.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_config.py @@ -4,8 +4,12 @@ import pytest -from sqlbuild._native import ResolvedDbtConfig, resolve_dbt_config, resolve_dbt_vars_mapping -from sqlbuild.integrations.dbt.exceptions import DbtInteropConfigError +from sqlbuild._native import ( + DbtInteropConfigError, + ResolvedDbtConfig, + resolve_dbt_config, + resolve_dbt_vars_mapping, +) from sqlbuild.spec.contracts.models import DbtConfig, LocalDbtConfig from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtConfigErrorTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_debug.py b/tests/unit/src/sqlbuild/integrations/dbt/test_debug.py index 3f76d490da..7f68f7a5ff 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_debug.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_debug.py @@ -5,8 +5,7 @@ import pytest -from sqlbuild._native import DbtRunner, debug_dbt_from_project -from sqlbuild.integrations.dbt.models import DbtCommandResult +from sqlbuild._native import DbtCommandResult, DbtRunner, debug_dbt_from_project from tests.unit.src.sqlbuild.integrations.dbt._test_types import DbtDebugPipelineTestCase from tests.unit.src.sqlbuild.integrations.dbt.helpers import RecordingDbtInvoker diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_event_stream.py b/tests/unit/src/sqlbuild/integrations/dbt/test_event_stream.py index 4fb1ecf21c..18a0258d17 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_event_stream.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_event_stream.py @@ -10,10 +10,10 @@ from sqlbuild._native import ( DbtEventStream, DbtEventStreamUpdate, + DbtInteropRuntimeError, + DbtNodeExecutionResult, execute_dbt_json_event_stream, ) -from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError -from sqlbuild.integrations.dbt.models import DbtNodeExecutionResult from sqlbuild.observability import ( EventDispatcher, LifecycleEvent, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py index 4a6051fd9f..6396fd1f88 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py @@ -13,6 +13,7 @@ from sqlbuild import _native as phases_module from sqlbuild import _native as plan_module from sqlbuild._native import ( + DbtCliOptions, DbtCombinedGraph, DbtCommandExecutionResult, DbtInteropCommand, @@ -25,6 +26,7 @@ DbtInteropSelectionResult, DbtLsNode, DbtManifestIndex, + DbtNodeExecutionResult, DbtRunner, build_dbt_interop_plan, build_failed_sqlbuild_model_names, @@ -34,7 +36,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtNodeExecutionResult from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtCompileFullRefreshPipelineTestCase, DbtExecutionSelectionStatusTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py b/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py index 60aa3dd288..989a060164 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py @@ -5,9 +5,12 @@ import pytest from sqlbuild._native import ( + DbtCliOptions, DbtCombinedGraph, + DbtCommandResult, DbtInteropCommandArgs, DbtInteropPlan, + DbtInteropRuntimeError, DbtManifestIndex, DbtRunner, build_dbt_combined_graph, @@ -16,8 +19,6 @@ plan_dbt_interop_command, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError -from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtCommandResult from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtPlanOrchestrationErrorTestCase, DbtPlanOrchestrationTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_runner.py b/tests/unit/src/sqlbuild/integrations/dbt/test_runner.py index 0aad6d5cb8..041f4b0407 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_runner.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_runner.py @@ -6,7 +6,11 @@ import pytest from sqlbuild._native import ( + DbtCliOptions, + DbtCommandResult, + DbtInteropRuntimeError, DbtLsNode, + DbtLsResult, DbtRunner, build_dbt_compile_argv, build_dbt_debug_argv, @@ -15,12 +19,6 @@ dbt_fqn_selector_term, parse_dbt_ls_json_lines, ) -from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError -from sqlbuild.integrations.dbt.models import ( - DbtCliOptions, - DbtCommandResult, - DbtLsResult, -) from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtArgvTestCase, DbtLsParseTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py b/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py index 8f96f16465..370ecb87c9 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py @@ -4,6 +4,7 @@ from sqlbuild._native import ( DbtCombinedGraph, + DbtInteropArgumentError, DbtInteropSelectionResult, DbtManifestIndex, build_dbt_combined_graph, @@ -11,7 +12,6 @@ resolve_dbt_interop_sqlbuild_selection, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.integrations.dbt.exceptions import DbtInteropArgumentError from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtSelectionErrorTestCase, DbtSelectionTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dlt/_test_types.py b/tests/unit/src/sqlbuild/integrations/dlt/_test_types.py index ee1d73ea38..5df8511ae4 100644 --- a/tests/unit/src/sqlbuild/integrations/dlt/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/dlt/_test_types.py @@ -8,9 +8,9 @@ @dataclass(frozen=True) class DltLoaderDiscoveryTestCase: description: str + repo_files: dict[str, str] expected_loader_names: tuple[str, ...] - expected_relative_path: str - expected_function_name: str + expected_relative_paths: tuple[str, ...] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/integrations/dlt/test_dlt_loaders.py b/tests/unit/src/sqlbuild/integrations/dlt/test_dlt_loaders.py index 86be6f37f5..564aa3cf45 100644 --- a/tests/unit/src/sqlbuild/integrations/dlt/test_dlt_loaders.py +++ b/tests/unit/src/sqlbuild/integrations/dlt/test_dlt_loaders.py @@ -2,15 +2,15 @@ from __future__ import annotations +from collections.abc import Callable from pathlib import Path from typing import Any, cast import pytest -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction, DiscoveredSourceFile -from sqlbuild.integrations.dlt.main.loaders import build_dlt_loader_functions -from sqlbuild.integrations.dlt.models import DltResourceConfig, DltSourceConfig -from sqlbuild.spec.contracts.models import IntegrationLoaderConfig, SourceEntry +from sqlbuild._native import discover_project_inputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.discovery.types import LoaderConnectionMode from tests.unit.src.sqlbuild.integrations.dlt._test_types import DltLoaderDiscoveryTestCase @@ -18,44 +18,45 @@ "test_case", [ DltLoaderDiscoveryTestCase( - description="builds deterministic synthetic loader", - expected_loader_names=("raw_orders",), - expected_relative_path="sources/raw.yml", - expected_function_name="raw_orders", + description="builds deterministic synthetic loaders per resource", + repo_files={ + "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', + "sources/raw.yml": ( + "dlt_sources:\n" + " - type: sql_database\n" + " config:\n" + ' credentials: "duckdb:///source.duckdb"\n' + " resources:\n" + " - name: raw_orders\n" + " table: orders\n" + " - name: raw_customers\n" + " table: customers\n" + ), + }, + expected_loader_names=("raw_orders", "raw_customers"), + expected_relative_paths=("sources/raw.yml", "sources/raw.yml"), ) ], ids=lambda case: case.description, ) -def test_given_dlt_source_when_building_loader_functions_then_name_is_deterministic( +def test_given_dlt_source_when_discovering_inputs_then_loader_functions_are_deterministic( test_case: DltLoaderDiscoveryTestCase, + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], ) -> None: - source_file: DiscoveredSourceFile = DiscoveredSourceFile( - file_path=Path("sources/raw.yml"), - relative_path=Path("sources/raw.yml"), - contents="", - source_entries=( - SourceEntry( - name="raw_orders", - loader="raw_orders", - integration_loader=IntegrationLoaderConfig( - kind="dlt", - config=DltSourceConfig( - source_type="sql_database", - config={"credentials": "duckdb:///source.duckdb"}, - destination={}, - resource=DltResourceConfig( - name="raw_orders", dlt_name="orders", raw_config={} - ), - group_index=0, - ), - ), - ), - ), - ) + write_repo_files(tmp_path, test_case.repo_files) - loaders: tuple[DiscoveredLoaderFunction, ...] = build_dlt_loader_functions((source_file,)) + inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - assert tuple(loader.name for loader in loaders) == test_case.expected_loader_names - assert loaders[0].relative_path == Path(test_case.expected_relative_path) - loader_function: Any = cast(Any, loaders[0].function) - assert loader_function.__name__ == test_case.expected_function_name + assert tuple(loader.name for loader in inputs.loader_functions) == ( + test_case.expected_loader_names + ) + assert tuple(str(loader.relative_path) for loader in inputs.loader_functions) == ( + test_case.expected_relative_paths + ) + assert tuple(cast(Any, loader.function).__name__ for loader in inputs.loader_functions) == ( + test_case.expected_loader_names + ) + assert {loader.connection_mode for loader in inputs.loader_functions} == { + LoaderConnectionMode.EXTERNAL + } diff --git a/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py b/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py index 57b25606d9..591cdf8683 100644 --- a/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py @@ -16,9 +16,9 @@ class IngestrCommandTestCase: @dataclass(frozen=True) class IngestrLoaderDiscoveryTestCase: description: str + repo_files: dict[str, str] expected_loader_names: tuple[str, ...] - expected_relative_path: str - expected_function_name: str + expected_relative_paths: tuple[str, ...] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/integrations/ingestr/test_loaders.py b/tests/unit/src/sqlbuild/integrations/ingestr/test_loaders.py index e23585aef2..29d71abcd3 100644 --- a/tests/unit/src/sqlbuild/integrations/ingestr/test_loaders.py +++ b/tests/unit/src/sqlbuild/integrations/ingestr/test_loaders.py @@ -2,54 +2,83 @@ from __future__ import annotations +from collections.abc import Callable from pathlib import Path from typing import Any, cast import pytest -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction, DiscoveredSourceFile -from sqlbuild.integrations.ingestr.main.loaders import build_ingestr_loader_functions -from sqlbuild.integrations.ingestr.models import IngestrSourceConfig -from sqlbuild.spec.contracts.models import IntegrationLoaderConfig, SourceEntry +from sqlbuild._native import discover_project_inputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.discovery.types import LoaderConnectionMode from tests.unit.src.sqlbuild.integrations.ingestr._test_types import ( IngestrLoaderDiscoveryTestCase, ) +_INGESTR_SOURCES: str = ( + "sources:\n" + " - name: raw_orders\n" + " ingestr:\n" + ' source_uri: "stripe://token"\n' + " source_table: charges\n" + " - name: plain_orders\n" + " schema: public\n" + " table: orders\n" +) +_DLT_SOURCES: str = ( + "dlt_sources:\n" + " - type: sql_database\n" + " config:\n" + ' credentials: "duckdb:///source.duckdb"\n' + " resources:\n" + " - name: dlt_orders\n" + " table: orders\n" +) + @pytest.mark.parametrize( "test_case", [ IngestrLoaderDiscoveryTestCase( description="builds deterministic synthetic loader", + repo_files={ + "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', + "sources/raw.yml": _INGESTR_SOURCES, + }, expected_loader_names=("ingestr__raw_orders",), - expected_relative_path="sources/raw.yml", - expected_function_name="ingestr__raw_orders", - ) + expected_relative_paths=("sources/raw.yml",), + ), + IngestrLoaderDiscoveryTestCase( + description="orders every ingestr loader before every dlt loader", + repo_files={ + "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', + "sources/a_dlt.yml": _DLT_SOURCES, + "sources/b_raw.yml": _INGESTR_SOURCES, + }, + expected_loader_names=("ingestr__raw_orders", "dlt_orders"), + expected_relative_paths=("sources/b_raw.yml", "sources/a_dlt.yml"), + ), ], ids=lambda case: case.description, ) -def test_given_ingestr_source_when_building_loader_functions_then_name_is_deterministic( +def test_given_ingestr_source_when_discovering_inputs_then_loader_functions_are_deterministic( test_case: IngestrLoaderDiscoveryTestCase, + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], ) -> None: - source_file: DiscoveredSourceFile = DiscoveredSourceFile( - file_path=Path("sources/raw.yml"), - relative_path=Path("sources/raw.yml"), - contents="", - source_entries=( - SourceEntry( - name="raw_orders", - loader="ingestr__raw_orders", - integration_loader=IntegrationLoaderConfig( - kind="ingestr", - config=IngestrSourceConfig(source_uri="stripe://token", source_table="charges"), - ), - ), - ), - ) + write_repo_files(tmp_path, test_case.repo_files) - loaders: tuple[DiscoveredLoaderFunction, ...] = build_ingestr_loader_functions((source_file,)) + inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - assert tuple(loader.name for loader in loaders) == test_case.expected_loader_names - assert loaders[0].relative_path == Path(test_case.expected_relative_path) - loader_function: Any = cast(Any, loaders[0].function) - assert loader_function.__name__ == test_case.expected_function_name + assert tuple(loader.name for loader in inputs.loader_functions) == ( + test_case.expected_loader_names + ) + assert tuple(str(loader.relative_path) for loader in inputs.loader_functions) == ( + test_case.expected_relative_paths + ) + assert tuple(cast(Any, loader.function).__name__ for loader in inputs.loader_functions) == ( + test_case.expected_loader_names + ) + assert {loader.connection_mode for loader in inputs.loader_functions} == { + LoaderConnectionMode.EXTERNAL + } diff --git a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py index 2d9277d037..fedae56d34 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py @@ -2,27 +2,25 @@ from __future__ import annotations +from dataclasses import astuple from pathlib import Path from typing import TYPE_CHECKING from sqlbuild._native import ( - lint_native_sql_bodies, - neutralize_lint_interpolation, - restore_lint_interpolation, - scan_headers, - sql_body_ranges, -) -from sqlbuild.lint.models import ( HeaderSpan, InterpolationSite, LintBody, LintConfig, LintViolation, + lint_native_sql_bodies, + neutralize_lint_interpolation, + restore_lint_interpolation, + scan_headers, + sql_body_ranges, ) if TYPE_CHECKING: - from sqlbuild._native import NativePreparedSql - from sqlbuild.lint.types import NativeLintPreparationRequest + from sqlbuild._native import NativeLintPreparationRequest, NativePreparedSql def lint_bodies_for(*, file_path: Path, contents: str) -> tuple[LintBody, ...]: @@ -161,7 +159,7 @@ def neutralize_interpolation( def restore_interpolation(*, fixed: str, sites: tuple[InterpolationSite, ...]) -> tuple[bool, str]: """Restore neutralized interpolation natively; returns whether it succeeded and the text.""" - return restore_lint_interpolation(fixed, [tuple(vars(site).values()) for site in sites]) + return restore_lint_interpolation(fixed, [astuple(site) for site in sites]) def run_native_sql_lint( diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py b/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py index 87595805d7..754ff94dc8 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py @@ -7,9 +7,7 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import run_lint -from sqlbuild.lint.exceptions import ProjectCompileError -from sqlbuild.lint.models import LintConfig, LintRunResult +from sqlbuild._native import LintConfig, LintRunResult, ProjectCompileError, run_lint from tests.unit.src.sqlbuild.lint._helpers._test_types import ( BatchedPreparationFailureTestCase, BatchedPreparationTestCase, diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py b/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py index 126ad59e2f..67505128c8 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import format_native_headers -from sqlbuild.lint.models import LintConfig +from sqlbuild._native import LintConfig, format_native_headers from tests.unit.src.sqlbuild.lint._helpers._test_types import HeaderAuditWrapTestCase _BODY: str = "\nSELECT 1 AS status\n" diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_native.py b/tests/unit/src/sqlbuild/lint/_helpers/test_native.py index 2f5a603626..5fe9d28cf7 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_native.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_native.py @@ -6,9 +6,13 @@ import pytest -from sqlbuild._native import format_native_headers, lint_native_headers, scan_headers -from sqlbuild.compiler.discovery.exceptions import SqlScenarioParseError -from sqlbuild.lint.models import LintConfig +from sqlbuild._native import ( + LintConfig, + SqlScenarioParseError, + format_native_headers, + lint_native_headers, + scan_headers, +) from tests.unit.src.sqlbuild.compiler.discovery.helpers import ( parse_model_sql, parse_sql_scenario_file, diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py b/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py index f38b7fff12..f3585609cc 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild.compiler.compile.models import ExpansionSpan -from sqlbuild.lint.models import LintBody, LintConfig, LintViolation +from sqlbuild._native import ExpansionSpan, LintBody, LintConfig, LintViolation from tests.unit.src.sqlbuild.lint._helpers._test_types import ( ManyViolationLocationTestCase, NativeSqlFindingTestCase, diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py index 25c07dd5c7..111ccd092b 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py @@ -6,9 +6,14 @@ import pytest -from sqlbuild._native import collect_lint_project_files, discover_project_inputs, run_lint +from sqlbuild._native import ( + LintConfig, + LintRunResult, + collect_lint_project_files, + discover_project_inputs, + run_lint, +) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.lint.models import LintConfig, LintRunResult from tests.unit.src.sqlbuild.lint._helpers._test_types import ( CollectProjectFilesTestCase, SelectedProjectFilesTestCase, diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py b/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py index d214b5b1fb..c2d5283bc3 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py @@ -5,7 +5,7 @@ import pytest import sqlbuild._native as _native -from sqlbuild.lint.models import InterpolationSite +from sqlbuild._native import InterpolationSite from tests.unit.src.sqlbuild.lint._helpers._test_types import ( DialectNeutralizeInterpolationTestCase, DollarQuotePreparationTestCase, diff --git a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py index b7d5eea802..74b0cb46d6 100644 --- a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py @@ -9,6 +9,12 @@ import sqlbuild._native as native_module from sqlbuild._native import ( + LintBody, + LintConfig, + LintRunResult, + MappedOffset, + ProjectCompileError, + SqlExpansionContext, build_lint_expansion_context, expand_sql_with_spans, lint_file_role, @@ -16,10 +22,7 @@ prepare_lint_body, run_lint, ) -from sqlbuild.compiler.compile.models import MappedOffset, SqlExpansionContext from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.lint.exceptions import ProjectCompileError -from sqlbuild.lint.models import LintBody, LintConfig, LintRunResult from tests.unit.src.sqlbuild.lint._test_types import ( ExpandedLintTestCase, ExpandedTypedConstantTestCase, diff --git a/tests/unit/src/sqlbuild/lint/test_run_lint.py b/tests/unit/src/sqlbuild/lint/test_run_lint.py index c741a3e7d7..a9e9193d7a 100644 --- a/tests/unit/src/sqlbuild/lint/test_run_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_run_lint.py @@ -6,9 +6,14 @@ import pytest -from sqlbuild._native import apply_lint_suppressions, run_lint -from sqlbuild.lint.constants import VIOLATION_SEVERITY_FAULT -from sqlbuild.lint.models import LintConfig, LintRunResult, LintViolation +from sqlbuild._native import ( + LintConfig, + LintRunResult, + LintSeverity, + LintViolation, + apply_lint_suppressions, + run_lint, +) from tests.unit.src.sqlbuild.lint._test_types import ( FixtureCeremonyFormatTestCase, FormatDescriptionResolutionTestCase, @@ -528,7 +533,7 @@ def test_given_header_fault_suppression_when_linting_then_mandatory_fault_remain column=1, code="description-present", message="MODEL header must include a description", - severity=VIOLATION_SEVERITY_FAULT, + severity=LintSeverity.FAULT, engine=LINT_ENGINE_SQLBUILD, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py index 61b2d9ab41..b125b43287 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field -from sqlbuild.rule_engine.models import RuleExemption, RuleIgnore +from sqlbuild._native import RuleExemption, RuleIgnore @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index 26da58fa3c..2359be6808 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -11,36 +11,29 @@ import sqlbuild._native as native from sqlbuild._native import ( - NativeRulesRequest, - build_rule_catalogue, - install_rules_skills, - project_with, -) -from sqlbuild._native import evaluate_custom_rules as evaluate_custom_rules_cached -from sqlbuild._native import evaluate_project_rules as evaluate -from sqlbuild.compiler.compile.models import ( CompiledDirectLogicSqlTestPayload, CompiledModelSqlTestPayload, - CompiledObjectKey, - CompiledSqlTest, CompiledSqlTestResource, CompileSqlTestCte, -) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile -from sqlbuild.compiler.scopes.models import ScopeIndex -from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES -from sqlbuild.rule_engine.models import ( CustomRulesOutcome, - Finding, + DiscoveredSqlTestBlock, NativeRulesEvaluation, - Rule, + NativeRulesRequest, RuleExemption, RuleIgnore, RulesCacheConfig, - RulesConfig, - RulesResult, + build_rule_catalogue, + install_rules_skills, + project_with, ) +from sqlbuild._native import evaluate_custom_rules as evaluate_custom_rules_cached +from sqlbuild._native import evaluate_project_rules as evaluate +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlTest +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode +from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile +from sqlbuild.compiler.scopes.models import ScopeIndex +from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES +from sqlbuild.rule_engine.models import Finding, Rule, RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import CustomRuleTestCase from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py index 292607f670..d82902f8a9 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py @@ -5,20 +5,16 @@ import pytest from sqlbuild._native import ( + RulesCacheConfig, + SqlTestRulesConfig, + ThresholdOverride, build_rule_catalogue, format_rule, install_rules_skills, select_catalogue_rules, ) from sqlbuild._native import evaluate_project_rules as evaluate -from sqlbuild.rule_engine.models import ( - Rule, - RulesCacheConfig, - RulesConfig, - RulesResult, - SqlTestRulesConfig, - ThresholdOverride, -) +from sqlbuild.rule_engine.models import Rule, RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( PolicyGuidanceTestCase, PolicySelectionTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py index a584d47f1c..38255e913c 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py @@ -4,9 +4,9 @@ import pytest -from sqlbuild._native import load_rules_config +from sqlbuild._native import SqlTestRulesConfig, load_rules_config from sqlbuild.rule_engine.exceptions import RulesConfigError, RulesError -from sqlbuild.rule_engine.models import LayoutConfig, RulesConfig, SqlTestRulesConfig +from sqlbuild.rule_engine.models import LayoutConfig, RulesConfig from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( AllowExceptionsConfigTestCase, PolicyLayoutConfigTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py index ca53f4685a..a7ecb72a6d 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py @@ -7,9 +7,9 @@ import pytest import sqlbuild._native as native_module +from sqlbuild._native import CustomRulesOutcome from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.engine import fact_replay -from sqlbuild.rule_engine.models import CustomRulesOutcome from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( CustomRuleProjectFileTestCase, CustomRuleReadTrackingTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py index 9f1ea679d4..4f65dae842 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py @@ -6,6 +6,8 @@ import pytest from sqlbuild._native import ( + RuleExemption, + RuleIgnore, compiled_model_record, project_with, ) @@ -13,12 +15,7 @@ from sqlbuild.compiler.compile.models import CompactLineageFacts from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import ( - RuleExemption, - RuleIgnore, - RulesConfig, - RulesResult, -) +from sqlbuild.rule_engine.models import RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( CustomRuleCacheDependencyTestCase, CustomRuleSuppressionTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_incremental_native_rules.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_incremental_native_rules.py index 85aa97e09d..40c3583455 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_incremental_native_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_incremental_native_rules.py @@ -3,7 +3,8 @@ import pytest -from sqlbuild.rule_engine.models import RuleExemption, RuleIgnore, RulesResult +from sqlbuild._native import RuleExemption, RuleIgnore +from sqlbuild.rule_engine.models import RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( IncrementalCacheCountsTestCase, IncrementalRulesModeTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py index 827be6fac4..bbdb0ebc62 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py @@ -17,6 +17,8 @@ import sqlbuild._native as native_module from sqlbuild._native import ( CustomHostObserver, + CustomRulesOutcome, + RulesCacheConfig, build_rule_catalogue, project_with, ) @@ -25,7 +27,7 @@ from sqlbuild.rule_engine._helpers.host.module_state import module_state_token, rule_namespaces from sqlbuild.rule_engine.classes.module_state_monitor import ModuleStateMonitor from sqlbuild.rule_engine.exceptions import OpaqueModuleStateError -from sqlbuild.rule_engine.models import CustomRulesOutcome, Rule, RulesCacheConfig, RulesConfig +from sqlbuild.rule_engine.models import Rule, RulesConfig from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project _MODEL_RULE: str = "XSQBRT101" diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py index 1260855e7c..528da9bb3f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py @@ -5,12 +5,11 @@ import pytest -from sqlbuild._native import CustomHostObserver, rules_bulk_cache_path +from sqlbuild._native import CustomHostObserver, CustomRulesOutcome, rules_bulk_cache_path from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.host import custom_host from sqlbuild.rule_engine.constants import CUSTOM_RULES_CACHE_FILE from sqlbuild.rule_engine.exceptions import HostCancelledError, RulesError -from sqlbuild.rule_engine.models import CustomRulesOutcome from tests.unit.src.sqlbuild.rule_engine._helpers.host._test_types import ( AwaitInputsTestCase, AwaitPublishedInputsTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py index 064cf94d71..6a7b516f4f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py @@ -4,8 +4,7 @@ import sqlbuild._native as native from sqlbuild import _native -from sqlbuild._native import NativeRulesRequest -from sqlbuild.rule_engine.models import NativeRulesEvaluation +from sqlbuild._native import NativeRulesEvaluation, NativeRulesRequest def record_reuse(*, monkeypatch: pytest.MonkeyPatch) -> list[bool]: diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py index e18b5f3d7f..03ec9b4be7 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py @@ -5,9 +5,10 @@ import pytest import sqlbuild._native as native +from sqlbuild._native import RulesCacheConfig from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import RulesCacheConfig, RulesConfig, RulesResult +from sqlbuild.rule_engine.models import RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import evaluate_contract_rule from tests.unit.src.sqlbuild.rule_engine._helpers.run._test_types import ( NativeRowsEncodeErrorTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py index 62b22ad19a..52e4b5abfa 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py @@ -1,8 +1,8 @@ from pathlib import Path +from sqlbuild._native import DiscoveredAuditBlock, DiscoveredAuditFile from sqlbuild.compiler.compile.models import CompiledAudit, CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredAuditBlock, DiscoveredAuditFile def build_audit(*, name: str, attached_target_name: str | None) -> CompiledAuditView: diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py index 1fe8015782..ed377026b0 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py @@ -1,9 +1,10 @@ from pathlib import Path from sqlbuild._native import ( + CompiledSource, project_with, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSource +from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import ( CompiledProjectView, CompiledResourceType, diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py index 07a697acca..2d882a3330 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py @@ -2,12 +2,12 @@ from pathlib import Path +from sqlbuild._native import CompileModelConfig, EnumMember from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, - CompileModelConfig, CompileSqlReference, ) from sqlbuild.compiler.compile.types import ( @@ -15,7 +15,7 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.discovery.models import EnumDeclaration, EnumMember +from sqlbuild.compiler.discovery.models import EnumDeclaration _ENUM_DECLARATION: EnumDeclaration = EnumDeclaration( name="status", diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py index b17e2cb6fa..350169e5c8 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py @@ -6,13 +6,15 @@ import pytest from sqlbuild._native import ( + RuleExemption, + RuleIgnore, compiled_model_record, project_with, ) from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import RuleExemption, RuleIgnore, RulesConfig, RulesResult +from sqlbuild.rule_engine.models import RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine.main.evaluate._test_types import PolicyBehaviorTestCase from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_evaluate.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_evaluate.py index 2cec008958..15e0370529 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_evaluate.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_evaluate.py @@ -5,14 +5,10 @@ import pytest +from sqlbuild._native import SelectStarAllow from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.rule_engine.models import ( - Finding, - RulesConfig, - RulesResult, - SelectStarAllow, -) +from sqlbuild.rule_engine.models import Finding, RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine.main.evaluate._test_types import ( PolicyEvaluationTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_authoring_imports.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_authoring_imports.py index e2e0395869..53e4ef3c4c 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_authoring_imports.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_authoring_imports.py @@ -19,7 +19,7 @@ imported_module="sqlbuild.rules", checked_modules=( "sqlbuild.rule_engine._helpers.engine.rule_harness", - "sqlbuild.compiler.discovery._helpers.filesystem.core", + "sqlbuild.compiler.discovery._helpers.filesystem.project_modules", "sqlbuild.adapters.duckdb.classes.duckdb_adapter", ), expected_loaded_modules=(), @@ -29,7 +29,7 @@ imported_module="sqlbuild.rule_engine._helpers.host.custom_host", checked_modules=( "sqlbuild.rule_engine._helpers.engine.rule_harness", - "sqlbuild.compiler.discovery._helpers.filesystem.core", + "sqlbuild.compiler.discovery._helpers.filesystem.project_modules", ), expected_loaded_modules=(), ), diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py index ebdf10c00f..897f0cf4cc 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py @@ -6,11 +6,12 @@ import pytest +from sqlbuild._native import RulesCacheConfig from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import RulesCacheConfig, RulesConfig, RulesResult +from sqlbuild.rule_engine.models import RulesConfig, RulesResult from sqlbuild.rule_engine.types import RuleOptionValue from sqlbuild.rules.testing import RuleCase, RuleResult, evaluate_rule from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project diff --git a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py index 0a2994db2d..090b25dd4f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py @@ -5,6 +5,7 @@ import pytest from sqlbuild._native import ( + RulesRunResult, build_project_graph, discover_project_inputs, load_rules_config, @@ -14,7 +15,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.rule_engine.models import RulesRunResult from tests.unit.src.sqlbuild.rule_engine.main.run_rules._test_types import ( TypeProofRuleSkipTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py b/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py index 5a7e853bba..33f78a0237 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py @@ -5,9 +5,10 @@ import pytest +from sqlbuild._native import RuleExemption from sqlbuild._native import install_rules_skills as install_skills from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import RuleExemption, RulesConfig +from sqlbuild.rule_engine.models import RulesConfig from tests.unit.src.sqlbuild.rule_engine.main.skills._test_types import ( SkillDivergenceTestCase, SkillFreshnessTestCase, diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/_test_types.py b/tests/unit/src/sqlbuild/runtime/event_exporting/_test_types.py index 3210501fc8..0c30d6abbb 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/_test_types.py @@ -11,3 +11,12 @@ class SinkApiTestCase: class EventExportPolicyTestCase: description: str expected_event_count: int + + +@dataclass(frozen=True) +class SeverityNarrowingTestCase: + description: str + first: str + second: str | None + expected_severity: str + expected_first_meets_second: bool diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py index b88ef37733..577e9b4db1 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py @@ -22,17 +22,17 @@ class SharedSinkProviderTestCase: @dataclass(frozen=True) class PriorityPairTestCase: description: str - queued_priority: int - incoming_priority: int - expected_inserted: bool - expected_displaced_sequence: int | None - expected_retained_sequence: int + queued_type: str + incoming_type: str + expected_delivered: tuple[str, ...] + expected_dropped: int @dataclass(frozen=True) class PriorityQueueTestCase: description: str - expected_sequences: tuple[int, ...] + event_types: tuple[str, ...] + expected_delivered: tuple[str, ...] @dataclass(frozen=True) @@ -40,3 +40,10 @@ class HealthIntervalTestCase: description: str interval: object expected_error: str + + +@dataclass(frozen=True) +class InterpreterExitTestCase: + description: str + script: str + expected_returncode: int diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py index 07d5cf434f..6ce4b02087 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py @@ -3,6 +3,7 @@ from threading import Event from typing import ClassVar +from sqlbuild._native import BoundEventExporter, EventExporterDispatcher from sqlbuild.compiler.discovery.models import ( DiscoveredEventExporter, DiscoveredProvider, @@ -10,11 +11,6 @@ ) from sqlbuild.observability import LifecycleEvent from sqlbuild.providers import Provider -from sqlbuild.runtime.event_exporting.models import ( - LifecycleExportPolicy, - QueuedLifecycleEvent, -) -from sqlbuild.spec.contracts.types import EventExportSeverity class BlockingProvider(Provider): @@ -86,20 +82,50 @@ def lifecycle_event(index: int = 1, event_type: str = "invocation_started") -> L ) -def queued_event( - *, - sequence: int, - priority: int, - eligible: tuple[int, ...] = (0,), - event_type: str = "invocation_started", -) -> QueuedLifecycleEvent: - return QueuedLifecycleEvent( - sequence, - lifecycle_event(sequence, event_type=event_type), - LifecycleExportPolicy("invocation", EventExportSeverity.DEBUG, priority), - eligible, +def correlated_event(index: int, event_type: str) -> LifecycleEvent: + return LifecycleEvent( + event_id=f"event-{index}", + event_type=event_type, + schema_version=2, + producer="test", + producer_version="1", + occurred_at=datetime.now(UTC), + invocation_id="invocation", + invocation_sequence=index, + run_id="run", + resource_id="resource", + resource_attempt_id="attempt", + operation_id="operation", + statement_id="statement", ) -def queued_sequence(event: QueuedLifecycleEvent | None) -> int | None: - return getattr(event, "sequence", None) +class BlockedDispatcher: + """A dispatcher whose only sink holds its first event until released.""" + + def __init__(self, *, queue_capacity: int) -> None: + self.delivered: list[str] = [] + self.entered: Event = Event() + self.release: Event = Event() + + def record(*, event: LifecycleEvent) -> None: + self.delivered.append(event.event_id) + self.entered.set() + _ = self.release.wait() + + self.dispatcher: EventExporterDispatcher = EventExporterDispatcher( + exporters=(BoundEventExporter("recorder", record, {}),), + queue_capacity=queue_capacity, + invocation_timeout_seconds=5.0, + ) + self.dispatcher.enqueue(correlated_event(0, "statement_submitted")) + assert self.entered.wait(timeout=1.0) + + def drain(self, event_types: tuple[str, ...]) -> tuple[tuple[str, ...], int]: + """Queue events behind the held one, release it, and return delivery order and drops.""" + + for index, event_type in enumerate(event_types, start=1): + self.dispatcher.enqueue(correlated_event(index, event_type)) + self.release.set() + dropped: int = self.dispatcher.shutdown().dropped + return tuple(self.delivered[1:]), dropped diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py index c21974caaf..d509b8f65a 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py @@ -7,6 +7,11 @@ import pytest +from sqlbuild._native import ( + EventExporterCommandScope, + EventExporterDispatcher, + EventExportSummary, +) from sqlbuild.compiler.discovery.models import ( DiscoveredCommandOutputSink, DiscoveredEventExporter, @@ -14,9 +19,6 @@ DiscoveredProviderUsage, ) from sqlbuild.providers import Provider -from sqlbuild.runtime.event_exporting.classes.command_scope import EventExporterCommandScope -from sqlbuild.runtime.event_exporting.classes.dispatcher import EventExporterDispatcher -from sqlbuild.runtime.event_exporting.models import EventExportSummary from sqlbuild.sinks import CommandOutputRecord, CommandOutputStream from tests.unit.src.sqlbuild.runtime.event_exporting.classes._test_types import ( EventExporterTeardownTestCase, diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py index 39fba30500..d46ed5d96d 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py @@ -8,14 +8,14 @@ import pytest -from sqlbuild.observability import LifecycleEvent -from sqlbuild.runtime.event_exporting.classes.dispatcher import EventExporterDispatcher -from sqlbuild.runtime.event_exporting.exceptions import EventExporterInputError -from sqlbuild.runtime.event_exporting.models import ( +from sqlbuild._native import ( BoundEventExporter, EventExporterCounts, + EventExporterDispatcher, EventExportSummary, ) +from sqlbuild.observability import LifecycleEvent +from sqlbuild.runtime.event_exporting.exceptions import EventExporterInputError from sqlbuild.runtime.output_capture.constants import COMMAND_OUTPUT_LOSS_RECORD_TYPE from sqlbuild.runtime.output_capture.models import BoundCommandOutputSink from sqlbuild.sinks import CommandOutputRecord, CommandOutputStream diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_interpreter_exit.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_interpreter_exit.py new file mode 100644 index 0000000000..f73489d8f4 --- /dev/null +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_interpreter_exit.py @@ -0,0 +1,104 @@ +import subprocess +import sys +from pathlib import Path +from textwrap import dedent + +import pytest + +from tests.unit.src.sqlbuild.runtime.event_exporting.classes._test_types import ( + InterpreterExitTestCase, +) + +_FINALIZING_WORK: str = dedent( + """ + import sys + import time + import types + + + class SlowFinalizer: + def __del__(self): + print("finalizing", sys.is_finalizing(), file=sys.stderr, flush=True) + time.sleep(0.6) + + + holder = types.ModuleType("holder") + holder.finalizer = SlowFinalizer() + sys.modules["holder"] = holder + """ +) + + +@pytest.mark.parametrize( + "test_case", + ( + InterpreterExitTestCase( + "summary callback still running at exit", + dedent( + """ + from sqlbuild._native import EventExporterDispatcher + + def summarize(summary): + time.sleep(0.2) + + EventExporterDispatcher(exporters=(), summary_callback=summarize).shutdown() + """ + ), + 0, + ), + InterpreterExitTestCase( + "timed-out sink still running at exit", + dedent( + """ + from types import SimpleNamespace + + from sqlbuild._native import BoundEventExporter, EventExporterDispatcher + + def slow(*, event): + time.sleep(0.3) + + dispatcher = EventExporterDispatcher( + exporters=(BoundEventExporter("slow", slow, {}),), + shutdown_timeout_seconds=0.0, + failure_callback=lambda failure: None, + ) + dispatcher.enqueue(SimpleNamespace(event_type="statement_submitted")) + time.sleep(0.05) + dispatcher.shutdown() + """ + ), + 0, + ), + InterpreterExitTestCase( + "dispatcher never shut down", + dedent( + """ + from sqlbuild._native import EventExporterDispatcher + + dispatcher = EventExporterDispatcher( + exporters=(), summary_callback=print, health_interval_seconds=0.01 + ) + """ + ), + 0, + ), + ), + ids=lambda case: case.description, +) +def test_given_live_exporter_threads_when_interpreter_finalizes_then_process_exits_cleanly( + test_case: InterpreterExitTestCase, + tmp_path: Path, +) -> None: + script: Path = tmp_path / "exporter_exit.py" + script.write_text(_FINALIZING_WORK + test_case.script, encoding="utf-8") + + result: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, str(script)], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + + assert result.returncode == test_case.expected_returncode, result.stderr + assert "finalizing True" in result.stderr diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_priority_queue.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_priority_queue.py index 47b45e8599..10cb6afcb4 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_priority_queue.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_priority_queue.py @@ -1,129 +1,79 @@ import pytest -from sqlbuild.runtime.event_exporting.classes.finite_priority_event_queue import ( - FinitePriorityEventQueue, -) -from sqlbuild.runtime.event_exporting.models import QueuedLifecycleEvent from tests.unit.src.sqlbuild.runtime.event_exporting.classes._test_types import ( PriorityPairTestCase, PriorityQueueTestCase, ) -from tests.unit.src.sqlbuild.runtime.event_exporting.classes.helpers import ( - queued_event, - queued_sequence, -) +from tests.unit.src.sqlbuild.runtime.event_exporting.classes.helpers import BlockedDispatcher @pytest.mark.parametrize( "test_case", ( - PriorityPairTestCase("0 then 0", 0, 0, False, None, 1), - PriorityPairTestCase("0 then 1", 0, 1, True, 1, 2), - PriorityPairTestCase("0 then 2", 0, 2, True, 1, 2), - PriorityPairTestCase("0 then 3", 0, 3, True, 1, 2), - PriorityPairTestCase("1 then 0", 1, 0, False, None, 1), - PriorityPairTestCase("1 then 1", 1, 1, False, None, 1), - PriorityPairTestCase("1 then 2", 1, 2, True, 1, 2), - PriorityPairTestCase("1 then 3", 1, 3, True, 1, 2), - PriorityPairTestCase("2 then 0", 2, 0, False, None, 1), - PriorityPairTestCase("2 then 1", 2, 1, False, None, 1), - PriorityPairTestCase("2 then 2", 2, 2, False, None, 1), - PriorityPairTestCase("2 then 3", 2, 3, True, 1, 2), - PriorityPairTestCase("3 then 0", 3, 0, False, None, 1), - PriorityPairTestCase("3 then 1", 3, 1, False, None, 1), - PriorityPairTestCase("3 then 2", 3, 2, False, None, 1), - PriorityPairTestCase("3 then 3", 3, 3, False, None, 1), + PriorityPairTestCase( + "0 then 0", "statement_submitted", "statement_submitted", ("event-1",), 1 + ), + PriorityPairTestCase( + "0 then 2", "statement_submitted", "statement_completed", ("event-2",), 1 + ), + PriorityPairTestCase( + "2 then 2", "statement_completed", "resource_attempt_completed", ("event-1",), 1 + ), + PriorityPairTestCase( + "2 then 3", "statement_completed", "operation_failed", ("event-2",), 1 + ), + PriorityPairTestCase( + "3 then 2", "operation_failed", "statement_completed", ("event-1",), 1 + ), + PriorityPairTestCase("3 then 3", "operation_failed", "run_failed", ("event-1",), 1), ), ids=lambda case: case.description, ) def test_given_full_queue_when_priority_pair_arrives_then_displacement_policy_is_exact( test_case: PriorityPairTestCase, ) -> None: - event_queue: FinitePriorityEventQueue = FinitePriorityEventQueue(capacity=1) - queued: QueuedLifecycleEvent = queued_event( - sequence=1, priority=test_case.queued_priority, eligible=(0, 1) - ) - incoming: QueuedLifecycleEvent = queued_event( - sequence=2, priority=test_case.incoming_priority, eligible=(1,) - ) - assert event_queue.put_nowait(queued) == (True, None) - - inserted: bool - displaced: QueuedLifecycleEvent | None - inserted, displaced = event_queue.put_nowait(incoming) - - assert inserted is test_case.expected_inserted - assert queued_sequence(displaced) == test_case.expected_displaced_sequence - assert event_queue.get(timeout=0.01).sequence == test_case.expected_retained_sequence - - -@pytest.mark.parametrize( - "test_case", - (PriorityQueueTestCase("equal priority FIFO", (0, 1, 2)),), - ids=lambda case: case.description, -) -def test_given_equal_priority_items_when_dequeuing_then_fifo_is_preserved( - test_case: PriorityQueueTestCase, -) -> None: - event_queue: FinitePriorityEventQueue = FinitePriorityEventQueue(capacity=3) - items: tuple[QueuedLifecycleEvent, ...] = tuple( - queued_event(sequence=sequence, priority=2) for sequence in range(3) - ) - for item in items: - assert event_queue.put_nowait(item) == (True, None) + blocked: BlockedDispatcher = BlockedDispatcher(queue_capacity=1) - assert tuple(event_queue.get(timeout=0.01).sequence for _ in items) == ( - test_case.expected_sequences - ) + delivered: tuple[str, ...] + dropped: int + delivered, dropped = blocked.drain((test_case.queued_type, test_case.incoming_type)) - -@pytest.mark.parametrize( - "test_case", - (PriorityQueueTestCase("invocation terminal remains last", (1, 2)),), - ids=lambda case: case.description, -) -def test_given_queued_work_before_invocation_terminal_when_dequeuing_then_terminal_is_last( - test_case: PriorityQueueTestCase, -) -> None: - event_queue: FinitePriorityEventQueue = FinitePriorityEventQueue(capacity=3) - statement: QueuedLifecycleEvent = queued_event( - sequence=1, - priority=0, - event_type="invocation_started", - ) - terminal: QueuedLifecycleEvent = queued_event( - sequence=2, - priority=3, - event_type="invocation_completed", - ) - for item in (statement, terminal): - assert event_queue.put_nowait(item) == (True, None) - - assert tuple(event_queue.get(timeout=0.01).sequence for _ in (statement, terminal)) == ( - test_case.expected_sequences - ) + assert delivered == test_case.expected_delivered + assert dropped == test_case.expected_dropped @pytest.mark.parametrize( "test_case", - (PriorityQueueTestCase("oldest lowest displacement", (1,)),), + ( + PriorityQueueTestCase( + "equal priority FIFO", + ("statement_completed", "statement_completed", "statement_completed"), + ("event-1", "event-2", "event-3"), + ), + PriorityQueueTestCase( + "invocation terminal remains last", + ("invocation_completed", "statement_submitted"), + ("event-2", "event-1"), + ), + PriorityQueueTestCase( + "highest priority first", + ("statement_submitted", "statement_completed", "operation_failed"), + ("event-3", "event-2", "event-1"), + ), + PriorityQueueTestCase( + "oldest lowest displacement", + ("statement_submitted", "statement_submitted", "statement_completed", "run_failed"), + ("event-4", "event-3", "event-2"), + ), + ), ids=lambda case: case.description, ) -def test_given_multiple_lower_items_when_higher_arrives_then_oldest_lowest_is_displaced( +def test_given_held_dispatcher_when_events_queue_then_delivery_order_is_priority_then_fifo( test_case: PriorityQueueTestCase, ) -> None: - event_queue: FinitePriorityEventQueue = FinitePriorityEventQueue(capacity=3) - oldest_low: QueuedLifecycleEvent = queued_event(sequence=1, priority=0, eligible=(0,)) - newer_low: QueuedLifecycleEvent = queued_event(sequence=2, priority=0, eligible=(1,)) - middle: QueuedLifecycleEvent = queued_event(sequence=3, priority=1, eligible=(0, 1)) - for item in (oldest_low, newer_low, middle): - event_queue.put_nowait(item) + blocked: BlockedDispatcher = BlockedDispatcher(queue_capacity=3) - inserted: bool - displaced: QueuedLifecycleEvent | None - inserted, displaced = event_queue.put_nowait( - queued_event(sequence=4, priority=3, eligible=(1,)) - ) + delivered: tuple[str, ...] + delivered, _ = blocked.drain(test_case.event_types) - assert inserted - assert (queued_sequence(displaced),) == test_case.expected_sequences + assert delivered == test_case.expected_delivered diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/test_policy.py b/tests/unit/src/sqlbuild/runtime/event_exporting/test_policy.py index dec74484d6..4f82779a20 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/test_policy.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/test_policy.py @@ -1,16 +1,17 @@ -from collections.abc import Mapping - import pytest -from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORT_KINDS -from sqlbuild.runtime.event_exporting.main._lifecycle_export_policy_catalog import ( +from sqlbuild._native import ( + event_export_severity_at_least, lifecycle_export_policy_catalog, + stricter_event_export_severity, ) -from sqlbuild.runtime.event_exporting.models import LifecycleExportPolicy +from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORT_KINDS from sqlbuild.runtime.event_exporting.types import LifecycleEventKind from sqlbuild.runtime.observability.constants import LIFECYCLE_EVENT_CATALOG +from sqlbuild.spec.contracts.types import EventExportSeverity from tests.unit.src.sqlbuild.runtime.event_exporting._test_types import ( EventExportPolicyTestCase, + SeverityNarrowingTestCase, ) @@ -22,19 +23,20 @@ def test_given_canonical_catalog_when_building_export_policy_then_every_type_is_mapped_once( test_case: EventExportPolicyTestCase, ) -> None: - policy: Mapping[str, LifecycleExportPolicy] = lifecycle_export_policy_catalog() + policy: dict[str, tuple[str, EventExportSeverity, int]] = lifecycle_export_policy_catalog() assert len(policy) == test_case.expected_event_count assert policy.keys() == LIFECYCLE_EVENT_CATALOG.keys() - assert {item.kind for item in policy.values()} == EVENT_EXPORT_KINDS - assert policy["invocation_failed"].severity == "error" - assert policy["resource_attempt_skipped"].severity == "info" - assert policy["retry_scheduled"].severity == "warning" - assert policy["statement_submitted"].severity == "debug" - assert policy["audit_completed"].kind == LifecycleEventKind.AUDIT - assert policy["audit_completed"].severity == "info" - assert policy["microbatch_started"].kind == LifecycleEventKind.MICROBATCH - assert policy["microbatch_started"].severity == "debug" - assert policy["microbatch_completed"].severity == "info" + assert {kind for kind, _, _ in policy.values()} == EVENT_EXPORT_KINDS + assert policy["invocation_failed"] == ("invocation", EventExportSeverity.ERROR, 3) + assert policy["resource_attempt_skipped"] == ("resource", EventExportSeverity.INFO, 2) + assert policy["retry_scheduled"] == ("retry", EventExportSeverity.WARNING, 1) + assert policy["statement_submitted"] == ("statement", EventExportSeverity.DEBUG, 0) + assert policy["audit_completed"][:2] == (LifecycleEventKind.AUDIT, EventExportSeverity.INFO) + assert policy["microbatch_started"][:2] == ( + LifecycleEventKind.MICROBATCH, + EventExportSeverity.DEBUG, + ) + assert policy["microbatch_completed"][1] == EventExportSeverity.INFO @pytest.mark.parametrize( @@ -45,7 +47,34 @@ def test_given_canonical_catalog_when_building_export_policy_then_every_type_is_ def test_given_lifecycle_catalog_when_deriving_export_dimensions_then_kind_enum_is_exhaustive( test_case: EventExportPolicyTestCase, ) -> None: - policy: Mapping[str, LifecycleExportPolicy] = lifecycle_export_policy_catalog() + policy: dict[str, tuple[str, EventExportSeverity, int]] = lifecycle_export_policy_catalog() - assert {item.kind for item in policy.values()} == {kind.value for kind in LifecycleEventKind} + assert {kind for kind, _, _ in policy.values()} == {kind.value for kind in LifecycleEventKind} assert len(LifecycleEventKind) == test_case.expected_event_count + + +@pytest.mark.parametrize( + "test_case", + ( + SeverityNarrowingTestCase("absent optional keeps required", "info", None, "info", True), + SeverityNarrowingTestCase("stricter optional wins", "info", "error", "error", False), + SeverityNarrowingTestCase( + "looser optional is ignored", "warning", "debug", "warning", True + ), + ), + ids=lambda case: case.description, +) +def test_given_severity_minimums_when_narrowing_then_stricter_wins_as_typed_severity( + test_case: SeverityNarrowingTestCase, +) -> None: + narrowed: EventExportSeverity = stricter_event_export_severity( + first=test_case.first, second=test_case.second + ) + + assert narrowed is EventExportSeverity(test_case.expected_severity) + assert ( + event_export_severity_at_least( + severity=test_case.first, minimum=test_case.second or "debug" + ) + is test_case.expected_first_meets_second + ) diff --git a/tests/unit/src/sqlbuild/runtime/output_capture/classes/_test_types.py b/tests/unit/src/sqlbuild/runtime/output_capture/classes/_test_types.py index 8a628cc4fd..ff9a042bbd 100644 --- a/tests/unit/src/sqlbuild/runtime/output_capture/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/output_capture/classes/_test_types.py @@ -19,3 +19,12 @@ class OutputCaptureTestCase: description: str expected_success: bool + + +@dataclass(frozen=True) +class InterpreterExitTestCase: + """A script left delivering output when the interpreter finalizes, and its exit code.""" + + description: str + script: str + expected_returncode: int diff --git a/tests/unit/src/sqlbuild/runtime/output_capture/classes/helpers.py b/tests/unit/src/sqlbuild/runtime/output_capture/classes/helpers.py index 22856d5dcb..438cffa3c9 100644 --- a/tests/unit/src/sqlbuild/runtime/output_capture/classes/helpers.py +++ b/tests/unit/src/sqlbuild/runtime/output_capture/classes/helpers.py @@ -6,8 +6,13 @@ from collections.abc import Callable, Mapping from datetime import UTC, datetime -from sqlbuild._native import OutputCaptureDispatcher -from sqlbuild.runtime.output_capture.models import CommandOutputRecord +from sqlbuild._native import ( + EventExporterCommandScope, + EventExporterDispatcher, + OutputCaptureDispatcher, +) +from sqlbuild.runtime.output_capture.models import BoundCommandOutputSink, CommandOutputRecord +from sqlbuild.runtime.output_capture.types import CommandOutputStream class RecordingOutputExporter: @@ -88,3 +93,22 @@ def make_dispatcher( clock=fixed_clock, failure_callback=failure_callback, ) + + +def sink_scope( + function: Callable[..., object], +) -> EventExporterCommandScope: + """A native exporter scope delivering every stream to one command-output sink.""" + + dispatcher: EventExporterDispatcher = EventExporterDispatcher(exporters=()) + dispatcher.bind_command_output_sinks( + ( + BoundCommandOutputSink( + name="recorder", + function=function, + provider_arguments={}, + streams=frozenset(CommandOutputStream), + ), + ) + ) + return EventExporterCommandScope(dispatcher=dispatcher) diff --git a/tests/unit/src/sqlbuild/runtime/output_capture/classes/test_interpreter_exit.py b/tests/unit/src/sqlbuild/runtime/output_capture/classes/test_interpreter_exit.py new file mode 100644 index 0000000000..625d07fcde --- /dev/null +++ b/tests/unit/src/sqlbuild/runtime/output_capture/classes/test_interpreter_exit.py @@ -0,0 +1,93 @@ +import subprocess +import sys +from pathlib import Path +from textwrap import dedent + +import pytest + +from tests.unit.src.sqlbuild.runtime.output_capture.classes._test_types import ( + InterpreterExitTestCase, +) + +_FINALIZING_WORK: str = dedent( + """ + import sys + import time + import types + + + class SlowFinalizer: + def __del__(self): + print("finalizing", sys.is_finalizing(), file=sys.stderr, flush=True) + time.sleep(1.5) + + + holder = types.ModuleType("holder") + holder.finalizer = SlowFinalizer() + sys.modules["holder"] = holder + """ +) +_SLOW_SINK: str = dedent( + """ + from sqlbuild._native import ( + EventExporterCommandScope, + EventExporterDispatcher, + OutputCaptureDispatcher, + ) + from sqlbuild.runtime.output_capture.models import BoundCommandOutputSink + from sqlbuild.runtime.output_capture.types import CommandOutputStream + + def slow(*, record): + time.sleep(1.0) + + exporters = EventExporterDispatcher(exporters=()) + exporters.bind_command_output_sinks( + ( + BoundCommandOutputSink( + name="slow", + function=slow, + provider_arguments={}, + streams=frozenset(CommandOutputStream), + ), + ) + ) + output = OutputCaptureDispatcher( + exporter=EventExporterCommandScope(dispatcher=exporters), + invocation_id="invocation", + shutdown_timeout_seconds=0.0, + ) + output.append(stream=CommandOutputStream.STDOUT, text="hello\\n") + time.sleep(0.35) + """ +) + + +@pytest.mark.parametrize( + "test_case", + ( + InterpreterExitTestCase( + "slow sink still running after close", + _SLOW_SINK + "output.close()\n", + 0, + ), + InterpreterExitTestCase("capture never closed", _SLOW_SINK, 0), + ), + ids=lambda case: case.description, +) +def test_given_slow_command_output_sink_when_interpreter_finalizes_then_process_exits_cleanly( + test_case: InterpreterExitTestCase, + tmp_path: Path, +) -> None: + script: Path = tmp_path / "output_exit.py" + script.write_text(_FINALIZING_WORK + test_case.script, encoding="utf-8") + + result: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, str(script)], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + + assert result.returncode == test_case.expected_returncode, result.stderr + assert "finalizing True" in result.stderr diff --git a/tests/unit/src/sqlbuild/runtime/output_capture/classes/test_output_capture_dispatcher.py b/tests/unit/src/sqlbuild/runtime/output_capture/classes/test_output_capture_dispatcher.py index c371e52fe7..dcbd83eb7c 100644 --- a/tests/unit/src/sqlbuild/runtime/output_capture/classes/test_output_capture_dispatcher.py +++ b/tests/unit/src/sqlbuild/runtime/output_capture/classes/test_output_capture_dispatcher.py @@ -2,16 +2,22 @@ from __future__ import annotations +import functools import io +import sqlite3 import sys import threading import time +from collections.abc import Callable import pytest from sqlbuild._native import OutputCaptureDispatcher from sqlbuild.runtime.output_capture.classes.text_tee import TextOutputTee -from sqlbuild.runtime.output_capture.models import CommandOutputCaptureSummary +from sqlbuild.runtime.output_capture.models import ( + CommandOutputCaptureSummary, + CommandOutputRecord, +) from sqlbuild.runtime.output_capture.types import CommandOutputStream from tests.unit.src.sqlbuild.runtime.output_capture.classes._test_types import ( ChunkingTestCase, @@ -23,6 +29,7 @@ FailingOutputExporter, RecordingOutputExporter, make_dispatcher, + sink_scope, ) @@ -331,5 +338,72 @@ def test_given_exporter_diagnostic_when_exporting_then_diagnostic_is_not_recaptu assert summary.failed == 1 +@pytest.mark.parametrize( + "test_case", + (OutputCaptureTestCase(description="sink_output_not_recaptured", expected_success=True),), + ids=lambda case: case.description, +) +def test_given_printing_command_output_sink_when_exporting_then_sink_output_is_not_recaptured( + test_case: OutputCaptureTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + assert test_case.expected_success is True + sink: io.StringIO = io.StringIO() + messages: list[str] = [] + + def record(*, record: CommandOutputRecord) -> None: + messages.append(record.message) + print("sink diagnostic", file=sys.stderr) + + dispatcher: OutputCaptureDispatcher = make_dispatcher(exporter=sink_scope(record)) + tee: TextOutputTee = TextOutputTee( + sink=sink, dispatcher=dispatcher, stream=CommandOutputStream.STDERR + ) + monkeypatch.setattr(sys, "stderr", tee) + + tee.write("command output\n") + summary: CommandOutputCaptureSummary = dispatcher.close() + + assert messages == ["command output\n"] + assert sink.getvalue() == "command output\nsink diagnostic\n" + assert summary.accepted == 1 + assert summary.delivered == 1 + + +@pytest.mark.parametrize( + "test_case", + (OutputCaptureTestCase(description="thread_bound_sink_resource", expected_success=True),), + ids=lambda case: case.description, +) +def test_given_thread_bound_sink_resource_when_exporting_many_records_then_one_thread_calls_sink( + test_case: OutputCaptureTestCase, +) -> None: + assert test_case.expected_success is True + connection: Callable[[], sqlite3.Connection] = functools.cache( + lambda: sqlite3.connect(":memory:") + ) + threads: list[int] = [] + row_counts: list[int] = [] + + def record(*, record: CommandOutputRecord) -> None: + threads.append(threading.get_ident()) + _ = connection().execute("CREATE TABLE IF NOT EXISTS lines (message TEXT)") + _ = connection().execute("INSERT INTO lines VALUES (?)", (record.message,)) + row_counts.append(connection().execute("SELECT COUNT(*) FROM lines").fetchone()[0]) + + dispatcher: OutputCaptureDispatcher = make_dispatcher( + exporter=sink_scope(record), queue_capacity=128, batch_size=4, max_record_bytes=8 + ) + for index in range(40): + dispatcher.append(stream=CommandOutputStream.STDOUT, text=f"line {index:02d}\n") + summary: CommandOutputCaptureSummary = dispatcher.close() + + assert summary.failed == 0 + assert summary.dropped == 0 + assert summary.delivered == summary.accepted == len(threads) == 40 + assert len(set(threads)) == 1 + assert threading.get_ident() not in threads + assert row_counts == list(range(1, 41)) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/spec/contracts/_helpers/test_config_values.py b/tests/unit/src/sqlbuild/spec/contracts/_helpers/test_config_values.py index 54e13cee3f..f830595feb 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/_helpers/test_config_values.py +++ b/tests/unit/src/sqlbuild/spec/contracts/_helpers/test_config_values.py @@ -3,13 +3,13 @@ import pytest from sqlbuild._native import ( + ConfigValueTypeError, get_config_bool, get_config_cursor_bound, get_config_int, get_config_str, get_config_string_tuple, ) -from sqlbuild.spec.contracts.exceptions import ConfigValueTypeError from tests.unit.src.sqlbuild.spec.contracts._helpers._test_types import ( ConfigValueErrorTestCase, ConfigValueSuccessTestCase, diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py b/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py index 60700b0045..a9cb661203 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py @@ -2,6 +2,7 @@ from dataclasses import dataclass, field +from sqlbuild._native import CollectionRendering from sqlbuild.spec.contracts.models import ( AuthoredTimeTravelRetention, ExecutionLimitsConfig, @@ -12,7 +13,6 @@ TargetWarehousesConfig, ) from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy -from sqlbuild.sql_values.types import CollectionRendering @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py index 77842a9979..641d236bfd 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py @@ -2,9 +2,8 @@ import pytest -from sqlbuild._native import resolve_effective_collection_rendering +from sqlbuild._native import CollectionRendering, resolve_effective_collection_rendering from sqlbuild.spec.contracts.models import ConstantsConfig, ProjectConfig -from sqlbuild.sql_values.types import CollectionRendering from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( EffectiveCollectionRenderingResolutionTestCase, ) diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py index 359873a767..7001a5e9e4 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import loader_destination_parts -from sqlbuild.spec.contracts.exceptions import SpecConfigError +from sqlbuild._native import SpecConfigError, loader_destination_parts from sqlbuild.spec.contracts.models import LoaderDestinationParts from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( InvalidLoaderDestinationTestCase, diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py index ce278ee9a6..9c55b71425 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py @@ -6,8 +6,11 @@ import pytest -from sqlbuild._native import discover_project_inputs, resolve_effective_scenario_config -from sqlbuild.compiler.discovery.exceptions import ProjectConfigError +from sqlbuild._native import ( + ProjectConfigError, + discover_project_inputs, + resolve_effective_scenario_config, +) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( InvalidScenarioNamespaceConfigTestCase, diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py index f3ce3f24f0..88594ac3da 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py @@ -2,8 +2,7 @@ import pytest -from sqlbuild._native import resolve_target_config -from sqlbuild.spec.contracts.exceptions import SpecConfigError +from sqlbuild._native import SpecConfigError, resolve_target_config from sqlbuild.spec.contracts.models import ( AuthoredTimeTravelRetention, ClonePolicy,